Refactoring: API for defining new frame sources

examples/slideInText.json5

@@ -1,12 +1,21 @@
 {
-  outPath: './slideInText.mp4',
+  outPath: "./slideInText.mp4",
   defaults: {
-    layer: { fontPath: './assets/Patua_One/PatuaOne-Regular.ttf' },
+    layer: { fontPath: "./assets/Patua_One/PatuaOne-Regular.ttf" },
   },
   clips: [
-    { duration: 3, layers: [
-      { type: 'image', path: 'assets/img2.jpg' },
-      { type: 'slide-in-text', text: 'Text that slides in', color: '#fff', position: { x: 0.04, y: 0.93, originY: 'bottom', originX: 'left' }, fontSize: 0.05 },
-    ] },
+    {
+      duration: 3,
+      layers: [
+        { type: "image", path: "assets/img2.jpg" },
+        {
+          type: "slide-in-text",
+          text: "Text that slides in",
+          textColor: "#fff",
+          position: { x: 0.04, y: 0.93, originY: "bottom", originX: "left" },
+          fontSize: 0.05,
+        },
+      ],
+    },
   ],
 }

src/api/defineFrameSource.ts

@@ -0,0 +1,66 @@
+import type { BaseLayer, DebugOptions, OptionalPromise } from "../types.js";
+import type { StaticCanvas } from "fabric/node";
+
+/**
+ * A public API for defining new frame sources.
+ */
+export function defineFrameSource<T extends BaseLayer>(type: T["type"], setup: FrameSourceSetupFunction<T>): FrameSourceFactory<T> {
+  return {
+    type,
+    async setup(options: CreateFrameSourceOptions<T>) {
+      return new FrameSource<T>(options, await setup(options));
+    }
+  }
+}
+
+export type CreateFrameSourceOptions<T> = DebugOptions & {
+  width: number,
+  height: number,
+  duration: number,
+  channels: number,
+  framerateStr: string,
+  params: T,
+};
+
+export interface FrameSourceFactory<T extends BaseLayer> {
+  type: T["type"];
+  setup: (fn: CreateFrameSourceOptions<T>) => Promise<FrameSource<T>>;
+}
+
+export interface FrameSourceImplementation {
+  readNextFrame(progress: number, canvas: StaticCanvas, offsetTime: number): OptionalPromise<Buffer | void>;
+  close?(): OptionalPromise<void | undefined>;
+}
+
+export type FrameSourceSetupFunction<T> = (fn: CreateFrameSourceOptions<T>) => Promise<FrameSourceImplementation>;
+
+export class FrameSource<T extends BaseLayer> {
+  options: CreateFrameSourceOptions<T>;
+  implementation: FrameSourceImplementation;
+
+  constructor(options: CreateFrameSourceOptions<T>, implementation: FrameSourceImplementation) {
+    this.options = options;
+    this.implementation = implementation;
+  }
+
+  async readNextFrame(time: number, canvas: StaticCanvas) {
+    const { start, layerDuration } = this.layer;
+
+    const offsetTime = time - (start ?? 0);
+    const offsetProgress = offsetTime / layerDuration!;
+    const shouldDrawLayer = offsetProgress >= 0 && offsetProgress <= 1;
+
+    // Skip drawing if the layer has not started or has already ended
+    if (!shouldDrawLayer) return;
+
+    return await this.implementation.readNextFrame(offsetProgress, canvas, offsetTime);
+  }
+
+  async close() {
+    await this.implementation.close?.();
+  }
+
+  get layer() {
+    return this.options.params;
+  }
+}

src/api/index.ts

@@ -0,0 +1,2 @@
+export type * from './defineFrameSource.js';
+export { defineFrameSource } from './defineFrameSource.js';

src/frameSource.ts

@@ -0,0 +1,70 @@
+import pMap from 'p-map';
+
+import { rgbaToFabricImage, createFabricCanvas, renderFabricCanvas } from './sources/fabric.js';
+import type { DebugOptions } from './types.js';
+import type { ProcessedClip } from './parseConfig.js';
+import { createLayerSource } from './sources/index.js';
+
+type FrameSourceOptions = DebugOptions & {
+  clip: ProcessedClip;
+  clipIndex: number;
+  width: number,
+  height: number,
+  channels: number,
+  framerateStr: string,
+}
+
+export async function createFrameSource({ clip, clipIndex, width, height, channels, verbose, logTimes, framerateStr }: FrameSourceOptions) {
+  const { layers, duration } = clip;
+
+  const visualLayers = layers.filter((layer) => layer.type !== 'audio');
+
+  const layerFrameSources = await pMap(visualLayers, async (layer, layerIndex) => {
+    if (verbose) console.log('createFrameSource', layer.type, 'clip', clipIndex, 'layer', layerIndex);
+    const options = { width, height, duration, channels, verbose, logTimes, framerateStr, params: layer };
+    return createLayerSource(options)
+  }, { concurrency: 1 });
+
+  async function readNextFrame({ time }: { time: number }) {
+    const canvas = createFabricCanvas({ width, height });
+
+    for (const frameSource of layerFrameSources) {
+      if (logTimes) console.time('frameSource.readNextFrame');
+      const rgba = await frameSource.readNextFrame(time, canvas);
+      if (logTimes) console.timeEnd('frameSource.readNextFrame');
+
+      // Frame sources can either render to the provided canvas and return nothing
+      // OR return an raw RGBA blob which will be drawn onto the canvas
+      if (rgba) {
+        // Optimization: Don't need to draw to canvas if there's only one layer
+        if (layerFrameSources.length === 1) return rgba;
+
+        if (logTimes) console.time('rgbaToFabricImage');
+        const img = await rgbaToFabricImage({ width, height, rgba });
+        if (logTimes) console.timeEnd('rgbaToFabricImage');
+        canvas.add(img);
+      } else {
+        // Assume this frame source has drawn its content to the canvas
+      }
+    }
+    // if (verbose) console.time('Merge frames');
+
+    if (logTimes) console.time('renderFabricCanvas');
+    const rgba = await renderFabricCanvas(canvas);
+    if (logTimes) console.timeEnd('renderFabricCanvas');
+    return rgba;
+  }
+
+  async function close() {
+    await pMap(layerFrameSources, frameSource => frameSource.close?.());
+  }
+
+  return {
+    readNextFrame,
+    close,
+  };
+}
+
+export default {
+  createFrameSource,
+};

src/index.ts

@@ -8,7 +8,7 @@ import { nanoid } from 'nanoid';
 import { configureFf, ffmpeg, parseFps } from './ffmpeg.js';
 import { multipleOf2, assertFileValid, checkTransition } from './util.js';
 import { createFabricCanvas, rgbaToFabricImage } from './sources/fabric.js';
-import { createFrameSource } from './sources/frameSource.js';
+import { createFrameSource } from './frameSource.js';
 import parseConfig, { ProcessedClip } from './parseConfig.js';
 import GlTransitions, { type RunTransitionOptions } from './glTransitions.js';
 import Audio from './audio.js';

src/sources/canvas.ts

@@ -0,0 +1,25 @@
+import { createCanvas } from 'canvas';
+import type { CanvasLayer } from '../types.js';
+import { canvasToRgba } from './fabric.js';
+import { defineFrameSource } from '../api/index.js';
+
+export default defineFrameSource<CanvasLayer>('canvas', async ({ width, height, params }) => {
+  const canvas = createCanvas(width, height);
+  const context = canvas.getContext('2d');
+
+  const { onClose, onRender } = await params.func(({ width, height, canvas }));
+
+  async function readNextFrame(progress: number) {
+    context.clearRect(0, 0, canvas.width, canvas.height);
+    await onRender(progress);
+    // require('fs').writeFileSync(`${new Date().getTime()}.png`, canvas.toBuffer('image/png'));
+    // I don't know any way to draw a node-canvas as a layer on a fabric.js canvas, other than converting to rgba first:
+    return canvasToRgba(context);
+  }
+
+  return {
+    readNextFrame,
+    // Node canvas needs no cleanup https://github.com/Automattic/node-canvas/issues/1216#issuecomment-412390668
+    close: onClose,
+  };
+});

src/sources/fabric.ts

@@ -1,10 +1,8 @@
 import * as fabric from 'fabric/node';
 import { type CanvasRenderingContext2D, createCanvas, ImageData } from 'canvas';
 import { boxBlurImage } from '../BoxBlur.js';
-import type { CreateFrameSourceOptions, FrameSource, CanvasLayer, CustomFabricFunctionCallbacks, Layer, OptionalPromise } from '../types.js';
-
-export type FabricFrameSourceOptions<T> = CreateFrameSourceOptions<T> & { fabric: typeof fabric };
-export type FabricFrameSourceCallback<T> = (options: FabricFrameSourceOptions<T>) => OptionalPromise<CustomFabricFunctionCallbacks>;
+import { defineFrameSource } from '../api/index.js';
+import type { FabricLayer } from '../types.js';
 
 // Fabric is used as a fundament for compositing layers in editly
 
@@ -20,9 +18,6 @@ export function fabricCanvasToRgba(fabricCanvas: fabric.StaticCanvas) {
   const internalCanvas = fabricCanvas.getNodeCanvas();
   const ctx = internalCanvas.getContext('2d');
 
-  // require('fs').writeFileSync(`${Math.floor(Math.random() * 1e12)}.png`, internalCanvas.toBuffer('image/png'));
-  // throw new Error('abort');
-
   return canvasToRgba(ctx);
 }
 
@@ -67,38 +62,6 @@ export async function rgbaToFabricImage({ width, height, rgba }: { width: number
   return new fabric.FabricImage(canvas);
 }
 
-export async function createFabricFrameSource<T extends Layer>(
-  func: FabricFrameSourceCallback<T>,
-  options: CreateFrameSourceOptions<T>
-): Promise<FrameSource> {
-  const { onRender = () => { }, onClose = () => { } } = await func({ fabric, ...options }) || {};
-
-  return {
-    readNextFrame: onRender,
-    close: onClose,
-  };
-}
-
-export async function createCustomCanvasFrameSource({ width, height, params }: Pick<CreateFrameSourceOptions<CanvasLayer>, "width" | "height" | "params">): Promise<FrameSource> {
-  const canvas = createCanvas(width, height);
-  const context = canvas.getContext('2d');
-
-  const { onClose, onRender } = await params.func(({ width, height, canvas }));
-
-  async function readNextFrame(progress: number) {
-    context.clearRect(0, 0, canvas.width, canvas.height);
-    await onRender(progress);
-    // require('fs').writeFileSync(`${new Date().getTime()}.png`, canvas.toBuffer('image/png'));
-    // I don't know any way to draw a node-canvas as a layer on a fabric.js canvas, other than converting to rgba first:
-    return canvasToRgba(context);
-  }
-
-  return {
-    readNextFrame,
-    // Node canvas needs no cleanup https://github.com/Automattic/node-canvas/issues/1216#issuecomment-412390668
-    close: onClose,
-  };
-}
 export type BlurImageOptions = {
   mutableImg: fabric.FabricImage,
   width: number,
@@ -116,4 +79,13 @@ export async function blurImage({ mutableImg, width, height }: BlurImageOptions)
   boxBlurImage(ctx, width, height, blurAmount, false, passes);
 
   return new fabric.FabricImage(canvas);
-}
+}// http://fabricjs.com/kitchensink
+
+export default defineFrameSource<FabricLayer>('fabric', async ({ width, height, params }) => {
+  const { onRender, onClose } = await params.func(({ width, height, fabric, params }));
+
+  return {
+    readNextFrame: onRender,
+    close: onClose
+  }
+})