diff --git a/.editorconfig b/.editorconfig index d3408845..0268ff77 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,4 +2,9 @@ indent_style = tab [*.{cs,js,ts,json}] -indent_size = 4 \ No newline at end of file +indent_size = 4 + +[*.{yml,yaml}] +indent_size = 2 +tab_width = 2 +indent_style = space \ No newline at end of file diff --git a/.github/workflows/publish-prerelease-docker.yaml b/.github/workflows/publish-prerelease-docker.yaml index 85758575..dcf54354 100644 --- a/.github/workflows/publish-prerelease-docker.yaml +++ b/.github/workflows/publish-prerelease-docker.yaml @@ -62,7 +62,7 @@ jobs: strategy: matrix: - repo: [package-manager, workforce, http-server, quantel-http-transformer-proxy] + repo: [package-manager, workforce, http-server, quantel-http-transformer-proxy, worker] steps: - uses: actions/checkout@v5 diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts index fa4cddca..c19d41b8 100644 --- a/apps/appcontainer-node/packages/generic/src/appContainer.ts +++ b/apps/appcontainer-node/packages/generic/src/appContainer.ts @@ -101,12 +101,12 @@ export class AppContainer { throw new Error(`Unknown app "${clientId}" just connected to the appContainer`) } client.once('close', () => { - this.logger.warn(`Connection to Worker "${clientId}" closed`) + this.logger.warn(`Connection from Worker "${clientId}" closed`) app.workerAgentApi = null this.workerStorage.releaseLockForTag(unprotectString(clientId)) }) - this.logger.info(`Connection to Worker "${client.clientId}" established`) + this.logger.info(`Connection from Worker "${client.clientId}" established`) app.workerAgentApi = api // Set upp the app for pinging and automatic spin-down: diff --git a/apps/http-server/app/Dockerfile b/apps/http-server/app/Dockerfile index 1a2d907d..f438b435 100644 --- a/apps/http-server/app/Dockerfile +++ b/apps/http-server/app/Dockerfile @@ -54,5 +54,4 @@ ENV HTTP_SERVER_PORT=8080 ENV HTTP_SERVER_BASE_PATH="/data" EXPOSE 8080 -ENTRYPOINT ["/usr/bin/dumb-init", "--"] -CMD ["yarn", "start"] +ENTRYPOINT ["/usr/bin/dumb-init", "--", "node", "dist/index.js"] diff --git a/apps/package-manager/app/Dockerfile b/apps/package-manager/app/Dockerfile index 42d82868..48eb7266 100644 --- a/apps/package-manager/app/Dockerfile +++ b/apps/package-manager/app/Dockerfile @@ -51,5 +51,4 @@ WORKDIR /src/apps/package-manager/app ENV package-manager_PORT=8070 EXPOSE 8070 -ENTRYPOINT ["/usr/bin/dumb-init", "--"] -CMD ["yarn", "start"] +ENTRYPOINT ["/usr/bin/dumb-init", "--", "node", "dist/index.js"] diff --git a/apps/quantel-http-transformer-proxy/app/Dockerfile b/apps/quantel-http-transformer-proxy/app/Dockerfile index 2dd7aa95..570906f8 100644 --- a/apps/quantel-http-transformer-proxy/app/Dockerfile +++ b/apps/quantel-http-transformer-proxy/app/Dockerfile @@ -52,4 +52,4 @@ ENV QUANTEL_HTTP_TRANSFORMER_PROXY_PORT=8080 # ENV QUANTEL_HTTP_TRANSFORMER_RATE_LIMIT_MAX= EXPOSE 8080 -ENTRYPOINT ["yarn", "start"] +ENTRYPOINT ["/usr/bin/dumb-init", "--", "node", "dist/index.js"] diff --git a/apps/worker/app/Dockerfile b/apps/worker/app/Dockerfile new file mode 100644 index 00000000..12759bc8 --- /dev/null +++ b/apps/worker/app/Dockerfile @@ -0,0 +1,58 @@ +FROM node:18-alpine as builder + +# Note: Build this from the root directory: +# cd package-manager +# docker build -f apps/worker/app/Dockerfile -t pm-worker . +# docker build -t pm-worker ../../../.. + +# Environment + +WORKDIR /src + +# Common + +COPY package.json tsconfig.json tsconfig.build.json yarn.lock lerna.json commonPackage.json .yarnrc.yml ./ +COPY scripts ./scripts +COPY .yarn ./.yarn + +# Shared dependencies +COPY shared ./shared + + +# App dependencies +RUN mkdir -p apps/worker +COPY apps/worker/packages apps/worker/packages + +# App +COPY apps/worker/app apps/worker/app + +# Install +RUN yarn install + +# Build +RUN yarn build + +# Purge dev-dependencies: +RUN yarn workspaces focus -A --production + +RUN rm -r scripts + + +# Create deploy-image: +FROM node:18-alpine + +COPY --from=builder /src /src + +RUN apk add --no-cache dumb-init ffmpeg + +# Run as non-root user +USER 1000 +WORKDIR /src/apps/worker/app +# ENV APP_CONTAINER_PORT=8090 +# # ENV HTTP_SERVER_API_KEY_READ= +# # ENV HTTP_SERVER_API_KEY_WRITE= +# ENV HTTP_SERVER_BASE_PATH="/data" +# EXPOSE 8090 + +ENTRYPOINT ["/usr/bin/dumb-init", "--", "./docker-entrypoint.sh"] + diff --git a/apps/worker/app/docker-entrypoint.sh b/apps/worker/app/docker-entrypoint.sh new file mode 100755 index 00000000..b0910dbd --- /dev/null +++ b/apps/worker/app/docker-entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -e + +HOSTNAME=$(hostname) + +# Inject a unique worker ID based on the hostname, and disable the appContainer connection +node dist/index.js --workerId=$HOSTNAME --appContainerURL="" $* \ No newline at end of file diff --git a/apps/workforce/app/Dockerfile b/apps/workforce/app/Dockerfile index a8f6cd8a..8b414869 100644 --- a/apps/workforce/app/Dockerfile +++ b/apps/workforce/app/Dockerfile @@ -51,5 +51,4 @@ WORKDIR /src/apps/workforce/app ENV WORKFORCE_PORT=8070 EXPOSE 8070 -ENTRYPOINT ["/usr/bin/dumb-init", "--"] -CMD ["yarn", "start"] +ENTRYPOINT ["/usr/bin/dumb-init", "--", "node", "dist/index.js"] diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts index 4753b954..eccf8405 100644 --- a/shared/packages/api/src/config.ts +++ b/shared/packages/api/src/config.ts @@ -29,6 +29,11 @@ const workforceArguments = defineArguments({ default: parseInt(process.env.WORKFORCE_PORT || '', 10) || 8070, describe: 'The port number to start the Workforce websocket server on', }, + allowNoAppContainers: { + type: 'boolean', + default: process.env.WORKFORCE_ALLOW_NO_APP_CONTAINERS === '1' || false, + describe: 'If true, the workforce will not check if it has no appContainers connected', + }, }) /** CLI-argument-definitions for the HTTP-Server process */ const httpServerArguments = defineArguments({ @@ -227,7 +232,7 @@ const appContainerArguments = defineArguments({ // These are passed-through to the spun-up workers: resourceId: { type: 'string', - default: process.env.WORKER_NETWORK_ID || 'default', + default: process.env.WORKER_RESOURCE_ID || 'default', describe: 'Identifier of the local resource/computer this worker runs on', }, networkIds: { @@ -334,6 +339,7 @@ export interface WorkforceConfig { process: ProcessConfig workforce: { port: number | null + allowNoAppContainers: boolean } } @@ -349,6 +355,7 @@ export async function getWorkforceConfig(): Promise { process: getProcessConfig(argv), workforce: { port: argv.port, + allowNoAppContainers: argv.allowNoAppContainers, }, } } diff --git a/shared/packages/expectationManager/src/internalManager/lib/expectationManagerServer.ts b/shared/packages/expectationManager/src/internalManager/lib/expectationManagerServer.ts index 94efdecc..251a4abd 100644 --- a/shared/packages/expectationManager/src/internalManager/lib/expectationManagerServer.ts +++ b/shared/packages/expectationManager/src/internalManager/lib/expectationManagerServer.ts @@ -50,7 +50,7 @@ export class ExpectationManagerServer { }) this.manager.workerAgents.upsert(clientId, { api, connected: true }) client.once('close', () => { - this.logger.warn(`Connection to Worker "${clientId}" closed`) + this.logger.warn(`Connection from Worker "${clientId}" closed`) const workerAgent = this.manager.workerAgents.get(clientId) if (workerAgent) { @@ -58,7 +58,7 @@ export class ExpectationManagerServer { this.manager.workerAgents.remove(clientId) } }) - this.logger.info(`Connection to Worker "${clientId}" established`) + this.logger.info(`Connection from Worker "${clientId}" established`) this.manager.tracker.triggerEvaluationNow() break } diff --git a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts index d8437a6e..d8f44ab9 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/accessor.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/accessor.ts @@ -46,7 +46,6 @@ export function getAccessorStaticHandle(accessor: AccessorOnPackage.Any) { } else if (accessor.type === Accessor.AccessType.HTTP_PROXY) { return HTTPProxyAccessorHandle } else if (accessor.type === Accessor.AccessType.FILE_SHARE) { - if (process.platform !== 'win32') throw new Error(`FileShareAccessor: not supported on ${process.platform}`) return FileShareAccessorHandle } else if (accessor.type === Accessor.AccessType.QUANTEL) { return QuantelAccessorHandle diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts index ee0df98b..f03b2b4f 100644 --- a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts +++ b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts @@ -31,6 +31,7 @@ import { MonitorId, betterPathResolve, betterPathIsAbsolute, + isRunningInTest, } from '@sofie-package-manager/api' import { BaseWorker } from '../worker' import { GenericWorker } from '../workers/genericWorker/genericWorker' @@ -111,12 +112,25 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle return this.getFullPath(this.filePath) } static doYouSupportAccess(worker: BaseWorker, accessor: AccessorOnPackage.Any): boolean { + if (!isFileShareSupportedOnCurrentPlatform()) return false + return defaultDoYouSupportAccess(worker, accessor) } get packageName(): string { return this.fullPath } checkHandleBasic(): AccessorHandlerCheckHandleBasicResult { + if (!isFileShareSupportedOnCurrentPlatform()) { + return { + success: false, + knownReason: true, + reason: { + user: `File share is not supported on this worker`, + tech: `File share is not supported on ${process.platform}`, + }, + } + } + if (this.accessor.type !== Accessor.AccessType.FILE_SHARE) { return { success: false, @@ -168,14 +182,46 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle return { success: true } } checkCompatibilityWithAccessor(): AccessorHandlerCheckHandleCompatibilityResult { + if (!isFileShareSupportedOnCurrentPlatform()) { + return { + success: false, + knownReason: true, + reason: { + user: `File share is not supported on this worker`, + tech: `File share is not supported on ${process.platform}`, + }, + } + } return { success: true } // no special compatibility checks } checkHandleRead(): AccessorHandlerCheckHandleReadResult { + if (!isFileShareSupportedOnCurrentPlatform()) { + return { + success: false, + knownReason: true, + reason: { + user: `File share is not supported on this worker`, + tech: `File share is not supported on ${process.platform}`, + }, + } + } + const defaultResult = defaultCheckHandleRead(this.accessor) if (defaultResult) return defaultResult return { success: true } } checkHandleWrite(): AccessorHandlerCheckHandleWriteResult { + if (!isFileShareSupportedOnCurrentPlatform()) { + return { + success: false, + knownReason: true, + reason: { + user: `File share is not supported on this worker`, + tech: `File share is not supported on ${process.platform}`, + }, + } + } + const defaultResult = defaultCheckHandleWrite(this.accessor) if (defaultResult) return defaultResult return { success: true } @@ -199,6 +245,18 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle return { success: true } } async tryPackageRead(): Promise { + if (!isFileShareSupportedOnCurrentPlatform()) { + return { + success: false, + knownReason: true, + reason: { + user: `File share is not supported on this worker`, + tech: `File share is not supported on ${process.platform}`, + }, + packageExists: false, + } + } + try { // Check if we can open the file for reading: const fd = await fsOpen(this.fullPath, 'r') @@ -232,6 +290,17 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle return { success: true } } private async _checkPackageReadAccess(): Promise { + if (!isFileShareSupportedOnCurrentPlatform()) { + return { + success: false, + knownReason: true, + reason: { + user: `File share is not supported on this worker`, + tech: `File share is not supported on ${process.platform}`, + }, + } + } + await this.prepareFileAccess() try { @@ -364,6 +433,17 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle await this.unlinkIfExists(this.metadataPath) } async runCronJob(packageContainerExp: PackageContainerExpectation): Promise { + if (!isFileShareSupportedOnCurrentPlatform()) { + return { + success: false, + knownReason: true, + reason: { + user: `File share is not supported on this worker`, + tech: `File share is not supported on ${process.platform}`, + }, + } + } + // Always check read/write access first: const checkRead = await this.checkPackageContainerReadAccess() if (!checkRead.success) return checkRead @@ -396,6 +476,17 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle async setupPackageContainerMonitors( packageContainerExp: PackageContainerExpectation ): Promise { + if (!isFileShareSupportedOnCurrentPlatform()) { + return { + success: false, + knownReason: true, + reason: { + user: `File share is not supported on this worker`, + tech: `File share is not supported on ${process.platform}`, + }, + } + } + const resultingMonitors: Record = {} const monitorIds = Object.keys( packageContainerExp.monitors @@ -419,6 +510,9 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle operationName: string, source: string | GenericAccessorHandle ): Promise { + if (!isFileShareSupportedOnCurrentPlatform()) + throw new Error(`FileShareAccessor: not supported on ${process.platform}`) + await this.fileHandler.clearPackageRemoval(this.filePath) return this.logWorkOperation(operationName, source, this.packageName) } @@ -443,6 +537,9 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle * This method should be called prior to any file access being made. */ async prepareFileAccess(forceRemount = false): Promise { + if (!isFileShareSupportedOnCurrentPlatform()) + throw new Error(`FileShareAccessor: not supported on ${process.platform}`) + if (!this.originalFolderPath) throw new Error(`FileShareAccessor: accessor.folderPath not set!`) const folderPath = this.originalFolderPath @@ -680,3 +777,8 @@ export class FileShareAccessorHandle extends GenericFileAccessorHandle interface MappedDriveLetters { [driveLetter: string]: string } + +function isFileShareSupportedOnCurrentPlatform(): boolean { + // This is only supported on windows currently + return isRunningInTest() || process.platform === 'win32' +} diff --git a/shared/packages/workforce/src/workforce.ts b/shared/packages/workforce/src/workforce.ts index b17cd325..0408f1e8 100644 --- a/shared/packages/workforce/src/workforce.ts +++ b/shared/packages/workforce/src/workforce.ts @@ -36,6 +36,8 @@ import { WorkerHandler } from './workerHandler' * and mediates connections between the two. */ export class Workforce { + private readonly config: WorkforceConfig + public workerAgents: Map< WorkerAgentId, { @@ -76,6 +78,8 @@ export class Workforce { constructor(logger: LoggerInstance, config: WorkforceConfig) { this.logger = logger.category('Workforce') + this.config = config + if (config.workforce.port !== null) { this.websocketServer = new WebsocketServer( config.workforce.port, @@ -95,11 +99,11 @@ export class Workforce { }) this.workerAgents.set(clientId, { api }) client.once('close', () => { - this.logger.warn(`Workforce: Connection to Worker "${clientId}" closed`) + this.logger.warn(`Workforce: Connection from Worker "${clientId}" closed`) this.workerAgents.delete(clientId) this.triggerEvaluateStatus() }) - this.logger.info(`Workforce: Connection to Worker "${clientId}" established`) + this.logger.info(`Workforce: Connection from Worker "${clientId}" established`) this.triggerEvaluateStatus() break } @@ -227,16 +231,18 @@ export class Workforce { message: '', } - statuses['any-appContainers'] = - this.appContainers.size === 0 - ? { - statusCode: StatusCode.BAD, - message: 'No appContainers connected to workforce', - } - : { - statusCode: StatusCode.GOOD, - message: '', - } + if (!this.config.workforce.allowNoAppContainers) { + statuses['any-appContainers'] = + this.appContainers.size === 0 + ? { + statusCode: StatusCode.BAD, + message: 'No appContainers connected to workforce', + } + : { + statusCode: StatusCode.GOOD, + message: '', + } + } const statusHash = hashObj(statuses) diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index 77e39aa5..70dbca89 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -52,6 +52,7 @@ export const defaultTestConfig: SingleAppConfig = { }, workforce: { port: null, + allowNoAppContainers: false, }, httpServer: { port: 0, @@ -274,7 +275,8 @@ export async function prepareTestEnvironment(debugLogging: boolean): Promise | null ) => { - if (debugLogging) console.log('reportPackageContainerPackageStatus', containerId, packageId, packageStatus) + if (debugLogging) + console.log('reportPackageContainerPackageStatus', containerId, packageId, packageStatus) let container = containerStatuses[containerId] if (!container) {