diff --git a/package-lock.json b/package-lock.json index 33a0c1eb..be507519 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@apify/utilities": "^2.18.0", "@crawlee/types": "^3.3.0", "agentkeepalive": "^4.2.1", + "ansi-colors": "^4.1.1", "async-retry": "^1.3.3", "axios": "^1.6.7", "content-type": "^1.0.5", diff --git a/package.json b/package.json index 8a4899ff..4d98d92c 100644 --- a/package.json +++ b/package.json @@ -43,10 +43,6 @@ "url": "https://github.com/apify/apify-client-js/issues" }, "homepage": "https://docs.apify.com/api/client/js/", - "files": [ - "dist", - "!dist/*.tsbuildinfo" - ], "scripts": { "build": "npm run clean && npm run build:node && npm run build:browser", "postbuild": "gen-esm-wrapper dist/index.js dist/index.mjs", @@ -67,6 +63,7 @@ "@apify/utilities": "^2.18.0", "@crawlee/types": "^3.3.0", "agentkeepalive": "^4.2.1", + "ansi-colors": "^4.1.1", "async-retry": "^1.3.3", "axios": "^1.6.7", "content-type": "^1.0.5", diff --git a/src/resource_clients/actor.ts b/src/resource_clients/actor.ts index 919ce30f..9650ae4e 100644 --- a/src/resource_clients/actor.ts +++ b/src/resource_clients/actor.ts @@ -3,6 +3,7 @@ import ow from 'ow'; import type { RUN_GENERAL_ACCESS } from '@apify/consts'; import { ACT_JOB_STATUSES, ACTOR_PERMISSION_LEVEL, META_ORIGINS } from '@apify/consts'; +import { Log } from '@apify/log'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; @@ -139,18 +140,28 @@ export class ActorClient extends ResourceClient { webhooks: ow.optional.array.ofType(ow.object), maxItems: ow.optional.number.not.negative, maxTotalChargeUsd: ow.optional.number.not.negative, + log: ow.optional.any(ow.null, ow.object.instanceOf(Log), ow.string.equals('default')), restartOnError: ow.optional.boolean, forcePermissionLevel: ow.optional.string.oneOf(Object.values(ACTOR_PERMISSION_LEVEL)), }), ); - const { waitSecs, ...startOptions } = options; + const { waitSecs, log, ...startOptions } = options; const { id } = await this.start(input, startOptions); // Calling root client because we need access to top level API. // Creating a new instance of RunClient here would only allow // setting it up as a nested route under actor API. - return this.apifyClient.run(id).waitForFinish({ waitSecs }); + const newRunClient = this.apifyClient.run(id); + + const streamedLog = await newRunClient.getStreamedLog({ toLog: options?.log }); + streamedLog?.start(); + return this.apifyClient + .run(id) + .waitForFinish({ waitSecs }) + .finally(async () => { + await streamedLog?.stop(); + }); } /** @@ -425,7 +436,16 @@ export interface ActorStartOptions { } export interface ActorCallOptions extends Omit { + /** + * Wait time in seconds for the actor run to finish. + */ waitSecs?: number; + /** + * `Log` instance that should be used to redirect actor run logs to. + * If `undefined` or `'default'` the pre-defined `Log` will be created and used. + * If `null`, no log redirection will occur. + */ + log?: Log | null | 'default'; } export interface ActorRunListItem { diff --git a/src/resource_clients/log.ts b/src/resource_clients/log.ts index b9589a19..c77b43bb 100644 --- a/src/resource_clients/log.ts +++ b/src/resource_clients/log.ts @@ -1,5 +1,11 @@ +// eslint-disable-next-line max-classes-per-file import type { Readable } from 'node:stream'; +import c from 'ansi-colors'; + +import type { Log } from '@apify/log'; +import { Logger, LogLevel } from '@apify/log'; + import type { ApifyApiError } from '../apify_api_error'; import type { ApiClientSubResourceOptions } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; @@ -20,11 +26,11 @@ export class LogClient extends ResourceClient { /** * https://docs.apify.com/api/v2#/reference/logs/log/get-log */ - async get(): Promise { + async get(options: LogOptions = {}): Promise { const requestOpts: ApifyRequestConfig = { url: this._url(), method: 'GET', - params: this._params(), + params: this._params(options), }; try { @@ -41,9 +47,10 @@ export class LogClient extends ResourceClient { * Gets the log in a Readable stream format. Only works in Node.js. * https://docs.apify.com/api/v2#/reference/logs/log/get-log */ - async stream(): Promise { + async stream(options: LogOptions = {}): Promise { const params = { stream: true, + raw: options.raw, }; const requestOpts: ApifyRequestConfig = { @@ -63,3 +70,170 @@ export class LogClient extends ResourceClient { return undefined; } } + +export interface LogOptions { + /** @default false */ + raw?: boolean; +} + +/** + * Logger for redirected actor logs. + */ +export class LoggerActorRedirect extends Logger { + constructor(options = {}) { + super({ skipTime: true, level: LogLevel.DEBUG, ...options }); + } + + override _log(level: LogLevel, message: string, data?: any, exception?: unknown, opts: Record = {}) { + if (level > this.options.level) { + return; + } + if (data || exception) { + throw new Error('Redirect logger does not use other arguments than level and message'); + } + let { prefix } = opts; + prefix = prefix ? `${prefix}` : ''; + + let maybeDate = ''; + if (!this.options.skipTime) { + maybeDate = `${new Date().toISOString().replace('Z', '').replace('T', ' ')} `; + } + + const line = `${c.gray(maybeDate)}${c.cyan(prefix)}${message || ''}`; + + // All redirected logs are logged at info level to avid any console specific formating for non-info levels, + // which have already been applied once to the original log. (For example error stack traces etc.) + this._outputWithConsole(LogLevel.INFO, line); + return line; + } +} + +/** + * Helper class for redirecting streamed Actor logs to another log. + */ +export class StreamedLog { + private destinationLog: Log; + private streamBuffer: Buffer[] = []; + private splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g; + private relevancyTimeLimit: Date | null; + + private logClient: LogClient; + private streamingTask: Promise | null = null; + private stopLogging = false; + + constructor(options: StreamedLogOptions) { + const { toLog, logClient, fromStart = true } = options; + this.destinationLog = toLog; + this.logClient = logClient; + this.relevancyTimeLimit = fromStart ? null : new Date(); + } + + /** + * Start log redirection. + */ + public start(): void { + if (this.streamingTask) { + throw new Error('Streaming task already active'); + } + this.stopLogging = false; + this.streamingTask = this.streamLog(); + } + + /** + * Stop log redirection. + */ + public async stop(): Promise { + if (!this.streamingTask) { + throw new Error('Streaming task is not active'); + } + this.stopLogging = true; + try { + await this.streamingTask; + } catch (err) { + if (!(err instanceof Error && err.name === 'AbortError')) { + throw err; + } + } finally { + this.streamingTask = null; + } + } + + /** + * Get log stream from response and redirect it to another log. + */ + private async streamLog(): Promise { + const logStream = await this.logClient.stream({ raw: true }); + if (!logStream) { + return; + } + const lastChunkRemainder = await this.logStreamChunks(logStream); + // Process whatever is left when exiting. Maybe it is incomplete, maybe it is last log without EOL. + const lastMessage = Buffer.from(lastChunkRemainder).toString().trim(); + if (lastMessage.length) { + this.destinationLog.info(lastMessage); + } + } + + private async logStreamChunks(logStream: Readable): Promise { + // Chunk may be incomplete. Keep remainder for next chunk. + let previousChunkRemainder: Uint8Array = new Uint8Array(); + + for await (const chunk of logStream) { + // Handle possible leftover incomplete line from previous chunk. + // Everything before last end of line is complete. + const chunkWithPreviousRemainder = new Uint8Array(previousChunkRemainder.length + chunk.length); + chunkWithPreviousRemainder.set(previousChunkRemainder, 0); + chunkWithPreviousRemainder.set(chunk, previousChunkRemainder.length); + + const lastCompleteMessageIndex = chunkWithPreviousRemainder.lastIndexOf(0x0a); + previousChunkRemainder = chunkWithPreviousRemainder.slice(lastCompleteMessageIndex); + + // Push complete part of the chunk to the buffer + this.streamBuffer.push(Buffer.from(chunkWithPreviousRemainder.slice(0, lastCompleteMessageIndex))); + this.logBufferContent(); + + // Keep processing the new data until stopped + if (this.stopLogging) { + break; + } + } + return previousChunkRemainder; + } + + /** + * Parse the buffer and log complete messages. + */ + private logBufferContent(): void { + const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker).slice(1); + // Parse the buffer parts into complete messages + const messageMarkers = allParts.filter((_, i) => i % 2 === 0); + const messageContents = allParts.filter((_, i) => i % 2 !== 0); + this.streamBuffer = []; + + messageMarkers.forEach((marker, index) => { + const decodedMarker = marker; + const decodedContent = messageContents[index]; + if (this.relevancyTimeLimit) { + // Log only relevant messages. Ignore too old log messages. + const logTime = new Date(decodedMarker); + if (logTime < this.relevancyTimeLimit) { + return; + } + } + const message = decodedMarker + decodedContent; + + // Original log level information is not available. Log all on info level. Log level could be guessed for + // some logs, but for any multiline logs such guess would be probably correct only for the first line. + this.destinationLog.info(message.trim()); + }); + } +} + +export interface StreamedLogOptions { + /** Log client used to communicate with the Apify API. */ + logClient: LogClient; + /** Log to which the Actor run logs will be redirected. */ + toLog: Log; + /** Whether to redirect all logs from Actor run start (even logs from the past). */ + fromStart?: boolean; +} diff --git a/src/resource_clients/run.ts b/src/resource_clients/run.ts index 0e517a83..8883e176 100644 --- a/src/resource_clients/run.ts +++ b/src/resource_clients/run.ts @@ -2,15 +2,16 @@ import type { AxiosRequestConfig } from 'axios'; import ow from 'ow'; import type { RUN_GENERAL_ACCESS } from '@apify/consts'; +import { LEVELS, Log } from '@apify/log'; import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client'; import { ResourceClient } from '../base/resource_client'; import type { ApifyResponse } from '../http_client'; -import { cast, parseDateFields, pluckData } from '../utils'; +import { cast, isNode, parseDateFields, pluckData } from '../utils'; import type { ActorRun } from './actor'; import { DatasetClient } from './dataset'; import { KeyValueStoreClient } from './key_value_store'; -import { LogClient } from './log'; +import { LogClient, LoggerActorRedirect, StreamedLog } from './log'; import { RequestQueueClient } from './request_queue'; const RUN_CHARGE_IDEMPOTENCY_HEADER = 'idempotency-key'; @@ -266,6 +267,39 @@ export class RunClient extends ResourceClient { }), ); } + + /** + * Get StreamedLog for convenient streaming of the run log and their redirection. + */ + async getStreamedLog(options: GetStreamedLogOptions = {}): Promise { + const { fromStart = true } = options; + let { toLog } = options; + if (toLog === null || !isNode()) { + // Explicitly no logging or not in Node.js + return undefined; + } + if (toLog === undefined || toLog === 'default') { + // Create default StreamedLog + // Get actor name and run id + const runData = await this.get(); + const runId = runData?.id ?? ''; + + const actorId = runData?.actId ?? ''; + const actorData = (await this.apifyClient.actor(actorId).get()) || { name: '' }; + + const actorName = actorData?.name ?? ''; + const name = [actorName, `runId:${runId}`].filter(Boolean).join(' '); + + toLog = new Log({ level: LEVELS.DEBUG, prefix: `${name} -> `, logger: new LoggerActorRedirect() }); + } + + return new StreamedLog({ logClient: this.log(), toLog, fromStart }); + } +} + +export interface GetStreamedLogOptions { + toLog?: Log | null | 'default'; + fromStart?: boolean; } export interface RunGetOptions { diff --git a/test/_helper.js b/test/_helper.js index 61f68e9c..951ecb38 100644 --- a/test/_helper.js +++ b/test/_helper.js @@ -1,6 +1,6 @@ const { launchPuppeteer, puppeteerUtils } = require('@crawlee/puppeteer'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); class Browser { async start() { diff --git a/test/actors.test.js b/test/actors.test.js index 1ab99dc9..40724109 100644 --- a/test/actors.test.js +++ b/test/actors.test.js @@ -1,7 +1,12 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); -const { ActorListSortBy, ApifyClient } = require('apify-client'); +const { ActorListSortBy, ApifyClient, LoggerActorRedirect } = require('apify-client'); const { stringifyWebhooksToBase64 } = require('../src/utils'); -const mockServer = require('./mock_server/server'); +const { mockServer, createDefaultApp } = require('./mock_server/server'); +const c = require('ansi-colors'); +const { MOCKED_ACTOR_LOGS_PROCESSED, StatusGenerator } = require('./mock_server/test_utils'); +const { Log, LEVELS } = require('@apify/log'); +const express = require('express'); +const { setTimeout } = require('node:timers/promises'); describe('Actor methods', () => { let baseUrl; @@ -251,6 +256,7 @@ describe('Actor methods', () => { timeout, build, waitSecs, + log: null, }); expect(res).toEqual(data); @@ -301,7 +307,7 @@ describe('Actor methods', () => { const maxItems = 100; mockServer.setResponse({ body }); - const res = await client.actor(actorId).call(undefined, { waitSecs, maxItems }); + const res = await client.actor(actorId).call(undefined, { waitSecs, maxItems, log: null }); expect(res).toEqual(data); validateRequest({ waitForFinish: waitSecs }, { runId }); @@ -669,3 +675,88 @@ describe('Actor methods', () => { }); }); }); + +describe('Run actor with redirected logs', () => { + let baseUrl; + let client; + const statusGenerator = new StatusGenerator(); + + beforeAll(async () => { + // Use custom router for the tests + const router = express.Router(); + // Set up a status generator to simulate run status changes. It will be reset for each test. + router.get('/actor-runs/redirect-run-id', async (req, res) => { + // Delay the response to give the actor time to run and produce expected logs + await setTimeout(10); + + const [status, statusMessage] = statusGenerator.next().value; + res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status, statusMessage } }); + }); + const app = createDefaultApp(router); + const server = await mockServer.start(undefined, app); + baseUrl = `http://localhost:${server.address().port}`; + }); + + afterAll(async () => { + await Promise.all([mockServer.close()]); + }); + + beforeEach(async () => { + client = new ApifyClient({ + baseUrl, + maxRetries: 0, + ...DEFAULT_OPTIONS, + }); + }); + afterEach(async () => { + // Reset the generator to so that the next test starts fresh + statusGenerator.reset(); + client = null; + }); + + describe('actor.call - redirected logs', () => { + test('default log', async () => { + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + + const defaultPrefix = 'redirect-actor-name runId:redirect-run-id -> '; + await client.actor('redirect-actor-id').call(); + + const loggerPrefix = c.cyan(defaultPrefix); + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); + logSpy.mockRestore(); + }); + + test('explicit default log', async () => { + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + + const defaultPrefix = 'redirect-actor-name runId:redirect-run-id -> '; + await client.actor('redirect-actor-id').call(undefined, { log: 'default' }); + + const loggerPrefix = c.cyan(defaultPrefix); + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); + logSpy.mockRestore(); + }); + + test('custom log', async () => { + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + + const customPrefix = 'custom prefix...'; + await client.actor('redirect-actor-id').call(undefined, { + log: new Log({ level: LEVELS.DEBUG, prefix: customPrefix, logger: new LoggerActorRedirect() }), + }); + + const loggerPrefix = c.cyan(customPrefix); + expect(logSpy.mock.calls).toEqual(MOCKED_ACTOR_LOGS_PROCESSED.map((item) => [loggerPrefix + item])); + logSpy.mockRestore(); + }); + + test('no log', async () => { + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + + await client.actor('redirect-actor-id').call(undefined, { log: null }); + + expect(logSpy.mock.calls).toEqual([]); + logSpy.mockRestore(); + }); + }); +}); diff --git a/test/apify_api_error.test.js b/test/apify_api_error.test.js index 2d086bb7..7f60e813 100644 --- a/test/apify_api_error.test.js +++ b/test/apify_api_error.test.js @@ -1,5 +1,5 @@ const { Browser, DEFAULT_OPTIONS } = require('./_helper'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); const { ApifyClient } = require('apify-client'); describe('ApifyApiError', () => { diff --git a/test/builds.test.js b/test/builds.test.js index fa9627fb..44310231 100644 --- a/test/builds.test.js +++ b/test/builds.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Build methods', () => { let baseUrl; diff --git a/test/datasets.test.js b/test/datasets.test.js index e9c1d160..e698f47a 100644 --- a/test/datasets.test.js +++ b/test/datasets.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Dataset methods', () => { let baseUrl; diff --git a/test/http_client.test.js b/test/http_client.test.js index db7c5e0d..c9c70bff 100644 --- a/test/http_client.test.js +++ b/test/http_client.test.js @@ -1,5 +1,5 @@ const { Browser } = require('./_helper'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); const { ApifyClient } = require('apify-client'); describe('HttpClient', () => { diff --git a/test/key_value_stores.test.js b/test/key_value_stores.test.js index 7a4ba98d..201da1af 100644 --- a/test/key_value_stores.test.js +++ b/test/key_value_stores.test.js @@ -2,7 +2,7 @@ const { Readable } = require('node:stream'); const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Key-Value Store methods', () => { let baseUrl; diff --git a/test/logs.test.js b/test/logs.test.js index 386f986e..03a60b34 100644 --- a/test/logs.test.js +++ b/test/logs.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Log methods', () => { let baseUrl; diff --git a/test/mock_server/server.js b/test/mock_server/server.js index 0e0576a7..f9d1b60e 100644 --- a/test/mock_server/server.js +++ b/test/mock_server/server.js @@ -21,12 +21,16 @@ const userRouter = require('./routes/users'); const webhookDispatches = require('./routes/webhook_dispatches'); const webhooks = require('./routes/webhooks'); -const app = express(); -const v2Router = express.Router(); +// Consts +const { MOCKED_ACTOR_LOGS } = require('./test_utils'); + +const defaultApp = createDefaultApp(); + const mockServer = { requests: [], response: null, - async start(port = 0) { + async start(port = 0, app = defaultApp) { + app.set('mockServer', this); this.server = http.createServer(app); return new Promise((resolve, reject) => { this.server.on('error', reject); @@ -55,52 +59,77 @@ const mockServer = { }, }; -// Debugging middleware -app.use((req, res, next) => { - next(); -}); -app.use(express.text()); -app.use(express.json({ limit: '9mb' })); -app.use(express.urlencoded({ extended: false })); -app.use(bodyParser.raw()); -app.use(express.static(path.join(__dirname, 'public'))); -app.use(compression()); +function createDefaultApp(v2Router = express.Router()) { + async function streamLogChunks(req, res) { + // Asynchronously write each chunk to the response stream + for (const chunk of MOCKED_ACTOR_LOGS) { + res.write(chunk); + res.flush(); // Flush the buffer and send the chunk immediately + // Wait for a short period to simulate work being done on the server + await new Promise((resolve) => { + setTimeout(resolve, 1); + }); + } -app.use('/', (req, res, next) => { - mockServer.requests.push(req); - next(); -}); -app.set('mockServer', mockServer); -app.use('/v2', v2Router); -app.use('/external', external); + // End the response stream once all chunks have been sent + res.end(); + } + const app = express(); + // Debugging middleware + app.use((req, res, next) => { + next(); + }); + app.use(express.text()); + app.use(express.json({ limit: '9mb' })); + app.use(express.urlencoded({ extended: false })); + app.use(bodyParser.raw()); + app.use(express.static(path.join(__dirname, 'public'))); + app.use(compression()); + app.use('/', (req, res, next) => { + mockServer.requests.push(req); + next(); + }); + app.use('/v2', v2Router); + app.use('/external', external); -// Attaching V2 routers -v2Router.use('/acts', actorRouter); -v2Router.use('/actor-builds', buildRouter); -v2Router.use('/actor-runs', runRouter); -v2Router.use('/actor-tasks', taskRouter); -v2Router.use('/users', userRouter); -v2Router.use('/logs', logRouter); -v2Router.use('/datasets', datasetRouter); -v2Router.use('/key-value-stores', keyValueStores); -v2Router.use('/request-queues', requestQueues); -v2Router.use('/webhooks', webhooks); -v2Router.use('/schedules', schedules); -v2Router.use('/webhook-dispatches', webhookDispatches); -v2Router.use('/store', store); + // Attaching V2 routers + v2Router.use('/acts/redirect-actor-id', async (req, res) => { + res.json({ data: { name: 'redirect-actor-name', id: 'redirect-run-id' } }); + }); + v2Router.use('/acts', actorRouter); + v2Router.use('/actor-builds', buildRouter); + v2Router.use('/actor-runs/redirect-run-id/log', streamLogChunks); + v2Router.use('/actor-runs/redirect-run-id', async (req, res) => { + res.json({ data: { id: 'redirect-run-id', actId: 'redirect-actor-id', status: 'SUCCEEDED' } }); + }); + + v2Router.use('/actor-runs', runRouter); + v2Router.use('/actor-tasks', taskRouter); + v2Router.use('/users', userRouter); + v2Router.use('/logs/redirect-log-id', streamLogChunks); + v2Router.use('/logs', logRouter); + v2Router.use('/datasets', datasetRouter); + v2Router.use('/key-value-stores', keyValueStores); + v2Router.use('/request-queues', requestQueues); + v2Router.use('/webhooks', webhooks); + v2Router.use('/schedules', schedules); + v2Router.use('/webhook-dispatches', webhookDispatches); + v2Router.use('/store', store); -// Debugging middleware -app.use((err, req, res, _next) => { - res.status(500).json({ error: { message: err.message } }); -}); + // Debugging middleware + app.use((err, req, res, _next) => { + res.status(500).json({ error: { message: err.message } }); + }); -app.use((req, res) => { - res.status(404).json({ - error: { - type: 'page-not-found', - message: 'Nothing to do here.', - }, + app.use((req, res) => { + res.status(404).json({ + error: { + type: 'page-not-found', + message: 'Nothing to do here.', + }, + }); }); -}); + return app; +} -module.exports = mockServer; +module.exports = { mockServer, createDefaultApp }; diff --git a/test/mock_server/test_utils.js b/test/mock_server/test_utils.js new file mode 100644 index 00000000..6296059e --- /dev/null +++ b/test/mock_server/test_utils.js @@ -0,0 +1,75 @@ +const MOCKED_ACTOR_LOGS = [ + '2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.\n', + '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.\n', + '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.\n', // Several logs merged into one chunk + Buffer.from('2025-05-13T07:26:14.132Z [apify] DEBUG \xc3', 'binary'), // Chunked log split in the middle of the 2-byte character + Buffer.from('\xa1\x0a', 'binary'), + Buffer.from('2025-05-13T07:26:14.132Z [apify] DEBUG \xE2', 'binary'), // Chunked log split in the middle of the 3-byte character + Buffer.from('\x82\xAC\x0a', 'binary'), + Buffer.from('2025-05-13T07:26:14.132Z [apify] DEBUG \xF0\x9F', 'binary'), // Chunked log split in the middle of the4-byte character + Buffer.from('\x98\x80\x0a', 'binary'), + '2025-05-13T07:24:14.132Z [apify] INFO multiline \n log\n', + '2025-05-13T07:25:14.132Z [apify] WARNING some warning\n', + '2025-05-13T07:26:14.132Z [apify] DEBUG c\n', + '2025-05-13T0', // Chunked log that got split in the marker + '7:26:14.132Z [apify] DEBUG d \n', + '2025-05-13T07:27:14.132Z [apify] DEB', // Chunked log that got split outside of marker + 'UG e\n', + '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...\n', // Already redirected message +]; + +const MOCKED_ACTOR_LOGS_PROCESSED = [ + '2025-05-13T07:24:12.588Z ACTOR: Pulling Docker image of build.', + '2025-05-13T07:24:12.686Z ACTOR: Creating Docker container.', + '2025-05-13T07:24:12.745Z ACTOR: Starting Docker container.', + '2025-05-13T07:26:14.132Z [apify] DEBUG á', + '2025-05-13T07:26:14.132Z [apify] DEBUG €', + '2025-05-13T07:26:14.132Z [apify] DEBUG 😀', + '2025-05-13T07:24:14.132Z [apify] INFO multiline \n log', + '2025-05-13T07:25:14.132Z [apify] WARNING some warning', + '2025-05-13T07:26:14.132Z [apify] DEBUG c', + '2025-05-13T07:26:14.132Z [apify] DEBUG d', + '2025-05-13T07:27:14.132Z [apify] DEBUG e', + '2025-05-13T07:28:14.132Z [apify.redirect-logger runId:4U1oAnKau6jpzjUuA] -> 2025-05-13T07:27:14.132Z ACTOR:...', +]; + +const MOCKED_ACTOR_STATUSES = [ + ['RUNNING', 'Actor Started'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['RUNNING', 'Doing some stuff'], + ['SUCCEEDED', 'Actor Finished'], +]; + +/** + * Helper class to allow iterating over defined list of statuses for each test case. + */ +class StatusGenerator { + constructor() { + this.reset(); + } + + reset() { + function* getStatusGenerator() { + for (const status of MOCKED_ACTOR_STATUSES) { + yield status; + } + // After exhausting, keep yielding the last status + while (true) { + yield MOCKED_ACTOR_STATUSES[MOCKED_ACTOR_STATUSES.length - 1]; + } + } + this.generator = getStatusGenerator(); + } + + next() { + return this.generator.next(); + } +} + +module.exports = { MOCKED_ACTOR_LOGS, MOCKED_ACTOR_LOGS_PROCESSED, MOCKED_ACTOR_STATUSES, StatusGenerator }; diff --git a/test/request_queues.test.js b/test/request_queues.test.js index 06739f0d..2834a505 100644 --- a/test/request_queues.test.js +++ b/test/request_queues.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Request Queue methods', () => { let baseUrl; diff --git a/test/runs.test.js b/test/runs.test.js index f3865bd0..c8a05032 100644 --- a/test/runs.test.js +++ b/test/runs.test.js @@ -1,6 +1,9 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); +const c = require('ansi-colors'); +const { MOCKED_ACTOR_LOGS_PROCESSED } = require('./mock_server/test_utils'); +const { setTimeout: setTimeoutNode } = require('node:timers/promises'); describe('Run methods', () => { let baseUrl; @@ -377,3 +380,55 @@ describe('Run methods', () => { }); }); }); + +describe('Redirect run logs', () => { + let baseUrl; + + beforeAll(async () => { + // Ensure that the tests that use characters like á are correctly decoded in console. + process.stdout.setDefaultEncoding('utf8'); + const server = await mockServer.start(); + baseUrl = `http://localhost:${server.address().port}`; + }); + + afterAll(async () => { + await Promise.all([mockServer.close()]); + }); + + let client; + beforeEach(async () => { + client = new ApifyClient({ + baseUrl, + maxRetries: 0, + ...DEFAULT_OPTIONS, + }); + }); + afterEach(async () => { + client = null; + }); + + const testCases = [ + { fromStart: true, expected: MOCKED_ACTOR_LOGS_PROCESSED }, + { fromStart: false, expected: MOCKED_ACTOR_LOGS_PROCESSED.slice(1) }, + ]; + + describe('run.getStreamedLog', () => { + test.each(testCases)('getStreamedLog fromStart:$fromStart', async ({ fromStart, expected }) => { + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + // Set fake time in constructor to skip the first redirected log entry// fromStart=True should redirect all logs + jest.useFakeTimers(); + jest.setSystemTime(new Date('2025-05-13T07:24:12.686Z')); + const streamedLog = await client.run('redirect-run-id').getStreamedLog({ fromStart }); + jest.useRealTimers(); + + streamedLog.start(); + // Wait some time to accumulate logs + await setTimeoutNode(1000); + await streamedLog.stop(); + + const loggerPrefix = c.cyan('redirect-actor-name runId:redirect-run-id -> '); + expect(logSpy.mock.calls).toEqual(expected.map((item) => [loggerPrefix + item])); + logSpy.mockRestore(); + }); + }); +}); diff --git a/test/schedules.test.js b/test/schedules.test.js index 498c6869..8427a564 100644 --- a/test/schedules.test.js +++ b/test/schedules.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Schedule methods', () => { let baseUrl; diff --git a/test/store.test.ts b/test/store.test.ts index 5e5592a8..daa07da9 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -2,7 +2,7 @@ import type { StoreCollectionListOptions } from 'apify-client'; import { ApifyClient } from 'apify-client'; import { Browser, DEFAULT_OPTIONS, validateRequest } from './_helper'; -import mockServer from './mock_server/server'; +import { mockServer } from './mock_server/server'; describe('Store', () => { let baseUrl: string | undefined; diff --git a/test/tasks.test.js b/test/tasks.test.js index 75c808ce..f151cce3 100644 --- a/test/tasks.test.js +++ b/test/tasks.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); const { stringifyWebhooksToBase64 } = require('../src/utils'); describe('Task methods', () => { diff --git a/test/users.test.js b/test/users.test.js index 7fb18f0c..42c5a25e 100644 --- a/test/users.test.js +++ b/test/users.test.js @@ -2,7 +2,7 @@ const { ME_USER_NAME_PLACEHOLDER } = require('@apify/consts'); const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('User methods', () => { let baseUrl; diff --git a/test/webhook_dispatches.test.js b/test/webhook_dispatches.test.js index 7b735b20..fb5c2dc2 100644 --- a/test/webhook_dispatches.test.js +++ b/test/webhook_dispatches.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Webhook Dispatch methods', () => { let baseUrl; diff --git a/test/webhooks.test.js b/test/webhooks.test.js index bf2c33ca..0ec5486d 100644 --- a/test/webhooks.test.js +++ b/test/webhooks.test.js @@ -1,6 +1,6 @@ const { Browser, validateRequest, DEFAULT_OPTIONS } = require('./_helper'); const { ApifyClient } = require('apify-client'); -const mockServer = require('./mock_server/server'); +const { mockServer } = require('./mock_server/server'); describe('Webhook methods', () => { let baseUrl;