diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index b9dee3aa524e..86641c0a85b3 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -17,9 +17,7 @@ import type { TestContext, WriteableTestContext, } from './types/tasks' -import { processError } from '@vitest/utils/error' // TODO: load dynamically -import { shuffle } from '@vitest/utils/helpers' -import { getSafeTimers } from '@vitest/utils/timers' + import { collectTests } from './collect' import { abortContextSignal, getFileContext } from './context' import { PendingError, TestRunAbortError } from './errors' @@ -299,9 +297,10 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { const suite = test.suite || test.file const repeats = test.repeats ?? 0 + const retryConfig = normalizeRetryConfig(test.retry ?? runner.config.retry) + for (let repeatCount = 0; repeatCount <= repeats; repeatCount++) { - const retry = test.retry ?? 0 - for (let retryCount = 0; retryCount <= retry; retryCount++) { + for (let retryCount = 0; retryCount <= retryConfig.count; retryCount++) { let beforeEachCleanups: unknown[] = [] try { await runner.onBeforeTryTask?.(test, { @@ -341,13 +340,17 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { if (!test.repeats) { test.result.state = 'pass' } - else if (test.repeats && retry === retryCount) { + else if (test.repeats && retryConfig.count === retryCount) { test.result.state = 'pass' } } } catch (e) { failTask(test.result, e, runner.config.diffOptions) + + if (e instanceof Error && !shouldRetryOnError(e, retryConfig.condition)) { + break + } } try { @@ -402,13 +405,16 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { break } - if (retryCount < retry) { - // reset state when retry test + if (retryCount < retryConfig.count) { test.result.state = 'run' test.result.retryCount = (test.result.retryCount ?? 0) + 1 + + if (retryConfig.delay > 0) { + const delayMs = calculateBackoffDelay(retryConfig.delay, retryCount) + await sleep(delayMs) + } } - // update retry info updateTask('test-retried', test, runner) } } diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index 917ec7fe610e..0f96eca220ef 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -315,7 +315,7 @@ function createSuiteCollector( type: 'test', file: undefined!, timeout, - retry: options.retry ?? runner.config.retry, + retry: options.retry ?? runner.config.retry ?? 0, repeats: options.repeats, mode: options.only ? 'only' diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index 6ed48ea4fbb8..1ae7c3f306bf 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -35,7 +35,7 @@ export interface VitestRunnerConfig { maxConcurrency: number testTimeout: number hookTimeout: number - retry: number + retry: number | import('./tasks').RetryConfig includeTaskLocation?: boolean diffOptions?: DiffOptions } diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index b88a4bb2599c..30c550c87b75 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -442,18 +442,19 @@ type ChainableTestAPI = ChainableFunction< type TestCollectorOptions = Omit +export interface RetryConfig { + count?: number + strategy?: 'immediate' | 'test-file' | 'deferred' + delay?: number + condition?: string | RegExp | ((error: Error) => boolean) +} + export interface TestOptions { /** * Test timeout. */ timeout?: number - /** - * Times to retry the test if fails. Useful for making flaky tests more stable. - * When retries is up, the last test error will be thrown. - * - * @default 0 - */ - retry?: number + retry?: number | RetryConfig /** * How many times the test will run again. * Only inner tests will repeat if set on `describe()`, nested `describe()` will inherit parent's repeat by default. diff --git a/packages/runner/src/utils/retry.ts b/packages/runner/src/utils/retry.ts new file mode 100644 index 000000000000..ce6456cbd334 --- /dev/null +++ b/packages/runner/src/utils/retry.ts @@ -0,0 +1,72 @@ +import type { RetryConfig } from '../types/tasks' + +export function normalizeRetryConfig( + retry: number | RetryConfig | undefined, +): { + count: number + strategy: 'immediate' | 'test-file' | 'deferred' + delay: number + condition?: string | RegExp | ((error: Error) => boolean) +} { + if (typeof retry === 'number') { + return { + count: retry, + strategy: 'immediate', + delay: 0, + } + } + + if (!retry) { + return { + count: 0, + strategy: 'immediate', + delay: 0, + } + } + + return { + count: retry.count ?? 0, + strategy: retry.strategy ?? 'immediate', + delay: retry.delay ?? 0, + condition: retry.condition, + } +} + +export function shouldRetryOnError( + error: Error, + condition?: string | RegExp | ((error: Error) => boolean), +): boolean { + if (!condition) { + return true + } + + if (typeof condition === 'string') { + return error.message.includes(condition) + } + + if (condition instanceof RegExp) { + return condition.test(error.message) + } + + if (typeof condition === 'function') { + try { + return condition(error) + } + catch { + return false + } + } + + return true +} + +export function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +export function calculateBackoffDelay( + baseDelay: number, + retryCount: number, +): number { + return baseDelay * 2 ** retryCount +} diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 9f52ccb53605..720fbfe4e237 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -602,8 +602,26 @@ export const cliOptionsConfig: VitestCLIOptions = { }, retry: { description: - 'Retry the test specific number of times if it fails (default: `0`)', - argument: '', + 'Retry configuration. Can be a number or advanced options (default: `0`)', + argument: '', + subcommands: { + count: { + description: 'Number of times to retry the test if it fails (default: 0)', + argument: '', + }, + strategy: { + description: 'Retry strategy: immediate, test-file, or deferred (default: immediate)', + argument: '', + }, + delay: { + description: 'Delay in milliseconds between retries (default: 0)', + argument: '', + }, + condition: { + description: 'Condition for retrying: string pattern to match error messages', + argument: '', + } as any, + }, }, diff: { description: diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index ebd4c3239259..4e0b63f03f81 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -742,11 +742,40 @@ export interface InlineConfig { bail?: number /** - * Retry the test specific number of times if it fails. + * Retry configuration for tests. + * Can be a number (for backward compatibility) or an object with advanced options. * * @default 0 */ - retry?: number + retry?: number | { + /** + * Number of times to retry the test if it fails. + * @default 0 + */ + count?: number + + /** + * Strategy for when retries are executed. + * - "immediate": retry failed test immediately (default) + * - "test-file": run retries until the end of the test file + * - "deferred": defer retries after all other tests have run across all test files + * @default "immediate" + */ + strategy?: 'immediate' | 'test-file' | 'deferred' + + /** + * Delay in milliseconds between retries. + * @default 0 + */ + delay?: number + + /** + * Condition for when to retry. Can be a RegExp pattern to match error messages + * or a function that receives the error and returns boolean. + * @default undefined (retry on any error) + */ + condition?: string | RegExp | ((error: Error) => boolean) + } /** * Show full diff when snapshot fails instead of a patch. diff --git a/packages/vitest/src/runtime/config.ts b/packages/vitest/src/runtime/config.ts index 98acd25e33f9..0c61f7332de9 100644 --- a/packages/vitest/src/runtime/config.ts +++ b/packages/vitest/src/runtime/config.ts @@ -92,7 +92,12 @@ export interface SerializedConfig { truncateThreshold?: number } | undefined diff: string | SerializedDiffOptions | undefined - retry: number + retry: number | { + count?: number + strategy?: 'immediate' | 'test-file' | 'deferred' + delay?: number + condition?: string | RegExp | ((error: Error) => boolean) + } includeTaskLocation: boolean | undefined inspect: boolean | string | undefined inspectBrk: boolean | string | undefined diff --git a/packages/vitest/src/utils/retry.ts b/packages/vitest/src/utils/retry.ts new file mode 100644 index 000000000000..f5f7f64b448f --- /dev/null +++ b/packages/vitest/src/utils/retry.ts @@ -0,0 +1,70 @@ +import type { SerializedConfig } from '../runtime/config' + +export function normalizeRetryConfig( + retry: SerializedConfig['retry'] +): { + count: number + strategy: 'immediate' | 'test-file' | 'deferred' + delay: number + condition?: string | RegExp | ((error: Error) => boolean) +} { + if (typeof retry === 'number') { + return { + count: retry, + strategy: 'immediate', + delay: 0, + } + } + + return { + count: retry.count ?? 0, + strategy: retry.strategy ?? 'immediate', + delay: retry.delay ?? 0, + condition: retry.condition, + } +} + +export function shouldRetryOnError( + error: Error, + condition?: string | RegExp | ((error: Error) => boolean) +): boolean { + if (!condition) { + return true + } + + if (typeof condition === 'string') { + return error.message.includes(condition) + } + + if (condition instanceof RegExp) { + return condition.test(error.message) + } + + if (typeof condition === 'function') { + try { + return condition(error) + } catch { + return false + } + } + + return true +} + +export function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +export function calculateBackoffDelay( + baseDelay: number, + retryCount: number, + algorithm: 'linear' | 'exponential' = 'linear' +): number { + switch (algorithm) { + case 'exponential': + return baseDelay * Math.pow(2, retryCount) + case 'linear': + default: + return baseDelay + } +} diff --git a/test/core/test/cli-test.test.ts b/test/core/test/cli-test.test.ts index ba38a3ae7265..5f21af5dd685 100644 --- a/test/core/test/cli-test.test.ts +++ b/test/core/test/cli-test.test.ts @@ -511,3 +511,25 @@ test('should include builtin reporters list', () => { const expected = Object.keys(ReportersMap) expect(new Set(listed)).toEqual(new Set(expected)) }) + +test('retry options work correctly', async () => { + expect(getCLIOptions('--retry 3')).toEqual({ + retry: 3, + }) + + expect(getCLIOptions('--retry.count 2 --retry.delay 1000 --retry.strategy immediate --retry.condition "NetworkError"')).toEqual({ + retry: { + count: 2, + delay: 1000, + strategy: 'immediate', + condition: 'NetworkError', + }, + }) + + expect(getCLIOptions('--retry.count 1 --retry.delay 500')).toEqual({ + retry: { + count: 1, + delay: 500, + }, + }) +})