Bring back --repeat CLI option

src/Event/Value/TestSuite/TestSuiteBuilder.php

@@ -18,6 +18,7 @@
 use PHPUnit\Event\Code\TestCollection;
 use PHPUnit\Event\RuntimeException;
 use PHPUnit\Framework\DataProviderTestSuite;
+use PHPUnit\Framework\RepeatTestSuite;
 use PHPUnit\Framework\TestCase;
 use PHPUnit\Framework\TestSuite as FrameworkTestSuite;
 use PHPUnit\Runner\Phpt\TestCase as PhptTestCase;
@@ -107,7 +108,9 @@ private static function process(FrameworkTestSuite $testSuite, array &$tests): v
                 continue;
             }
 
-            if ($test instanceof TestCase || $test instanceof PhptTestCase) {
+            if ($test instanceof TestCase ||
+                $test instanceof PhptTestCase ||
+                $test instanceof RepeatTestSuite) {
                 $tests[] = $test->valueObjectForEvents();
             }
         }

src/Framework/RepeatTestSuite.php

@@ -0,0 +1,138 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\Framework;
+
+use function count;
+use LogicException;
+use PHPUnit\Event;
+use PHPUnit\Event\Facade as EventFacade;
+use PHPUnit\Runner\Phpt\TestCase as PhptTestCase;
+use PHPUnit\TestRunner\TestResult\PassedTests;
+
+/**
+ * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit
+ *
+ * @internal This class is not covered by the backward compatibility promise for PHPUnit
+ */
+final readonly class RepeatTestSuite implements Reorderable, Test
+{
+    /**
+     * @var non-empty-list<PhptTestCase>|non-empty-list<TestCase>
+     */
+    private array $tests;
+
+    /**
+     * @param positive-int $times
+     */
+    public function __construct(PhptTestCase|TestCase $test, int $times)
+    {
+        $tests = [];
+
+        for ($i = 0; $i < $times; $i++) {
+            $tests[] = $test;
+        }
+
+        $this->tests = $tests;
+    }
+
+    public function count(): int
+    {
+        return count($this->tests);
+    }
+
+    public function run(): void
+    {
+        if ($this->isPhptTestCase()) {
+            $this->runPhptTestCase();
+        } else {
+            $this->runTestCase();
+        }
+    }
+
+    public function sortId(): string
+    {
+        return $this->tests[0]->sortId();
+    }
+
+    public function provides(): array
+    {
+        return $this->tests[0]->provides();
+    }
+
+    public function requires(): array
+    {
+        return $this->tests[0]->requires();
+    }
+
+    public function name(): string
+    {
+        if ($this->isPhptTestCase()) {
+            throw new LogicException('Cannot call RepeatTestSuite::nameWithDataSet() on a PhptTestCase.');
+        }
+
+        return $this->tests[0]::class . '::' . $this->tests[0]->nameWithDataSet();
+    }
+
+    public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod
+    {
+        return $this->tests[0]->valueObjectForEvents();
+    }
+
+    /**
+     * @phpstan-assert-if-true non-empty-list<PhptTestCase> $this->tests
+     */
+    public function isPhptTestCase(): bool
+    {
+        return $this->tests[0] instanceof PhptTestCase;
+    }
+
+    private function runTestCase(): void
+    {
+        $defectOccurred = false;
+
+        foreach ($this->tests as $test) {
+            if ($defectOccurred) {
+                $test->markSkippedForErrorInPreviousRepetition();
+
+                continue;
+            }
+
+            $test->run();
+
+            if ($test->status()->isFailure() || $test->status()->isError()) {
+                $defectOccurred = true;
+
+                PassedTests::instance()->testMethodDidNotPass($test::class . '::' . $test->name());
+            }
+        }
+    }
+
+    private function runPhptTestCase(): void
+    {
+        $defectOccurred = false;
+
+        foreach ($this->tests as $test) {
+            if ($defectOccurred) {
+                EventFacade::emitter()->testSkipped(
+                    $this->valueObjectForEvents(),
+                    'Test repetition failure',
+                );
+
+                continue;
+            }
+
+            $test->run();
+
+            if (!$test->passed()) {
+                $defectOccurred = true;
+            }
+        }
+    }
+}

src/Framework/TestBuilder.php

@@ -36,10 +36,11 @@
      * @param ReflectionClass<TestCase> $theClass
      * @param non-empty-string          $methodName
      * @param list<non-empty-string>    $groups
+     * @param positive-int              $repeatTimes
      *
      * @throws InvalidDataProviderException
      */
-    public function build(ReflectionClass $theClass, string $methodName, array $groups = []): Test
+    public function build(ReflectionClass $theClass, string $methodName, array $groups = [], int $repeatTimes = 1): Test
     {
         $className = $theClass->getName();
 
@@ -64,6 +65,7 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou
                 $this->shouldGlobalStateBePreserved($className, $methodName),
                 $this->backupSettings($className, $methodName),
                 $groups,
+                $repeatTimes,
             );
         }
 
@@ -85,8 +87,9 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou
      * @param array<ProvidedData>                                                                                                                                               $data
      * @param array{backupGlobals: ?true, backupGlobalsExcludeList: list<string>, backupStaticProperties: ?true, backupStaticPropertiesExcludeList: array<string,list<string>>} $backupSettings
      * @param list<non-empty-string>                                                                                                                                            $groups
+     * @param positive-int                                                                                                                                                      $repeatTimes
      */
-    private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups): DataProviderTestSuite
+    private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups, int $repeatTimes = 1): DataProviderTestSuite
     {
         $dataProviderTestSuite = DataProviderTestSuite::empty(
             $className . '::' . $methodName,
@@ -109,7 +112,7 @@ private function buildDataProviderTestSuite(string $methodName, string $classNam
                 $backupSettings,
             );
 
-            $dataProviderTestSuite->addTest($_test, $groups);
+            $dataProviderTestSuite->addTest($_test, $groups, $repeatTimes);
         }
 
         return $dataProviderTestSuite;

src/Framework/TestCase.php

@@ -965,6 +965,21 @@ final public function wasPrepared(): bool
         return $this->wasPrepared;
     }
 
+    /**
+     * @internal This method is not covered by the backward compatibility promise for PHPUnit
+     */
+    public function markSkippedForErrorInPreviousRepetition(): void
+    {
+        $message = 'Test repetition failure';
+
+        Event\Facade::emitter()->testSkipped(
+            $this->valueObjectForEvents(),
+            $message,
+        );
+
+        $this->status = TestStatus::skipped($message);
+    }
+
     /**
      * Returns a matcher that matches when the method is executed
      * zero or more times.

src/Framework/TestSuite.php

@@ -96,8 +96,9 @@ public static function empty(string $name): static
     /**
      * @param ReflectionClass<TestCase> $class
      * @param list<non-empty-string>    $groups
+     * @param positive-int              $repeatTimes
      */
-    public static function fromClassReflector(ReflectionClass $class, array $groups = []): static
+    public static function fromClassReflector(ReflectionClass $class, array $groups = [], int $repeatTimes = 1): static
     {
         $testSuite = new static($class->getName());
 
@@ -118,7 +119,7 @@ public static function fromClassReflector(ReflectionClass $class, array $groups
                 continue;
             }
 
-            $testSuite->addTestMethod($class, $method, $groups);
+            $testSuite->addTestMethod($class, $method, $groups, $repeatTimes);
         }
 
         if ($testSuite->isEmpty()) {
@@ -145,8 +146,9 @@ final private function __construct(string $name)
      * Adds a test to the suite.
      *
      * @param list<non-empty-string> $groups
+     * @param positive-int           $repeatTimes
      */
-    public function addTest(Test $test, array $groups = []): void
+    public function addTest(Test $test, array $groups = [], int $repeatTimes = 1): void
     {
         if ($test instanceof self) {
             $this->tests[] = $test;
@@ -158,7 +160,11 @@ public function addTest(Test $test, array $groups = []): void
 
         assert($test instanceof TestCase || $test instanceof PhptTestCase);
 
-        $this->tests[] = $test;
+        if ($repeatTimes === 1) {
+            $this->tests[] = $test;
+        } else {
+            $this->tests[] = new RepeatTestSuite($test, $repeatTimes);
+        }
 
         $this->clearCaches();
 
@@ -188,10 +194,11 @@ public function addTest(Test $test, array $groups = []): void
      *
      * @param ReflectionClass<TestCase> $testClass
      * @param list<non-empty-string>    $groups
+     * @param positive-int              $repeatTimes
      *
      * @throws Exception
      */
-    public function addTestSuite(ReflectionClass $testClass, array $groups = []): void
+    public function addTestSuite(ReflectionClass $testClass, array $groups = [], int $repeatTimes = 1): void
     {
         if ($testClass->isAbstract()) {
             throw new Exception(
@@ -212,7 +219,7 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = []): vo
             );
         }
 
-        $this->addTest(self::fromClassReflector($testClass, $groups), $groups);
+        $this->addTest(self::fromClassReflector($testClass, $groups, $repeatTimes), $groups, $repeatTimes);
     }
 
     /**
@@ -224,18 +231,20 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = []): vo
      * leaving the current test run untouched.
      *
      * @param list<non-empty-string> $groups
+     * @param positive-int           $repeatTimes
      *
      * @throws Exception
      */
-    public function addTestFile(string $filename, array $groups = []): void
+    public function addTestFile(string $filename, array $groups = [], int $repeatTimes = 1): void
     {
         try {
             if (str_ends_with($filename, '.phpt') && is_file($filename)) {
-                $this->addTest(new PhptTestCase($filename));
+                $this->addTest(new PhptTestCase($filename), [], $repeatTimes);
             } else {
                 $this->addTestSuite(
                     (new TestSuiteLoader)->load($filename),
                     $groups,
+                    $repeatTimes,
                 );
             }
         } catch (RunnerException $e) {
@@ -249,13 +258,14 @@ public function addTestFile(string $filename, array $groups = []): void
      * Wrapper for addTestFile() that adds multiple test files.
      *
      * @param iterable<string> $fileNames
+     * @param positive-int     $repeatTimes
      *
      * @throws Exception
      */
-    public function addTestFiles(iterable $fileNames): void
+    public function addTestFiles(iterable $fileNames, int $repeatTimes = 1): void
     {
         foreach ($fileNames as $filename) {
-            $this->addTestFile((string) $filename);
+            $this->addTestFile((string) $filename, [], $repeatTimes);
         }
     }
 
@@ -503,16 +513,17 @@ public function isForTestClass(): bool
     /**
      * @param ReflectionClass<TestCase> $class
      * @param list<non-empty-string>    $groups
+     * @param positive-int              $repeatTimes
      *
      * @throws Exception
      */
-    protected function addTestMethod(ReflectionClass $class, ReflectionMethod $method, array $groups): void
+    protected function addTestMethod(ReflectionClass $class, ReflectionMethod $method, array $groups, int $repeatTimes): void
     {
         $className  = $class->getName();
         $methodName = $method->getName();
 
         try {
-            $test = (new TestBuilder)->build($class, $methodName, $groups);
+            $test = (new TestBuilder)->build($class, $methodName, $groups, $repeatTimes);
         } catch (InvalidDataProviderException $e) {
             if ($e->getProviderLabel() === null) {
                 $message = sprintf(
@@ -562,6 +573,7 @@ protected function addTestMethod(ReflectionClass $class, ReflectionMethod $metho
                 $groups,
                 (new Groups)->groups($class->getName(), $methodName),
             ),
+            $repeatTimes,
         );
     }