test: Add Flaky Test Plan (#6534)

Add an extra test plan, Sentry_Flaky, that reruns failing tests 3 times before the test officially fails.

Author: philipphofmann

.github/workflows/test.yml

@@ -373,7 +373,7 @@ jobs:
         run: ./scripts/ci-boot-simulator.sh --xcode ${{ matrix.xcode }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}"
 
       # We split building and running tests in two steps so we know how long running the tests takes.
-      - name: Build tests
+      - name: Build Tests
         id: build_tests
         run: |
           ./scripts/sentry-xcodebuild.sh \
@@ -385,6 +385,25 @@ jobs:
             --configuration TestCI \
             --scheme ${{matrix.scheme}}
 
+      # Run Flaky Tests TestPlan which has a retry mechanism on failure.
+      # We intentionally run these before the other test plan to fail early.
+      # Use a separate result bundle name to avoid conflicts with the regular test run.
+      # xcodebuild fails if a result bundle already exists at the target path.
+      - name: Run Flaky Tests
+        # Only the Sentry Scheme has the Flaky TestPlan.
+        if: ${{ matrix.scheme == 'Sentry' }}
+        run: |
+          ./scripts/sentry-xcodebuild.sh \
+            --platform ${{matrix.platform}} \
+            --os ${{matrix.test-destination-os}} \
+            --ref ${{ github.ref_name }} \
+            --command test-without-building \
+            --device "${{matrix.device}}" \
+            --configuration TestCI \
+            --scheme ${{matrix.scheme}} \
+            --test-plan Sentry_Flaky \
+            --result-bundle flaky-results.xcresult
+
       - name: Run tests
         # We call a script with the platform so the destination
         # passed to xcodebuild doesn't end up in the job name,
@@ -398,7 +417,8 @@ jobs:
             --command test-without-building \
             --device "${{matrix.device}}" \
             --configuration TestCI \
-            --scheme ${{matrix.scheme}}
+            --scheme ${{matrix.scheme}} \
+            --result-bundle results.xcresult
 
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@3585e9575db828022551b4231f165eb59a0e74e3 # v5.6.2
@@ -442,6 +462,7 @@ jobs:
           name: result-bundle-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
           path: |
             results.xcresult
+            flaky-results.xcresult
 
       - name: Gather code coverage information via slather
         run: slather coverage --configuration TestCI --scheme Sentry

Plans/Sentry_Base.xctestplan

@@ -35,6 +35,7 @@
         "SentryCrashReportStore_Tests\/testCrashReportCount1",
         "SentryCrashReportStore_Tests\/testDeleteAllReports",
         "SentryCrashReportStore_Tests\/testStoresLoadsMultipleReports",
+        "SentryHttpTransportFlushIntegrationTests",
         "SentryHttpTransportTests\/testFlush_WhenNoInternet_BlocksAndFinishes()",
         "SentryNetworkTrackerIntegrationTestServerTests",
         "SentryNetworkTrackerIntegrationTestServerTests\/testGetRequest_SpanCreatedAndBaggageHeaderAdded()",

Plans/Sentry_Flaky.xctestplan

@@ -0,0 +1,29 @@
+{
+  "configurations" : [
+    {
+      "id" : "A451EF21-C702-4095-835C-4F189D10635A",
+      "name" : "Configuration 1",
+      "options" : {
+
+      }
+    }
+  ],
+  "defaultOptions" : {
+    "testExecutionOrdering" : "random",
+    "testRepetitionMode" : "retryOnFailure",
+    "testTimeoutsEnabled" : true
+  },
+  "testTargets" : [
+    {
+      "selectedTests" : [
+        "SentryHttpTransportFlushIntegrationTests"
+      ],
+      "target" : {
+        "containerPath" : "container:Sentry.xcodeproj",
+        "identifier" : "63AA76641EB8CB2F00D153DE",
+        "name" : "SentryTests"
+      }
+    }
+  ],
+  "version" : 1
+}

Sentry.xcodeproj/project.pbxproj

@@ -1389,6 +1389,7 @@
 		6293F5742D422A8A002BC3BD /* SentryStacktraceCodable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryStacktraceCodable.swift; sourceTree = "<group>"; };
 		62950F0F29E7FE0100A42624 /* SentryTransactionContextTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryTransactionContextTests.swift; sourceTree = "<group>"; };
 		629690522AD3E060000185FA /* SentryReachabilitySwiftTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryReachabilitySwiftTests.swift; sourceTree = "<group>"; };
+		629B24502EAB50C70050052C /* Sentry_Flaky.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = Sentry_Flaky.xctestplan; sourceTree = "<group>"; };
 		62A3C7BD2B7E2A6A00C75227 /* SentrySpotlightTransportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentrySpotlightTransportTests.swift; sourceTree = "<group>"; };
 		62A456E02B03704A003F19A1 /* SentryUIEventTrackerMode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = SentryUIEventTrackerMode.h; path = include/SentryUIEventTrackerMode.h; sourceTree = "<group>"; };
 		62A456E22B0370AA003F19A1 /* SentryUIEventTrackerTransactionMode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = SentryUIEventTrackerTransactionMode.h; path = include/SentryUIEventTrackerTransactionMode.h; sourceTree = "<group>"; };
@@ -4310,6 +4311,7 @@
 			isa = PBXGroup;
 			children = (
 				D46D45E12D5F3FD600A1CB35 /* Sentry_Base.xctestplan */,
+				629B24502EAB50C70050052C /* Sentry_Flaky.xctestplan */,
 				6214366A2E7A7D6600A2CB8F /* Sentry_TestServer.xctestplan */,
 				D46D45E92D5F411700A1CB35 /* SentrySwiftUI_Base.xctestplan */,
 				D4A2360A2D5F84FA00D55C58 /* SwiftUITestSample_Base.xctestplan */,

Sentry.xcodeproj/xcshareddata/xcschemes/Sentry.xcscheme

@@ -52,6 +52,9 @@
          <TestPlanReference
             reference = "container:Plans/Sentry_TestServer.xctestplan">
          </TestPlanReference>
+         <TestPlanReference
+            reference = "container:Plans/Sentry_Flaky.xctestplan">
+         </TestPlanReference>
       </TestPlans>
       <Testables>
          <TestableReference

develop-docs/TEST.md

@@ -133,7 +133,9 @@ Once daily and for every PR via [Github action](../.github/workflows/benchmarkin
 
 ### Flaky tests
 
-If you see a test being flaky, you should ideally fix it immediately. If that's not feasible, you can disable the test in the test scheme by unchecking it in the associated test plan:
+If you see a test being flaky, you should ideally fix it immediately. If that's not feasible, you can add it to the Sentry_Flaky test plan and remove it from the SentryBase test plan. The Sentry_Flaky test plan has a retry mechanism of 3 times until a test officially fails.
+
+Or you disable the test in the test scheme by unchecking it in the associated test plan:
 
 ![Disabling test cases via the Xcode Tests navigator](./develop-docs/disabling_tests_xcode_test_plan.png)
 

scripts/sentry-xcodebuild.sh

@@ -22,6 +22,7 @@ CONFIGURATION_OVERRIDE=""
 DERIVED_DATA_PATH=""
 TEST_SCHEME="Sentry"
 TEST_PLAN=""
+RESULT_BUNDLE_PATH="results.xcresult"
 
 usage() {
     echo "Usage: $0"
@@ -34,6 +35,7 @@ usage() {
     echo "  -D|--derived-data <path>        Derived data path"
     echo "  -s|--scheme <scheme>            Test scheme (default: Sentry)"
     echo "  -t|--test-plan <plan>           Test plan name (default: empty)"
+    echo "  -R|--result-bundle <path>       Result bundle path (default: results.xcresult)"
     exit 1
 }
 
@@ -76,6 +78,10 @@ while [[ $# -gt 0 ]]; do
             TEST_PLAN="$2"
             shift 2
             ;;
+        -R|--result-bundle)
+            RESULT_BUNDLE_PATH="$2"
+            shift 2
+            ;;
         *)
             echo "Unknown option: $1"
             usage
@@ -188,7 +194,7 @@ if [ $RUN_TEST_WITHOUT_BUILDING == true ]; then
         "${TEST_PLAN_ARGS[@]+${TEST_PLAN_ARGS[@]}}" \
         -configuration "$CONFIGURATION" \
         -destination "$DESTINATION" \
-        -resultBundlePath "results.xcresult" \
+        -resultBundlePath "$RESULT_BUNDLE_PATH" \
         test-without-building 2>&1 |
         tee raw-test-output.log |
         xcbeautify --report junit