diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 524131ec80b..5d996e7792d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -384,7 +384,7 @@ jobs: run: ./scripts/ci-boot-simulator.sh --xcode ${{ matrix.xcode }} --device "${{ matrix.device }}" --os-version "${{ matrix.test-destination-os }}" # We split building and running tests in two steps so we know how long running the tests takes. - - name: Build tests + - name: Build Tests id: build_tests run: | ./scripts/sentry-xcodebuild.sh \ @@ -396,6 +396,25 @@ jobs: --configuration TestCI \ --scheme ${{matrix.scheme}} + # Run Flaky Tests TestPlan which has a retry mechanism on failure. + # We intentionally run these before the other test plan to fail early. + # Use a separate result bundle name to avoid conflicts with the regular test run. + # xcodebuild fails if a result bundle already exists at the target path. + - name: Run Flaky Tests + # Only the Sentry Scheme has the Flaky TestPlan. + if: ${{ matrix.scheme == 'Sentry' }} + run: | + ./scripts/sentry-xcodebuild.sh \ + --platform ${{matrix.platform}} \ + --os ${{matrix.test-destination-os}} \ + --ref ${{ github.ref_name }} \ + --command test-without-building \ + --device "${{matrix.device}}" \ + --configuration TestCI \ + --scheme ${{matrix.scheme}} \ + --test-plan Sentry_Flaky \ + --result-bundle flaky-results.xcresult + - name: Run tests # We call a script with the platform so the destination # passed to xcodebuild doesn't end up in the job name, @@ -409,7 +428,8 @@ jobs: --command test-without-building \ --device "${{matrix.device}}" \ --configuration TestCI \ - --scheme ${{matrix.scheme}} + --scheme ${{matrix.scheme}} \ + --result-bundle results.xcresult - name: Publish Test Report uses: mikepenz/action-junit-report@3585e9575db828022551b4231f165eb59a0e74e3 # v5.6.2 @@ -453,6 +473,7 @@ jobs: name: result-bundle-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} path: | results.xcresult + flaky-results.xcresult - name: Gather code coverage information via slather run: slather coverage --configuration TestCI --scheme Sentry diff --git a/Plans/Sentry_Base.xctestplan b/Plans/Sentry_Base.xctestplan index a9915e115ea..9a36338301d 100644 --- a/Plans/Sentry_Base.xctestplan +++ b/Plans/Sentry_Base.xctestplan @@ -35,6 +35,7 @@ "SentryCrashReportStore_Tests\/testCrashReportCount1", "SentryCrashReportStore_Tests\/testDeleteAllReports", "SentryCrashReportStore_Tests\/testStoresLoadsMultipleReports", + "SentryHttpTransportFlushIntegrationTests", "SentryHttpTransportTests\/testFlush_WhenNoInternet_BlocksAndFinishes()", "SentryNetworkTrackerIntegrationTestServerTests", "SentryNetworkTrackerIntegrationTestServerTests\/testGetRequest_SpanCreatedAndBaggageHeaderAdded()", diff --git a/Plans/Sentry_Flaky.xctestplan b/Plans/Sentry_Flaky.xctestplan new file mode 100644 index 00000000000..ddbcfddbbee --- /dev/null +++ b/Plans/Sentry_Flaky.xctestplan @@ -0,0 +1,29 @@ +{ + "configurations" : [ + { + "id" : "A451EF21-C702-4095-835C-4F189D10635A", + "name" : "Configuration 1", + "options" : { + + } + } + ], + "defaultOptions" : { + "testExecutionOrdering" : "random", + "testRepetitionMode" : "retryOnFailure", + "testTimeoutsEnabled" : true + }, + "testTargets" : [ + { + "selectedTests" : [ + "SentryHttpTransportFlushIntegrationTests" + ], + "target" : { + "containerPath" : "container:Sentry.xcodeproj", + "identifier" : "63AA76641EB8CB2F00D153DE", + "name" : "SentryTests" + } + } + ], + "version" : 1 +} diff --git a/Sentry.xcodeproj/project.pbxproj b/Sentry.xcodeproj/project.pbxproj index 745edaac8d9..b9a229f2790 100644 --- a/Sentry.xcodeproj/project.pbxproj +++ b/Sentry.xcodeproj/project.pbxproj @@ -1389,6 +1389,7 @@ 6293F5742D422A8A002BC3BD /* SentryStacktraceCodable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryStacktraceCodable.swift; sourceTree = ""; }; 62950F0F29E7FE0100A42624 /* SentryTransactionContextTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryTransactionContextTests.swift; sourceTree = ""; }; 629690522AD3E060000185FA /* SentryReachabilitySwiftTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryReachabilitySwiftTests.swift; sourceTree = ""; }; + 629B24502EAB50C70050052C /* Sentry_Flaky.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = Sentry_Flaky.xctestplan; sourceTree = ""; }; 62A3C7BD2B7E2A6A00C75227 /* SentrySpotlightTransportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentrySpotlightTransportTests.swift; sourceTree = ""; }; 62A456E02B03704A003F19A1 /* SentryUIEventTrackerMode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = SentryUIEventTrackerMode.h; path = include/SentryUIEventTrackerMode.h; sourceTree = ""; }; 62A456E22B0370AA003F19A1 /* SentryUIEventTrackerTransactionMode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = SentryUIEventTrackerTransactionMode.h; path = include/SentryUIEventTrackerTransactionMode.h; sourceTree = ""; }; @@ -4310,6 +4311,7 @@ isa = PBXGroup; children = ( D46D45E12D5F3FD600A1CB35 /* Sentry_Base.xctestplan */, + 629B24502EAB50C70050052C /* Sentry_Flaky.xctestplan */, 6214366A2E7A7D6600A2CB8F /* Sentry_TestServer.xctestplan */, D46D45E92D5F411700A1CB35 /* SentrySwiftUI_Base.xctestplan */, D4A2360A2D5F84FA00D55C58 /* SwiftUITestSample_Base.xctestplan */, diff --git a/Sentry.xcodeproj/xcshareddata/xcschemes/Sentry.xcscheme b/Sentry.xcodeproj/xcshareddata/xcschemes/Sentry.xcscheme index cd383f436cd..66337df16a2 100644 --- a/Sentry.xcodeproj/xcshareddata/xcschemes/Sentry.xcscheme +++ b/Sentry.xcodeproj/xcshareddata/xcschemes/Sentry.xcscheme @@ -52,6 +52,9 @@ + + Derived data path" echo " -s|--scheme Test scheme (default: Sentry)" echo " -t|--test-plan Test plan name (default: empty)" + echo " -R|--result-bundle Result bundle path (default: results.xcresult)" exit 1 } @@ -76,6 +78,10 @@ while [[ $# -gt 0 ]]; do TEST_PLAN="$2" shift 2 ;; + -R|--result-bundle) + RESULT_BUNDLE_PATH="$2" + shift 2 + ;; *) echo "Unknown option: $1" usage @@ -188,7 +194,7 @@ if [ $RUN_TEST_WITHOUT_BUILDING == true ]; then "${TEST_PLAN_ARGS[@]+${TEST_PLAN_ARGS[@]}}" \ -configuration "$CONFIGURATION" \ -destination "$DESTINATION" \ - -resultBundlePath "results.xcresult" \ + -resultBundlePath "$RESULT_BUNDLE_PATH" \ test-without-building 2>&1 | tee raw-test-output.log | xcbeautify --report junit