diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index eb19e5c93..1d5fd62d1 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -167,6 +167,20 @@ jobs: make flutter/android/release flutter/android/apk flutter/android/test-apk gsutil mv output/android-apks/test-main.apk $GCLOUD_BUCKET_PATH/test-main-samsung.apk gsutil mv output/android-apks/test-helper.apk $GCLOUD_BUCKET_PATH/test-helper-samsung.apk + - name: Build and upload test APK for all backends + env: + OFFICIAL_BUILD: false + PERF_TEST: true + WITH_TFLITE: 1 + WITH_PIXEL: 1 + WITH_MEDIATEK: 1 + WITH_QTI: 1 + WITH_SAMSUNG: 1 + WITH_APPLE: 0 + run: | + make flutter/android/release flutter/android/apk flutter/android/test-apk + gsutil mv output/android-apks/test-main.apk $GCLOUD_BUCKET_PATH/test-main-unified.apk + gsutil mv output/android-apks/test-helper.apk $GCLOUD_BUCKET_PATH/test-helper-unified.apk - name: Build Android release APK with all backends env: OFFICIAL_BUILD: true @@ -231,113 +245,13 @@ jobs: retention-days: 28 if-no-files-found: error - test-android-apk-tflite: - needs: build-android-apk - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - project_id: mobile-app-build-290400 - - name: Trigger Firebase Test Lab for [Google Pixel 6](oriole) - run: | - gcloud firebase test android run \ - --type instrumentation \ - --app $GCLOUD_BUCKET_PATH/test-main-tflite.apk \ - --test $GCLOUD_BUCKET_PATH/test-helper-tflite.apk \ - --timeout 30m \ - --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=oriole,version=31,locale=en,orientation=portrait \ - --client-details=buildNumber=${{ github.run_number }} - - test-android-apk-pixel: - needs: build-android-apk - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - project_id: mobile-app-build-290400 - - name: Trigger Firebase Test Lab for [Google Pixel 6](oriole) - run: | - gcloud firebase test android run \ - --type instrumentation \ - --app $GCLOUD_BUCKET_PATH/test-main-pixel.apk \ - --test $GCLOUD_BUCKET_PATH/test-helper-pixel.apk \ - --timeout 30m \ - --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=oriole,version=32,locale=en,orientation=portrait \ - --client-details=buildNumber=${{ github.run_number }} - - test-android-apk-qti: - needs: build-android-apk - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - project_id: mobile-app-build-290400 - - name: Trigger Firebase Test Lab for [Samsung Galaxy S22 Ultra](b0q) - run: | - gcloud firebase test android run \ - --type instrumentation \ - --app $GCLOUD_BUCKET_PATH/test-main-qti.apk \ - --test $GCLOUD_BUCKET_PATH/test-helper-qti.apk \ - --timeout 30m \ - --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=b0q,version=33,locale=en,orientation=portrait \ - --client-details=buildNumber=${{ github.run_number }} - - test-android-apk-mtk: - needs: build-android-apk - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - project_id: mobile-app-build-290400 - - name: Trigger Firebase Test Lab for [OnePlus DN2103](OP515BL1) - run: | - gcloud firebase test android run \ - --type instrumentation \ - --app $GCLOUD_BUCKET_PATH/test-main-mtk.apk \ - --test $GCLOUD_BUCKET_PATH/test-helper-mtk.apk \ - --timeout 30m \ - --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=OP515BL1,version=33,locale=en,orientation=portrait \ - --client-details=buildNumber=${{ github.run_number }} - - test-android-apk-samsung: + test-android-apk-unified: needs: build-android-apk runs-on: ubuntu-22.04 - timeout-minutes: 30 + timeout-minutes: 60 env: - MAIN_APK_NAME: test-main-samsung-${{ github.run_number }}.apk - HELPER_APK_NAME: test-helper-samsung-${{ github.run_number }}.apk + MAIN_APK_NAME: test-main-unified-${{ github.run_number }}.apk + HELPER_APK_NAME: test-helper-unified-${{ github.run_number }}.apk steps: - uses: actions/checkout@v4 - name: Set up authentication for Google Cloud SDK @@ -350,8 +264,8 @@ jobs: version: '>= 363.0.0' - name: Download test APK run: | - gsutil cp $GCLOUD_BUCKET_PATH/test-main-samsung.apk /tmp/$MAIN_APK_NAME - gsutil cp $GCLOUD_BUCKET_PATH/test-helper-samsung.apk /tmp/$HELPER_APK_NAME + gsutil cp $GCLOUD_BUCKET_PATH/test-main-unified.apk /tmp/$MAIN_APK_NAME + gsutil cp $GCLOUD_BUCKET_PATH/test-helper-unified.apk /tmp/$HELPER_APK_NAME - name: Upload main app run: | curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \ @@ -364,19 +278,25 @@ jobs: -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/test-suite" \ -F "file=@/tmp/$HELPER_APK_NAME" \ -F "custom_id=$HELPER_APK_NAME" - - name: Trigger App Automate for [Samsung Galaxy S24] + - name: Trigger App Automate env: BROWSERSTACK_CREDENTIALS: ${{ secrets.BROWSERSTACK_CREDENTIALS }} BROWSERSTACK_APP: ${{ env.MAIN_APK_NAME }} BROWSERSTACK_TEST_SUITE: ${{ env.HELPER_APK_NAME }} BROWSERSTACK_BUILD_TAG: ${{ github.run_number }} + BROWSERSTACK_DEVICES: >- + ["Samsung Galaxy M32-11.0", + "Samsung Galaxy S24-14.0", + "Samsung Galaxy S24 Ultra-14.0", + "Samsung Galaxy Tab S10 Plus-14.0", + "Google Pixel 9 Pro-15.0"] run: | bash .github/workflows/scripts/browserstack-app-automate.sh deploy-playstore: needs: - build-android-apk - - test-android-apk-tflite + - test-android-apk-unified runs-on: ubuntu-22.04 # if: github.ref == 'refs/heads/master' steps: diff --git a/.github/workflows/ios-build-test.yml b/.github/workflows/ios-build-test.yml index 4dbc67bef..57e2898a0 100644 --- a/.github/workflows/ios-build-test.yml +++ b/.github/workflows/ios-build-test.yml @@ -59,7 +59,7 @@ jobs: - name: Build iOS integration tests run: | cd flutter && flutter --no-version-check build ios --simulator integration_test/first_test.dart - - uses: nick-fields/retry@v2 + - uses: nick-fields/retry@v3 name: Run iOS integration tests with: timeout_minutes: 45 diff --git a/.github/workflows/scripts/browserstack-app-automate.sh b/.github/workflows/scripts/browserstack-app-automate.sh index 3543d1776..ed6b2c438 100644 --- a/.github/workflows/scripts/browserstack-app-automate.sh +++ b/.github/workflows/scripts/browserstack-app-automate.sh @@ -2,10 +2,7 @@ # This script is used to trigger a build on BrowserStack App Automate and monitor its status. -set -e - # Build parameters -DEVICES='["Samsung Galaxy S24-14.0"]' PROJECT="mobile-app-build-290400" DEVICE_LOGS=true RETRY_INTERVAL=10 @@ -19,6 +16,7 @@ CREDENTIALS="${BROWSERSTACK_CREDENTIALS:-}" APP="${BROWSERSTACK_APP:-}" TEST_SUITE="${BROWSERSTACK_TEST_SUITE:-}" BUILD_TAG="${BROWSERSTACK_BUILD_TAG:-}" +DEVICES="${BROWSERSTACK_DEVICES:-}" # Validate required environment variables if [[ -z "$CREDENTIALS" ]]; then @@ -26,8 +24,10 @@ if [[ -z "$CREDENTIALS" ]]; then exit 1 fi -if [[ -z "$APP" || -z "$TEST_SUITE" || -z "$BUILD_TAG" ]]; then - echo "Error: Environment variables BROWSERSTACK_APP, BROWSERSTACK_TEST_SUITE and BROWSERSTACK_BUILD_TAG must be set." +if [[ -z "$APP" || -z "$TEST_SUITE" || -z "$BUILD_TAG" || -z "$DEVICES" ]]; then + echo "Error: Environment variables"\ + "BROWSERSTACK_APP, BROWSERSTACK_TEST_SUITE, BROWSERSTACK_BUILD_TAG and BROWSERSTACK_DEVICES"\ + "must be set." exit 1 fi @@ -49,10 +49,11 @@ trigger_build() { if [[ "$build_id" == "null" || -z "$build_id" ]]; then echo "Failed to trigger the build. Response: $response" - exit 1 + return 1 + else + echo "$build_id" + return 0 fi - - echo "$build_id" } # Function to check build status @@ -60,7 +61,21 @@ check_build_status() { local build_id=$1 local response=$(curl -s -u "$CREDENTIALS" -X GET "$STATUS_URL/$build_id") local status=$(echo "$response" | jq -r '.status') + echo "$(date +'%Y-%m-%d %H:%M:%S') Build Status: $status" + + # Display device status + if [[ "$status" != "running" ]]; then + echo "$response" | jq -r ' + .devices[] | + "Device: " + .device + + ", OS Version: " + .os_version + + ", Duration: " + (.sessions[0].duration | tostring) + "s" + + ", Status: " + (.sessions[0].status) + ' + fi + + # Display build status if [[ "$status" == "passed" ]]; then echo "Build completed successfully!" exit 0 @@ -71,7 +86,11 @@ check_build_status() { } # Main -BUILD_ID=$(trigger_build) +if ! BUILD_ID=$(trigger_build); then + echo "Trigger build failed. Message: $BUILD_ID" + exit 1 +fi + echo "Build triggered successfully. Build ID: $BUILD_ID" echo "See the build status at: https://app-automate.browserstack.com/dashboard/v2/builds/$BUILD_ID" diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index d27ed14ef..c1d37a2e0 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -19,13 +19,13 @@ key: OR | const Map _imageClassificationV2 = { 'cpu': Interval(min: 0.88, max: 0.91), - 'npu': Interval(min: 0.88, max: 0.91), + 'npu': Interval(min: 0.69, max: 0.91), 'tpu': Interval(min: 0.88, max: 0.91), 'ane': Interval(min: 0.69, max: 0.91), 'cpu&gpu&ane': Interval(min: 0.69, max: 0.91), 'snpe_dsp': Interval(min: 0.88, max: 0.91), 'psnpe_dsp': Interval(min: 0.88, max: 0.91), - 'neuron-mdla': Interval(min: 0.79, max: 0.81), + 'neuron-mdla': Interval(min: 0.79, max: 0.91), 'samsung_npu': Interval(min: 0.99, max: 1.0), }; @@ -38,7 +38,7 @@ const Map _objectDetection = { 'cpu&gpu&ane': Interval(min: 0.45, max: 0.46), 'snpe_dsp': Interval(min: 0.32, max: 0.35), 'psnpe_dsp': Interval(min: 0.32, max: 0.35), - 'neuron': Interval(min: 0.28, max: 0.31), + 'neuron': Interval(min: 0.28, max: 0.35), 'samsung_npu': Interval(min: 0.36, max: 0.39), }; @@ -78,7 +78,7 @@ const Map _superResolution = { 'snpe_dsp': Interval(min: 0.32, max: 0.35), 'psnpe_dsp': Interval(min: 0.32, max: 0.35), 'neuron': Interval(min: 0.32, max: 0.35), - 'samsung_npu': Interval(min: 0.08, max: 0.11), + 'samsung_npu': Interval(min: 0.08, max: 0.12), }; // TODO (anhappdev): update expected accuracy for stable diffusion diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index bc2b36c72..1b9e416a1 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -20,12 +20,18 @@ const _kCloudBuildX23 = 'Intel Xeon 2.30GHz'; const _kCloudBuildX28 = 'Intel Xeon 2.80GHz'; const _kRyzen5600 = 'AMD Ryzen 5 5600X 6-Core'; -// Android +// Android devices on Firebase TestLab const _kPixel5 = 'Pixel 5'; // Google Pixel 5 const _kPixel6 = 'Pixel 6'; // Google Pixel 6 -const _kS22Ultra = 'SM-S908U1'; // Galaxy S22 Ultra +const _kS22Ultra = 'SM-S908U1'; // Samsung Galaxy S22 Ultra const _kDN2103 = 'DN2103'; // OnePlus DN2103 -const _kS24 = 'SM-S921B'; // Galaxy S24 + +// Android devices on BrowserStack App Automate +const _kPixel9Pro = 'Pixel 9 Pro'; // Google Pixel 9 Pro +const _kS24 = 'SM-S921B'; // Samsung Galaxy S24 +const _kS24Ultra = 'SM-S928B'; // Samsung Galaxy S24 Ultra +const _kS10Plus = 'SM-X826B'; // Samsung Galaxy Tab S10 Plus +const _kM32 = 'SM-M326B'; // Samsung Galaxy M32 // iOS const _kIphoneOnGitHubAction = 'iPhone16,2'; @@ -38,6 +44,7 @@ const Map> _imageClassificationV2 = { _kRyzen5600: Interval(min: 1, max: 37), _kPixel5: Interval(min: 20, max: 75), _kPixel6: Interval(min: 100, max: 600), + _kM32: Interval(min: 5, max: 15), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 9), _kIphoneOnMacbookM1: Interval(min: 10, max: 27), }, @@ -46,15 +53,18 @@ const Map> _imageClassificationV2 = { }, _kPixelBackend: { _kPixel6: Interval(min: 100, max: 600), + _kPixel9Pro: Interval(min: 100, max: 600), }, _kQtiBackend: { _kS22Ultra: Interval(min: 250, max: 400), + _kS24Ultra: Interval(min: 800, max: 1500), }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 90), + _kS10Plus: Interval(min: 400, max: 800) }, _kSamsungBackend: { - _kS24: Interval(min: 700, max: 1000), + _kS24: Interval(min: 600, max: 1000), }, }; @@ -65,6 +75,7 @@ const Map> _objectDetection = { _kRyzen5600: Interval(min: 14, max: 22), _kPixel5: Interval(min: 40, max: 60), _kPixel6: Interval(min: 100, max: 450), + _kM32: Interval(min: 20, max: 40), _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4), _kIphoneOnMacbookM1: Interval(min: 9, max: 16), }, @@ -73,15 +84,18 @@ const Map> _objectDetection = { }, _kPixelBackend: { _kPixel6: Interval(min: 200, max: 500), + _kPixel9Pro: Interval(min: 200, max: 500), }, _kQtiBackend: { _kS22Ultra: Interval(min: 700, max: 1400), + _kS24Ultra: Interval(min: 1800, max: 2500), }, _kMediatekBackend: { _kDN2103: Interval(min: 120, max: 210), + _kS10Plus: Interval(min: 1200, max: 1800) }, _kSamsungBackend: { - _kS24: Interval(min: 1500, max: 2400), + _kS24: Interval(min: 1400, max: 2400), }, }; @@ -92,6 +106,7 @@ const Map> _imageSegmentationV2 = { _kRyzen5600: Interval(min: 5, max: 7), _kPixel5: Interval(min: 25, max: 40), _kPixel6: Interval(min: 80, max: 180), + _kM32: Interval(min: 2, max: 10), _kIphoneOnGitHubAction: Interval(min: 0.1, max: 2.5), _kIphoneOnMacbookM1: Interval(min: 3, max: 6), }, @@ -100,15 +115,18 @@ const Map> _imageSegmentationV2 = { }, _kPixelBackend: { _kPixel6: Interval(min: 100, max: 190), + _kPixel9Pro: Interval(min: 200, max: 500), }, _kQtiBackend: { _kS22Ultra: Interval(min: 400, max: 750), + _kS24Ultra: Interval(min: 1200, max: 1600), }, _kMediatekBackend: { _kDN2103: Interval(min: 45, max: 70), + _kS10Plus: Interval(min: 800, max: 1400) }, _kSamsungBackend: { - _kS24: Interval(min: 1000, max: 1500), + _kS24: Interval(min: 800, max: 1500), }, }; @@ -119,6 +137,7 @@ const Map> _naturalLanguageProcessing = { _kRyzen5600: Interval(min: 2.8, max: 3.2), _kPixel5: Interval(min: 2.3, max: 3.0), _kPixel6: Interval(min: 2, max: 75), + _kM32: Interval(min: 2, max: 5), _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1), _kIphoneOnMacbookM1: Interval(min: 1.8, max: 3), }, @@ -127,12 +146,15 @@ const Map> _naturalLanguageProcessing = { }, _kPixelBackend: { _kPixel6: Interval(min: 2, max: 85), + _kPixel9Pro: Interval(min: 60, max: 150), }, _kQtiBackend: { _kS22Ultra: Interval(min: 100, max: 200), + _kS24Ultra: Interval(min: 250, max: 450), }, _kMediatekBackend: { _kDN2103: Interval(min: 1, max: 6), + _kS10Plus: Interval(min: 100, max: 300) }, _kSamsungBackend: { _kS24: Interval(min: 220, max: 350), @@ -146,6 +168,7 @@ const Map> _superResolution = { _kRyzen5600: Interval(min: 0.1, max: 3), _kPixel5: Interval(min: 4, max: 8), _kPixel6: Interval(min: 7, max: 14), + _kM32: Interval(min: 0.1, max: 3), _kIphoneOnGitHubAction: Interval(min: 0.02, max: 1.0), _kIphoneOnMacbookM1: Interval(min: 0.1, max: 10), }, @@ -154,15 +177,18 @@ const Map> _superResolution = { }, _kPixelBackend: { _kPixel6: Interval(min: 7, max: 17), + _kPixel9Pro: Interval(min: 30, max: 70), }, _kQtiBackend: { _kS22Ultra: Interval(min: 25, max: 75), + _kS24Ultra: Interval(min: 120, max: 180), }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 15), + _kS10Plus: Interval(min: 150, max: 280) }, _kSamsungBackend: { - _kS24: Interval(min: 100, max: 180), + _kS24: Interval(min: 90, max: 180), }, }; @@ -174,6 +200,7 @@ const Map> _stableDiffusion = { _kRyzen5600: Interval(min: 0, max: 100), _kPixel5: Interval(min: 0, max: 100), _kPixel6: Interval(min: 0, max: 100), + _kM32: Interval(min: 0, max: 100), _kIphoneOnGitHubAction: Interval(min: 0, max: 100), _kIphoneOnMacbookM1: Interval(min: 0, max: 100), }, @@ -182,12 +209,15 @@ const Map> _stableDiffusion = { }, _kPixelBackend: { _kPixel6: Interval(min: 0, max: 100), + _kPixel9Pro: Interval(min: 0, max: 100), }, _kQtiBackend: { _kS22Ultra: Interval(min: 0, max: 100), + _kS24Ultra: Interval(min: 0, max: 100), }, _kMediatekBackend: { _kDN2103: Interval(min: 0, max: 100), + _kS10Plus: Interval(min: 0, max: 100) }, _kSamsungBackend: { _kS24: Interval(min: 0, max: 100), @@ -201,6 +231,7 @@ const Map> _imageClassificationOfflineV2 = { _kRyzen5600: Interval(min: 20, max: 60), _kPixel5: Interval(min: 20, max: 180), _kPixel6: Interval(min: 100, max: 700), + _kM32: Interval(min: 8, max: 18), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), _kIphoneOnMacbookM1: Interval(min: 10, max: 45), }, @@ -209,15 +240,18 @@ const Map> _imageClassificationOfflineV2 = { }, _kPixelBackend: { _kPixel6: Interval(min: 100, max: 700), + _kPixel9Pro: Interval(min: 300, max: 700), }, _kQtiBackend: { _kS22Ultra: Interval(min: 250, max: 450), + _kS24Ultra: Interval(min: 900, max: 1600), }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 90), + _kS10Plus: Interval(min: 700, max: 1200) }, _kSamsungBackend: { - _kS24: Interval(min: 900, max: 1200), + _kS24: Interval(min: 800, max: 1200), }, };