From 40cbba56186f7e0fca2f5235567ef7c6b1641692 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 14:17:01 +0700 Subject: [PATCH 01/19] Build unified test app --- .github/workflows/android-build-test.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index eb19e5c93..93377044e 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -167,6 +167,20 @@ jobs: make flutter/android/release flutter/android/apk flutter/android/test-apk gsutil mv output/android-apks/test-main.apk $GCLOUD_BUCKET_PATH/test-main-samsung.apk gsutil mv output/android-apks/test-helper.apk $GCLOUD_BUCKET_PATH/test-helper-samsung.apk + - name: Build and upload test APK for all backends + env: + OFFICIAL_BUILD: false + PERF_TEST: true + WITH_TFLITE: 1 + WITH_PIXEL: 1 + WITH_MEDIATEK: 1 + WITH_QTI: 1 + WITH_SAMSUNG: 1 + WITH_APPLE: 0 + run: | + make flutter/android/release flutter/android/apk flutter/android/test-apk + gsutil mv output/android-apks/test-main.apk $GCLOUD_BUCKET_PATH/test-main-unified.apk + gsutil mv output/android-apks/test-helper.apk $GCLOUD_BUCKET_PATH/test-helper-unified.apk - name: Build Android release APK with all backends env: OFFICIAL_BUILD: true From c027bf69a17f18a3d41fb78088f7a7453bf0c2a2 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 14:37:03 +0700 Subject: [PATCH 02/19] Add env BROWSERSTACK_DEVICES --- .github/workflows/android-build-test.yml | 1 + .github/workflows/scripts/browserstack-app-automate.sh | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 93377044e..5450d3bbf 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -384,6 +384,7 @@ jobs: BROWSERSTACK_APP: ${{ env.MAIN_APK_NAME }} BROWSERSTACK_TEST_SUITE: ${{ env.HELPER_APK_NAME }} BROWSERSTACK_BUILD_TAG: ${{ github.run_number }} + BROWSERSTACK_DEVICES: '["Samsung Galaxy S24-14.0"]' run: | bash .github/workflows/scripts/browserstack-app-automate.sh diff --git a/.github/workflows/scripts/browserstack-app-automate.sh b/.github/workflows/scripts/browserstack-app-automate.sh index 3543d1776..86059a277 100644 --- a/.github/workflows/scripts/browserstack-app-automate.sh +++ b/.github/workflows/scripts/browserstack-app-automate.sh @@ -5,7 +5,6 @@ set -e # Build parameters -DEVICES='["Samsung Galaxy S24-14.0"]' PROJECT="mobile-app-build-290400" DEVICE_LOGS=true RETRY_INTERVAL=10 @@ -19,6 +18,7 @@ CREDENTIALS="${BROWSERSTACK_CREDENTIALS:-}" APP="${BROWSERSTACK_APP:-}" TEST_SUITE="${BROWSERSTACK_TEST_SUITE:-}" BUILD_TAG="${BROWSERSTACK_BUILD_TAG:-}" +DEVICES="${BROWSERSTACK_DEVICES:-}" # Validate required environment variables if [[ -z "$CREDENTIALS" ]]; then @@ -26,8 +26,10 @@ if [[ -z "$CREDENTIALS" ]]; then exit 1 fi -if [[ -z "$APP" || -z "$TEST_SUITE" || -z "$BUILD_TAG" ]]; then - echo "Error: Environment variables BROWSERSTACK_APP, BROWSERSTACK_TEST_SUITE and BROWSERSTACK_BUILD_TAG must be set." +if [[ -z "$APP" || -z "$TEST_SUITE" || -z "$BUILD_TAG" || -z "$DEVICES" ]]; then + echo "Error: Environment variables"\ + "BROWSERSTACK_APP, BROWSERSTACK_TEST_SUITE, BROWSERSTACK_BUILD_TAG and BROWSERSTACK_DEVICES"\ + "must be set." exit 1 fi From 0a9908744a5b4e2100421d3611f499194e948d78 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 14:40:11 +0700 Subject: [PATCH 03/19] Add android-test-browserstack.yml --- .../workflows/android-test-browserstack.yml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/android-test-browserstack.yml diff --git a/.github/workflows/android-test-browserstack.yml b/.github/workflows/android-test-browserstack.yml new file mode 100644 index 000000000..1ad3d8308 --- /dev/null +++ b/.github/workflows/android-test-browserstack.yml @@ -0,0 +1,51 @@ +name: Android Test + +on: + workflow_dispatch: + inputs: + build_number: + description: 'Build or run number' + required: true + type: number + +jobs: + test-android-apk-unfied: + runs-on: ubuntu-22.04 + env: + MAIN_APK_NAME: test-main-unified-${{ inputs.build_number }}.apk + HELPER_APK_NAME: test-helper-unified-${{ inputs.build_number }}.apk + steps: + - uses: actions/checkout@v4 + - name: Set up authentication for Google Cloud SDK + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} + - name: Set up Google Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + with: + version: '>= 363.0.0' + - name: Download test APK + run: | + gsutil cp $GCLOUD_BUCKET_PATH/test-main-unified.apk /tmp/$MAIN_APK_NAME + gsutil cp $GCLOUD_BUCKET_PATH/test-helper-unified.apk /tmp/$HELPER_APK_NAME + - name: Upload main app + run: | + curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \ + -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/app" \ + -F "file=@/tmp/$MAIN_APK_NAME" \ + -F "custom_id=$MAIN_APK_NAME" + - name: Upload test suite + run: | + curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \ + -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/test-suite" \ + -F "file=@/tmp/$HELPER_APK_NAME" \ + -F "custom_id=$HELPER_APK_NAME" + - name: Trigger App Automate + env: + BROWSERSTACK_CREDENTIALS: ${{ secrets.BROWSERSTACK_CREDENTIALS }} + BROWSERSTACK_APP: ${{ env.MAIN_APK_NAME }} + BROWSERSTACK_TEST_SUITE: ${{ env.HELPER_APK_NAME }} + BROWSERSTACK_BUILD_TAG: ${{ inputs.build_number }} + BROWSERSTACK_DEVICES: '["Samsung Galaxy S24-14.0", "Samsung Galaxy S24 Ultra-14.0", "Google Pixel 9 Pro-15.0"]' + run: | + bash .github/workflows/scripts/browserstack-app-automate.sh From bb43c647b79431fa678a6617dd0d1d1e3a09e183 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 15:41:03 +0700 Subject: [PATCH 04/19] Move test-android-apk-unified --- .github/workflows/android-build-test.yml | 44 ++++++++++++++++ .../workflows/android-test-browserstack.yml | 51 ------------------- 2 files changed, 44 insertions(+), 51 deletions(-) delete mode 100644 .github/workflows/android-test-browserstack.yml diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 5450d3bbf..bb0a25dc9 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -388,6 +388,50 @@ jobs: run: | bash .github/workflows/scripts/browserstack-app-automate.sh + test-android-apk-unified: + runs-on: ubuntu-22.04 + env: + MAIN_APK_NAME: test-main-unified-${{ github.run_number }}.apk + HELPER_APK_NAME: test-helper-unified-${{ github.run_number }}.apk + steps: + - uses: actions/checkout@v4 + - name: Set up authentication for Google Cloud SDK + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} + - name: Set up Google Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + with: + version: '>= 363.0.0' + - name: Download test APK + run: | + gsutil cp $GCLOUD_BUCKET_PATH/test-main-unified.apk /tmp/$MAIN_APK_NAME + gsutil cp $GCLOUD_BUCKET_PATH/test-helper-unified.apk /tmp/$HELPER_APK_NAME + - name: Upload main app + run: | + curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \ + -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/app" \ + -F "file=@/tmp/$MAIN_APK_NAME" \ + -F "custom_id=$MAIN_APK_NAME" + - name: Upload test suite + run: | + curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \ + -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/test-suite" \ + -F "file=@/tmp/$HELPER_APK_NAME" \ + -F "custom_id=$HELPER_APK_NAME" + - name: Trigger App Automate + env: + BROWSERSTACK_CREDENTIALS: ${{ secrets.BROWSERSTACK_CREDENTIALS }} + BROWSERSTACK_APP: ${{ env.MAIN_APK_NAME }} + BROWSERSTACK_TEST_SUITE: ${{ env.HELPER_APK_NAME }} + BROWSERSTACK_BUILD_TAG: ${{ github.run_number }} + BROWSERSTACK_DEVICES: >- + ["Samsung Galaxy S24-14.0", + "Samsung Galaxy S24 Ultra-14.0", + "Google Pixel 9 Pro-15.0"] + run: | + bash .github/workflows/scripts/browserstack-app-automate.sh + deploy-playstore: needs: - build-android-apk diff --git a/.github/workflows/android-test-browserstack.yml b/.github/workflows/android-test-browserstack.yml deleted file mode 100644 index 1ad3d8308..000000000 --- a/.github/workflows/android-test-browserstack.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Android Test - -on: - workflow_dispatch: - inputs: - build_number: - description: 'Build or run number' - required: true - type: number - -jobs: - test-android-apk-unfied: - runs-on: ubuntu-22.04 - env: - MAIN_APK_NAME: test-main-unified-${{ inputs.build_number }}.apk - HELPER_APK_NAME: test-helper-unified-${{ inputs.build_number }}.apk - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - - name: Download test APK - run: | - gsutil cp $GCLOUD_BUCKET_PATH/test-main-unified.apk /tmp/$MAIN_APK_NAME - gsutil cp $GCLOUD_BUCKET_PATH/test-helper-unified.apk /tmp/$HELPER_APK_NAME - - name: Upload main app - run: | - curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \ - -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/app" \ - -F "file=@/tmp/$MAIN_APK_NAME" \ - -F "custom_id=$MAIN_APK_NAME" - - name: Upload test suite - run: | - curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \ - -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/test-suite" \ - -F "file=@/tmp/$HELPER_APK_NAME" \ - -F "custom_id=$HELPER_APK_NAME" - - name: Trigger App Automate - env: - BROWSERSTACK_CREDENTIALS: ${{ secrets.BROWSERSTACK_CREDENTIALS }} - BROWSERSTACK_APP: ${{ env.MAIN_APK_NAME }} - BROWSERSTACK_TEST_SUITE: ${{ env.HELPER_APK_NAME }} - BROWSERSTACK_BUILD_TAG: ${{ inputs.build_number }} - BROWSERSTACK_DEVICES: '["Samsung Galaxy S24-14.0", "Samsung Galaxy S24 Ultra-14.0", "Google Pixel 9 Pro-15.0"]' - run: | - bash .github/workflows/scripts/browserstack-app-automate.sh From a9d06b7362f09ac441601517634e7e44e55bfb77 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 15:49:29 +0700 Subject: [PATCH 05/19] Fix test-android-apk-unified --- .github/workflows/android-build-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index bb0a25dc9..b2f9ebe90 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -389,7 +389,9 @@ jobs: bash .github/workflows/scripts/browserstack-app-automate.sh test-android-apk-unified: + needs: build-android-apk runs-on: ubuntu-22.04 + timeout-minutes: 60 env: MAIN_APK_NAME: test-main-unified-${{ github.run_number }}.apk HELPER_APK_NAME: test-helper-unified-${{ github.run_number }}.apk From e0a32ae28d0185afdda803c8d87d3f5c3008ff95 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 17:34:44 +0700 Subject: [PATCH 06/19] Retry step trigger app automate --- .github/workflows/android-build-test.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index b2f9ebe90..bacbfb684 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -378,15 +378,19 @@ jobs: -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/test-suite" \ -F "file=@/tmp/$HELPER_APK_NAME" \ -F "custom_id=$HELPER_APK_NAME" - - name: Trigger App Automate for [Samsung Galaxy S24] + - uses: nick-fields/retry@v3 + name: Trigger App Automate for [Samsung Galaxy S24] env: BROWSERSTACK_CREDENTIALS: ${{ secrets.BROWSERSTACK_CREDENTIALS }} BROWSERSTACK_APP: ${{ env.MAIN_APK_NAME }} BROWSERSTACK_TEST_SUITE: ${{ env.HELPER_APK_NAME }} BROWSERSTACK_BUILD_TAG: ${{ github.run_number }} BROWSERSTACK_DEVICES: '["Samsung Galaxy S24-14.0"]' - run: | - bash .github/workflows/scripts/browserstack-app-automate.sh + with: + timeout_minutes: 45 + max_attempts: 2 + command: | + bash .github/workflows/scripts/browserstack-app-automate.sh test-android-apk-unified: needs: build-android-apk From befab3e5eca0c1be3ad01771f8be35a317da033f Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 17:40:19 +0700 Subject: [PATCH 07/19] Add expected throughput and accuracy for _kPixel9Pro --- flutter/integration_test/expected_throughput.dart | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index bc2b36c72..85d70f8ac 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -23,6 +23,7 @@ const _kRyzen5600 = 'AMD Ryzen 5 5600X 6-Core'; // Android const _kPixel5 = 'Pixel 5'; // Google Pixel 5 const _kPixel6 = 'Pixel 6'; // Google Pixel 6 +const _kPixel9Pro = 'Pixel 9 Pro'; // Google Pixel 9 Pro const _kS22Ultra = 'SM-S908U1'; // Galaxy S22 Ultra const _kDN2103 = 'DN2103'; // OnePlus DN2103 const _kS24 = 'SM-S921B'; // Galaxy S24 @@ -46,6 +47,7 @@ const Map> _imageClassificationV2 = { }, _kPixelBackend: { _kPixel6: Interval(min: 100, max: 600), + _kPixel9Pro: Interval(min: 100, max: 600), }, _kQtiBackend: { _kS22Ultra: Interval(min: 250, max: 400), @@ -73,6 +75,7 @@ const Map> _objectDetection = { }, _kPixelBackend: { _kPixel6: Interval(min: 200, max: 500), + _kPixel9Pro: Interval(min: 200, max: 500), }, _kQtiBackend: { _kS22Ultra: Interval(min: 700, max: 1400), @@ -100,6 +103,7 @@ const Map> _imageSegmentationV2 = { }, _kPixelBackend: { _kPixel6: Interval(min: 100, max: 190), + _kPixel9Pro: Interval(min: 200, max: 500), }, _kQtiBackend: { _kS22Ultra: Interval(min: 400, max: 750), @@ -127,6 +131,7 @@ const Map> _naturalLanguageProcessing = { }, _kPixelBackend: { _kPixel6: Interval(min: 2, max: 85), + _kPixel9Pro: Interval(min: 60, max: 150), }, _kQtiBackend: { _kS22Ultra: Interval(min: 100, max: 200), @@ -154,6 +159,7 @@ const Map> _superResolution = { }, _kPixelBackend: { _kPixel6: Interval(min: 7, max: 17), + _kPixel9Pro: Interval(min: 30, max: 70), }, _kQtiBackend: { _kS22Ultra: Interval(min: 25, max: 75), @@ -182,6 +188,7 @@ const Map> _stableDiffusion = { }, _kPixelBackend: { _kPixel6: Interval(min: 0, max: 100), + _kPixel9Pro: Interval(min: 0, max: 100), }, _kQtiBackend: { _kS22Ultra: Interval(min: 0, max: 100), @@ -209,6 +216,7 @@ const Map> _imageClassificationOfflineV2 = { }, _kPixelBackend: { _kPixel6: Interval(min: 100, max: 700), + _kPixel9Pro: Interval(min: 300, max: 700), }, _kQtiBackend: { _kS22Ultra: Interval(min: 250, max: 450), From 23ed0a2a1e86d614d33fc78b288213cb909d1a13 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 17:40:33 +0700 Subject: [PATCH 08/19] Update expected throughput for _kS24 --- flutter/integration_test/expected_throughput.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 85d70f8ac..f7970cbe9 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -84,7 +84,7 @@ const Map> _objectDetection = { _kDN2103: Interval(min: 120, max: 210), }, _kSamsungBackend: { - _kS24: Interval(min: 1500, max: 2400), + _kS24: Interval(min: 1400, max: 2400), }, }; @@ -112,7 +112,7 @@ const Map> _imageSegmentationV2 = { _kDN2103: Interval(min: 45, max: 70), }, _kSamsungBackend: { - _kS24: Interval(min: 1000, max: 1500), + _kS24: Interval(min: 900, max: 1500), }, }; From ff09759da4b1894f6c0ce65a3696712ef22f415b Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 18:06:57 +0700 Subject: [PATCH 09/19] Add expected throughput and accuracy for _kS10Plus --- .github/workflows/android-build-test.yml | 1 + .../integration_test/expected_accuracy.dart | 4 ++-- .../integration_test/expected_throughput.dart | 18 ++++++++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index bacbfb684..3b0f86f38 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -434,6 +434,7 @@ jobs: BROWSERSTACK_DEVICES: >- ["Samsung Galaxy S24-14.0", "Samsung Galaxy S24 Ultra-14.0", + "Samsung Galaxy Tab S10 Plus-14.0", "Google Pixel 9 Pro-15.0"] run: | bash .github/workflows/scripts/browserstack-app-automate.sh diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index d27ed14ef..34581c1a3 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -25,7 +25,7 @@ const Map _imageClassificationV2 = { 'cpu&gpu&ane': Interval(min: 0.69, max: 0.91), 'snpe_dsp': Interval(min: 0.88, max: 0.91), 'psnpe_dsp': Interval(min: 0.88, max: 0.91), - 'neuron-mdla': Interval(min: 0.79, max: 0.81), + 'neuron-mdla': Interval(min: 0.79, max: 0.91), 'samsung_npu': Interval(min: 0.99, max: 1.0), }; @@ -38,7 +38,7 @@ const Map _objectDetection = { 'cpu&gpu&ane': Interval(min: 0.45, max: 0.46), 'snpe_dsp': Interval(min: 0.32, max: 0.35), 'psnpe_dsp': Interval(min: 0.32, max: 0.35), - 'neuron': Interval(min: 0.28, max: 0.31), + 'neuron': Interval(min: 0.28, max: 0.35), 'samsung_npu': Interval(min: 0.36, max: 0.39), }; diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index f7970cbe9..ac9aee116 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -20,13 +20,16 @@ const _kCloudBuildX23 = 'Intel Xeon 2.30GHz'; const _kCloudBuildX28 = 'Intel Xeon 2.80GHz'; const _kRyzen5600 = 'AMD Ryzen 5 5600X 6-Core'; -// Android +// Android devices on Firebase TestLab const _kPixel5 = 'Pixel 5'; // Google Pixel 5 const _kPixel6 = 'Pixel 6'; // Google Pixel 6 -const _kPixel9Pro = 'Pixel 9 Pro'; // Google Pixel 9 Pro -const _kS22Ultra = 'SM-S908U1'; // Galaxy S22 Ultra +const _kS22Ultra = 'SM-S908U1'; // Samsung Galaxy S22 Ultra const _kDN2103 = 'DN2103'; // OnePlus DN2103 -const _kS24 = 'SM-S921B'; // Galaxy S24 + +// Android devices on BrowserStack App Automate +const _kPixel9Pro = 'Pixel 9 Pro'; // Google Pixel 9 Pro +const _kS24 = 'SM-S921B'; // Samsung Galaxy S24 +const _kS10Plus = 'SM-X826B'; // Samsung Galaxy Tab S10 Plus // iOS const _kIphoneOnGitHubAction = 'iPhone16,2'; @@ -54,6 +57,7 @@ const Map> _imageClassificationV2 = { }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 90), + _kS10Plus: Interval(min: 400, max: 800) }, _kSamsungBackend: { _kS24: Interval(min: 700, max: 1000), @@ -82,6 +86,7 @@ const Map> _objectDetection = { }, _kMediatekBackend: { _kDN2103: Interval(min: 120, max: 210), + _kS10Plus: Interval(min: 1200, max: 1800) }, _kSamsungBackend: { _kS24: Interval(min: 1400, max: 2400), @@ -110,6 +115,7 @@ const Map> _imageSegmentationV2 = { }, _kMediatekBackend: { _kDN2103: Interval(min: 45, max: 70), + _kS10Plus: Interval(min: 800, max: 1400) }, _kSamsungBackend: { _kS24: Interval(min: 900, max: 1500), @@ -138,6 +144,7 @@ const Map> _naturalLanguageProcessing = { }, _kMediatekBackend: { _kDN2103: Interval(min: 1, max: 6), + _kS10Plus: Interval(min: 100, max: 300) }, _kSamsungBackend: { _kS24: Interval(min: 220, max: 350), @@ -166,6 +173,7 @@ const Map> _superResolution = { }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 15), + _kS10Plus: Interval(min: 150, max: 250) }, _kSamsungBackend: { _kS24: Interval(min: 100, max: 180), @@ -195,6 +203,7 @@ const Map> _stableDiffusion = { }, _kMediatekBackend: { _kDN2103: Interval(min: 0, max: 100), + _kS10Plus: Interval(min: 0, max: 100) }, _kSamsungBackend: { _kS24: Interval(min: 0, max: 100), @@ -223,6 +232,7 @@ const Map> _imageClassificationOfflineV2 = { }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 90), + _kS10Plus: Interval(min: 700, max: 1200) }, _kSamsungBackend: { _kS24: Interval(min: 900, max: 1200), From 83172ce0c5bf21afc8cb6f35ab9454a6cff47ebd Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 18:25:39 +0700 Subject: [PATCH 10/19] Add expected throughput and accuracy for _kS24Ultra --- flutter/integration_test/expected_throughput.dart | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index ac9aee116..ed400d596 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -29,6 +29,7 @@ const _kDN2103 = 'DN2103'; // OnePlus DN2103 // Android devices on BrowserStack App Automate const _kPixel9Pro = 'Pixel 9 Pro'; // Google Pixel 9 Pro const _kS24 = 'SM-S921B'; // Samsung Galaxy S24 +const _kS24Ultra = 'SM-S928B'; // Samsung Galaxy S24 Ultra const _kS10Plus = 'SM-X826B'; // Samsung Galaxy Tab S10 Plus // iOS @@ -54,6 +55,7 @@ const Map> _imageClassificationV2 = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 250, max: 400), + _kS24Ultra: Interval(min: 800, max: 1400), }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 90), @@ -83,6 +85,7 @@ const Map> _objectDetection = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 700, max: 1400), + _kS24Ultra: Interval(min: 1800, max: 2400), }, _kMediatekBackend: { _kDN2103: Interval(min: 120, max: 210), @@ -112,6 +115,7 @@ const Map> _imageSegmentationV2 = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 400, max: 750), + _kS24Ultra: Interval(min: 1200, max: 1600), }, _kMediatekBackend: { _kDN2103: Interval(min: 45, max: 70), @@ -141,6 +145,7 @@ const Map> _naturalLanguageProcessing = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 100, max: 200), + _kS24Ultra: Interval(min: 250, max: 450), }, _kMediatekBackend: { _kDN2103: Interval(min: 1, max: 6), @@ -170,6 +175,7 @@ const Map> _superResolution = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 25, max: 75), + _kS24Ultra: Interval(min: 120, max: 180), }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 15), @@ -200,6 +206,7 @@ const Map> _stableDiffusion = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 0, max: 100), + _kS24Ultra: Interval(min: 0, max: 100), }, _kMediatekBackend: { _kDN2103: Interval(min: 0, max: 100), @@ -229,6 +236,7 @@ const Map> _imageClassificationOfflineV2 = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 250, max: 450), + _kS24Ultra: Interval(min: 900, max: 1400), }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 90), From c44ca63fbb46bd2d0b07aa0c69e1bb5d3a8c3ab3 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 19:41:32 +0700 Subject: [PATCH 11/19] Add expected throughput and accuracy for _kM32 --- .github/workflows/android-build-test.yml | 3 ++- flutter/integration_test/expected_accuracy.dart | 2 +- flutter/integration_test/expected_throughput.dart | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 3b0f86f38..0531b0361 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -435,7 +435,8 @@ jobs: ["Samsung Galaxy S24-14.0", "Samsung Galaxy S24 Ultra-14.0", "Samsung Galaxy Tab S10 Plus-14.0", - "Google Pixel 9 Pro-15.0"] + "Google Pixel 9 Pro-15.0", + "Samsung Galaxy M32-11.0"] run: | bash .github/workflows/scripts/browserstack-app-automate.sh diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index 34581c1a3..eb24327b2 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -19,7 +19,7 @@ key: OR | const Map _imageClassificationV2 = { 'cpu': Interval(min: 0.88, max: 0.91), - 'npu': Interval(min: 0.88, max: 0.91), + 'npu': Interval(min: 0.69, max: 0.91), 'tpu': Interval(min: 0.88, max: 0.91), 'ane': Interval(min: 0.69, max: 0.91), 'cpu&gpu&ane': Interval(min: 0.69, max: 0.91), diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index ed400d596..16dc19167 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -31,6 +31,7 @@ const _kPixel9Pro = 'Pixel 9 Pro'; // Google Pixel 9 Pro const _kS24 = 'SM-S921B'; // Samsung Galaxy S24 const _kS24Ultra = 'SM-S928B'; // Samsung Galaxy S24 Ultra const _kS10Plus = 'SM-X826B'; // Samsung Galaxy Tab S10 Plus +const _kM32 = 'SM-M326B'; // Samsung Galaxy M32 // iOS const _kIphoneOnGitHubAction = 'iPhone16,2'; @@ -43,6 +44,7 @@ const Map> _imageClassificationV2 = { _kRyzen5600: Interval(min: 1, max: 37), _kPixel5: Interval(min: 20, max: 75), _kPixel6: Interval(min: 100, max: 600), + _kM32: Interval(min: 5, max: 15), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 9), _kIphoneOnMacbookM1: Interval(min: 10, max: 27), }, @@ -73,6 +75,7 @@ const Map> _objectDetection = { _kRyzen5600: Interval(min: 14, max: 22), _kPixel5: Interval(min: 40, max: 60), _kPixel6: Interval(min: 100, max: 450), + _kM32: Interval(min: 20, max: 40), _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4), _kIphoneOnMacbookM1: Interval(min: 9, max: 16), }, @@ -103,6 +106,7 @@ const Map> _imageSegmentationV2 = { _kRyzen5600: Interval(min: 5, max: 7), _kPixel5: Interval(min: 25, max: 40), _kPixel6: Interval(min: 80, max: 180), + _kM32: Interval(min: 2, max: 10), _kIphoneOnGitHubAction: Interval(min: 0.1, max: 2.5), _kIphoneOnMacbookM1: Interval(min: 3, max: 6), }, @@ -133,6 +137,7 @@ const Map> _naturalLanguageProcessing = { _kRyzen5600: Interval(min: 2.8, max: 3.2), _kPixel5: Interval(min: 2.3, max: 3.0), _kPixel6: Interval(min: 2, max: 75), + _kM32: Interval(min: 2, max: 5), _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1), _kIphoneOnMacbookM1: Interval(min: 1.8, max: 3), }, @@ -163,6 +168,7 @@ const Map> _superResolution = { _kRyzen5600: Interval(min: 0.1, max: 3), _kPixel5: Interval(min: 4, max: 8), _kPixel6: Interval(min: 7, max: 14), + _kM32: Interval(min: 0.1, max: 3), _kIphoneOnGitHubAction: Interval(min: 0.02, max: 1.0), _kIphoneOnMacbookM1: Interval(min: 0.1, max: 10), }, @@ -194,6 +200,7 @@ const Map> _stableDiffusion = { _kRyzen5600: Interval(min: 0, max: 100), _kPixel5: Interval(min: 0, max: 100), _kPixel6: Interval(min: 0, max: 100), + _kM32: Interval(min: 0, max: 100), _kIphoneOnGitHubAction: Interval(min: 0, max: 100), _kIphoneOnMacbookM1: Interval(min: 0, max: 100), }, @@ -224,6 +231,7 @@ const Map> _imageClassificationOfflineV2 = { _kRyzen5600: Interval(min: 20, max: 60), _kPixel5: Interval(min: 20, max: 180), _kPixel6: Interval(min: 100, max: 700), + _kM32: Interval(min: 8, max: 18), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), _kIphoneOnMacbookM1: Interval(min: 10, max: 45), }, From ba52d180f1c0459e380a450247dd98be07de1141 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 10 Dec 2024 19:43:47 +0700 Subject: [PATCH 12/19] Update retry@v2 to retry@v3 --- .github/workflows/ios-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ios-build-test.yml b/.github/workflows/ios-build-test.yml index 4dbc67bef..57e2898a0 100644 --- a/.github/workflows/ios-build-test.yml +++ b/.github/workflows/ios-build-test.yml @@ -59,7 +59,7 @@ jobs: - name: Build iOS integration tests run: | cd flutter && flutter --no-version-check build ios --simulator integration_test/first_test.dart - - uses: nick-fields/retry@v2 + - uses: nick-fields/retry@v3 name: Run iOS integration tests with: timeout_minutes: 45 From 947750b590124f0c95b76f050966bfe9a6c49ea3 Mon Sep 17 00:00:00 2001 From: Anh Date: Wed, 11 Dec 2024 08:31:02 +0700 Subject: [PATCH 13/19] Update expected_throughput --- flutter/integration_test/expected_throughput.dart | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 16dc19167..3159c5872 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -57,14 +57,14 @@ const Map> _imageClassificationV2 = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 250, max: 400), - _kS24Ultra: Interval(min: 800, max: 1400), + _kS24Ultra: Interval(min: 800, max: 1500), }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 90), _kS10Plus: Interval(min: 400, max: 800) }, _kSamsungBackend: { - _kS24: Interval(min: 700, max: 1000), + _kS24: Interval(min: 600, max: 1000), }, }; @@ -88,7 +88,7 @@ const Map> _objectDetection = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 700, max: 1400), - _kS24Ultra: Interval(min: 1800, max: 2400), + _kS24Ultra: Interval(min: 1800, max: 2500), }, _kMediatekBackend: { _kDN2103: Interval(min: 120, max: 210), @@ -126,7 +126,7 @@ const Map> _imageSegmentationV2 = { _kS10Plus: Interval(min: 800, max: 1400) }, _kSamsungBackend: { - _kS24: Interval(min: 900, max: 1500), + _kS24: Interval(min: 800, max: 1500), }, }; @@ -188,7 +188,7 @@ const Map> _superResolution = { _kS10Plus: Interval(min: 150, max: 250) }, _kSamsungBackend: { - _kS24: Interval(min: 100, max: 180), + _kS24: Interval(min: 90, max: 180), }, }; @@ -244,14 +244,14 @@ const Map> _imageClassificationOfflineV2 = { }, _kQtiBackend: { _kS22Ultra: Interval(min: 250, max: 450), - _kS24Ultra: Interval(min: 900, max: 1400), + _kS24Ultra: Interval(min: 900, max: 1600), }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 90), _kS10Plus: Interval(min: 700, max: 1200) }, _kSamsungBackend: { - _kS24: Interval(min: 900, max: 1200), + _kS24: Interval(min: 800, max: 1200), }, }; From 5c60cf071b657a2086559268c354ee4511444387 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 17 Dec 2024 21:25:40 +0700 Subject: [PATCH 14/19] Remove single backend APK test --- .github/workflows/android-build-test.yml | 149 +---------------------- 1 file changed, 1 insertion(+), 148 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 0531b0361..a095d3f53 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -245,153 +245,6 @@ jobs: retention-days: 28 if-no-files-found: error - test-android-apk-tflite: - needs: build-android-apk - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - project_id: mobile-app-build-290400 - - name: Trigger Firebase Test Lab for [Google Pixel 6](oriole) - run: | - gcloud firebase test android run \ - --type instrumentation \ - --app $GCLOUD_BUCKET_PATH/test-main-tflite.apk \ - --test $GCLOUD_BUCKET_PATH/test-helper-tflite.apk \ - --timeout 30m \ - --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=oriole,version=31,locale=en,orientation=portrait \ - --client-details=buildNumber=${{ github.run_number }} - - test-android-apk-pixel: - needs: build-android-apk - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - project_id: mobile-app-build-290400 - - name: Trigger Firebase Test Lab for [Google Pixel 6](oriole) - run: | - gcloud firebase test android run \ - --type instrumentation \ - --app $GCLOUD_BUCKET_PATH/test-main-pixel.apk \ - --test $GCLOUD_BUCKET_PATH/test-helper-pixel.apk \ - --timeout 30m \ - --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=oriole,version=32,locale=en,orientation=portrait \ - --client-details=buildNumber=${{ github.run_number }} - - test-android-apk-qti: - needs: build-android-apk - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - project_id: mobile-app-build-290400 - - name: Trigger Firebase Test Lab for [Samsung Galaxy S22 Ultra](b0q) - run: | - gcloud firebase test android run \ - --type instrumentation \ - --app $GCLOUD_BUCKET_PATH/test-main-qti.apk \ - --test $GCLOUD_BUCKET_PATH/test-helper-qti.apk \ - --timeout 30m \ - --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=b0q,version=33,locale=en,orientation=portrait \ - --client-details=buildNumber=${{ github.run_number }} - - test-android-apk-mtk: - needs: build-android-apk - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - project_id: mobile-app-build-290400 - - name: Trigger Firebase Test Lab for [OnePlus DN2103](OP515BL1) - run: | - gcloud firebase test android run \ - --type instrumentation \ - --app $GCLOUD_BUCKET_PATH/test-main-mtk.apk \ - --test $GCLOUD_BUCKET_PATH/test-helper-mtk.apk \ - --timeout 30m \ - --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=OP515BL1,version=33,locale=en,orientation=portrait \ - --client-details=buildNumber=${{ github.run_number }} - - test-android-apk-samsung: - needs: build-android-apk - runs-on: ubuntu-22.04 - timeout-minutes: 30 - env: - MAIN_APK_NAME: test-main-samsung-${{ github.run_number }}.apk - HELPER_APK_NAME: test-helper-samsung-${{ github.run_number }}.apk - steps: - - uses: actions/checkout@v4 - - name: Set up authentication for Google Cloud SDK - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - with: - version: '>= 363.0.0' - - name: Download test APK - run: | - gsutil cp $GCLOUD_BUCKET_PATH/test-main-samsung.apk /tmp/$MAIN_APK_NAME - gsutil cp $GCLOUD_BUCKET_PATH/test-helper-samsung.apk /tmp/$HELPER_APK_NAME - - name: Upload main app - run: | - curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \ - -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/app" \ - -F "file=@/tmp/$MAIN_APK_NAME" \ - -F "custom_id=$MAIN_APK_NAME" - - name: Upload test suite - run: | - curl -u "${{ secrets.BROWSERSTACK_CREDENTIALS }}" \ - -X POST "https://api-cloud.browserstack.com/app-automate/flutter-integration-tests/v2/android/test-suite" \ - -F "file=@/tmp/$HELPER_APK_NAME" \ - -F "custom_id=$HELPER_APK_NAME" - - uses: nick-fields/retry@v3 - name: Trigger App Automate for [Samsung Galaxy S24] - env: - BROWSERSTACK_CREDENTIALS: ${{ secrets.BROWSERSTACK_CREDENTIALS }} - BROWSERSTACK_APP: ${{ env.MAIN_APK_NAME }} - BROWSERSTACK_TEST_SUITE: ${{ env.HELPER_APK_NAME }} - BROWSERSTACK_BUILD_TAG: ${{ github.run_number }} - BROWSERSTACK_DEVICES: '["Samsung Galaxy S24-14.0"]' - with: - timeout_minutes: 45 - max_attempts: 2 - command: | - bash .github/workflows/scripts/browserstack-app-automate.sh - test-android-apk-unified: needs: build-android-apk runs-on: ubuntu-22.04 @@ -443,7 +296,7 @@ jobs: deploy-playstore: needs: - build-android-apk - - test-android-apk-tflite + - test-android-apk-unified runs-on: ubuntu-22.04 # if: github.ref == 'refs/heads/master' steps: From 6190c5d8312637d966dda26d484f5fbd558f70c5 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 17 Dec 2024 22:12:29 +0700 Subject: [PATCH 15/19] Fix issue with not showing error message --- .../scripts/browserstack-app-automate.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scripts/browserstack-app-automate.sh b/.github/workflows/scripts/browserstack-app-automate.sh index 86059a277..779de276d 100644 --- a/.github/workflows/scripts/browserstack-app-automate.sh +++ b/.github/workflows/scripts/browserstack-app-automate.sh @@ -2,8 +2,6 @@ # This script is used to trigger a build on BrowserStack App Automate and monitor its status. -set -e - # Build parameters PROJECT="mobile-app-build-290400" DEVICE_LOGS=true @@ -51,10 +49,11 @@ trigger_build() { if [[ "$build_id" == "null" || -z "$build_id" ]]; then echo "Failed to trigger the build. Response: $response" - exit 1 + return 1 + else + echo "$build_id" + return 0 fi - - echo "$build_id" } # Function to check build status @@ -73,7 +72,11 @@ check_build_status() { } # Main -BUILD_ID=$(trigger_build) +if ! BUILD_ID=$(trigger_build); then + echo "Trigger build failed. Message: $BUILD_ID" + exit 1 +fi + echo "Build triggered successfully. Build ID: $BUILD_ID" echo "See the build status at: https://app-automate.browserstack.com/dashboard/v2/builds/$BUILD_ID" From 5c0e09a574b702c141f61e2700069af369f43a72 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 17 Dec 2024 22:34:06 +0700 Subject: [PATCH 16/19] Display device status --- .../workflows/scripts/browserstack-app-automate.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/scripts/browserstack-app-automate.sh b/.github/workflows/scripts/browserstack-app-automate.sh index 779de276d..a3400aac5 100644 --- a/.github/workflows/scripts/browserstack-app-automate.sh +++ b/.github/workflows/scripts/browserstack-app-automate.sh @@ -61,7 +61,19 @@ check_build_status() { local build_id=$1 local response=$(curl -s -u "$CREDENTIALS" -X GET "$STATUS_URL/$build_id") local status=$(echo "$response" | jq -r '.status') + echo "$(date +'%Y-%m-%d %H:%M:%S') Build Status: $status" + + # Display device status + echo "$response" | jq -r ' + .devices[] | + "Device: " + .device + + ", OS Version: " + .os_version + + ", Duration: " + (.sessions[0].duration | tostring) + "s" + + ", Status: " + (.sessions[0].status) + ' + + # Display build status if [[ "$status" == "passed" ]]; then echo "Build completed successfully!" exit 0 From 5fea493068624f14bcd19957e5df078655a47131 Mon Sep 17 00:00:00 2001 From: Anh Date: Sat, 28 Dec 2024 08:17:45 +0700 Subject: [PATCH 17/19] Change run order --- .github/workflows/android-build-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index a095d3f53..1d5fd62d1 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -285,11 +285,11 @@ jobs: BROWSERSTACK_TEST_SUITE: ${{ env.HELPER_APK_NAME }} BROWSERSTACK_BUILD_TAG: ${{ github.run_number }} BROWSERSTACK_DEVICES: >- - ["Samsung Galaxy S24-14.0", + ["Samsung Galaxy M32-11.0", + "Samsung Galaxy S24-14.0", "Samsung Galaxy S24 Ultra-14.0", "Samsung Galaxy Tab S10 Plus-14.0", - "Google Pixel 9 Pro-15.0", - "Samsung Galaxy M32-11.0"] + "Google Pixel 9 Pro-15.0"] run: | bash .github/workflows/scripts/browserstack-app-automate.sh From 8c01417afd3aa336db8be40a5d5a597886914648 Mon Sep 17 00:00:00 2001 From: Anh Date: Sat, 28 Dec 2024 09:24:41 +0700 Subject: [PATCH 18/19] Update expected values --- flutter/integration_test/expected_accuracy.dart | 2 +- flutter/integration_test/expected_throughput.dart | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index eb24327b2..c1d37a2e0 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -78,7 +78,7 @@ const Map _superResolution = { 'snpe_dsp': Interval(min: 0.32, max: 0.35), 'psnpe_dsp': Interval(min: 0.32, max: 0.35), 'neuron': Interval(min: 0.32, max: 0.35), - 'samsung_npu': Interval(min: 0.08, max: 0.11), + 'samsung_npu': Interval(min: 0.08, max: 0.12), }; // TODO (anhappdev): update expected accuracy for stable diffusion diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 3159c5872..1b9e416a1 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -185,7 +185,7 @@ const Map> _superResolution = { }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 15), - _kS10Plus: Interval(min: 150, max: 250) + _kS10Plus: Interval(min: 150, max: 280) }, _kSamsungBackend: { _kS24: Interval(min: 90, max: 180), From 8f2cd637fe2386806894d3467a5c7a06b2c69974 Mon Sep 17 00:00:00 2001 From: Anh Date: Sat, 28 Dec 2024 09:33:53 +0700 Subject: [PATCH 19/19] Update browserstack-app-automate.sh --- .../scripts/browserstack-app-automate.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/scripts/browserstack-app-automate.sh b/.github/workflows/scripts/browserstack-app-automate.sh index a3400aac5..ed6b2c438 100644 --- a/.github/workflows/scripts/browserstack-app-automate.sh +++ b/.github/workflows/scripts/browserstack-app-automate.sh @@ -65,13 +65,15 @@ check_build_status() { echo "$(date +'%Y-%m-%d %H:%M:%S') Build Status: $status" # Display device status - echo "$response" | jq -r ' - .devices[] | - "Device: " + .device + - ", OS Version: " + .os_version + - ", Duration: " + (.sessions[0].duration | tostring) + "s" + - ", Status: " + (.sessions[0].status) - ' + if [[ "$status" != "running" ]]; then + echo "$response" | jq -r ' + .devices[] | + "Device: " + .device + + ", OS Version: " + .os_version + + ", Duration: " + (.sessions[0].duration | tostring) + "s" + + ", Status: " + (.sessions[0].status) + ' + fi # Display build status if [[ "$status" == "passed" ]]; then