chore: changes for submission v4.1 (#916)

* Update settings for stable_diffusion task

* Remove image_classification V1 task

* Remove BenchmarkId: imageClassification and imageClassificationOffline

Author: anhappdev

flutter/assets/tasks.pbtxt

@@ -216,86 +216,12 @@ task {
   }
 }
 
-task {
-  id: "image_classification"
-  name: "Image Classification"
-  min_query_count: 1024
-  min_duration: 60
-  max_duration: 600
-  max_throughput: 1000
-  max_accuracy: 1.0
-  scenario: "SingleStream"
-  datasets {
-    type: IMAGENET
-    full {
-      name: "Imagenet classification validation set"
-      input_path: "local:///mlperf_datasets/imagenet/img"
-      groundtruth_path: "local:///mlperf_datasets/imagenet/imagenet_val_full.txt"
-    }
-    lite {
-      name: "Open images subset for classification"
-      input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet.zip"
-      groundtruth_path: ""
-    }
-    tiny {
-      name: "Imagenet dataset for integration test"
-      input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet_tiny.zip"
-      groundtruth_path: "https://github.com/mlcommons/mobile_models/raw/main/v3_0/assets/imagenet_tiny-groundtruth.txt"
-    }
-  }
-  model {
-    id: "mobilenetedgetpu_224_v1_0"
-    name: "MobileNetEdgeTPU 224 1.0"
-    offset: 1
-    image_width: 224
-    image_height: 224
-    num_classes: 1000
-  }
-}
-
-task {
-  id: "image_classification_offline"
-  name: "Image Classification (Offline)"
-  min_query_count: 24576
-  min_duration: 0
-  max_duration: 0
-  max_throughput: 2000
-  max_accuracy: 1.0
-  scenario: "Offline"
-  datasets {
-    type: IMAGENET
-    full {
-      name: "Imagenet classification validation set"
-      input_path: "local:///mlperf_datasets/imagenet/img"
-      groundtruth_path: "local:///mlperf_datasets/imagenet/imagenet_val_full.txt"
-    }
-    lite {
-      name: "Open images subset for classification"
-      input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet.zip"
-      groundtruth_path: ""
-    }
-    tiny {
-      name: "Imagenet dataset for integration test"
-      input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet_tiny.zip"
-      groundtruth_path: "https://github.com/mlcommons/mobile_models/raw/main/v3_0/assets/imagenet_tiny-groundtruth.txt"
-    }
-  }
-  model {
-    id: "mobilenetedgetpu_224_v1_0"
-    name: "MobileNetEdgeTPU 224 1.0"
-    offset: 1
-    image_width: 224
-    image_height: 224
-    num_classes: 1000
-  }
-}
-
 task {
   id: "stable_diffusion"
   name: "Stable Diffusion"
-  min_query_count: 5
+  min_query_count: 1024
   min_duration: 60
-  max_duration: 600
+  max_duration: 300
   max_throughput: 2000
   max_accuracy: 1.0
   scenario: "SingleStream"

flutter/integration_test/expected_accuracy.dart

@@ -16,17 +16,6 @@ key: <accelerator> OR <accelerator>|<backendName>
 - neuron > Android MediaTek
 */
 
-const Map<String, Interval> _imageClassification = {
-  'cpu': Interval(min: 1.00, max: 1.00),
-  'npu': Interval(min: 0.89, max: 0.91),
-  'tpu': Interval(min: 0.89, max: 0.91),
-  'ane': Interval(min: 1.00, max: 1.00),
-  'cpu&gpu&ane': Interval(min: 1.00, max: 1.00),
-  'snpe_dsp': Interval(min: 0.78, max: 0.82),
-  'psnpe_dsp': Interval(min: 0.78, max: 0.82),
-  'neuron-mdla': Interval(min: 0.88, max: 0.91),
-};
-
 const Map<String, Interval> _imageClassificationV2 = {
   'cpu': Interval(min: 0.88, max: 0.91),
   'npu': Interval(min: 0.88, max: 0.91),
@@ -100,13 +89,11 @@ const Map<String, Interval> _stableDiffusion = {
 };
 
 const benchmarkExpectedAccuracy = {
-  BenchmarkId.imageClassification: _imageClassification,
   BenchmarkId.imageClassificationV2: _imageClassificationV2,
   BenchmarkId.objectDetection: _objectDetection,
   BenchmarkId.imageSegmentationV2: _imageSegmentation,
   BenchmarkId.naturalLanguageProcessing: _naturalLanguageProcessing,
   BenchmarkId.superResolution: _superResolution,
   BenchmarkId.stableDiffusion: _stableDiffusion,
-  BenchmarkId.imageClassificationOffline: _imageClassification,
   BenchmarkId.imageClassificationOfflineV2: _imageClassificationV2,
 };

flutter/integration_test/expected_throughput.dart

@@ -29,30 +29,6 @@ const _kDN2103 = 'DN2103'; // OnePlus DN2103
 const _kIphoneOnGitHubAction = 'iPhone15,3';
 const _kIphoneOnMacbookM1 = 'iPhone14,7';
 
-const Map<String, Map<String, Interval>> _imageClassification = {
-  _kTFLiteBackend: {
-    _kCloudBuildX23: Interval(min: 4, max: 12),
-    _kCloudBuildX28: Interval(min: 4, max: 13),
-    _kRyzen5600: Interval(min: 31, max: 37),
-    _kPixel5: Interval(min: 80, max: 130),
-    _kPixel6: Interval(min: 600, max: 1100),
-    _kIphoneOnGitHubAction: Interval(min: 1, max: 8),
-    _kIphoneOnMacbookM1: Interval(min: 19, max: 27),
-  },
-  _kCoreMLBackend: {
-    _kIphoneOnGitHubAction: Interval(min: 1, max: 12),
-  },
-  _kPixelBackend: {
-    _kPixel6: Interval(min: 700, max: 1200),
-  },
-  _kQtiBackend: {
-    _kS22Ultra: Interval(min: 1600, max: 2400),
-  },
-  _kMediatekBackend: {
-    _kDN2103: Interval(min: 30, max: 55),
-  },
-};
-
 const Map<String, Map<String, Interval>> _imageClassificationV2 = {
   _kTFLiteBackend: {
     _kCloudBuildX23: Interval(min: 1, max: 9),
@@ -198,30 +174,6 @@ const Map<String, Map<String, Interval>> _stableDiffusion = {
   },
 };
 
-const Map<String, Map<String, Interval>> _imageClassificationOffline = {
-  _kTFLiteBackend: {
-    _kCloudBuildX23: Interval(min: 8, max: 14),
-    _kCloudBuildX28: Interval(min: 7, max: 16),
-    _kRyzen5600: Interval(min: 45, max: 60),
-    _kPixel5: Interval(min: 120, max: 190),
-    _kPixel6: Interval(min: 800, max: 1700),
-    _kIphoneOnGitHubAction: Interval(min: 2, max: 15),
-    _kIphoneOnMacbookM1: Interval(min: 30, max: 45),
-  },
-  _kCoreMLBackend: {
-    _kIphoneOnGitHubAction: Interval(min: 2, max: 20),
-  },
-  _kPixelBackend: {
-    _kPixel6: Interval(min: 900, max: 1800),
-  },
-  _kQtiBackend: {
-    _kS22Ultra: Interval(min: 2400, max: 3500),
-  },
-  _kMediatekBackend: {
-    _kDN2103: Interval(min: 75, max: 140),
-  },
-};
-
 const Map<String, Map<String, Interval>> _imageClassificationOfflineV2 = {
   _kTFLiteBackend: {
     _kCloudBuildX23: Interval(min: 1, max: 9),
@@ -247,13 +199,11 @@ const Map<String, Map<String, Interval>> _imageClassificationOfflineV2 = {
 };
 
 const benchmarkExpectedThroughput = {
-  BenchmarkId.imageClassification: _imageClassification,
   BenchmarkId.imageClassificationV2: _imageClassificationV2,
   BenchmarkId.objectDetection: _objectDetection,
   BenchmarkId.imageSegmentationV2: _imageSegmentationV2,
   BenchmarkId.naturalLanguageProcessing: _naturalLanguageProcessing,
   BenchmarkId.superResolution: _superResolution,
   BenchmarkId.stableDiffusion: _stableDiffusion,
-  BenchmarkId.imageClassificationOffline: _imageClassificationOffline,
   BenchmarkId.imageClassificationOfflineV2: _imageClassificationOfflineV2,
 };

flutter/lib/app_constants.dart

@@ -16,23 +16,19 @@ class WidgetKeys {
 }
 
 class BenchmarkId {
-  static const imageClassification = 'image_classification';
   static const objectDetection = 'object_detection';
   static const imageSegmentationV2 = 'image_segmentation_v2';
   static const naturalLanguageProcessing = 'natural_language_processing';
   static const superResolution = 'super_resolution';
-  static const imageClassificationOffline = 'image_classification_offline';
   static const imageClassificationV2 = 'image_classification_v2';
   static const imageClassificationOfflineV2 = 'image_classification_offline_v2';
   static const stableDiffusion = 'stable_diffusion';
 
   static const allIds = [
-    imageClassification,
     objectDetection,
     imageSegmentationV2,
     naturalLanguageProcessing,
     superResolution,
-    imageClassificationOffline,
     imageClassificationV2,
     imageClassificationOfflineV2,
     stableDiffusion,

flutter/lib/benchmark/info.dart

@@ -27,18 +27,6 @@ class BenchmarkInfo {
 
   BenchmarkLocalizationInfo getLocalizedInfo(AppLocalizations stringResources) {
     switch (task.id) {
-      case (BenchmarkId.imageClassification):
-        return BenchmarkLocalizationInfo(
-          name: stringResources.benchNameImageClassification,
-          detailsTitle: stringResources.benchInfoImageClassification,
-          detailsContent: stringResources.benchInfoImageClassificationDesc,
-        );
-      case (BenchmarkId.imageClassificationOffline):
-        return BenchmarkLocalizationInfo(
-          name: stringResources.benchNameImageClassificationOffline,
-          detailsTitle: stringResources.benchInfoImageClassification,
-          detailsContent: stringResources.benchInfoImageClassificationDesc,
-        );
       case (BenchmarkId.imageClassificationV2):
         return BenchmarkLocalizationInfo(
           name: stringResources.benchNameImageClassification,

flutter/lib/ui/icons.dart

@@ -66,26 +66,21 @@ class AppIcons {
 
 class BenchmarkIcons {
   static final darkSet = {
-    BenchmarkId.imageClassification: AppIcons.imageClassification,
     BenchmarkId.imageClassificationV2: AppIcons.imageClassification,
     BenchmarkId.objectDetection: AppIcons.objectDetection,
     BenchmarkId.imageSegmentationV2: AppIcons.imageSegmentation,
     BenchmarkId.naturalLanguageProcessing: AppIcons.languageProcessing,
     BenchmarkId.superResolution: AppIcons.superResolution,
-    BenchmarkId.imageClassificationOffline: AppIcons.imageClassificationOffline,
     BenchmarkId.imageClassificationOfflineV2:
         AppIcons.imageClassificationOffline,
   };
 
   static final lightSet = {
-    BenchmarkId.imageClassification: AppIcons.imageClassificationWhite,
     BenchmarkId.imageClassificationV2: AppIcons.imageClassificationWhite,
     BenchmarkId.objectDetection: AppIcons.objectDetectionWhite,
     BenchmarkId.imageSegmentationV2: AppIcons.imageSegmentationWhite,
     BenchmarkId.naturalLanguageProcessing: AppIcons.languageProcessingWhite,
     BenchmarkId.superResolution: AppIcons.superResolutionWhite,
-    BenchmarkId.imageClassificationOffline:
-        AppIcons.imageClassificationOfflineWhite,
     BenchmarkId.imageClassificationOfflineV2:
         AppIcons.imageClassificationOfflineWhite,
   };