[camera_android_camerax] Fix NV21 Format

packages/camera/camera_android_camerax/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.6.22
+
+* Converts NV21-compatible streamed images to NV21 when requested.
+
 ## 0.6.21
 
 * Implements NV21 support for image streaming.

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtils.java

@@ -0,0 +1,75 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// Note: code in this file is directly inspired by the official Google MLKit example:
+// https://github.com/googlesamples/mlkit
+
+package io.flutter.plugins.camerax;
+
+import androidx.annotation.NonNull;
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/* Utitlities for working with {@code ImageProxy}s. */
+public class ImageProxyUtils {
+
+  /**
+   * Converts PlaneProxy[] in YUV_420_888 format (with VU planes in NV21 layout) to a single NV21
+   * ByteBuffer.
+   */
+  @NonNull
+  public static ByteBuffer planesToNV21(@NonNull List<PlaneProxy> planes, int width, int height) {
+    if (!areUVPlanesNV21(planes, width, height)) {
+      throw new IllegalArgumentException(
+          "Provided UV planes are not in NV21 layout and thus cannot be converted.");
+    }
+
+    int imageSize = width * height;
+    int nv21Size = imageSize + 2 * (imageSize / 4);
+    byte[] nv21Bytes = new byte[nv21Size];
+
+    // Copy Y plane.
+    ByteBuffer yBuffer = planes.get(0).getBuffer();
+    yBuffer.rewind();
+    yBuffer.get(nv21Bytes, 0, imageSize);
+
+    // Copy interleaved VU plane (NV21 layout).
+    ByteBuffer vBuffer = planes.get(2).getBuffer();
+    ByteBuffer uBuffer = planes.get(1).getBuffer();
+
+    vBuffer.rewind();
+    uBuffer.rewind();
+    vBuffer.get(nv21Bytes, imageSize, 1);
+    uBuffer.get(nv21Bytes, imageSize + 1, 2 * imageSize / 4 - 1);
+
+    return ByteBuffer.wrap(nv21Bytes);
+  }
+
+  public static boolean areUVPlanesNV21(@NonNull List<PlaneProxy> planes, int width, int height) {
+    int imageSize = width * height;
+
+    ByteBuffer uBuffer = planes.get(1).getBuffer();
+    ByteBuffer vBuffer = planes.get(2).getBuffer();
+
+    // Backup buffer properties.
+    int vBufferPosition = vBuffer.position();
+    int uBufferLimit = uBuffer.limit();
+
+    // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value.
+    vBuffer.position(vBufferPosition + 1);
+    // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value.
+    uBuffer.limit(uBufferLimit - 1);
+
+    // Check that the buffers are equal and have the expected number of elements.
+    boolean areNV21 =
+        (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0);
+
+    // Restore buffers to their initial state.
+    vBuffer.position(vBufferPosition);
+    uBuffer.limit(uBufferLimit);
+
+    return areNV21;
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtilsProxyApi.java

@@ -0,0 +1,37 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.plugins.camerax;
+
+import androidx.annotation.NonNull;
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/**
+ * ProxyApi implementation for {@link DeviceOrientationManager}. This class may handle instantiating
+ * native object instances that are attached to a Dart instance or handle method calls on the
+ * associated native class or an instance of that class.
+ */
+public class ImageProxyUtilsProxyApi extends PigeonApiImageProxyUtils {
+  ImageProxyUtilsProxyApi(@NonNull ProxyApiRegistrar pigeonRegistrar) {
+    super(pigeonRegistrar);
+  }
+
+  // List<? extends PlaneProxy> can be considered the same as List<PlaneProxy>.
+  @SuppressWarnings("unchecked")
+  @NonNull
+  @Override
+  public byte[] getNv21Buffer(
+      long imageWidth, long imageHeight, @NonNull List<? extends PlaneProxy> planes) {
+    final ByteBuffer nv21Buffer =
+        ImageProxyUtils.planesToNV21(
+            (List<PlaneProxy>) planes, (int) imageWidth, (int) imageHeight);
+
+    byte[] bytes = new byte[nv21Buffer.remaining()];
+    nv21Buffer.get(bytes, 0, bytes.length);
+
+    return bytes;
+  }
+}

packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ProxyApiRegistrar.java

@@ -425,4 +425,10 @@ public PigeonApiMeteringPointFactory getPigeonApiMeteringPointFactory() {
   public CameraPermissionsErrorProxyApi getPigeonApiCameraPermissionsError() {
     return new CameraPermissionsErrorProxyApi(this);
   }
+
+  @NonNull
+  @Override
+  public PigeonApiImageProxyUtils getPigeonApiImageProxyUtils() {
+    return new ImageProxyUtilsProxyApi(this);
+  }
 }

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsApiTest.java

@@ -0,0 +1,44 @@
+package io.flutter.plugins.camerax;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.mockito.Mockito.mockStatic;
+
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import org.junit.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+public class ImageProxyUtilsApiTest {
+
+  @Test
+  public void getNv21Buffer_returnsExpectedBytes() {
+    final PigeonApiImageProxyUtils api = new TestProxyApiRegistrar().getPigeonApiImageProxyUtils();
+
+    List<PlaneProxy> planes =
+        Arrays.asList(
+            Mockito.mock(PlaneProxy.class),
+            Mockito.mock(PlaneProxy.class),
+            Mockito.mock(PlaneProxy.class));
+    long width = 4;
+    long height = 2;
+    byte[] expectedBytes = new byte[] {1, 2, 3, 4, 5};
+    ByteBuffer mockBuffer = ByteBuffer.wrap(expectedBytes);
+
+    try (MockedStatic<ImageProxyUtils> mockedStatic = mockStatic(ImageProxyUtils.class)) {
+      mockedStatic
+          .when(
+              () ->
+                  ImageProxyUtils.planesToNV21(
+                      Mockito.anyList(), Mockito.anyInt(), Mockito.anyInt()))
+          .thenReturn(mockBuffer);
+
+      byte[] result = api.getNv21Buffer(width, height, planes);
+
+      assertArrayEquals(expectedBytes, result);
+      mockedStatic.verify(() -> ImageProxyUtils.planesToNV21(planes, (int) width, (int) height));
+    }
+  }
+}

packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsTest.java

@@ -0,0 +1,99 @@
+package io.flutter.plugins.camerax;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertThrows;
+
+import androidx.camera.core.ImageProxy.PlaneProxy;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class ImageProxyUtilsTest {
+
+  @Test
+  public void planesToNV21_throwsExceptionForNonNV21Layout() {
+    int width = 4;
+    int height = 2;
+    byte[] y = new byte[] {0, 1, 2, 3, 4, 5, 6, 7};
+
+    // U and V planes: not in NV21 layout (all values are the same)
+    byte[] u = new byte[] {20, 20, 20, 20};
+    byte[] v = new byte[] {30, 30, 30, 30};
+
+    PlaneProxy yPlane = mockPlaneProxyWithData(y);
+    PlaneProxy uPlane = mockPlaneProxyWithData(u);
+    PlaneProxy vPlane = mockPlaneProxyWithData(v);
+
+    List<PlaneProxy> planes = Arrays.asList(yPlane, uPlane, vPlane);
+
+    assertThrows(
+        IllegalArgumentException.class, () -> ImageProxyUtils.planesToNV21(planes, width, height));
+  }
+
+  @Test
+  public void planesToNV21_returnsExpectedBufferWhenPlanesAreNV21Compatible() {
+    int width = 4;
+    int height = 2;
+    int imageSize = width * height; // 8
+
+    // Y plane.
+    byte[] y = new byte[] {0, 1, 2, 3, 4, 5, 6, 7};
+    PlaneProxy yPlane = mockPlaneProxyWithData(y);
+
+    // U and V planes in NV21 format. Both have 2 bytes that are identical (5, 7).
+    ByteBuffer vBuffer = ByteBuffer.wrap(new byte[] {9, 5, 7});
+    ByteBuffer uBuffer = ByteBuffer.wrap(new byte[] {5, 7, 33});
+
+    PlaneProxy uPlane = Mockito.mock(PlaneProxy.class);
+    PlaneProxy vPlane = Mockito.mock(PlaneProxy.class);
+
+    Mockito.when(uPlane.getBuffer()).thenReturn(uBuffer);
+    Mockito.when(vPlane.getBuffer()).thenReturn(vBuffer);
+
+    List<PlaneProxy> planes = Arrays.asList(yPlane, uPlane, vPlane);
+
+    ByteBuffer nv21Buffer = ImageProxyUtils.planesToNV21(planes, width, height);
+    byte[] nv21 = new byte[nv21Buffer.remaining()];
+    nv21Buffer.get(nv21);
+
+    // The planesToNV21 method copies:
+    // 1. All of the Y plane bytes.
+    // 2. The first byte of the V plane.
+    // 3. The first three (2 * 8 / 4 - 1) bytes of the U plane.
+    byte[] expected =
+        new byte[] {
+          0,
+          1,
+          2,
+          3,
+          4,
+          5,
+          6,
+          7, // Y
+          9,
+          5,
+          7,
+          33 // V0, U0, U1, U2
+        };
+
+    assertArrayEquals(expected, nv21);
+  }
+
+  // Creates a mock PlaneProxy with a buffer (of zeroes) of the given size.
+  private PlaneProxy mockPlaneProxy(int bufferSize) {
+    PlaneProxy plane = Mockito.mock(PlaneProxy.class);
+    ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
+    Mockito.when(plane.getBuffer()).thenReturn(buffer);
+    return plane;
+  }
+
+  // Creates a mock PlaneProxy with specific data.
+  private PlaneProxy mockPlaneProxyWithData(byte[] data) {
+    PlaneProxy plane = Mockito.mock(PlaneProxy.class);
+    ByteBuffer buffer = ByteBuffer.wrap(Arrays.copyOf(data, data.length));
+    Mockito.when(plane.getBuffer()).thenReturn(buffer);
+    return plane;
+  }
+}

packages/camera/camera_android_camerax/lib/src/android_camera_camerax.dart

@@ -7,6 +7,7 @@ import 'dart:math' show Point;
 
 import 'package:async/async.dart';
 import 'package:camera_platform_interface/camera_platform_interface.dart';
+import 'package:flutter/foundation.dart' show Uint8List;
 import 'package:flutter/services.dart'
     show DeviceOrientation, PlatformException;
 import 'package:flutter/widgets.dart' show Texture, Widget, visibleForTesting;
@@ -290,6 +291,9 @@ class AndroidCameraCameraX extends CameraPlatform {
   /// The ID of the surface texture that the camera preview is drawn to.
   late int _flutterSurfaceTextureId;
 
+  /// The format of outputted images from image streaming.
+  int? _imageAnalysisOutputImageFormat;
+
   /// Returns list of all available cameras and their descriptions.
   @override
   Future<List<CameraDescription>> availableCameras() async {
@@ -472,11 +476,11 @@ class AndroidCameraCameraX extends CameraPlatform {
     }
     // Configure ImageAnalysis instance.
     // Defaults to YUV_420_888 image format.
+    _imageAnalysisOutputImageFormat =
+        _imageAnalysisOutputFormatFromImageFormatGroup(imageFormatGroup);
     imageAnalysis = proxy.newImageAnalysis(
       resolutionSelector: _presetResolutionSelector,
-      outputImageFormat: _imageAnalysisOutputFormatFromImageFormatGroup(
-        imageFormatGroup,
-      ),
+      outputImageFormat: _imageAnalysisOutputImageFormat,
       /* use CameraX default target rotation */ targetRotation: null,
     );
 
@@ -1293,22 +1297,58 @@ class AndroidCameraCameraX extends CameraPlatform {
     Future<void> analyze(ImageProxy imageProxy) async {
       final List<PlaneProxy> planes = await imageProxy.getPlanes();
       final List<CameraImagePlane> cameraImagePlanes = <CameraImagePlane>[];
-      for (final PlaneProxy plane in planes) {
+
+      // Determine image planes.
+      if (_imageAnalysisOutputImageFormat ==
+          imageAnalysisOutputImageFormatNv21) {
+        // Convert three generically YUV_420_888 formatted image planes into one singular
+        // NV21 formatted image plane if NV21 was requested for image streaming. The conversion
+        // should be null safe.
+        final Uint8List? bytes = await proxy.getNv21BufferImageProxyUtils(
+          imageProxy.width,
+          imageProxy.height,
+          planes,
+        );
+
         cameraImagePlanes.add(
           CameraImagePlane(
-            bytes: plane.buffer,
-            bytesPerRow: plane.rowStride,
-            bytesPerPixel: plane.pixelStride,
+            bytes: bytes!,
+            bytesPerRow: imageProxy.width,
+            bytesPerPixel: 1,
           ),
         );
+      } else {
+        for (final PlaneProxy plane in planes) {
+          cameraImagePlanes.add(
+            CameraImagePlane(
+              bytes: plane.buffer,
+              bytesPerRow: plane.rowStride,
+              bytesPerPixel: plane.pixelStride,
+            ),
+          );
+        }
       }
 
-      final int format = imageProxy.format;
-      final CameraImageFormat cameraImageFormat = CameraImageFormat(
-        _imageFormatGroupFromPlatformData(format),
-        raw: format,
-      );
+      // Determine image format.
+      CameraImageFormat? cameraImageFormat;
+
+      if (_imageAnalysisOutputImageFormat ==
+          imageAnalysisOutputImageFormatNv21) {
+        // Manually override ImageFormat to NV21 if set for image streaming as CameraX
+        // still reports YUV_420_888 if the underlying format is NV21.
+        cameraImageFormat = const CameraImageFormat(
+          ImageFormatGroup.nv21,
+          raw: imageProxyFormatNv21,
+        );
+      } else {
+        final int imageRawFormat = imageProxy.format;
+        cameraImageFormat = CameraImageFormat(
+          _imageFormatGroupFromPlatformData(imageRawFormat),
+          raw: imageRawFormat,
+        );
+      }
 
+      // Send out CameraImageData.
       final CameraImageData cameraImageData = CameraImageData(
         format: cameraImageFormat,
         planes: cameraImagePlanes,