diff --git a/packages/camera/camera_android_camerax/CHANGELOG.md b/packages/camera/camera_android_camerax/CHANGELOG.md index 40159b68c97..91e14471b66 100644 --- a/packages/camera/camera_android_camerax/CHANGELOG.md +++ b/packages/camera/camera_android_camerax/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.6.22 + +* Converts NV21-compatible streamed images to NV21 when requested. + ## 0.6.21 * Implements NV21 support for image streaming. diff --git a/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXLibrary.g.kt b/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXLibrary.g.kt index f4a84496acc..216e984ab17 100644 --- a/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXLibrary.g.kt +++ b/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/CameraXLibrary.g.kt @@ -597,6 +597,12 @@ abstract class CameraXLibraryPigeonProxyApiRegistrar(val binaryMessenger: Binary */ abstract fun getPigeonApiImageProxy(): PigeonApiImageProxy + /** + * An implementation of [PigeonApiImageProxyUtils] used to add a new Dart instance of + * `ImageProxyUtils` to the Dart `InstanceManager`. + */ + abstract fun getPigeonApiImageProxyUtils(): PigeonApiImageProxyUtils + /** * An implementation of [PigeonApiPlaneProxy] used to add a new Dart instance of `PlaneProxy` to * the Dart `InstanceManager`. @@ -738,6 +744,7 @@ abstract class CameraXLibraryPigeonProxyApiRegistrar(val binaryMessenger: Binary PigeonApiAnalyzer.setUpMessageHandlers(binaryMessenger, getPigeonApiAnalyzer()) PigeonApiLiveData.setUpMessageHandlers(binaryMessenger, getPigeonApiLiveData()) PigeonApiImageProxy.setUpMessageHandlers(binaryMessenger, getPigeonApiImageProxy()) + PigeonApiImageProxyUtils.setUpMessageHandlers(binaryMessenger, getPigeonApiImageProxyUtils()) PigeonApiQualitySelector.setUpMessageHandlers(binaryMessenger, getPigeonApiQualitySelector()) PigeonApiFallbackStrategy.setUpMessageHandlers(binaryMessenger, getPigeonApiFallbackStrategy()) PigeonApiCameraControl.setUpMessageHandlers(binaryMessenger, getPigeonApiCameraControl()) @@ -785,6 +792,7 @@ abstract class CameraXLibraryPigeonProxyApiRegistrar(val binaryMessenger: Binary PigeonApiAnalyzer.setUpMessageHandlers(binaryMessenger, null) PigeonApiLiveData.setUpMessageHandlers(binaryMessenger, null) PigeonApiImageProxy.setUpMessageHandlers(binaryMessenger, null) + PigeonApiImageProxyUtils.setUpMessageHandlers(binaryMessenger, null) PigeonApiQualitySelector.setUpMessageHandlers(binaryMessenger, null) PigeonApiFallbackStrategy.setUpMessageHandlers(binaryMessenger, null) PigeonApiCameraControl.setUpMessageHandlers(binaryMessenger, null) @@ -916,6 +924,8 @@ private class CameraXLibraryPigeonProxyApiBaseCodec( registrar.getPigeonApiLiveData().pigeon_newInstance(value) {} } else if (value is androidx.camera.core.ImageProxy) { registrar.getPigeonApiImageProxy().pigeon_newInstance(value) {} + } else if (value is ImageProxyUtils) { + registrar.getPigeonApiImageProxyUtils().pigeon_newInstance(value) {} } else if (value is androidx.camera.core.ImageProxy.PlaneProxy) { registrar.getPigeonApiPlaneProxy().pigeon_newInstance(value) {} } else if (value is androidx.camera.video.QualitySelector) { @@ -5334,6 +5344,83 @@ abstract class PigeonApiImageProxy( } } } +/** Utils for working with [ImageProxy]s. */ +@Suppress("UNCHECKED_CAST") +abstract class PigeonApiImageProxyUtils( + open val pigeonRegistrar: CameraXLibraryPigeonProxyApiRegistrar +) { + /** + * Returns a single Byte Buffer that is representative of the [planes] that are NV21 compatible. + */ + abstract fun getNv21Buffer( + imageWidth: Long, + imageHeight: Long, + planes: List + ): ByteArray + + companion object { + @Suppress("LocalVariableName") + fun setUpMessageHandlers(binaryMessenger: BinaryMessenger, api: PigeonApiImageProxyUtils?) { + val codec = api?.pigeonRegistrar?.codec ?: CameraXLibraryPigeonCodec() + run { + val channel = + BasicMessageChannel( + binaryMessenger, + "dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.getNv21Buffer", + codec) + if (api != null) { + channel.setMessageHandler { message, reply -> + val args = message as List + val imageWidthArg = args[0] as Long + val imageHeightArg = args[1] as Long + val planesArg = args[2] as List + val wrapped: List = + try { + listOf(api.getNv21Buffer(imageWidthArg, imageHeightArg, planesArg)) + } catch (exception: Throwable) { + CameraXLibraryPigeonUtils.wrapError(exception) + } + reply.reply(wrapped) + } + } else { + channel.setMessageHandler(null) + } + } + } + } + + @Suppress("LocalVariableName", "FunctionName") + /** Creates a Dart instance of ImageProxyUtils and attaches it to [pigeon_instanceArg]. */ + fun pigeon_newInstance(pigeon_instanceArg: ImageProxyUtils, callback: (Result) -> Unit) { + if (pigeonRegistrar.ignoreCallsToDart) { + callback( + Result.failure( + CameraXError("ignore-calls-error", "Calls to Dart are being ignored.", ""))) + } else if (pigeonRegistrar.instanceManager.containsInstance(pigeon_instanceArg)) { + callback(Result.success(Unit)) + } else { + val pigeon_identifierArg = + pigeonRegistrar.instanceManager.addHostCreatedInstance(pigeon_instanceArg) + val binaryMessenger = pigeonRegistrar.binaryMessenger + val codec = pigeonRegistrar.codec + val channelName = + "dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.pigeon_newInstance" + val channel = BasicMessageChannel(binaryMessenger, channelName, codec) + channel.send(listOf(pigeon_identifierArg)) { + if (it is List<*>) { + if (it.size > 1) { + callback( + Result.failure(CameraXError(it[0] as String, it[1] as String, it[2] as String?))) + } else { + callback(Result.success(Unit)) + } + } else { + callback(Result.failure(CameraXLibraryPigeonUtils.createConnectionError(channelName))) + } + } + } + } +} /** * A plane proxy which has an analogous interface as `android.media.Image.Plane`. * diff --git a/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtils.java b/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtils.java new file mode 100644 index 00000000000..6d9f38c8888 --- /dev/null +++ b/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtils.java @@ -0,0 +1,75 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Note: code in this file is directly inspired by the official Google MLKit example: +// https://github.com/googlesamples/mlkit + +package io.flutter.plugins.camerax; + +import androidx.annotation.NonNull; +import androidx.camera.core.ImageProxy.PlaneProxy; +import java.nio.ByteBuffer; +import java.util.List; + +/* Utilities for working with {@code ImageProxy}s. */ +public class ImageProxyUtils { + + /** + * Converts list of {@link PlaneProxy}s in YUV_420_888 format (with VU planes in NV21 layout) to a + * single NV21 {@code ByteBuffer}. + */ + @NonNull + public static ByteBuffer planesToNV21(@NonNull List planes, int width, int height) { + if (!areUVPlanesNV21(planes, width, height)) { + throw new IllegalArgumentException( + "Provided UV planes are not in NV21 layout and thus cannot be converted."); + } + + int imageSize = width * height; + int nv21Size = imageSize + 2 * (imageSize / 4); + byte[] nv21Bytes = new byte[nv21Size]; + + // Copy Y plane. + ByteBuffer yBuffer = planes.get(0).getBuffer(); + yBuffer.rewind(); + yBuffer.get(nv21Bytes, 0, imageSize); + + // Copy interleaved VU plane (NV21 layout). + ByteBuffer vBuffer = planes.get(2).getBuffer(); + ByteBuffer uBuffer = planes.get(1).getBuffer(); + + vBuffer.rewind(); + uBuffer.rewind(); + vBuffer.get(nv21Bytes, imageSize, 1); + uBuffer.get(nv21Bytes, imageSize + 1, 2 * imageSize / 4 - 1); + + return ByteBuffer.wrap(nv21Bytes); + } + + public static boolean areUVPlanesNV21(@NonNull List planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes.get(1).getBuffer(); + ByteBuffer vBuffer = planes.get(2).getBuffer(); + + // Backup buffer properties. + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. + vBuffer.position(vBufferPosition + 1); + // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. + uBuffer.limit(uBufferLimit - 1); + + // Check that the buffers are equal and have the expected number of elements. + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + // Restore buffers to their initial state. + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } +} diff --git a/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtilsProxyApi.java b/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtilsProxyApi.java new file mode 100644 index 00000000000..e7a249db100 --- /dev/null +++ b/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ImageProxyUtilsProxyApi.java @@ -0,0 +1,37 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camerax; + +import androidx.annotation.NonNull; +import androidx.camera.core.ImageProxy.PlaneProxy; +import java.nio.ByteBuffer; +import java.util.List; + +/** + * ProxyApi implementation for {@link ImageProxyUtils}. This class may handle instantiating native + * object instances that are attached to a Dart instance or handle method calls on the associated + * native class or an instance of that class. + */ +public class ImageProxyUtilsProxyApi extends PigeonApiImageProxyUtils { + ImageProxyUtilsProxyApi(@NonNull ProxyApiRegistrar pigeonRegistrar) { + super(pigeonRegistrar); + } + + // List can be considered the same as List. + @SuppressWarnings("unchecked") + @NonNull + @Override + public byte[] getNv21Buffer( + long imageWidth, long imageHeight, @NonNull List planes) { + final ByteBuffer nv21Buffer = + ImageProxyUtils.planesToNV21( + (List) planes, (int) imageWidth, (int) imageHeight); + + byte[] bytes = new byte[nv21Buffer.remaining()]; + nv21Buffer.get(bytes, 0, bytes.length); + + return bytes; + } +} diff --git a/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ProxyApiRegistrar.java b/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ProxyApiRegistrar.java index 436e850f2ac..5d1b548f3be 100644 --- a/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ProxyApiRegistrar.java +++ b/packages/camera/camera_android_camerax/android/src/main/java/io/flutter/plugins/camerax/ProxyApiRegistrar.java @@ -425,4 +425,10 @@ public PigeonApiMeteringPointFactory getPigeonApiMeteringPointFactory() { public CameraPermissionsErrorProxyApi getPigeonApiCameraPermissionsError() { return new CameraPermissionsErrorProxyApi(this); } + + @NonNull + @Override + public PigeonApiImageProxyUtils getPigeonApiImageProxyUtils() { + return new ImageProxyUtilsProxyApi(this); + } } diff --git a/packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsApiTest.java b/packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsApiTest.java new file mode 100644 index 00000000000..a64585370f3 --- /dev/null +++ b/packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsApiTest.java @@ -0,0 +1,48 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camerax; + +import static org.junit.Assert.assertArrayEquals; +import static org.mockito.Mockito.mockStatic; + +import androidx.camera.core.ImageProxy.PlaneProxy; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +public class ImageProxyUtilsApiTest { + + @Test + public void getNv21Buffer_returnsExpectedBytes() { + final PigeonApiImageProxyUtils api = new TestProxyApiRegistrar().getPigeonApiImageProxyUtils(); + + List planes = + Arrays.asList( + Mockito.mock(PlaneProxy.class), + Mockito.mock(PlaneProxy.class), + Mockito.mock(PlaneProxy.class)); + long width = 4; + long height = 2; + byte[] expectedBytes = new byte[] {1, 2, 3, 4, 5}; + ByteBuffer mockBuffer = ByteBuffer.wrap(expectedBytes); + + try (MockedStatic mockedStatic = mockStatic(ImageProxyUtils.class)) { + mockedStatic + .when( + () -> + ImageProxyUtils.planesToNV21( + Mockito.anyList(), Mockito.anyInt(), Mockito.anyInt())) + .thenReturn(mockBuffer); + + byte[] result = api.getNv21Buffer(width, height, planes); + + assertArrayEquals(expectedBytes, result); + mockedStatic.verify(() -> ImageProxyUtils.planesToNV21(planes, (int) width, (int) height)); + } + } +} diff --git a/packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsTest.java b/packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsTest.java new file mode 100644 index 00000000000..3dd059d8f9b --- /dev/null +++ b/packages/camera/camera_android_camerax/android/src/test/java/io/flutter/plugins/camerax/ImageProxyUtilsTest.java @@ -0,0 +1,103 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camerax; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertThrows; + +import androidx.camera.core.ImageProxy.PlaneProxy; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import org.junit.Test; +import org.mockito.Mockito; + +public class ImageProxyUtilsTest { + + @Test + public void planesToNV21_throwsExceptionForNonNV21Layout() { + int width = 4; + int height = 2; + byte[] y = new byte[] {0, 1, 2, 3, 4, 5, 6, 7}; + + // U and V planes are not in NV21 layout (not interleaved). + byte[] u = new byte[] {20, 20, 20, 20}; + byte[] v = new byte[] {30, 30, 30, 30}; + + PlaneProxy yPlane = mockPlaneProxyWithData(y); + PlaneProxy uPlane = mockPlaneProxyWithData(u); + PlaneProxy vPlane = mockPlaneProxyWithData(v); + + List planes = Arrays.asList(yPlane, uPlane, vPlane); + + assertThrows( + IllegalArgumentException.class, () -> ImageProxyUtils.planesToNV21(planes, width, height)); + } + + @Test + public void planesToNV21_returnsExpectedBufferWhenPlanesAreNV21Compatible() { + int width = 4; + int height = 2; + int imageSize = width * height; // 8 + + // Y plane. + byte[] y = new byte[] {0, 1, 2, 3, 4, 5, 6, 7}; + PlaneProxy yPlane = mockPlaneProxyWithData(y); + + // U and V planes in NV21 format. Both have 2 bytes that are overlapping (5, 7). + ByteBuffer vBuffer = ByteBuffer.wrap(new byte[] {9, 5, 7}); + ByteBuffer uBuffer = ByteBuffer.wrap(new byte[] {5, 7, 33}); + + PlaneProxy uPlane = Mockito.mock(PlaneProxy.class); + PlaneProxy vPlane = Mockito.mock(PlaneProxy.class); + + Mockito.when(uPlane.getBuffer()).thenReturn(uBuffer); + Mockito.when(vPlane.getBuffer()).thenReturn(vBuffer); + + List planes = Arrays.asList(yPlane, uPlane, vPlane); + + ByteBuffer nv21Buffer = ImageProxyUtils.planesToNV21(planes, width, height); + byte[] nv21 = new byte[nv21Buffer.remaining()]; + nv21Buffer.get(nv21); + + // The planesToNV21 method copies: + // 1. All of the Y plane bytes. + // 2. The first byte of the V plane. + // 3. The first three (2 * 8 / 4 - 1) bytes of the U plane. + byte[] expected = + new byte[] { + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, // Y + 9, + 5, + 7, + 33 // V0, U0, U1, U2 + }; + + assertArrayEquals(expected, nv21); + } + + // Creates a mock PlaneProxy with a buffer (of zeroes) of the given size. + private PlaneProxy mockPlaneProxy(int bufferSize) { + PlaneProxy plane = Mockito.mock(PlaneProxy.class); + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + Mockito.when(plane.getBuffer()).thenReturn(buffer); + return plane; + } + + // Creates a mock PlaneProxy with specific data. + private PlaneProxy mockPlaneProxyWithData(byte[] data) { + PlaneProxy plane = Mockito.mock(PlaneProxy.class); + ByteBuffer buffer = ByteBuffer.wrap(Arrays.copyOf(data, data.length)); + Mockito.when(plane.getBuffer()).thenReturn(buffer); + return plane; + } +} diff --git a/packages/camera/camera_android_camerax/lib/src/android_camera_camerax.dart b/packages/camera/camera_android_camerax/lib/src/android_camera_camerax.dart index 5527ce65d64..b7385aa85b6 100644 --- a/packages/camera/camera_android_camerax/lib/src/android_camera_camerax.dart +++ b/packages/camera/camera_android_camerax/lib/src/android_camera_camerax.dart @@ -7,6 +7,7 @@ import 'dart:math' show Point; import 'package:async/async.dart'; import 'package:camera_platform_interface/camera_platform_interface.dart'; +import 'package:flutter/foundation.dart' show Uint8List; import 'package:flutter/services.dart' show DeviceOrientation, PlatformException; import 'package:flutter/widgets.dart' show Texture, Widget, visibleForTesting; @@ -290,6 +291,9 @@ class AndroidCameraCameraX extends CameraPlatform { /// The ID of the surface texture that the camera preview is drawn to. late int _flutterSurfaceTextureId; + /// The configured format of outputted images from image streaming. + int? _imageAnalysisOutputImageFormat; + /// Returns list of all available cameras and their descriptions. @override Future> availableCameras() async { @@ -472,11 +476,11 @@ class AndroidCameraCameraX extends CameraPlatform { } // Configure ImageAnalysis instance. // Defaults to YUV_420_888 image format. + _imageAnalysisOutputImageFormat = + _imageAnalysisOutputFormatFromImageFormatGroup(imageFormatGroup); imageAnalysis = proxy.newImageAnalysis( resolutionSelector: _presetResolutionSelector, - outputImageFormat: _imageAnalysisOutputFormatFromImageFormatGroup( - imageFormatGroup, - ), + outputImageFormat: _imageAnalysisOutputImageFormat, /* use CameraX default target rotation */ targetRotation: null, ); @@ -1293,22 +1297,60 @@ class AndroidCameraCameraX extends CameraPlatform { Future analyze(ImageProxy imageProxy) async { final List planes = await imageProxy.getPlanes(); final List cameraImagePlanes = []; - for (final PlaneProxy plane in planes) { + + // Determine image planes. + if (_imageAnalysisOutputImageFormat == + imageAnalysisOutputImageFormatNv21) { + // Convert three generically YUV_420_888 formatted image planes into one singular + // NV21 formatted image plane if NV21 was requested for image streaming. The conversion + // should be null safe. + final Uint8List? bytes = await proxy.getNv21BufferImageProxyUtils( + imageProxy.width, + imageProxy.height, + planes, + ); + cameraImagePlanes.add( CameraImagePlane( - bytes: plane.buffer, - bytesPerRow: plane.rowStride, - bytesPerPixel: plane.pixelStride, + bytes: bytes!, + bytesPerRow: imageProxy.width, + // NV21 has 1.5 bytes per pixel (Y plane has width * height; VU plane has width * height / 2), + // but this is rounded up because an int is expected. camera_android reports the same. + bytesPerPixel: 1, ), ); + } else { + for (final PlaneProxy plane in planes) { + cameraImagePlanes.add( + CameraImagePlane( + bytes: plane.buffer, + bytesPerRow: plane.rowStride, + bytesPerPixel: plane.pixelStride, + ), + ); + } } - final int format = imageProxy.format; - final CameraImageFormat cameraImageFormat = CameraImageFormat( - _imageFormatGroupFromPlatformData(format), - raw: format, - ); + // Determine image format. + CameraImageFormat? cameraImageFormat; + + if (_imageAnalysisOutputImageFormat == + imageAnalysisOutputImageFormatNv21) { + // Manually override ImageFormat to NV21 if set for image streaming as CameraX + // still reports YUV_420_888 if the underlying format is NV21. + cameraImageFormat = const CameraImageFormat( + ImageFormatGroup.nv21, + raw: imageProxyFormatNv21, + ); + } else { + final int imageRawFormat = imageProxy.format; + cameraImageFormat = CameraImageFormat( + _imageFormatGroupFromPlatformData(imageRawFormat), + raw: imageRawFormat, + ); + } + // Send out CameraImageData. final CameraImageData cameraImageData = CameraImageData( format: cameraImageFormat, planes: cameraImagePlanes, diff --git a/packages/camera/camera_android_camerax/lib/src/camerax_library.g.dart b/packages/camera/camera_android_camerax/lib/src/camerax_library.g.dart index b1e1f737ac3..6d4d56da8db 100644 --- a/packages/camera/camera_android_camerax/lib/src/camerax_library.g.dart +++ b/packages/camera/camera_android_camerax/lib/src/camerax_library.g.dart @@ -248,6 +248,9 @@ class PigeonInstanceManager { ImageProxy.pigeon_setUpMessageHandlers( pigeon_instanceManager: instanceManager, ); + ImageProxyUtils.pigeon_setUpMessageHandlers( + pigeon_instanceManager: instanceManager, + ); PlaneProxy.pigeon_setUpMessageHandlers( pigeon_instanceManager: instanceManager, ); @@ -6746,6 +6749,130 @@ class ImageProxy extends PigeonInternalProxyApiBaseClass { } } +/// Utils for working with [ImageProxy]s. +class ImageProxyUtils extends PigeonInternalProxyApiBaseClass { + /// Constructs [ImageProxyUtils] without creating the associated native object. + /// + /// This should only be used by subclasses created by this library or to + /// create copies for an [PigeonInstanceManager]. + @protected + ImageProxyUtils.pigeon_detached({ + super.pigeon_binaryMessenger, + super.pigeon_instanceManager, + }); + + late final _PigeonInternalProxyApiBaseCodec _pigeonVar_codecImageProxyUtils = + _PigeonInternalProxyApiBaseCodec(pigeon_instanceManager); + + static void pigeon_setUpMessageHandlers({ + bool pigeon_clearHandlers = false, + BinaryMessenger? pigeon_binaryMessenger, + PigeonInstanceManager? pigeon_instanceManager, + ImageProxyUtils Function()? pigeon_newInstance, + }) { + final _PigeonInternalProxyApiBaseCodec pigeonChannelCodec = + _PigeonInternalProxyApiBaseCodec( + pigeon_instanceManager ?? PigeonInstanceManager.instance, + ); + final BinaryMessenger? binaryMessenger = pigeon_binaryMessenger; + { + final BasicMessageChannel + pigeonVar_channel = BasicMessageChannel( + 'dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.pigeon_newInstance', + pigeonChannelCodec, + binaryMessenger: binaryMessenger, + ); + if (pigeon_clearHandlers) { + pigeonVar_channel.setMessageHandler(null); + } else { + pigeonVar_channel.setMessageHandler((Object? message) async { + assert( + message != null, + 'Argument for dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.pigeon_newInstance was null.', + ); + final List args = (message as List?)!; + final int? arg_pigeon_instanceIdentifier = (args[0] as int?); + assert( + arg_pigeon_instanceIdentifier != null, + 'Argument for dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.pigeon_newInstance was null, expected non-null int.', + ); + try { + (pigeon_instanceManager ?? PigeonInstanceManager.instance) + .addHostCreatedInstance( + pigeon_newInstance?.call() ?? + ImageProxyUtils.pigeon_detached( + pigeon_binaryMessenger: pigeon_binaryMessenger, + pigeon_instanceManager: pigeon_instanceManager, + ), + arg_pigeon_instanceIdentifier!, + ); + return wrapResponse(empty: true); + } on PlatformException catch (e) { + return wrapResponse(error: e); + } catch (e) { + return wrapResponse( + error: PlatformException(code: 'error', message: e.toString()), + ); + } + }); + } + } + } + + /// Returns a single Byte Buffer that is representative of the [planes] + /// that are NV21 compatible. + static Future getNv21Buffer( + int imageWidth, + int imageHeight, + List planes, { + BinaryMessenger? pigeon_binaryMessenger, + PigeonInstanceManager? pigeon_instanceManager, + }) async { + final _PigeonInternalProxyApiBaseCodec pigeonChannelCodec = + _PigeonInternalProxyApiBaseCodec( + pigeon_instanceManager ?? PigeonInstanceManager.instance, + ); + final BinaryMessenger? pigeonVar_binaryMessenger = pigeon_binaryMessenger; + const String pigeonVar_channelName = + 'dev.flutter.pigeon.camera_android_camerax.ImageProxyUtils.getNv21Buffer'; + final BasicMessageChannel pigeonVar_channel = + BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final Future pigeonVar_sendFuture = pigeonVar_channel.send( + [imageWidth, imageHeight, planes], + ); + final List? pigeonVar_replyList = + await pigeonVar_sendFuture as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else if (pigeonVar_replyList[0] == null) { + throw PlatformException( + code: 'null-error', + message: 'Host platform returned null value for non-null return value.', + ); + } else { + return (pigeonVar_replyList[0] as Uint8List?)!; + } + } + + @override + ImageProxyUtils pigeon_copy() { + return ImageProxyUtils.pigeon_detached( + pigeon_binaryMessenger: pigeon_binaryMessenger, + pigeon_instanceManager: pigeon_instanceManager, + ); + } +} + /// A plane proxy which has an analogous interface as /// `android.media.Image.Plane`. /// diff --git a/packages/camera/camera_android_camerax/lib/src/camerax_proxy.dart b/packages/camera/camera_android_camerax/lib/src/camerax_proxy.dart index dfda9a7bc6f..27ab775b684 100644 --- a/packages/camera/camera_android_camerax/lib/src/camerax_proxy.dart +++ b/packages/camera/camera_android_camerax/lib/src/camerax_proxy.dart @@ -70,6 +70,7 @@ class CameraXProxy { _infoSupportedHardwareLevelCameraCharacteristics, this.sensorOrientationCameraCharacteristics = _sensorOrientationCameraCharacteristics, + this.getNv21BufferImageProxyUtils = ImageProxyUtils.getNv21Buffer, }); /// Handles adding support for generic classes. @@ -376,6 +377,16 @@ class CameraXProxy { final CameraCharacteristicsKey Function() sensorOrientationCameraCharacteristics; + /// Calls to [ImageProxyUtils.getNv21Buffer]. + final Future Function( + int imageWidth, + int imageHeight, + List planes, { + BinaryMessenger? pigeon_binaryMessenger, + PigeonInstanceManager? pigeon_instanceManager, + }) + getNv21BufferImageProxyUtils; + static CameraSelector _defaultBackCameraCameraSelector() => CameraSelector.defaultBackCamera; diff --git a/packages/camera/camera_android_camerax/pigeons/camerax_library.dart b/packages/camera/camera_android_camerax/pigeons/camerax_library.dart index f9114df3f7c..d7b4b63b5af 100644 --- a/packages/camera/camera_android_camerax/pigeons/camerax_library.dart +++ b/packages/camera/camera_android_camerax/pigeons/camerax_library.dart @@ -923,6 +923,18 @@ abstract class ImageProxy { void close(); } +/// Utilities for working with [ImageProxy]s. +@ProxyApi() +abstract class ImageProxyUtils { + /// Returns a single buffer that is representative of three NV21-compatible [planes]. + @static + Uint8List getNv21Buffer( + int imageWidth, + int imageHeight, + List planes, + ); +} + /// A plane proxy which has an analogous interface as /// `android.media.Image.Plane`. /// diff --git a/packages/camera/camera_android_camerax/pubspec.yaml b/packages/camera/camera_android_camerax/pubspec.yaml index eeed6e6a4c1..b6c9172053f 100644 --- a/packages/camera/camera_android_camerax/pubspec.yaml +++ b/packages/camera/camera_android_camerax/pubspec.yaml @@ -2,7 +2,7 @@ name: camera_android_camerax description: Android implementation of the camera plugin using the CameraX library. repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_android_camerax issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 -version: 0.6.21 +version: 0.6.22 environment: sdk: ^3.8.1 diff --git a/packages/camera/camera_android_camerax/test/android_camera_camerax_test.dart b/packages/camera/camera_android_camerax/test/android_camera_camerax_test.dart index 766c4ddedfe..d3133d38833 100644 --- a/packages/camera/camera_android_camerax/test/android_camera_camerax_test.dart +++ b/packages/camera/camera_android_camerax/test/android_camera_camerax_test.dart @@ -162,6 +162,24 @@ void main() { int? targetRotation, })? newImageAnalysis, + Analyzer Function({ + required void Function(Analyzer, ImageProxy) analyze, + // ignore: non_constant_identifier_names + BinaryMessenger? pigeon_binaryMessenger, + // ignore: non_constant_identifier_names + PigeonInstanceManager? pigeon_instanceManager, + })? + newAnalyzer, + Future Function( + int imageWidth, + int imageHeight, + List planes, { + // ignore: non_constant_identifier_names + BinaryMessenger? pigeon_binaryMessenger, + // ignore: non_constant_identifier_names + PigeonInstanceManager? pigeon_instanceManager, + })? + getNv21BufferImageProxyUtils, }) { late final CameraXProxy proxy; final AspectRatioStrategy ratio_4_3FallbackAutoStrategyAspectRatioStrategy = @@ -467,6 +485,30 @@ void main() { }) { return MockFallbackStrategy(); }, + newAnalyzer: + newAnalyzer ?? + ({ + required void Function(Analyzer, ImageProxy) analyze, + // ignore: non_constant_identifier_names + BinaryMessenger? pigeon_binaryMessenger, + // ignore: non_constant_identifier_names + PigeonInstanceManager? pigeon_instanceManager, + }) { + return MockAnalyzer(); + }, + getNv21BufferImageProxyUtils: + getNv21BufferImageProxyUtils ?? + ( + int imageWidth, + int imageHeight, + List planes, { + // ignore: non_constant_identifier_names + BinaryMessenger? pigeon_binaryMessenger, + // ignore: non_constant_identifier_names + PigeonInstanceManager? pigeon_instanceManager, + }) { + return Future.value(Uint8List(0)); + }, ); return proxy; @@ -4789,6 +4831,122 @@ void main() { await onStreamedFrameAvailableSubscription.cancel(); }, ); + test( + 'onStreamedFrameAvailable emits NV21 CameraImageData with correct format and single plane when initialized with NV21', + () async { + final AndroidCameraCameraX camera = AndroidCameraCameraX(); + const int cameraId = 42; + final MockProcessCameraProvider mockProcessCameraProvider = + MockProcessCameraProvider(); + final MockImageAnalysis mockImageAnalysis = MockImageAnalysis(); + final MockCamera mockCamera = MockCamera(); + final MockCameraInfo mockCameraInfo = MockCameraInfo(); + final MockImageProxy mockImageProxy = MockImageProxy(); + final MockPlaneProxy mockPlane = MockPlaneProxy(); + final List mockPlanes = [ + mockPlane, + mockPlane, + mockPlane, + ]; + final Uint8List testNv21Buffer = Uint8List(10); + + // Mock use case bindings and related Camera objects. + when( + mockProcessCameraProvider.bindToLifecycle(any, any), + ).thenAnswer((_) async => mockCamera); + when(mockCamera.getCameraInfo()).thenAnswer((_) async => mockCameraInfo); + when( + mockCameraInfo.getCameraState(), + ).thenAnswer((_) async => MockLiveCameraState()); + + // Set up CameraXProxy with ImageAnalysis specifics needed for testing its Analyzer. + camera.proxy = getProxyForTestingUseCaseConfiguration( + mockProcessCameraProvider, + newAnalyzer: + ({ + required void Function(Analyzer, ImageProxy) analyze, + // ignore: non_constant_identifier_names + BinaryMessenger? pigeon_binaryMessenger, + // ignore: non_constant_identifier_names + PigeonInstanceManager? pigeon_instanceManager, + }) { + return Analyzer.pigeon_detached( + analyze: analyze, + pigeon_instanceManager: PigeonInstanceManager( + onWeakReferenceRemoved: (_) {}, + ), + ); + }, + newImageAnalysis: + ({ + int? outputImageFormat, + // ignore: non_constant_identifier_names + BinaryMessenger? pigeon_binaryMessenger, + // ignore: non_constant_identifier_names + PigeonInstanceManager? pigeon_instanceManager, + ResolutionSelector? resolutionSelector, + int? targetRotation, + }) => mockImageAnalysis, + getNv21BufferImageProxyUtils: + ( + int imageWidth, + int imageHeight, + List planes, { + // ignore: non_constant_identifier_names + BinaryMessenger? pigeon_binaryMessenger, + // ignore: non_constant_identifier_names + PigeonInstanceManager? pigeon_instanceManager, + }) => Future.value(testNv21Buffer), + ); + + // Create and initialize camera with NV21. + await camera.createCamera( + const CameraDescription( + name: 'test', + lensDirection: CameraLensDirection.back, + sensorOrientation: 0, + ), + ResolutionPreset.low, + ); + await camera.initializeCamera( + cameraId, + imageFormatGroup: ImageFormatGroup.nv21, + ); + + // Create mock ImageProxy with theoretical underlying NV21 format but with three + // planes still in YUV_420_888 format that should get transformed to testNv21Buffer. + when(mockImageProxy.height).thenReturn(100); + when(mockImageProxy.width).thenReturn(200); + when(mockImageProxy.getPlanes()).thenAnswer((_) async => mockPlanes); + when(mockPlane.buffer).thenReturn(Uint8List(33)); + when(mockPlane.rowStride).thenReturn(10); + when(mockPlane.pixelStride).thenReturn(22); + + // Set up listener to receive mock ImageProxy. + final Completer imageDataCompleter = + Completer(); + final StreamSubscription subscription = camera + .onStreamedFrameAvailable(cameraId) + .listen((CameraImageData imageData) { + imageDataCompleter.complete(imageData); + }); + + await untilCalled(mockImageAnalysis.setAnalyzer(any)); + final Analyzer capturedAnalyzer = + verify(mockImageAnalysis.setAnalyzer(captureAny)).captured.single + as Analyzer; + capturedAnalyzer.analyze(MockAnalyzer(), mockImageProxy); + + final CameraImageData imageData = await imageDataCompleter.future; + + expect(imageData.format.raw, AndroidCameraCameraX.imageProxyFormatNv21); + expect(imageData.format.group, ImageFormatGroup.nv21); + expect(imageData.planes.length, 1); + expect(imageData.planes[0].bytes, testNv21Buffer); + + await subscription.cancel(); + }, + ); test( 'onStreamedFrameAvailable returns stream that responds expectedly to being canceled',