Merge pull request #10081 from floe:java-camera2-view

* add (untested) JavaCamera2View class

* initial fixes

* minor cleanup

* exclude JavaCamera2View from build for older SDK version

* fix method name typo

* add asserts + sanity checks

* fix preview format checks

* fix the memory leak

* export cvtTwoPlaneYUVtoBGR for Java usage

* add handling for two-plane YUV frames (C wrapper still missing)

* add two-plane cvtColor helper function

* fix warnings

* actually use the new cvtColorTwoPlane helper func

* fix wrong output matrix size

* fix wrong conversion type

* simplify method signature, add error condition

* minor fixes to Mat types

* remove leftover semaphore from camera api 1

* android: JavaCamera2View minor refactoring

- re-apply Java code style
- imports cleanup
- dump exceptions information

* android: JavaCamera2View: pause/resume fixes

* android: JavaCamera2View: fix mScale

modules/imgproc/include/opencv2/imgproc.hpp

@@ -3694,6 +3694,8 @@ channels is derived automatically from src and code.
  */
 CV_EXPORTS_W void cvtColor( InputArray src, OutputArray dst, int code, int dstCn = 0 );
 
+CV_EXPORTS_W void cvtColorTwoPlane( InputArray src1, InputArray src2, OutputArray dst, int code );
+
 //! @} imgproc_misc
 
 // main function for all demosaicing processes

modules/imgproc/src/color.cpp

@@ -11050,6 +11050,42 @@ inline bool isFullRange(int code)
 
 } // namespace::
 
+// helper function for dual-plane modes
+void cv::cvtColorTwoPlane( InputArray _ysrc, InputArray _uvsrc, OutputArray _dst, int code )
+{
+    // only YUV420 is currently supported
+    switch (code) {
+        case COLOR_YUV2BGR_NV21:  case COLOR_YUV2RGB_NV21:  case COLOR_YUV2BGR_NV12:  case COLOR_YUV2RGB_NV12:
+        case COLOR_YUV2BGRA_NV21: case COLOR_YUV2RGBA_NV21: case COLOR_YUV2BGRA_NV12: case COLOR_YUV2RGBA_NV12:
+            break;
+        default:
+            CV_Error( CV_StsBadFlag, "Unknown/unsupported color conversion code" );
+            return;
+    }
+
+    int stype = _ysrc.type();
+    int depth = CV_MAT_DEPTH(stype), uidx, dcn;
+
+    Mat ysrc, uvsrc, dst;
+    ysrc = _ysrc.getMat();
+    uvsrc = _uvsrc.getMat();
+    Size ysz = _ysrc.size();
+    Size uvs = _uvsrc.size();
+
+    // http://www.fourcc.org/yuv.php#NV21 == yuv420sp -> a plane of 8 bit Y samples followed by an interleaved V/U plane containing 8 bit 2x2 subsampled chroma samples
+    // http://www.fourcc.org/yuv.php#NV12 -> a plane of 8 bit Y samples followed by an interleaved U/V plane containing 8 bit 2x2 subsampled colour difference samples
+    dcn = (code==COLOR_YUV420sp2BGRA || code==COLOR_YUV420sp2RGBA || code==COLOR_YUV2BGRA_NV12 || code==COLOR_YUV2RGBA_NV12) ? 4 : 3;
+    uidx = (code==COLOR_YUV2BGR_NV21 || code==COLOR_YUV2BGRA_NV21 || code==COLOR_YUV2RGB_NV21 || code==COLOR_YUV2RGBA_NV21) ? 1 : 0;
+    CV_Assert( dcn == 3 || dcn == 4 );
+    CV_Assert( ysz.width == uvs.width * 2 );
+    CV_Assert( ysz.width % 2 == 0 && depth == CV_8U );
+    CV_Assert( ysz.height == uvs.height * 2 );
+    _dst.create( ysz, CV_MAKETYPE(depth, dcn));
+    dst = _dst.getMat();
+    hal::cvtTwoPlaneYUVtoBGR(ysrc.data, uvsrc.data, ysrc.step, dst.data, dst.step, dst.cols, dst.rows, dcn, swapBlue(code), uidx);
+}
+
+
 //////////////////////////////////////////////////////////////////////////////////////////
 //                                   The main function                                  //
 //////////////////////////////////////////////////////////////////////////////////////////

modules/java/generator/android-21/java/org/opencv/android/JavaCamera2View.java

@@ -0,0 +1,374 @@
+package org.opencv.android;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import android.annotation.TargetApi;
+import android.content.Context;
+import android.graphics.ImageFormat;
+import android.hardware.camera2.CameraAccessException;
+import android.hardware.camera2.CameraCaptureSession;
+import android.hardware.camera2.CameraCharacteristics;
+import android.hardware.camera2.CameraDevice;
+import android.hardware.camera2.CameraManager;
+import android.hardware.camera2.CaptureRequest;
+import android.hardware.camera2.params.StreamConfigurationMap;
+import android.media.Image;
+import android.media.ImageReader;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.util.AttributeSet;
+import android.util.Log;
+import android.view.Surface;
+import android.view.ViewGroup.LayoutParams;
+
+import org.opencv.core.CvType;
+import org.opencv.core.Mat;
+import org.opencv.imgproc.Imgproc;
+
+/**
+ * This class is an implementation of the Bridge View between OpenCV and Java Camera.
+ * This class relays on the functionality available in base class and only implements
+ * required functions:
+ * connectCamera - opens Java camera and sets the PreviewCallback to be delivered.
+ * disconnectCamera - closes the camera and stops preview.
+ * When frame is delivered via callback from Camera - it processed via OpenCV to be
+ * converted to RGBA32 and then passed to the external callback for modifications if required.
+ */
+
+@TargetApi(21)
+public class JavaCamera2View extends CameraBridgeViewBase {
+
+    private static final String LOGTAG = "JavaCamera2View";
+
+    private ImageReader mImageReader;
+    private int mPreviewFormat = ImageFormat.YUV_420_888;
+
+    private CameraDevice mCameraDevice;
+    private CameraCaptureSession mCaptureSession;
+    private CaptureRequest.Builder mPreviewRequestBuilder;
+    private String mCameraID;
+    private android.util.Size mPreviewSize = new android.util.Size(-1, -1);
+
+    private HandlerThread mBackgroundThread;
+    private Handler mBackgroundHandler;
+
+    public JavaCamera2View(Context context, int cameraId) {
+        super(context, cameraId);
+    }
+
+    public JavaCamera2View(Context context, AttributeSet attrs) {
+        super(context, attrs);
+    }
+
+    private void startBackgroundThread() {
+        Log.i(LOGTAG, "startBackgroundThread");
+        stopBackgroundThread();
+        mBackgroundThread = new HandlerThread("OpenCVCameraBackground");
+        mBackgroundThread.start();
+        mBackgroundHandler = new Handler(mBackgroundThread.getLooper());
+    }
+
+    private void stopBackgroundThread() {
+        Log.i(LOGTAG, "stopBackgroundThread");
+        if (mBackgroundThread == null)
+            return;
+        mBackgroundThread.quitSafely();
+        try {
+            mBackgroundThread.join();
+            mBackgroundThread = null;
+            mBackgroundHandler = null;
+        } catch (InterruptedException e) {
+            Log.e(LOGTAG, "stopBackgroundThread", e);
+        }
+    }
+
+    protected boolean initializeCamera() {
+        Log.i(LOGTAG, "initializeCamera");
+        CameraManager manager = (CameraManager) getContext().getSystemService(Context.CAMERA_SERVICE);
+        try {
+            String camList[] = manager.getCameraIdList();
+            if (camList.length == 0) {
+                Log.e(LOGTAG, "Error: camera isn't detected.");
+                return false;
+            }
+            if (mCameraIndex == CameraBridgeViewBase.CAMERA_ID_ANY) {
+                mCameraID = camList[0];
+            } else {
+                for (String cameraID : camList) {
+                    CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraID);
+                    if ((mCameraIndex == CameraBridgeViewBase.CAMERA_ID_BACK &&
+                            characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_BACK) ||
+                        (mCameraIndex == CameraBridgeViewBase.CAMERA_ID_FRONT &&
+                            characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT)
+                    ) {
+                        mCameraID = cameraID;
+                        break;
+                    }
+                }
+            }
+            if (mCameraID != null) {
+                Log.i(LOGTAG, "Opening camera: " + mCameraID);
+                manager.openCamera(mCameraID, mStateCallback, mBackgroundHandler);
+            }
+            return true;
+        } catch (CameraAccessException e) {
+            Log.e(LOGTAG, "OpenCamera - Camera Access Exception", e);
+        } catch (IllegalArgumentException e) {
+            Log.e(LOGTAG, "OpenCamera - Illegal Argument Exception", e);
+        } catch (SecurityException e) {
+            Log.e(LOGTAG, "OpenCamera - Security Exception", e);
+        }
+        return false;
+    }
+
+    private final CameraDevice.StateCallback mStateCallback = new CameraDevice.StateCallback() {
+
+        @Override
+        public void onOpened(CameraDevice cameraDevice) {
+            mCameraDevice = cameraDevice;
+            createCameraPreviewSession();
+        }
+
+        @Override
+        public void onDisconnected(CameraDevice cameraDevice) {
+            cameraDevice.close();
+            mCameraDevice = null;
+        }
+
+        @Override
+        public void onError(CameraDevice cameraDevice, int error) {
+            cameraDevice.close();
+            mCameraDevice = null;
+        }
+
+    };
+
+    private void createCameraPreviewSession() {
+        final int w = mPreviewSize.getWidth(), h = mPreviewSize.getHeight();
+        Log.i(LOGTAG, "createCameraPreviewSession(" + w + "x" + h + ")");
+        if (w < 0 || h < 0)
+            return;
+        try {
+            if (null == mCameraDevice) {
+                Log.e(LOGTAG, "createCameraPreviewSession: camera isn't opened");
+                return;
+            }
+            if (null != mCaptureSession) {
+                Log.e(LOGTAG, "createCameraPreviewSession: mCaptureSession is already started");
+                return;
+            }
+
+            mImageReader = ImageReader.newInstance(w, h, mPreviewFormat, 2);
+            mImageReader.setOnImageAvailableListener(new ImageReader.OnImageAvailableListener() {
+                @Override
+                public void onImageAvailable(ImageReader reader) {
+                    Image image = reader.acquireLatestImage();
+                    if (image == null)
+                        return;
+
+                    // sanity checks - 3 planes
+                    Image.Plane[] planes = image.getPlanes();
+                    assert (planes.length == 3);
+                    assert (image.getFormat() == mPreviewFormat);
+
+                    // see also https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
+                    // Y plane (0) non-interleaved => stride == 1; U/V plane interleaved => stride == 2
+                    assert (planes[0].getPixelStride() == 1);
+                    assert (planes[1].getPixelStride() == 2);
+                    assert (planes[2].getPixelStride() == 2);
+
+                    ByteBuffer y_plane = planes[0].getBuffer();
+                    ByteBuffer uv_plane = planes[1].getBuffer();
+                    Mat y_mat = new Mat(h, w, CvType.CV_8UC1, y_plane);
+                    Mat uv_mat = new Mat(h / 2, w / 2, CvType.CV_8UC2, uv_plane);
+                    JavaCamera2Frame tempFrame = new JavaCamera2Frame(y_mat, uv_mat, w, h);
+                    deliverAndDrawFrame(tempFrame);
+                    tempFrame.release();
+                    image.close();
+                }
+            }, mBackgroundHandler);
+            Surface surface = mImageReader.getSurface();
+
+            mPreviewRequestBuilder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
+            mPreviewRequestBuilder.addTarget(surface);
+
+            mCameraDevice.createCaptureSession(Arrays.asList(surface),
+                new CameraCaptureSession.StateCallback() {
+                    @Override
+                    public void onConfigured(CameraCaptureSession cameraCaptureSession) {
+                        Log.i(LOGTAG, "createCaptureSession::onConfigured");
+                        if (null == mCameraDevice) {
+                            return; // camera is already closed
+                        }
+                        mCaptureSession = cameraCaptureSession;
+                        try {
+                            mPreviewRequestBuilder.set(CaptureRequest.CONTROL_AF_MODE,
+                                    CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
+                            mPreviewRequestBuilder.set(CaptureRequest.CONTROL_AE_MODE,
+                                    CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH);
+
+                            mCaptureSession.setRepeatingRequest(mPreviewRequestBuilder.build(), null, mBackgroundHandler);
+                            Log.i(LOGTAG, "CameraPreviewSession has been started");
+                        } catch (Exception e) {
+                            Log.e(LOGTAG, "createCaptureSession failed", e);
+                        }
+                    }
+
+                    @Override
+                    public void onConfigureFailed(CameraCaptureSession cameraCaptureSession) {
+                        Log.e(LOGTAG, "createCameraPreviewSession failed");
+                    }
+                },
+                null
+            );
+        } catch (CameraAccessException e) {
+            Log.e(LOGTAG, "createCameraPreviewSession", e);
+        }
+    }
+
+    @Override
+    protected void disconnectCamera() {
+        Log.i(LOGTAG, "closeCamera");
+        try {
+            CameraDevice c = mCameraDevice;
+            mCameraDevice = null;
+            if (null != mCaptureSession) {
+                mCaptureSession.close();
+                mCaptureSession = null;
+            }
+            if (null != c) {
+                c.close();
+            }
+            if (null != mImageReader) {
+                mImageReader.close();
+                mImageReader = null;
+            }
+        } finally {
+            stopBackgroundThread();
+        }
+    }
+
+    boolean calcPreviewSize(final int width, final int height) {
+        Log.i(LOGTAG, "calcPreviewSize: " + width + "x" + height);
+        if (mCameraID == null) {
+            Log.e(LOGTAG, "Camera isn't initialized!");
+            return false;
+        }
+        CameraManager manager = (CameraManager) getContext().getSystemService(Context.CAMERA_SERVICE);
+        try {
+            CameraCharacteristics characteristics = manager.getCameraCharacteristics(mCameraID);
+            StreamConfigurationMap map = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
+            int bestWidth = 0, bestHeight = 0;
+            float aspect = (float) width / height;
+            android.util.Size[] sizes = map.getOutputSizes(ImageReader.class);
+            bestWidth = sizes[0].getWidth();
+            bestHeight = sizes[0].getHeight();
+            for (android.util.Size sz : sizes) {
+                int w = sz.getWidth(), h = sz.getHeight();
+                Log.d(LOGTAG, "trying size: " + w + "x" + h);
+                if (width >= w && height >= h && bestWidth <= w && bestHeight <= h
+                        && Math.abs(aspect - (float) w / h) < 0.2) {
+                    bestWidth = w;
+                    bestHeight = h;
+                }
+            }
+            Log.i(LOGTAG, "best size: " + bestWidth + "x" + bestHeight);
+            assert(!(bestWidth == 0 || bestHeight == 0));
+            if (mPreviewSize.getWidth() == bestWidth && mPreviewSize.getHeight() == bestHeight)
+                return false;
+            else {
+                mPreviewSize = new android.util.Size(bestWidth, bestHeight);
+                return true;
+            }
+        } catch (CameraAccessException e) {
+            Log.e(LOGTAG, "calcPreviewSize - Camera Access Exception", e);
+        } catch (IllegalArgumentException e) {
+            Log.e(LOGTAG, "calcPreviewSize - Illegal Argument Exception", e);
+        } catch (SecurityException e) {
+            Log.e(LOGTAG, "calcPreviewSize - Security Exception", e);
+        }
+        return false;
+    }
+
+    @Override
+    protected boolean connectCamera(int width, int height) {
+        Log.i(LOGTAG, "setCameraPreviewSize(" + width + "x" + height + ")");
+        startBackgroundThread();
+        initializeCamera();
+        try {
+            boolean needReconfig = calcPreviewSize(width, height);
+            mFrameWidth = mPreviewSize.getWidth();
+            mFrameHeight = mPreviewSize.getHeight();
+
+            if ((getLayoutParams().width == LayoutParams.MATCH_PARENT) && (getLayoutParams().height == LayoutParams.MATCH_PARENT))
+                mScale = Math.min(((float)height)/mFrameHeight, ((float)width)/mFrameWidth);
+            else
+                mScale = 0;
+
+            AllocateCache();
+
+            if (needReconfig) {
+                if (null != mCaptureSession) {
+                    Log.d(LOGTAG, "closing existing previewSession");
+                    mCaptureSession.close();
+                    mCaptureSession = null;
+                }
+                createCameraPreviewSession();
+            }
+        } catch (RuntimeException e) {
+            throw new RuntimeException("Interrupted while setCameraPreviewSize.", e);
+        }
+        return true;
+    }
+
+    private class JavaCamera2Frame implements CvCameraViewFrame {
+        @Override
+        public Mat gray() {
+            return mYuvFrameData.submat(0, mHeight, 0, mWidth);
+        }
+
+        @Override
+        public Mat rgba() {
+            if (mPreviewFormat == ImageFormat.NV21)
+                Imgproc.cvtColor(mYuvFrameData, mRgba, Imgproc.COLOR_YUV2RGBA_NV21, 4);
+            else if (mPreviewFormat == ImageFormat.YV12)
+                Imgproc.cvtColor(mYuvFrameData, mRgba, Imgproc.COLOR_YUV2RGB_I420, 4); // COLOR_YUV2RGBA_YV12 produces inverted colors
+            else if (mPreviewFormat == ImageFormat.YUV_420_888) {
+                assert (mUVFrameData != null);
+                Imgproc.cvtColorTwoPlane(mYuvFrameData, mUVFrameData, mRgba, Imgproc.COLOR_YUV2RGBA_NV21);
+            } else
+                throw new IllegalArgumentException("Preview Format can be NV21 or YV12");
+
+            return mRgba;
+        }
+
+        public JavaCamera2Frame(Mat Yuv420sp, int width, int height) {
+            super();
+            mWidth = width;
+            mHeight = height;
+            mYuvFrameData = Yuv420sp;
+            mUVFrameData = null;
+            mRgba = new Mat();
+        }
+
+        public JavaCamera2Frame(Mat Y, Mat UV, int width, int height) {
+            super();
+            mWidth = width;
+            mHeight = height;
+            mYuvFrameData = Y;
+            mUVFrameData = UV;
+            mRgba = new Mat();
+        }
+
+        public void release() {
+            mRgba.release();
+        }
+
+        private Mat mYuvFrameData;
+        private Mat mUVFrameData;
+        private Mat mRgba;
+        private int mWidth;
+        private int mHeight;
+    };
+}