Merge pull request #15100 from Volskig:cam_multiplexing_function_v

Implement Camera Multiplexing API

* IdideoCapture + two wrong function

function waitAny

Add errors catcher

Stub for Python added.

Sifting warnings

One test added

Two tests for camera and Perf tests added

* Perf sync and async tests for waitAny() added, waitAnyInterior() deleted, getDeviceHandle() deleted

* Variable OPENCV_TEST_CAMERA_LIST added

* Without fps set

* ASSERT_FAILED for environment variable

* Perf tests is DISABLED_

* --Trailing whitespace

* Return false from cap.cpp deleted

* Two functions deleted from interface, +range for, +environment variable in test_camera

* Space deleted

* printf deleted, perror added

* CV_WRAP deleted, cv2 cleared from stubs

* -- space

* default timeout added

* @param changed

* place of waitAny changed

* --whitespace

* ++function description

* function description changed

* revert unused changes

* videoio: rework API for VideoCapture::waitAny()

modules/videoio/include/opencv2/videoio.hpp

@@ -581,6 +581,9 @@ enum { CAP_PROP_IMAGES_BASE = 18000,
 
 
 class IVideoCapture;
+//! @cond IGNORED
+namespace internal { class VideoCapturePrivateAccessor; }
+//! @endcond IGNORED
 
 /** @brief Class for video capturing from video files, image sequences or cameras.
 
@@ -790,10 +793,34 @@ public:
 
     /// query if exception mode is active
     CV_WRAP bool getExceptionMode() { return throwOnFail; }
+
+
+    /** @brief Wait for ready frames from VideoCapture.
+
+    @param streams input video streams
+    @param readyIndex stream indexes with grabbed frames (ready to use .retrieve() to fetch actual frame)
+    @param timeoutNs number of nanoseconds (0 - infinite)
+    @return `true` if streamReady is not empty
+
+    @throws Exception %Exception on stream errors (check .isOpened() to filter out malformed streams) or VideoCapture type is not supported
+
+    The primary use of the function is in multi-camera environments.
+    The method fills the ready state vector, grabbs video frame, if camera is ready.
+
+    After this call use VideoCapture::retrieve() to decode and fetch frame data.
+    */
+    static /*CV_WRAP*/
+    bool waitAny(
+            const std::vector<VideoCapture>& streams,
+            CV_OUT std::vector<int>& readyIndex,
+            int64 timeoutNs = 0);
+
 protected:
     Ptr<CvCapture> cap;
     Ptr<IVideoCapture> icap;
     bool throwOnFail;
+
+    friend class internal::VideoCapturePrivateAccessor;
 };
 
 class IVideoWriter;

modules/videoio/perf/perf_camera.impl.hpp

@@ -0,0 +1,76 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html
+
+// Not a standalone header.
+
+#include <opencv2/core/utils/configuration.private.hpp>
+
+namespace opencv_test {
+using namespace perf;
+
+static
+utils::Paths getTestCameras()
+{
+    static utils::Paths cameras = utils::getConfigurationParameterPaths("OPENCV_TEST_PERF_CAMERA_LIST");
+    return cameras;
+}
+
+PERF_TEST(VideoCapture_Camera, waitAny_V4L)
+{
+    auto cameraNames = getTestCameras();
+    if (cameraNames.empty())
+       throw SkipTestException("No list of tested cameras. Use OPENCV_TEST_PERF_CAMERA_LIST parameter");
+
+    const int totalFrames = 50; // number of expected frames (summary for all cameras)
+    const int64 timeoutNS = 100 * 1000000;
+
+    const Size frameSize(640, 480);
+    const int fpsDefaultEven = 30;
+    const int fpsDefaultOdd = 15;
+
+    std::vector<VideoCapture> cameras;
+    for (size_t i = 0; i < cameraNames.size(); ++i)
+    {
+        const auto& name = cameraNames[i];
+        int fps = (int)utils::getConfigurationParameterSizeT(cv::format("OPENCV_TEST_CAMERA%d_FPS", (int)i).c_str(), (i & 1) ? fpsDefaultOdd : fpsDefaultEven);
+        std::cout << "Camera[" << i << "] = '" << name << "', fps=" << fps << std::endl;
+        VideoCapture cap(name, CAP_V4L);
+        ASSERT_TRUE(cap.isOpened()) << name;
+        EXPECT_TRUE(cap.set(CAP_PROP_FRAME_WIDTH, frameSize.width)) << name;
+        EXPECT_TRUE(cap.set(CAP_PROP_FRAME_HEIGHT, frameSize.height)) << name;
+        EXPECT_TRUE(cap.set(CAP_PROP_FPS, fps)) << name;
+        //launch cameras
+        Mat firstFrame;
+        EXPECT_TRUE(cap.read(firstFrame));
+        EXPECT_EQ(frameSize.width, firstFrame.cols);
+        EXPECT_EQ(frameSize.height, firstFrame.rows);
+        cameras.push_back(cap);
+    }
+
+    TEST_CYCLE()
+    {
+        int counter = 0;
+        std::vector<int> cameraReady;
+        do
+        {
+            EXPECT_TRUE(VideoCapture::waitAny(cameras, cameraReady, timeoutNS));
+            EXPECT_FALSE(cameraReady.empty());
+            for (int idx : cameraReady)
+            {
+                VideoCapture& c = cameras[idx];
+                Mat frame;
+                ASSERT_TRUE(c.retrieve(frame));
+                EXPECT_EQ(frameSize.width, frame.cols);
+                EXPECT_EQ(frameSize.height, frame.rows);
+
+                ++counter;
+            }
+        }
+        while(counter < totalFrames);
+    }
+
+    SANITY_CHECK_NOTHING();
+}
+
+} // namespace

modules/videoio/perf/perf_input.cpp

@@ -3,6 +3,8 @@
 // of this distribution and at http://opencv.org/license.html
 #include "perf_precomp.hpp"
 
+#include "perf_camera.impl.hpp"
+
 namespace opencv_test
 {
 using namespace perf;

modules/videoio/src/cap.cpp

@@ -319,6 +319,29 @@ double VideoCapture::get(int propId) const
 }
 
 
+bool VideoCapture::waitAny(const std::vector<VideoCapture>& streams, CV_OUT std::vector<int>& readyIndex, int64 timeoutNs)
+{
+    CV_Assert(!streams.empty());
+
+    VideoCaptureAPIs backend = (VideoCaptureAPIs)streams[0].icap->getCaptureDomain();
+
+    for (size_t i = 1; i < streams.size(); ++i)
+    {
+        VideoCaptureAPIs backend_i = (VideoCaptureAPIs)streams[i].icap->getCaptureDomain();
+        CV_CheckEQ((int)backend, (int)backend_i, "All captures must have the same backend");
+    }
+
+#if (defined HAVE_CAMV4L2 || defined HAVE_VIDEOIO)  // see cap_v4l.cpp guard
+    if (backend == CAP_V4L2)
+        return VideoCapture_V4L_waitAny(streams, readyIndex, timeoutNs);
+#else
+    CV_UNUSED(readyIndex); CV_UNUSED(timeoutNs);
+#endif
+    CV_Error(Error::StsNotImplemented, "VideoCapture::waitAny() is supported by V4L backend only");
+}
+
+
+
 //=================================================================================================
 
 

modules/videoio/src/cap_interface.hpp

@@ -61,6 +61,15 @@ public:
     virtual int getCaptureDomain() const { return cv::CAP_ANY; } // Return the type of the capture object: CAP_FFMPEG, etc...
 };
 
+namespace internal {
+class VideoCapturePrivateAccessor
+{
+public:
+    static
+    IVideoCapture* getIVideoCapture(const VideoCapture& cap) { return cap.icap.get(); }
+};
+} // namespace
+
 //===================================================
 
 // Wrapper
@@ -116,6 +125,8 @@ public:
     {
         return cap ? cap->getCaptureDomain() : 0;
     }
+
+    CvCapture* getCvCapture() const { return cap; }
 };
 
 class LegacyWriter : public IVideoWriter
@@ -208,6 +219,11 @@ Ptr<IVideoCapture> createXINECapture(const std::string &filename);
 
 Ptr<IVideoCapture> createAndroidCapture_file(const std::string &filename);
 
+bool VideoCapture_V4L_waitAny(
+        const std::vector<VideoCapture>& streams,
+        CV_OUT std::vector<int>& ready,
+        int64 timeoutNs);
+
 } // cv::
 
 #endif // CAP_INTERFACE_HPP

modules/videoio/src/cap_v4l.cpp

@@ -228,6 +228,8 @@ make & enjoy!
 #include <sys/ioctl.h>
 #include <limits>
 
+#include <poll.h>
+
 #ifdef HAVE_CAMV4L2
 #include <asm/types.h>          /* for videodev2.h */
 #include <linux/videodev2.h>
@@ -372,6 +374,8 @@ struct CvCaptureCAM_V4L CV_FINAL : public CvCapture
     bool convertableToRgb() const;
     void convertToRgb(const Buffer &currentBuffer);
     void releaseFrame();
+
+    bool havePendingFrame;  // true if next .grab() should be noop, .retrive() resets this flag
 };
 
 /***********************   Implementations  ***************************************/
@@ -384,7 +388,8 @@ CvCaptureCAM_V4L::CvCaptureCAM_V4L() :
     bufferSize(DEFAULT_V4L_BUFFERS),
     fps(0), convert_rgb(0), frame_allocated(false), returnFrame(false),
     channelNumber(-1), normalizePropRange(false),
-    type(V4L2_BUF_TYPE_VIDEO_CAPTURE)
+    type(V4L2_BUF_TYPE_VIDEO_CAPTURE),
+    havePendingFrame(false)
 {
     frame = cvIplImage();
     memset(&timestamp, 0, sizeof(timestamp));
@@ -863,6 +868,7 @@ bool CvCaptureCAM_V4L::read_frame_v4l2()
 
 bool CvCaptureCAM_V4L::tryIoctl(unsigned long ioctlCode, void *parameter) const
 {
+    CV_LOG_DEBUG(NULL, "tryIoctl(handle=" << deviceHandle << ", ioctl=0x" << std::hex << ioctlCode << ", ...)")
     while (-1 == ioctl(deviceHandle, ioctlCode, parameter)) {
         if (!(errno == EBUSY || errno == EAGAIN))
             return false;
@@ -889,7 +895,13 @@ bool CvCaptureCAM_V4L::tryIoctl(unsigned long ioctlCode, void *parameter) const
 
 bool CvCaptureCAM_V4L::grabFrame()
 {
-    if (FirstCapture) {
+    if (havePendingFrame)  // frame has been already grabbed during preroll
+    {
+        return true;
+    }
+
+    if (FirstCapture)
+    {
         /* Some general initialization must take place the first time through */
 
         /* This is just a technicality, but all buffers must be filled up before any
@@ -1939,6 +1951,8 @@ bool CvCaptureCAM_V4L::streaming(bool startStream)
 
 IplImage *CvCaptureCAM_V4L::retrieveFrame(int)
 {
+    havePendingFrame = false;  // unlock .grab()
+
     if (bufferIndex < 0)
         return &frame;
 
@@ -1989,6 +2003,109 @@ Ptr<IVideoCapture> create_V4L_capture_file(const std::string &filename)
     return NULL;
 }
 
+static
+bool VideoCapture_V4L_deviceHandlePoll(const std::vector<int>& deviceHandles, std::vector<int>& ready, int64 timeoutNs)
+{
+    CV_Assert(!deviceHandles.empty());
+    const size_t N = deviceHandles.size();
+
+    ready.clear(); ready.reserve(N);
+
+    const auto poll_flags = POLLIN | POLLRDNORM | POLLERR;
+
+    std::vector<pollfd> fds; fds.reserve(N);
+
+    for (size_t i = 0; i < N; ++i)
+    {
+        int handle = deviceHandles[i];
+        CV_LOG_DEBUG(NULL, "camera" << i << ": handle = " << handle);
+        CV_Assert(handle != 0);
+        fds.push_back(pollfd{handle, poll_flags, 0});
+    }
+
+    int timeoutMs = -1;
+    if (timeoutNs > 0)
+    {
+        timeoutMs = saturate_cast<int>((timeoutNs + 999999) / 1000000);
+    }
+
+    int ret = poll(fds.data(), N, timeoutMs);
+    if (ret == -1)
+    {
+        perror("poll error");
+        return false;
+    }
+
+    if (ret == 0)
+        return 0;  // just timeout
+
+    for (size_t i = 0; i < N; ++i)
+    {
+        const auto& fd = fds[i];
+        CV_LOG_DEBUG(NULL, "camera" << i << ": fd.revents = 0x" << std::hex << fd.revents);
+        if ((fd.revents & (POLLIN | POLLRDNORM)) != 0)
+        {
+            ready.push_back(i);
+        }
+        else if ((fd.revents & POLLERR) != 0)
+        {
+            CV_Error_(Error::StsError, ("Error is reported for camera stream: %d (handle = %d)", (int)i, deviceHandles[i]));
+        }
+        else
+        {
+            // not ready
+        }
+    }
+    return true;
+}
+
+bool VideoCapture_V4L_waitAny(const std::vector<VideoCapture>& streams, CV_OUT std::vector<int>& ready, int64 timeoutNs)
+{
+    CV_Assert(!streams.empty());
+
+    const size_t N = streams.size();
+
+    // unwrap internal API
+    std::vector<CvCaptureCAM_V4L*> capPtr(N, NULL);
+    for (size_t i = 0; i < N; ++i)
+    {
+        IVideoCapture* iCap = internal::VideoCapturePrivateAccessor::getIVideoCapture(streams[i]);
+        LegacyCapture* legacyCapture = dynamic_cast<LegacyCapture*>(iCap);
+        CV_Assert(legacyCapture);
+        CvCapture* cvCap = legacyCapture->getCvCapture();
+        CV_Assert(cvCap);
+
+        CvCaptureCAM_V4L *ptr_CvCaptureCAM_V4L = dynamic_cast<CvCaptureCAM_V4L*>(cvCap);
+        CV_Assert(ptr_CvCaptureCAM_V4L);
+        capPtr[i] = ptr_CvCaptureCAM_V4L;
+    }
+
+    // initialize cameras streams and get handles
+    std::vector<int> deviceHandles; deviceHandles.reserve(N);
+    for (size_t i = 0; i < N; ++i)
+    {
+        CvCaptureCAM_V4L *ptr = capPtr[i];
+        if (ptr->FirstCapture)
+        {
+            ptr->havePendingFrame = ptr->grabFrame();
+            CV_Assert(ptr->havePendingFrame);
+            // TODO: Need to filter these cameras, because frame is available
+        }
+        CV_Assert(ptr->deviceHandle);
+        deviceHandles.push_back(ptr->deviceHandle);
+    }
+
+    bool res = VideoCapture_V4L_deviceHandlePoll(deviceHandles, ready, timeoutNs);
+    for (size_t i = 0; i < ready.size(); ++i)
+    {
+        int idx = ready[i];
+        CvCaptureCAM_V4L *ptr = capPtr[idx];
+        ptr->havePendingFrame = ptr->grabFrame();
+        CV_Assert(ptr->havePendingFrame);
+    }
+    return res;
+}
+
 } // cv::
 
 #endif

modules/videoio/test/test_camera.cpp

@@ -8,6 +8,7 @@
 // Usage: opencv_test_videoio --gtest_also_run_disabled_tests --gtest_filter=*videoio_camera*<tested case>*
 
 #include "test_precomp.hpp"
+#include <opencv2/core/utils/configuration.private.hpp>
 
 namespace opencv_test { namespace {
 
@@ -105,4 +106,79 @@ TEST(DISABLED_videoio_camera, v4l_read_framesize)
     capture.release();
 }
 
+
+static
+utils::Paths getTestCameras()
+{
+    static utils::Paths cameras = utils::getConfigurationParameterPaths("OPENCV_TEST_CAMERA_LIST");
+    return cameras;
+}
+
+TEST(DISABLED_videoio_camera, waitAny_V4L)
+{
+    auto cameraNames = getTestCameras();
+    if (cameraNames.empty())
+       throw SkipTestException("No list of tested cameras. Use OPENCV_TEST_CAMERA_LIST parameter");
+
+    const int totalFrames = 50; // number of expected frames (summary for all cameras)
+    const int64 timeoutNS = 100 * 1000000;
+
+    const Size frameSize(640, 480);
+    const int fpsDefaultEven = 30;
+    const int fpsDefaultOdd = 15;
+
+    std::vector<VideoCapture> cameras;
+    for (size_t i = 0; i < cameraNames.size(); ++i)
+    {
+        const auto& name = cameraNames[i];
+        int fps = (int)utils::getConfigurationParameterSizeT(cv::format("OPENCV_TEST_CAMERA%d_FPS", (int)i).c_str(), (i & 1) ? fpsDefaultOdd : fpsDefaultEven);
+        std::cout << "Camera[" << i << "] = '" << name << "', fps=" << fps << std::endl;
+        VideoCapture cap(name, CAP_V4L);
+        ASSERT_TRUE(cap.isOpened()) << name;
+        EXPECT_TRUE(cap.set(CAP_PROP_FRAME_WIDTH, frameSize.width)) << name;
+        EXPECT_TRUE(cap.set(CAP_PROP_FRAME_HEIGHT, frameSize.height)) << name;
+        EXPECT_TRUE(cap.set(CAP_PROP_FPS, fps)) << name;
+        //launch cameras
+        Mat firstFrame;
+        EXPECT_TRUE(cap.read(firstFrame));
+        EXPECT_EQ(frameSize.width, firstFrame.cols);
+        EXPECT_EQ(frameSize.height, firstFrame.rows);
+        cameras.push_back(cap);
+    }
+
+    std::vector<size_t> frameFromCamera(cameraNames.size(), 0);
+    {
+        int counter = 0;
+        std::vector<int> cameraReady;
+        do
+        {
+            EXPECT_TRUE(VideoCapture::waitAny(cameras, cameraReady, timeoutNS));
+            EXPECT_FALSE(cameraReady.empty());
+            for (int idx : cameraReady)
+            {
+                //std::cout << "Reading frame from camera: " << idx << std::endl;
+                ASSERT_TRUE(idx >= 0 && (size_t)idx < cameras.size()) << idx;
+                VideoCapture& c = cameras[idx];
+                Mat frame;
+#if 1
+                ASSERT_TRUE(c.retrieve(frame)) << idx;
+#else
+                ASSERT_TRUE(c.read(frame)) << idx;
+#endif
+                EXPECT_EQ(frameSize.width, frame.cols) << idx;
+                EXPECT_EQ(frameSize.height, frame.rows) << idx;
+
+                ++frameFromCamera[idx];
+                ++counter;
+            }
+        }
+        while(counter < totalFrames);
+    }
+
+    for (size_t i = 0; i < cameraNames.size(); ++i)
+    {
+        EXPECT_GT(frameFromCamera[i], (size_t)0) << i;
+    }
+}
+
 }} // namespace