Merge pull request #21202 from alalek:videoio_msmf_update_camera

modules/videoio/src/cap_msmf.cpp

@@ -32,6 +32,7 @@
 #endif
 #include <new>
 #include <map>
+#include <queue>
 #include <vector>
 #include <string>
 #include <algorithm>
@@ -395,8 +396,10 @@ private:
 class SourceReaderCB : public IMFSourceReaderCallback
 {
 public:
+    static const size_t MSMF_READER_MAX_QUEUE_SIZE = 3;
+
     SourceReaderCB() :
-        m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0), m_lastSampleTimestamp(0)
+        m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0)
     {
     }
 
@@ -441,12 +444,19 @@ public:
             if (pSample)
             {
                 CV_LOG_DEBUG(NULL, "videoio(MSMF): got frame at " << llTimestamp);
-                if (m_lastSample.Get())
+                if (m_capturedFrames.size() >= MSMF_READER_MAX_QUEUE_SIZE)
                 {
-                    CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed)");
+#if 0
+                    CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed). Timestamp=" << m_capturedFrames.front().timestamp);
+                    m_capturedFrames.pop();
+#else
+                    // this branch reduces latency if we drop frames due to slow processing.
+                    // avoid fetching of already outdated frames from the queue's front.
+                    CV_LOG_DEBUG(NULL, "videoio(MSMF): drop previous frames (not processed): " << m_capturedFrames.size());
+                    std::queue<CapturedFrameInfo>().swap(m_capturedFrames);  // similar to missing m_capturedFrames.clean();
+#endif
                 }
-                m_lastSampleTimestamp = llTimestamp;
+                m_capturedFrames.emplace(CapturedFrameInfo{ llTimestamp, _ComPtr<IMFSample>(pSample), hrStatus });
-                m_lastSample = pSample;
             }
         }
         else
@@ -483,30 +493,43 @@ public:
         return S_OK;
     }
 
-    HRESULT Wait(DWORD dwMilliseconds, _ComPtr<IMFSample>& mediaSample, BOOL& pbEOS)
+    HRESULT Wait(DWORD dwMilliseconds, _ComPtr<IMFSample>& mediaSample, LONGLONG& sampleTimestamp, BOOL& pbEOS)
     {
         pbEOS = FALSE;
 
-        DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds);
+        for (;;)
-        if (dwResult == WAIT_TIMEOUT)
-        {
-            return E_PENDING;
-        }
-        else if (dwResult != WAIT_OBJECT_0)
         {
-            return HRESULT_FROM_WIN32(GetLastError());
+            {
-        }
+                cv::AutoLock lock(m_mutex);
 
-        pbEOS = m_bEOS;
+                pbEOS = m_bEOS && m_capturedFrames.empty();
-        if (!pbEOS)
+                if (pbEOS)
-        {
+                    return m_hrStatus;
-            cv::AutoLock lock(m_mutex);
+
-            mediaSample = m_lastSample;
+                if (!m_capturedFrames.empty())
-            CV_Assert(mediaSample);
+                {
-            m_lastSample.Release();
+                    CV_Assert(!m_capturedFrames.empty());
-            ResetEvent(m_hEvent);  // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold.
+                    CapturedFrameInfo frameInfo = m_capturedFrames.front(); m_capturedFrames.pop();
+                    CV_LOG_DEBUG(NULL, "videoio(MSMF): handle frame at " << frameInfo.timestamp);
+                    mediaSample = frameInfo.sample;
+                    CV_Assert(mediaSample);
+                    sampleTimestamp = frameInfo.timestamp;
+                    ResetEvent(m_hEvent);  // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold.
+                    return frameInfo.hrStatus;
+                }
+            }
+
+            CV_LOG_DEBUG(NULL, "videoio(MSMF): waiting for frame... ");
+            DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds);
+            if (dwResult == WAIT_TIMEOUT)
+            {
+                return E_PENDING;
+            }
+            else if (dwResult != WAIT_OBJECT_0)
+            {
+                return HRESULT_FROM_WIN32(GetLastError());
+            }
         }
-        return m_hrStatus;
     }
 
 private:
@@ -525,8 +548,14 @@ public:
 
     IMFSourceReader *m_reader;
     DWORD m_dwStreamIndex;
-    LONGLONG m_lastSampleTimestamp;
+
-    _ComPtr<IMFSample>  m_lastSample;
+    struct CapturedFrameInfo {
+        LONGLONG timestamp;
+        _ComPtr<IMFSample> sample;
+        HRESULT hrStatus;
+    };
+
+    std::queue<CapturedFrameInfo> m_capturedFrames;
 };
 
 //==================================================================================================
@@ -1569,7 +1598,6 @@ bool CvCapture_MSMF::configureAudioFrame()
         }
         audioDataInUse.clear();
         audioDataInUse.shrink_to_fit();
-
         return true;
     }
     else
@@ -1690,6 +1718,7 @@ bool CvCapture_MSMF::grabFrame()
     if (grabIsDone)
     {
         grabIsDone = false;
+        CV_LOG_DEBUG(NULL, "videoio(MSMF): return pre-grabbed frame " << usedVideoSampleTime);
         return true;
     }
 
@@ -1717,7 +1746,8 @@ bool CvCapture_MSMF::grabFrame()
             }
         }
         BOOL bEOS = false;
-        if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], bEOS)))  // 10 sec
+        LONGLONG timestamp = 0;
+        if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], timestamp, bEOS)))  // 10 sec
         {
             CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr);
             return false;
@@ -1728,10 +1758,11 @@ bool CvCapture_MSMF::grabFrame()
             return false;
         }
         if (videoStream != -1)
-            usedVideoSampleTime = reader->m_lastSampleTimestamp;
+            usedVideoSampleTime = timestamp;
         if (audioStream != -1)
             return configureAudioFrame();
 
+        CV_LOG_DEBUG(NULL, "videoio(MSMF): grabbed frame " << usedVideoSampleTime);
         return true;
     }
     else if (isOpen)
@@ -1765,6 +1796,7 @@ bool CvCapture_MSMF::grabFrame()
 bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame)
 {
     CV_TRACE_FUNCTION();
+    CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame start...");
     do
     {
         if (!usedVideoSample)
@@ -1855,6 +1887,7 @@ bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame)
             buffer2d->Unlock2D();
         else
             buf->Unlock();
+        CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame done!");
         return !frame.empty();
     } while (0);
 

modules/videoio/test/test_camera.cpp

@@ -26,15 +26,26 @@ static void test_readFrames(/*const*/ VideoCapture& capture, const int N = 100,
     const bool validTickAndFps = cvTickFreq != 0 && fps != 0.;
     testTimestamps &= validTickAndFps;
 
+    double frame0ts = 0;
+
     for (int i = 0; i < N; i++)
     {
         SCOPED_TRACE(cv::format("frame=%d", i));
 
         capture >> frame;
-        const int64 sysTimeCurr = cv::getTickCount();
-        const double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC);
         ASSERT_FALSE(frame.empty());
 
+        const int64 sysTimeCurr = cv::getTickCount();
+        double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC);
+        if (i == 0)
+            frame0ts = camTimeCurr;
+        camTimeCurr -= frame0ts;  // normalized timestamp based on the first frame
+
+        if (cvtest::debugLevel > 0)
+        {
+            std::cout << i << ": " << camTimeCurr << std::endl;
+        }
+
         // Do we have a previous frame?
         if (i > 0 && testTimestamps)
         {