diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 68171ea7af..d78236913b 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -32,6 +32,7 @@ #endif #include #include +#include #include #include #include @@ -395,8 +396,10 @@ private: class SourceReaderCB : public IMFSourceReaderCallback { public: + static const size_t MSMF_READER_MAX_QUEUE_SIZE = 3; + SourceReaderCB() : - m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0), m_lastSampleTimestamp(0) + m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0) { } @@ -441,12 +444,19 @@ public: if (pSample) { CV_LOG_DEBUG(NULL, "videoio(MSMF): got frame at " << llTimestamp); - if (m_lastSample.Get()) + if (m_capturedFrames.size() >= MSMF_READER_MAX_QUEUE_SIZE) { - CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed)"); +#if 0 + CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed). Timestamp=" << m_capturedFrames.front().timestamp); + m_capturedFrames.pop(); +#else + // this branch reduces latency if we drop frames due to slow processing. + // avoid fetching of already outdated frames from the queue's front. + CV_LOG_DEBUG(NULL, "videoio(MSMF): drop previous frames (not processed): " << m_capturedFrames.size()); + std::queue().swap(m_capturedFrames); // similar to missing m_capturedFrames.clean(); +#endif } - m_lastSampleTimestamp = llTimestamp; - m_lastSample = pSample; + m_capturedFrames.emplace(CapturedFrameInfo{ llTimestamp, _ComPtr(pSample), hrStatus }); } } else @@ -483,30 +493,43 @@ public: return S_OK; } - HRESULT Wait(DWORD dwMilliseconds, _ComPtr& mediaSample, BOOL& pbEOS) + HRESULT Wait(DWORD dwMilliseconds, _ComPtr& mediaSample, LONGLONG& sampleTimestamp, BOOL& pbEOS) { pbEOS = FALSE; - DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds); - if (dwResult == WAIT_TIMEOUT) - { - return E_PENDING; - } - else if (dwResult != WAIT_OBJECT_0) + for (;;) { - return HRESULT_FROM_WIN32(GetLastError()); - } + { + cv::AutoLock lock(m_mutex); - pbEOS = m_bEOS; - if (!pbEOS) - { - cv::AutoLock lock(m_mutex); - mediaSample = m_lastSample; - CV_Assert(mediaSample); - m_lastSample.Release(); - ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. + pbEOS = m_bEOS && m_capturedFrames.empty(); + if (pbEOS) + return m_hrStatus; + + if (!m_capturedFrames.empty()) + { + CV_Assert(!m_capturedFrames.empty()); + CapturedFrameInfo frameInfo = m_capturedFrames.front(); m_capturedFrames.pop(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): handle frame at " << frameInfo.timestamp); + mediaSample = frameInfo.sample; + CV_Assert(mediaSample); + sampleTimestamp = frameInfo.timestamp; + ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. + return frameInfo.hrStatus; + } + } + + CV_LOG_DEBUG(NULL, "videoio(MSMF): waiting for frame... "); + DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds); + if (dwResult == WAIT_TIMEOUT) + { + return E_PENDING; + } + else if (dwResult != WAIT_OBJECT_0) + { + return HRESULT_FROM_WIN32(GetLastError()); + } } - return m_hrStatus; } private: @@ -525,8 +548,14 @@ public: IMFSourceReader *m_reader; DWORD m_dwStreamIndex; - LONGLONG m_lastSampleTimestamp; - _ComPtr m_lastSample; + + struct CapturedFrameInfo { + LONGLONG timestamp; + _ComPtr sample; + HRESULT hrStatus; + }; + + std::queue m_capturedFrames; }; //================================================================================================== @@ -1569,7 +1598,6 @@ bool CvCapture_MSMF::configureAudioFrame() } audioDataInUse.clear(); audioDataInUse.shrink_to_fit(); - return true; } else @@ -1690,6 +1718,7 @@ bool CvCapture_MSMF::grabFrame() if (grabIsDone) { grabIsDone = false; + CV_LOG_DEBUG(NULL, "videoio(MSMF): return pre-grabbed frame " << usedVideoSampleTime); return true; } @@ -1717,7 +1746,8 @@ bool CvCapture_MSMF::grabFrame() } } BOOL bEOS = false; - if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], bEOS))) // 10 sec + LONGLONG timestamp = 0; + if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], timestamp, bEOS))) // 10 sec { CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr); return false; @@ -1728,10 +1758,11 @@ bool CvCapture_MSMF::grabFrame() return false; } if (videoStream != -1) - usedVideoSampleTime = reader->m_lastSampleTimestamp; + usedVideoSampleTime = timestamp; if (audioStream != -1) return configureAudioFrame(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): grabbed frame " << usedVideoSampleTime); return true; } else if (isOpen) @@ -1765,6 +1796,7 @@ bool CvCapture_MSMF::grabFrame() bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame) { CV_TRACE_FUNCTION(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame start..."); do { if (!usedVideoSample) @@ -1855,6 +1887,7 @@ bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame) buffer2d->Unlock2D(); else buf->Unlock(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame done!"); return !frame.empty(); } while (0); diff --git a/modules/videoio/test/test_camera.cpp b/modules/videoio/test/test_camera.cpp index c8a5598dcc..fc269959c3 100644 --- a/modules/videoio/test/test_camera.cpp +++ b/modules/videoio/test/test_camera.cpp @@ -26,15 +26,26 @@ static void test_readFrames(/*const*/ VideoCapture& capture, const int N = 100, const bool validTickAndFps = cvTickFreq != 0 && fps != 0.; testTimestamps &= validTickAndFps; + double frame0ts = 0; + for (int i = 0; i < N; i++) { SCOPED_TRACE(cv::format("frame=%d", i)); capture >> frame; - const int64 sysTimeCurr = cv::getTickCount(); - const double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC); ASSERT_FALSE(frame.empty()); + const int64 sysTimeCurr = cv::getTickCount(); + double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC); + if (i == 0) + frame0ts = camTimeCurr; + camTimeCurr -= frame0ts; // normalized timestamp based on the first frame + + if (cvtest::debugLevel > 0) + { + std::cout << i << ": " << camTimeCurr << std::endl; + } + // Do we have a previous frame? if (i > 0 && testTimestamps) {