From 2fa624aef0bc681c37e8bb267401c54a9e4c1df9 Mon Sep 17 00:00:00 2001
From: Jaime Rivera <jaime.rivera.cs@gmail.com>
Date: Sun, 29 Nov 2020 21:17:24 -0800
Subject: [PATCH] Add Timestamps to MSMF Video Capture by index

Enable frame timestamp tests for MSMF

Add functional test for camera live timestamps

Remove trailing whitespace

Add timestamp test to all functional tests. Protect div by 0

Add Timestamps to MSMF Video Capture by index
---
 modules/videoio/src/cap_msmf.cpp       |  5 ++--
 modules/videoio/test/test_camera.cpp   | 34 ++++++++++++++++++++++++--
 modules/videoio/test/test_video_io.cpp |  2 +-
 3 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp
index 15b1d2ade7..7caa7c1ea0 100644
--- a/modules/videoio/src/cap_msmf.cpp
+++ b/modules/videoio/src/cap_msmf.cpp
@@ -346,8 +346,6 @@ public:
 
     STDMETHODIMP OnReadSample(HRESULT hrStatus, DWORD dwStreamIndex, DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample *pSample) CV_OVERRIDE
     {
-        CV_UNUSED(llTimestamp);
-
         HRESULT hr = 0;
         cv::AutoLock lock(m_mutex);
 
@@ -360,6 +358,7 @@ public:
                 {
                     CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed)");
                 }
+                m_lastSampleTimestamp = llTimestamp;
                 m_lastSample = pSample;
             }
         }
@@ -439,6 +438,7 @@ public:
 
     IMFSourceReader *m_reader;
     DWORD m_dwStreamIndex;
+    LONGLONG m_lastSampleTimestamp;
     _ComPtr<IMFSample>  m_lastSample;
 };
 
@@ -912,6 +912,7 @@ bool CvCapture_MSMF::grabFrame()
             CV_LOG_WARNING(NULL, "videoio(MSMF): EOS signal. Capture stream is lost");
             return false;
         }
+        sampleTime = reader->m_lastSampleTimestamp;
         return true;
     }
     else if (isOpen)
diff --git a/modules/videoio/test/test_camera.cpp b/modules/videoio/test/test_camera.cpp
index d816f637a7..e82285ad5e 100644
--- a/modules/videoio/test/test_camera.cpp
+++ b/modules/videoio/test/test_camera.cpp
@@ -11,21 +11,51 @@
 
 namespace opencv_test { namespace {
 
-static void test_readFrames(/*const*/ VideoCapture& capture, const int N = 100, Mat* lastFrame = NULL)
+static void test_readFrames(/*const*/ VideoCapture& capture, const int N = 100, Mat* lastFrame = NULL, bool testTimestamps = true)
 {
     Mat frame;
     int64 time0 = cv::getTickCount();
+    int64 sysTimePrev = time0;
+    const double cvTickFreq = cv::getTickFrequency();
+
+    double camTimePrev = 0.0;
+    const double fps = capture.get(cv::CAP_PROP_FPS);
+    const double framePeriod = fps == 0.0 ? 1. : 1.0 / fps;
+
+    const bool validTickAndFps = cvTickFreq != 0 && fps != 0.;
+    testTimestamps &= validTickAndFps;
+
     for (int i = 0; i < N; i++)
     {
         SCOPED_TRACE(cv::format("frame=%d", i));
 
         capture >> frame;
+        const int64 sysTimeCurr = cv::getTickCount();
+        const double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC);
         ASSERT_FALSE(frame.empty());
 
+        // Do we have a previous frame?
+        if (i > 0 && testTimestamps)
+        {
+            const double sysTimeElapsedSecs = (sysTimeCurr - sysTimePrev) / cvTickFreq;
+            const double camTimeElapsedSecs = (camTimeCurr - camTimePrev) / 1000.;
+
+            // Check that the time between two camera frames and two system time calls
+            // are within 1.5 frame periods of one another.
+            //
+            // 1.5x is chosen to accomodate for a dropped frame, and an additional 50%
+            // to account for drift in the scale of the camera and system time domains.
+            EXPECT_NEAR(sysTimeElapsedSecs, camTimeElapsedSecs, framePeriod * 1.5);
+        }
+
         EXPECT_GT(cvtest::norm(frame, NORM_INF), 0) << "Complete black image has been received";
+
+        sysTimePrev = sysTimeCurr;
+        camTimePrev = camTimeCurr;
     }
+
     int64 time1 = cv::getTickCount();
-    printf("Processed %d frames on %.2f FPS\n", N, (N * cv::getTickFrequency()) / (time1 - time0 + 1));
+    printf("Processed %d frames on %.2f FPS\n", N, (N * cvTickFreq) / (time1 - time0 + 1));
     if (lastFrame) *lastFrame = frame.clone();
 }
 
diff --git a/modules/videoio/test/test_video_io.cpp b/modules/videoio/test/test_video_io.cpp
index 3f5617d8ce..19fc32b53e 100644
--- a/modules/videoio/test/test_video_io.cpp
+++ b/modules/videoio/test/test_video_io.cpp
@@ -237,7 +237,7 @@ public:
         if (!isBackendAvailable(apiPref, cv::videoio_registry::getStreamBackends()))
             throw SkipTestException(cv::String("Backend is not available/disabled: ") + cv::videoio_registry::getBackendName(apiPref));
 
-        if ((apiPref == CAP_MSMF) || ((apiPref == CAP_FFMPEG) && ((ext == "h264") || (ext == "h265"))))
+        if (((apiPref == CAP_FFMPEG) && ((ext == "h264") || (ext == "h265"))))
             throw SkipTestException(cv::String("Backend ") +  cv::videoio_registry::getBackendName(apiPref) +
                     cv::String(" does not support CAP_PROP_POS_MSEC option"));