Merge pull request #22792 from tailsu:sd/avfoundation-orientation-meta

Add support for CAP_PROP_ORIENTATION_AUTO to AVFoundation backend

modules/videoio/include/opencv2/videoio.hpp

@@ -182,8 +182,8 @@ enum VideoCaptureProperties {
        CAP_PROP_WB_TEMPERATURE=45, //!< white-balance color temperature
        CAP_PROP_CODEC_PIXEL_FORMAT =46,    //!< (read-only) codec's pixel format. 4-character code - see VideoWriter::fourcc . Subset of [AV_PIX_FMT_*](https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/raw.c) or -1 if unknown
        CAP_PROP_BITRATE       =47, //!< (read-only) Video bitrate in kbits/s
-       CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only)
-       CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata  (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499)
+       CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg and AVFoundation back-ends only)
+       CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata  (applicable for FFmpeg and AVFoundation back-ends only) (https://github.com/opencv/opencv/issues/15499)
        CAP_PROP_HW_ACCELERATION=50, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in cv::VideoCapture constructor / .open() method. Default value is backend-specific.
        CAP_PROP_HW_DEVICE      =51, //!< (**open-only**) Hardware device index (select GPU if multiple available). Device enumeration is acceleration type specific.
        CAP_PROP_HW_ACCELERATION_USE_OPENCL=52, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between HW accelerated decoder and cv::UMat.

modules/videoio/src/cap.cpp

@@ -698,4 +698,29 @@ int VideoWriter::fourcc(char c1, char c2, char c3, char c4)
     return (c1 & 255) + ((c2 & 255) << 8) + ((c3 & 255) << 16) + ((c4 & 255) << 24);
 }
 
+
+void applyMetadataRotation(const IVideoCapture& cap, OutputArray mat)
+{
+    bool rotation_auto = 0 != cap.getProperty(CAP_PROP_ORIENTATION_AUTO);
+    int rotation_angle = static_cast<int>(cap.getProperty(CAP_PROP_ORIENTATION_META));
+
+    if(!rotation_auto || rotation_angle%360 == 0)
+    {
+        return;
+    }
+
+    cv::RotateFlags flag;
+    if(rotation_angle == 90 || rotation_angle == -270) { // Rotate clockwise 90 degrees
+        flag = cv::ROTATE_90_CLOCKWISE;
+    } else if(rotation_angle == 270 || rotation_angle == -90) { // Rotate clockwise 270 degrees
+        flag = cv::ROTATE_90_COUNTERCLOCKWISE;
+    } else if(rotation_angle == 180 || rotation_angle == -180) { // Rotate clockwise 180 degrees
+        flag = cv::ROTATE_180;
+    } else { // Unsupported rotation
+        return;
+    }
+
+    cv::rotate(mat, mat, flag);
+}
+
 } // namespace cv

modules/videoio/src/cap_avfoundation.mm

@@ -162,6 +162,7 @@ private:
 
     bool setupReadingAt(CMTime position);
     IplImage* retrieveFramePixelBuffer();
+    int getPreferredOrientationDegrees() const;
 
     CMTime mFrameTimestamp;
     size_t mFrameNum;
@@ -1098,6 +1099,13 @@ IplImage* CvCaptureFile::retrieveFramePixelBuffer() {
     return mOutImage;
 }
 
+int CvCaptureFile::getPreferredOrientationDegrees() const {
+    if (mAssetTrack == nil) return 0;
+
+    CGAffineTransform transform = mAssetTrack.preferredTransform;
+    double radians = atan2(transform.b, transform.a);
+    return static_cast<int>(round(radians * 180 / M_PI));
+}
 
 IplImage* CvCaptureFile::retrieveFrame(int) {
     return retrieveFramePixelBuffer();
@@ -1129,6 +1137,8 @@ double CvCaptureFile::getProperty(int property_id) const{
             return mFormat;
         case CV_CAP_PROP_FOURCC:
             return mMode;
+        case cv::CAP_PROP_ORIENTATION_META:
+            return getPreferredOrientationDegrees();
         default:
             break;
     }

modules/videoio/src/cap_avfoundation_mac.mm

@@ -169,6 +169,7 @@ private:
 
     bool setupReadingAt(CMTime position);
     IplImage* retrieveFramePixelBuffer();
+    int getPreferredOrientationDegrees() const;
 
     CMTime mFrameTimestamp;
     size_t mFrameNum;
@@ -1064,6 +1065,13 @@ IplImage* CvCaptureFile::retrieveFramePixelBuffer() {
     return mOutImage;
 }
 
+int CvCaptureFile::getPreferredOrientationDegrees() const {
+    if (mAssetTrack == nil) return 0;
+
+    CGAffineTransform transform = mAssetTrack.preferredTransform;
+    double radians = atan2(transform.b, transform.a);
+    return static_cast<int>(round(radians * 180 / M_PI));
+}
 
 IplImage* CvCaptureFile::retrieveFrame(int) {
     return retrieveFramePixelBuffer();
@@ -1095,6 +1103,8 @@ double CvCaptureFile::getProperty(int property_id) const{
             return mFormat;
         case CV_CAP_PROP_FOURCC:
             return mMode;
+        case cv::CAP_PROP_ORIENTATION_META:
+            return getPreferredOrientationDegrees();
         default:
             break;
     }

modules/videoio/src/cap_ffmpeg.cpp

@@ -112,7 +112,7 @@ public:
         }
 
         cv::Mat tmp(height, width, CV_MAKETYPE(CV_8U, cn), data, step);
-        this->rotateFrame(tmp);
+        applyMetadataRotation(*this, tmp);
         tmp.copyTo(frame);
 
         return true;
@@ -137,30 +137,6 @@ public:
 
 protected:
     CvCapture_FFMPEG* ffmpegCapture;
-
-    void rotateFrame(cv::Mat &mat) const
-    {
-        bool rotation_auto = 0 != getProperty(CAP_PROP_ORIENTATION_AUTO);
-        int rotation_angle = static_cast<int>(getProperty(CAP_PROP_ORIENTATION_META));
-
-        if(!rotation_auto || rotation_angle%360 == 0)
-        {
-            return;
-        }
-
-        cv::RotateFlags flag;
-        if(rotation_angle == 90 || rotation_angle == -270) { // Rotate clockwise 90 degrees
-            flag = cv::ROTATE_90_CLOCKWISE;
-        } else if(rotation_angle == 270 || rotation_angle == -90) { // Rotate clockwise 270 degrees
-            flag = cv::ROTATE_90_COUNTERCLOCKWISE;
-        } else if(rotation_angle == 180 || rotation_angle == -180) { // Rotate clockwise 180 degrees
-            flag = cv::ROTATE_180;
-        } else { // Unsupported rotation
-            return;
-        }
-
-        cv::rotate(mat, mat, flag);
-    }
 };
 
 } // namespace

modules/videoio/src/cap_interface.hpp

@@ -221,6 +221,8 @@ public:
     virtual int getCaptureDomain() { return CAP_ANY; } // Return the type of the capture object: CAP_DSHOW, etc...
 };
 
+void applyMetadataRotation(const IVideoCapture& cap, OutputArray mat);
+
 class IVideoWriter
 {
 public:
@@ -249,21 +251,58 @@ class LegacyCapture : public IVideoCapture
 {
 private:
     CvCapture * cap;
+    bool autorotate;
     LegacyCapture(const LegacyCapture &);
     LegacyCapture& operator=(const LegacyCapture &);
+
+    bool shouldSwapWidthHeight() const
+    {
+        if (!autorotate)
+            return false;
+        int rotation = static_cast<int>(cap->getProperty(cv::CAP_PROP_ORIENTATION_META));
+        return std::abs(rotation % 180) == 90;
+    }
+
 public:
-    LegacyCapture(CvCapture * cap_) : cap(cap_) {}
+    LegacyCapture(CvCapture * cap_) : cap(cap_), autorotate(true) {}
     ~LegacyCapture()
     {
         cvReleaseCapture(&cap);
     }
     double getProperty(int propId) const CV_OVERRIDE
     {
-        return cap ? cap->getProperty(propId) : 0;
+        if (!cap)
+            return 0;
+
+        switch(propId)
+        {
+            case cv::CAP_PROP_ORIENTATION_AUTO:
+                return static_cast<double>(autorotate);
+
+            case cv::CAP_PROP_FRAME_WIDTH:
+                return shouldSwapWidthHeight() ? cap->getProperty(cv::CAP_PROP_FRAME_HEIGHT) : cap->getProperty(cv::CAP_PROP_FRAME_WIDTH);
+
+            case cv::CAP_PROP_FRAME_HEIGHT:
+                return shouldSwapWidthHeight() ? cap->getProperty(cv::CAP_PROP_FRAME_WIDTH) : cap->getProperty(cv::CAP_PROP_FRAME_HEIGHT);
+
+            default:
+                return cap->getProperty(propId);
+        }
     }
     bool setProperty(int propId, double value) CV_OVERRIDE
     {
-        return cvSetCaptureProperty(cap, propId, value) != 0;
+        if (!cap)
+            return false;
+
+        switch(propId)
+        {
+            case cv::CAP_PROP_ORIENTATION_AUTO:
+                autorotate = (value != 0);
+                return true;
+
+            default:
+                return cvSetCaptureProperty(cap, propId, value) != 0;
+        }
     }
     bool grabFrame() CV_OVERRIDE
     {
@@ -286,6 +325,7 @@ public:
             Mat temp = cv::cvarrToMat(_img);
             flip(temp, image, 0);
         }
+        applyMetadataRotation(*this, image);
         return true;
     }
     bool isOpened() const CV_OVERRIDE

modules/videoio/test/test_ffmpeg.cpp

@@ -475,68 +475,6 @@ const ffmpeg_get_fourcc_param_t ffmpeg_get_fourcc_param[] =
 
 INSTANTIATE_TEST_CASE_P(videoio, ffmpeg_get_fourcc, testing::ValuesIn(ffmpeg_get_fourcc_param));
 
-// related issue: https://github.com/opencv/opencv/issues/15499
-TEST(videoio, mp4_orientation_meta_auto)
-{
-    if (!videoio_registry::hasBackend(CAP_FFMPEG))
-        throw SkipTestException("FFmpeg backend was not found");
-
-    string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/big_buck_bunny_rotated.mp4";
-
-    VideoCapture cap;
-    EXPECT_NO_THROW(cap.open(video_file, CAP_FFMPEG));
-    ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << CAP_FFMPEG << std::endl;
-
-    // related issue: https://github.com/opencv/opencv/issues/22088
-    EXPECT_EQ(90, cap.get(CAP_PROP_ORIENTATION_META));
-
-    cap.set(CAP_PROP_ORIENTATION_AUTO, true);
-    if (cap.get(CAP_PROP_ORIENTATION_AUTO) == 0)
-        throw SkipTestException("FFmpeg frame rotation metadata is not supported");
-
-    Size actual;
-    EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH),
-                                    (int)cap.get(CAP_PROP_FRAME_HEIGHT)));
-    EXPECT_EQ(384, actual.width);
-    EXPECT_EQ(672, actual.height);
-
-    Mat frame;
-
-    cap >> frame;
-
-    ASSERT_EQ(384, frame.cols);
-    ASSERT_EQ(672, frame.rows);
-}
-
-// related issue: https://github.com/opencv/opencv/issues/15499
-TEST(videoio, mp4_orientation_no_rotation)
-{
-    if (!videoio_registry::hasBackend(CAP_FFMPEG))
-        throw SkipTestException("FFmpeg backend was not found");
-
-    string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/big_buck_bunny_rotated.mp4";
-
-    VideoCapture cap;
-    EXPECT_NO_THROW(cap.open(video_file, CAP_FFMPEG));
-    cap.set(CAP_PROP_ORIENTATION_AUTO, 0);
-    ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << CAP_FFMPEG << std::endl;
-    ASSERT_FALSE(cap.get(CAP_PROP_ORIENTATION_AUTO));
-
-    Size actual;
-    EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH),
-                                    (int)cap.get(CAP_PROP_FRAME_HEIGHT)));
-    EXPECT_EQ(672, actual.width);
-    EXPECT_EQ(384, actual.height);
-
-    Mat frame;
-
-    cap >> frame;
-
-    ASSERT_EQ(672, frame.cols);
-    ASSERT_EQ(384, frame.rows);
-}
-
-
 static void ffmpeg_check_read_raw(VideoCapture& cap)
 {
     ASSERT_TRUE(cap.isOpened()) << "Can't open the video";

modules/videoio/test/test_orientation.cpp

@@ -0,0 +1,76 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "test_precomp.hpp"
+
+using namespace std;
+
+namespace opencv_test { namespace {
+
+typedef TestWithParam<cv::VideoCaptureAPIs> VideoCaptureAPITests;
+
+// related issue: https://github.com/opencv/opencv/issues/15499
+TEST_P(VideoCaptureAPITests, mp4_orientation_meta_auto)
+{
+    cv::VideoCaptureAPIs api = GetParam();
+    if (!videoio_registry::hasBackend(api))
+        throw SkipTestException("backend " + std::to_string(int(api)) + " was not found");
+
+    string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/rotated_metadata.mp4";
+
+    VideoCapture cap;
+    EXPECT_NO_THROW(cap.open(video_file, api));
+    ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << api << std::endl;
+
+    // related issue: https://github.com/opencv/opencv/issues/22088
+    EXPECT_EQ(90, cap.get(CAP_PROP_ORIENTATION_META));
+
+    EXPECT_TRUE(cap.set(CAP_PROP_ORIENTATION_AUTO, true));
+
+    Size actual;
+    EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH),
+                                    (int)cap.get(CAP_PROP_FRAME_HEIGHT)));
+    EXPECT_EQ(270, actual.width);
+    EXPECT_EQ(480, actual.height);
+
+    Mat frame;
+
+    cap >> frame;
+
+    ASSERT_EQ(270, frame.cols);
+    ASSERT_EQ(480, frame.rows);
+}
+
+// related issue: https://github.com/opencv/opencv/issues/15499
+TEST_P(VideoCaptureAPITests, mp4_orientation_no_rotation)
+{
+    cv::VideoCaptureAPIs api = GetParam();
+    if (!videoio_registry::hasBackend(api))
+        throw SkipTestException("backend " + std::to_string(int(api)) + " was not found");
+
+    string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/rotated_metadata.mp4";
+
+    VideoCapture cap;
+    EXPECT_NO_THROW(cap.open(video_file, api));
+    cap.set(CAP_PROP_ORIENTATION_AUTO, 0);
+    ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << api << std::endl;
+    ASSERT_FALSE(cap.get(CAP_PROP_ORIENTATION_AUTO));
+
+    Size actual;
+    EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH),
+                                    (int)cap.get(CAP_PROP_FRAME_HEIGHT)));
+    EXPECT_EQ(480, actual.width);
+    EXPECT_EQ(270, actual.height);
+
+    Mat frame;
+
+    cap >> frame;
+
+    ASSERT_EQ(480, frame.cols);
+    ASSERT_EQ(270, frame.rows);
+}
+
+INSTANTIATE_TEST_CASE_P(videoio, VideoCaptureAPITests, testing::Values(CAP_FFMPEG, CAP_AVFOUNDATION));
+
+}} // namespace