From e9982e856f2aab28d427d7fc9e04281ae60b63ee Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Thu, 26 Dec 2024 12:48:49 +0300 Subject: [PATCH] Merge pull request #25584 from dkurt:videocapture_from_buffer Open VideoCapture from data stream #25584 ### Pull Request Readiness Checklist Add VideoCapture option to read a raw binary video data from `std::streambuf`. There are multiple motivations: 1. Avoid disk file creation in case of video already in memory (received by network or from database). 2. Streaming mode. Frames decoding starts during sequential file transfer by chunks. Suppoted backends: * FFmpeg * MSMF (no streaming mode) Supporter interfaces: * C++ (std::streambuf) * Python (io.BufferedIOBase) resolves https://github.com/opencv/opencv/issues/24400 - [x] test h264 - [x] test IP camera like approach with no metadata but key frame only? - [x] C API plugin See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/videoio/include/opencv2/videoio.hpp | 40 +++++ .../include/opencv2/videoio/registry.hpp | 10 ++ .../include/opencv2/videoio/utils.private.hpp | 31 ++++ modules/videoio/misc/objc/gen_dict.json | 3 +- .../videoio/misc/python/pyopencv_videoio.hpp | 110 ++++++++++++++ .../videoio/misc/python/test/test_videoio.py | 66 +++++++++ modules/videoio/src/backend.hpp | 5 + modules/videoio/src/backend_plugin.cpp | 66 ++++++++- modules/videoio/src/backend_static.cpp | 43 ++++-- modules/videoio/src/cap.cpp | 140 +++++++++++++++++- modules/videoio/src/cap_ffmpeg.cpp | 67 ++++++++- modules/videoio/src/cap_ffmpeg_impl.hpp | 100 +++++++++++-- modules/videoio/src/cap_interface.hpp | 3 + modules/videoio/src/cap_msmf.cpp | 105 +++++++++++-- modules/videoio/src/plugin_capture_api.hpp | 37 ++++- modules/videoio/src/videoio_registry.cpp | 56 ++++++- modules/videoio/src/videoio_registry.hpp | 2 + modules/videoio/test/test_video_io.cpp | 130 ++++++++++++++++ 18 files changed, 966 insertions(+), 48 deletions(-) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 92fa5d39b7..6e50f87e44 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -719,6 +719,25 @@ enum VideoCaptureOBSensorProperties{ //! @} videoio_flags_others +/** @brief Read data stream interface + */ +class CV_EXPORTS_W IStreamReader +{ +public: + virtual ~IStreamReader(); + + /** @brief Read bytes from stream */ + virtual long long read(char* buffer, long long size) = 0; + + /** @brief Sets the stream position + * + * @param offset Seek offset + * @param origin SEEK_SET / SEEK_END / SEEK_CUR + * + * @see fseek + */ + virtual long long seek(long long offset, int origin) = 0; +}; class IVideoCapture; //! @cond IGNORED @@ -798,6 +817,14 @@ public: */ CV_WRAP explicit VideoCapture(int index, int apiPreference, const std::vector& params); + /** @overload + @brief Opens a video using data stream. + + The `params` parameter allows to specify extra parameters encoded as pairs `(paramId_1, paramValue_1, paramId_2, paramValue_2, ...)`. + See cv::VideoCaptureProperties + */ + CV_WRAP VideoCapture(const Ptr& source, int apiPreference, const std::vector& params); + /** @brief Default destructor The method first calls VideoCapture::release to close the already opened file or camera. @@ -852,6 +879,19 @@ public: */ CV_WRAP virtual bool open(int index, int apiPreference, const std::vector& params); + /** @brief Opens a video using data stream. + + @overload + + The `params` parameter allows to specify extra parameters encoded as pairs `(paramId_1, paramValue_1, paramId_2, paramValue_2, ...)`. + See cv::VideoCaptureProperties + + @return `true` if the file has been successfully opened + + The method first calls VideoCapture::release to close the already opened file or camera. + */ + CV_WRAP virtual bool open(const Ptr& source, int apiPreference, const std::vector& params); + /** @brief Returns true if video capturing has been initialized already. If the previous call to VideoCapture constructor or VideoCapture::open() succeeded, the method returns diff --git a/modules/videoio/include/opencv2/videoio/registry.hpp b/modules/videoio/include/opencv2/videoio/registry.hpp index cf72247b3f..47e94b1909 100644 --- a/modules/videoio/include/opencv2/videoio/registry.hpp +++ b/modules/videoio/include/opencv2/videoio/registry.hpp @@ -35,6 +35,9 @@ CV_EXPORTS_W std::vector getCameraBackends(); /** @brief Returns list of available backends which works via `cv::VideoCapture(filename)` */ CV_EXPORTS_W std::vector getStreamBackends(); +/** @brief Returns list of available backends which works via `cv::VideoCapture(buffer)` */ +CV_EXPORTS_W std::vector getStreamBufferedBackends(); + /** @brief Returns list of available backends which works via `cv::VideoWriter()` */ CV_EXPORTS_W std::vector getWriterBackends(); @@ -58,6 +61,13 @@ CV_EXPORTS_W std::string getStreamBackendPluginVersion( CV_OUT int& version_API ); +/** @brief Returns description and ABI/API version of videoio plugin's buffer capture interface */ +CV_EXPORTS_W std::string getStreamBufferedBackendPluginVersion( + VideoCaptureAPIs api, + CV_OUT int& version_ABI, + CV_OUT int& version_API +); + /** @brief Returns description and ABI/API version of videoio plugin's writer interface */ CV_EXPORTS_W std::string getWriterBackendPluginVersion( VideoCaptureAPIs api, diff --git a/modules/videoio/include/opencv2/videoio/utils.private.hpp b/modules/videoio/include/opencv2/videoio/utils.private.hpp index e331aaf2ac..0fa6d285b0 100644 --- a/modules/videoio/include/opencv2/videoio/utils.private.hpp +++ b/modules/videoio/include/opencv2/videoio/utils.private.hpp @@ -10,6 +10,37 @@ namespace cv { CV_EXPORTS std::string icvExtractPattern(const std::string& filename, unsigned *offset); + +class PluginStreamReader : public IStreamReader +{ +public: + PluginStreamReader(void* _opaque, + long long (*_read)(void* opaque, char* buffer, long long size), + long long (*_seek)(void* opaque, long long offset, int way)) + { + opaque = _opaque; + readCallback = _read; + seekCallback = _seek; + } + + virtual ~PluginStreamReader() {} + + long long read(char* buffer, long long size) override + { + return readCallback(opaque, buffer, size); + } + + long long seek(long long offset, int way) override + { + return seekCallback(opaque, offset, way); + } + +private: + void* opaque; + long long (*readCallback)(void* opaque, char* buffer, long long size); + long long (*seekCallback)(void* opaque, long long offset, int way); +}; + } #endif // OPENCV_VIDEOIO_UTILS_PRIVATE_HPP diff --git a/modules/videoio/misc/objc/gen_dict.json b/modules/videoio/misc/objc/gen_dict.json index 70dc844f0c..f42211a949 100644 --- a/modules/videoio/misc/objc/gen_dict.json +++ b/modules/videoio/misc/objc/gen_dict.json @@ -14,7 +14,8 @@ "func_arg_fix" : { "VideoCapture" : { "(BOOL)open:(int)index apiPreference:(int)apiPreference" : { "open" : {"name" : "openWithIndex"} }, - "(BOOL)open:(int)index apiPreference:(int)apiPreference params:(IntVector*)params" : { "open" : {"name" : "openWithIndexAndParameters"} } + "(BOOL)open:(int)index apiPreference:(int)apiPreference params:(IntVector*)params" : { "open" : {"name" : "openWithIndexAndParameters"} }, + "(BOOL)open:(IStreamReader*)source apiPreference:(int)apiPreference params:(IntVector*)params" : { "open" : {"name" : "openWithStreamReader"} } } } } diff --git a/modules/videoio/misc/python/pyopencv_videoio.hpp b/modules/videoio/misc/python/pyopencv_videoio.hpp index e729c8631f..ab26e00c82 100644 --- a/modules/videoio/misc/python/pyopencv_videoio.hpp +++ b/modules/videoio/misc/python/pyopencv_videoio.hpp @@ -31,4 +31,114 @@ template<> bool pyopencv_to(PyObject* obj, cv::VideoCapture& stream, const ArgIn return true; } +class PythonStreamReader : public cv::IStreamReader +{ +public: + PythonStreamReader(PyObject* _obj = nullptr) : obj(_obj) + { + if (obj) + Py_INCREF(obj); + } + + ~PythonStreamReader() + { + if (obj) + Py_DECREF(obj); + } + + long long read(char* buffer, long long size) CV_OVERRIDE + { + if (!obj) + return 0; + + PyObject* ioBase = reinterpret_cast(obj); + + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + PyObject* py_size = pyopencv_from(static_cast(size)); + + PyObject* res = PyObject_CallMethodObjArgs(ioBase, PyString_FromString("read"), py_size, NULL); + bool hasPyReadError = PyErr_Occurred() != nullptr; + char* src = PyBytes_AsString(res); + size_t len = static_cast(PyBytes_Size(res)); + bool hasPyBytesError = PyErr_Occurred() != nullptr; + if (src && len <= static_cast(size)) + { + std::memcpy(buffer, src, len); + } + Py_DECREF(res); + Py_DECREF(py_size); + + PyGILState_Release(gstate); + + if (hasPyReadError) + CV_Error(cv::Error::StsError, "Python .read() call error"); + if (hasPyBytesError) + CV_Error(cv::Error::StsError, "Python buffer access error"); + + CV_CheckLE(len, static_cast(size), "Stream chunk size should be less or equal than requested size"); + + return len; + } + + long long seek(long long offset, int way) CV_OVERRIDE + { + if (!obj) + return 0; + + PyObject* ioBase = reinterpret_cast(obj); + + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + PyObject* py_offset = pyopencv_from(static_cast(offset)); + PyObject* py_whence = pyopencv_from(way); + + PyObject* res = PyObject_CallMethodObjArgs(ioBase, PyString_FromString("seek"), py_offset, py_whence, NULL); + bool hasPySeekError = PyErr_Occurred() != nullptr; + long long pos = PyLong_AsLongLong(res); + bool hasPyConvertError = PyErr_Occurred() != nullptr; + Py_DECREF(res); + Py_DECREF(py_offset); + Py_DECREF(py_whence); + + PyGILState_Release(gstate); + + if (hasPySeekError) + CV_Error(cv::Error::StsError, "Python .seek() call error"); + if (hasPyConvertError) + CV_Error(cv::Error::StsError, "Python .seek() result => long long conversion error"); + return pos; + } + +private: + PyObject* obj; +}; + +template<> +bool pyopencv_to(PyObject* obj, Ptr& p, const ArgInfo&) +{ + if (!obj) + return false; + + PyObject* ioModule = PyImport_ImportModule("io"); + PyObject* type = PyObject_GetAttrString(ioModule, "BufferedIOBase"); + Py_DECREF(ioModule); + bool isValidPyType = PyObject_IsInstance(obj, type) == 1; + Py_DECREF(type); + + if (!isValidPyType) + { + PyErr_SetString(PyExc_TypeError, "Input stream should be derived from io.BufferedIOBase"); + return false; + } + + if (!PyErr_Occurred()) { + p = makePtr(obj); + return true; + } + return false; +} + #endif // HAVE_OPENCV_VIDEOIO diff --git a/modules/videoio/misc/python/test/test_videoio.py b/modules/videoio/misc/python/test/test_videoio.py index 2bbfeecda0..181ade769d 100644 --- a/modules/videoio/misc/python/test/test_videoio.py +++ b/modules/videoio/misc/python/test/test_videoio.py @@ -3,6 +3,8 @@ from __future__ import print_function import numpy as np import cv2 as cv +import io +import sys from tests_common import NewOpenCVTests @@ -21,5 +23,69 @@ class Bindings(NewOpenCVTests): for backend in backends: self.check_name(cv.videoio_registry.getBackendName(backend)) + def test_capture_stream_file(self): + if sys.version_info[0] < 3: + raise self.skipTest('Python 3.x required') + + api_pref = None + for backend in cv.videoio_registry.getStreamBufferedBackends(): + if not cv.videoio_registry.hasBackend(backend): + continue + if not cv.videoio_registry.isBackendBuiltIn(backend): + _, abi, api = cv.videoio_registry.getStreamBufferedBackendPluginVersion(backend) + if (abi < 1 or (abi == 1 and api < 2)): + continue + api_pref = backend + break + + if not api_pref: + raise self.skipTest("No available backends") + + with open(self.find_file("cv/video/768x576.avi"), "rb") as f: + cap = cv.VideoCapture(f, api_pref, []) + self.assertTrue(cap.isOpened()) + hasFrame, frame = cap.read() + self.assertTrue(hasFrame) + self.assertEqual(frame.shape, (576, 768, 3)) + + def test_capture_stream_buffer(self): + if sys.version_info[0] < 3: + raise self.skipTest('Python 3.x required') + + api_pref = None + for backend in cv.videoio_registry.getStreamBufferedBackends(): + if not cv.videoio_registry.hasBackend(backend): + continue + if not cv.videoio_registry.isBackendBuiltIn(backend): + _, abi, api = cv.videoio_registry.getStreamBufferedBackendPluginVersion(backend) + if (abi < 1 or (abi == 1 and api < 2)): + continue + api_pref = backend + break + + if not api_pref: + raise self.skipTest("No available backends") + + class BufferStream(io.BufferedIOBase): + def __init__(self, filepath): + self.f = open(filepath, "rb") + + def read(self, size=-1): + return self.f.read(size) + + def seek(self, offset, whence): + return self.f.seek(offset, whence) + + def __del__(self): + self.f.close() + + stream = BufferStream(self.find_file("cv/video/768x576.avi")) + + cap = cv.VideoCapture(stream, api_pref, []) + self.assertTrue(cap.isOpened()) + hasFrame, frame = cap.read() + self.assertTrue(hasFrame) + self.assertEqual(frame.shape, (576, 768, 3)) + if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/videoio/src/backend.hpp b/modules/videoio/src/backend.hpp index 2a95ec05aa..b75cc51ea8 100644 --- a/modules/videoio/src/backend.hpp +++ b/modules/videoio/src/backend.hpp @@ -18,6 +18,7 @@ public: virtual ~IBackend() {} virtual Ptr createCapture(int camera, const VideoCaptureParameters& params) const = 0; virtual Ptr createCapture(const std::string &filename, const VideoCaptureParameters& params) const = 0; + virtual Ptr createCapture(const Ptr&stream, const VideoCaptureParameters& params) const = 0; virtual Ptr createWriter(const std::string& filename, int fourcc, double fps, const cv::Size& sz, const VideoWriterParameters& params) const = 0; }; @@ -34,15 +35,19 @@ public: typedef Ptr (*FN_createCaptureFile)(const std::string & filename); typedef Ptr (*FN_createCaptureCamera)(int camera); +typedef Ptr (*FN_createCaptureStream)(const Ptr& stream); typedef Ptr (*FN_createCaptureFileWithParams)(const std::string & filename, const VideoCaptureParameters& params); typedef Ptr (*FN_createCaptureCameraWithParams)(int camera, const VideoCaptureParameters& params); +typedef Ptr (*FN_createCaptureStreamWithParams)(const Ptr& stream, const VideoCaptureParameters& params); typedef Ptr (*FN_createWriter)(const std::string& filename, int fourcc, double fps, const Size& sz, const VideoWriterParameters& params); Ptr createBackendFactory(FN_createCaptureFile createCaptureFile, FN_createCaptureCamera createCaptureCamera, + FN_createCaptureStream createCaptureStream, FN_createWriter createWriter); Ptr createBackendFactory(FN_createCaptureFileWithParams createCaptureFile, FN_createCaptureCameraWithParams createCaptureCamera, + FN_createCaptureStreamWithParams createCaptureStream, FN_createWriter createWriter); Ptr createPluginBackendFactory(VideoCaptureAPIs id, const char* baseName); diff --git a/modules/videoio/src/backend_plugin.cpp b/modules/videoio/src/backend_plugin.cpp index cc9b33b80c..a38537c778 100644 --- a/modules/videoio/src/backend_plugin.cpp +++ b/modules/videoio/src/backend_plugin.cpp @@ -208,6 +208,7 @@ public: Ptr createCapture(int camera, const VideoCaptureParameters& params) const CV_OVERRIDE; Ptr createCapture(const std::string &filename) const; Ptr createCapture(const std::string &filename, const VideoCaptureParameters& params) const CV_OVERRIDE; + Ptr createCapture(const Ptr& stream, const VideoCaptureParameters& params) const CV_OVERRIDE; Ptr createWriter(const std::string& filename, int fourcc, double fps, const cv::Size& sz, const VideoWriterParameters& params) const CV_OVERRIDE; @@ -447,16 +448,52 @@ class PluginCapture : public cv::IVideoCapture { const OpenCV_VideoIO_Capture_Plugin_API* plugin_api_; CvPluginCapture capture_; + Ptr readStream_; public: static Ptr create(const OpenCV_VideoIO_Capture_Plugin_API* plugin_api, - const std::string &filename, int camera, const VideoCaptureParameters& params) + const std::string &filename, const Ptr& stream, int camera, const VideoCaptureParameters& params) { CV_Assert(plugin_api); CV_Assert(plugin_api->v0.Capture_release); CvPluginCapture capture = NULL; + if (stream && plugin_api->api_header.api_version >= 2 && plugin_api->v2.Capture_open_stream) + { + std::vector vint_params = params.getIntVector(); + int* c_params = vint_params.data(); + unsigned n_params = (unsigned)(vint_params.size() / 2); + + if (CV_ERROR_OK == plugin_api->v2.Capture_open_stream( + stream.get(), + [](void* opaque, char* buffer, long long size) -> long long { + CV_LOG_VERBOSE(NULL, 0, "IStreamReader::read(" << size << ")..."); + auto is = reinterpret_cast(opaque); + try { + return is->read(buffer, size); + } catch (...) { + CV_LOG_WARNING(NULL, "IStreamReader::read(" << size << ") failed"); + return 0; + } + }, + [](void* opaque, long long offset, int way) -> long long { + CV_LOG_VERBOSE(NULL, 0, "IStreamReader::seek(" << offset << ", way=" << way << ")..."); + auto is = reinterpret_cast(opaque); + try { + return is->seek(offset, way); + } catch (...) { + CV_LOG_WARNING(NULL, "IStreamReader::seek(" << offset << ", way=" << way << ") failed"); + return -1; + } + }, c_params, n_params, &capture)) + { + CV_Assert(capture); + return makePtr(plugin_api, capture, stream); + } + } + else if (stream) + return Ptr(); if (plugin_api->api_header.api_version >= 1 && plugin_api->v1.Capture_open_with_params) { @@ -488,8 +525,8 @@ public: return Ptr(); } - PluginCapture(const OpenCV_VideoIO_Capture_Plugin_API* plugin_api, CvPluginCapture capture) - : plugin_api_(plugin_api), capture_(capture) + PluginCapture(const OpenCV_VideoIO_Capture_Plugin_API* plugin_api, CvPluginCapture capture, const Ptr& readStream = Ptr()) + : plugin_api_(plugin_api), capture_(capture), readStream_(readStream) { CV_Assert(plugin_api_); CV_Assert(capture_); } @@ -661,7 +698,7 @@ Ptr PluginBackend::createCapture(int camera, const VideoCapturePa try { if (capture_api_) - return PluginCapture::create(capture_api_, std::string(), camera, params); //.staticCast(); + return PluginCapture::create(capture_api_, std::string(), nullptr, camera, params); //.staticCast(); if (plugin_api_) { Ptr cap = legacy::PluginCapture::create(plugin_api_, std::string(), camera); //.staticCast(); @@ -685,7 +722,7 @@ Ptr PluginBackend::createCapture(const std::string &filename, con try { if (capture_api_) - return PluginCapture::create(capture_api_, filename, 0, params); //.staticCast(); + return PluginCapture::create(capture_api_, filename, nullptr, 0, params); //.staticCast(); if (plugin_api_) { Ptr cap = legacy::PluginCapture::create(plugin_api_, filename, 0); //.staticCast(); @@ -704,6 +741,25 @@ Ptr PluginBackend::createCapture(const std::string &filename, con return Ptr(); } +Ptr PluginBackend::createCapture(const Ptr& stream, const VideoCaptureParameters& params) const +{ + try + { + if (capture_api_) + return PluginCapture::create(capture_api_, std::string(), stream, 0, params); //.staticCast(); + if (plugin_api_) + { + CV_Error(Error::StsNotImplemented, "Legacy plugin API for stream capture"); + } + } + catch (...) + { + CV_LOG_DEBUG(NULL, "Video I/O: can't open stream capture"); + throw; + } + return Ptr(); +} + Ptr PluginBackend::createWriter(const std::string& filename, int fourcc, double fps, const cv::Size& sz, const VideoWriterParameters& params) const { diff --git a/modules/videoio/src/backend_static.cpp b/modules/videoio/src/backend_static.cpp index 3001906acf..511aff9ffa 100644 --- a/modules/videoio/src/backend_static.cpp +++ b/modules/videoio/src/backend_static.cpp @@ -36,10 +36,11 @@ class StaticBackend: public IBackend public: FN_createCaptureFile fn_createCaptureFile_; FN_createCaptureCamera fn_createCaptureCamera_; + FN_createCaptureStream fn_createCaptureStream_; FN_createWriter fn_createWriter_; - StaticBackend(FN_createCaptureFile fn_createCaptureFile, FN_createCaptureCamera fn_createCaptureCamera, FN_createWriter fn_createWriter) - : fn_createCaptureFile_(fn_createCaptureFile), fn_createCaptureCamera_(fn_createCaptureCamera), fn_createWriter_(fn_createWriter) + StaticBackend(FN_createCaptureFile fn_createCaptureFile, FN_createCaptureCamera fn_createCaptureCamera, FN_createCaptureStream fn_createCaptureStream, FN_createWriter fn_createWriter) + : fn_createCaptureFile_(fn_createCaptureFile), fn_createCaptureCamera_(fn_createCaptureCamera), fn_createCaptureStream_(fn_createCaptureStream), fn_createWriter_(fn_createWriter) { // nothing } @@ -72,6 +73,19 @@ public: } return Ptr(); } + Ptr createCapture(const Ptr& stream, const VideoCaptureParameters& params) const CV_OVERRIDE + { + if (fn_createCaptureStream_) + { + Ptr cap = fn_createCaptureStream_(stream); + if (cap && !params.empty()) + { + applyParametersFallback(cap, params); + } + return cap; + } + return Ptr(); + } Ptr createWriter(const std::string& filename, int fourcc, double fps, const cv::Size& sz, const VideoWriterParameters& params) const CV_OVERRIDE { @@ -87,8 +101,8 @@ protected: Ptr backend; public: - StaticBackendFactory(FN_createCaptureFile createCaptureFile, FN_createCaptureCamera createCaptureCamera, FN_createWriter createWriter) - : backend(makePtr(createCaptureFile, createCaptureCamera, createWriter)) + StaticBackendFactory(FN_createCaptureFile createCaptureFile, FN_createCaptureCamera createCaptureCamera, FN_createCaptureStream createCaptureStream, FN_createWriter createWriter) + : backend(makePtr(createCaptureFile, createCaptureCamera, createCaptureStream, createWriter)) { // nothing } @@ -106,9 +120,10 @@ public: Ptr createBackendFactory(FN_createCaptureFile createCaptureFile, FN_createCaptureCamera createCaptureCamera, + FN_createCaptureStream createCaptureStream, FN_createWriter createWriter) { - return makePtr(createCaptureFile, createCaptureCamera, createWriter).staticCast(); + return makePtr(createCaptureFile, createCaptureCamera, createCaptureStream, createWriter).staticCast(); } @@ -118,10 +133,11 @@ class StaticBackendWithParams: public IBackend public: FN_createCaptureFileWithParams fn_createCaptureFile_; FN_createCaptureCameraWithParams fn_createCaptureCamera_; + FN_createCaptureStreamWithParams fn_createCaptureStream_; FN_createWriter fn_createWriter_; - StaticBackendWithParams(FN_createCaptureFileWithParams fn_createCaptureFile, FN_createCaptureCameraWithParams fn_createCaptureCamera, FN_createWriter fn_createWriter) - : fn_createCaptureFile_(fn_createCaptureFile), fn_createCaptureCamera_(fn_createCaptureCamera), fn_createWriter_(fn_createWriter) + StaticBackendWithParams(FN_createCaptureFileWithParams fn_createCaptureFile, FN_createCaptureCameraWithParams fn_createCaptureCamera, FN_createCaptureStreamWithParams fn_createCaptureStream, FN_createWriter fn_createWriter) + : fn_createCaptureFile_(fn_createCaptureFile), fn_createCaptureCamera_(fn_createCaptureCamera), fn_createCaptureStream_(fn_createCaptureStream), fn_createWriter_(fn_createWriter) { // nothing } @@ -140,6 +156,12 @@ public: return fn_createCaptureFile_(filename, params); return Ptr(); } + Ptr createCapture(const Ptr& stream, const VideoCaptureParameters& params) const CV_OVERRIDE + { + if (fn_createCaptureStream_) + return fn_createCaptureStream_(stream, params); + return Ptr(); + } Ptr createWriter(const std::string& filename, int fourcc, double fps, const cv::Size& sz, const VideoWriterParameters& params) const CV_OVERRIDE { @@ -155,8 +177,8 @@ protected: Ptr backend; public: - StaticBackendWithParamsFactory(FN_createCaptureFileWithParams createCaptureFile, FN_createCaptureCameraWithParams createCaptureCamera, FN_createWriter createWriter) - : backend(makePtr(createCaptureFile, createCaptureCamera, createWriter)) + StaticBackendWithParamsFactory(FN_createCaptureFileWithParams createCaptureFile, FN_createCaptureCameraWithParams createCaptureCamera, FN_createCaptureStreamWithParams createCaptureStream, FN_createWriter createWriter) + : backend(makePtr(createCaptureFile, createCaptureCamera, createCaptureStream, createWriter)) { // nothing } @@ -174,9 +196,10 @@ public: Ptr createBackendFactory(FN_createCaptureFileWithParams createCaptureFile, FN_createCaptureCameraWithParams createCaptureCamera, + FN_createCaptureStreamWithParams createCaptureStream, FN_createWriter createWriter) { - return makePtr(createCaptureFile, createCaptureCamera, createWriter).staticCast(); + return makePtr(createCaptureFile, createCaptureCamera, createCaptureStream, createWriter).staticCast(); } diff --git a/modules/videoio/src/cap.cpp b/modules/videoio/src/cap.cpp index f8572aad08..97f5736052 100644 --- a/modules/videoio/src/cap.cpp +++ b/modules/videoio/src/cap.cpp @@ -65,6 +65,10 @@ static bool param_VIDEOWRITER_DEBUG = utils::getConfigurationParameterBool("OPEN void DefaultDeleter::operator ()(CvCapture* obj) const { cvReleaseCapture(&obj); } void DefaultDeleter::operator ()(CvVideoWriter* obj) const { cvReleaseVideoWriter(&obj); } +IStreamReader::~IStreamReader() +{ + // nothing +} VideoCapture::VideoCapture() : throwOnFail(false) {} @@ -82,6 +86,13 @@ VideoCapture::VideoCapture(const String& filename, int apiPreference, const std: open(filename, apiPreference, params); } +VideoCapture::VideoCapture(const Ptr& source, int apiPreference, const std::vector& params) + : throwOnFail(false) +{ + CV_TRACE_FUNCTION(); + open(source, apiPreference, params); +} + VideoCapture::VideoCapture(int index, int apiPreference) : throwOnFail(false) { CV_TRACE_FUNCTION(); @@ -188,7 +199,7 @@ bool VideoCapture::open(const String& filename, int apiPreference, const std::ve else { CV_CAPTURE_LOG_DEBUG(NULL, - cv::format("VIDEOIO(%s): backend is not available " + cv::format("VIDEOIO(%s): backend is not available " "(plugin is missing, or can't be loaded due " "dependencies or it is not compatible)", info.name)); @@ -228,6 +239,131 @@ bool VideoCapture::open(const String& filename, int apiPreference, const std::ve return false; } +bool VideoCapture::open(const Ptr& stream, int apiPreference, const std::vector& params) + +{ + CV_INSTRUMENT_REGION(); + + if (apiPreference == CAP_ANY) + { + CV_Error_(Error::StsBadArg, ("Avoid CAP_ANY - explicit backend expected to avoid read data stream reset")); + } + + if (isOpened()) + { + release(); + } + + const VideoCaptureParameters parameters(params); + const std::vector backends = cv::videoio_registry::getAvailableBackends_CaptureByStream(); + for (size_t i = 0; i < backends.size(); i++) + { + const VideoBackendInfo& info = backends[i]; + if (apiPreference != info.id) + continue; + + if (!info.backendFactory) + { + CV_LOG_DEBUG(NULL, "VIDEOIO(" << info.name << "): factory is not available (plugins require filesystem support)"); + continue; + } + CV_CAPTURE_LOG_DEBUG(NULL, + cv::format("VIDEOIO(%s): trying capture buffer ...", + info.name)); + CV_Assert(!info.backendFactory.empty()); + const Ptr backend = info.backendFactory->getBackend(); + if (!backend.empty()) + { + try + { + icap = backend->createCapture(stream, parameters); + if (!icap.empty()) + { + CV_CAPTURE_LOG_DEBUG(NULL, + cv::format("VIDEOIO(%s): created, isOpened=%d", + info.name, icap->isOpened())); + if (icap->isOpened()) + { + return true; + } + icap.release(); + } + else + { + CV_CAPTURE_LOG_DEBUG(NULL, + cv::format("VIDEOIO(%s): can't create capture", + info.name)); + } + } + catch (const cv::Exception& e) + { + if (throwOnFail) + { + throw; + } + CV_LOG_WARNING(NULL, + cv::format("VIDEOIO(%s): raised OpenCV exception:\n\n%s\n", + info.name, e.what())); + } + catch (const std::exception& e) + { + if (throwOnFail) + { + throw; + } + CV_LOG_WARNING(NULL, cv::format("VIDEOIO(%s): raised C++ exception:\n\n%s\n", + info.name, e.what())); + } + catch (...) + { + if (throwOnFail) + { + throw; + } + CV_LOG_WARNING(NULL, + cv::format("VIDEOIO(%s): raised unknown C++ exception!\n\n", + info.name)); + } + } + else + { + CV_CAPTURE_LOG_DEBUG(NULL, + cv::format("VIDEOIO(%s): backend is not available " + "(plugin is missing, or can't be loaded due " + "dependencies or it is not compatible)", + info.name)); + } + } + + bool found = cv::videoio_registry::isBackendBuiltIn(static_cast(apiPreference)); + if (found) + { + CV_LOG_WARNING(NULL, cv::format("VIDEOIO(%s): backend is generally available " + "but can't be used to capture by read data stream", + cv::videoio_registry::getBackendName(static_cast(apiPreference)).c_str())); + } + + if (throwOnFail) + { + CV_Error_(Error::StsError, ("could not open read data stream")); + } + + if (cv::videoio_registry::checkDeprecatedBackend(apiPreference)) + { + CV_LOG_DEBUG(NULL, + cv::format("VIDEOIO(%s): backend is removed from OpenCV", + cv::videoio_registry::getBackendName((VideoCaptureAPIs) apiPreference).c_str())); + } + else + { + CV_LOG_DEBUG(NULL, "VIDEOIO: choosen backend does not work or wrong. " + "Please make sure that your computer support chosen backend and OpenCV built " + "with right flags."); + } + + return false; +} + bool VideoCapture::open(int cameraNum, int apiPreference) { return open(cameraNum, apiPreference, std::vector()); @@ -326,7 +462,7 @@ bool VideoCapture::open(int cameraNum, int apiPreference, const std::vector else { CV_CAPTURE_LOG_DEBUG(NULL, - cv::format("VIDEOIO(%s): backend is not available " + cv::format("VIDEOIO(%s): backend is not available " "(plugin is missing, or can't be loaded due " "dependencies or it is not compatible)", info.name)); diff --git a/modules/videoio/src/cap_ffmpeg.cpp b/modules/videoio/src/cap_ffmpeg.cpp index 4b1f9bb857..62f64cb0f7 100644 --- a/modules/videoio/src/cap_ffmpeg.cpp +++ b/modules/videoio/src/cap_ffmpeg.cpp @@ -74,6 +74,11 @@ public: { open(filename, params); } + CvCapture_FFMPEG_proxy(const Ptr& stream, const cv::VideoCaptureParameters& params) + : ffmpegCapture(NULL) + { + open(stream, params); + } virtual ~CvCapture_FFMPEG_proxy() { close(); } virtual double getProperty_(int propId) const CV_OVERRIDE @@ -122,6 +127,14 @@ public: ffmpegCapture = cvCreateFileCaptureWithParams_FFMPEG(filename.c_str(), params); return ffmpegCapture != 0; } + bool open(const Ptr& stream, const cv::VideoCaptureParameters& params) + { + close(); + + readStream = stream; // Increase counter + ffmpegCapture = cvCreateStreamCaptureWithParams_FFMPEG(stream, params); + return ffmpegCapture != 0; + } void close() { if (ffmpegCapture) @@ -135,6 +148,7 @@ public: protected: CvCapture_FFMPEG* ffmpegCapture; + Ptr readStream; }; } // namespace @@ -147,6 +161,14 @@ cv::Ptr cvCreateFileCapture_FFMPEG_proxy(const std::string &f return cv::Ptr(); } +cv::Ptr cvCreateStreamCapture_FFMPEG_proxy(const Ptr& stream, const cv::VideoCaptureParameters& params) +{ + cv::Ptr capture = std::make_shared(stream, params); + if (capture && capture->isOpened()) + return capture; + return cv::Ptr(); +} + namespace { class CvVideoWriter_FFMPEG_proxy CV_FINAL : @@ -234,7 +256,7 @@ cv::Ptr cvCreateVideoWriter_FFMPEG_proxy(const std::string& fi #include "plugin_api.hpp" #else #define CAPTURE_ABI_VERSION 1 -#define CAPTURE_API_VERSION 1 +#define CAPTURE_API_VERSION 2 #include "plugin_capture_api.hpp" #define WRITER_ABI_VERSION 1 #define WRITER_API_VERSION 1 @@ -255,7 +277,7 @@ CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_ CvCapture_FFMPEG_proxy *cap = 0; try { - cap = new CvCapture_FFMPEG_proxy(filename, cv::VideoCaptureParameters()); + cap = new CvCapture_FFMPEG_proxy(String(filename), cv::VideoCaptureParameters()); if (cap->isOpened()) { *handle = (CvPluginCapture)cap; @@ -292,7 +314,43 @@ CvResult CV_API_CALL cv_capture_open_with_params( try { cv::VideoCaptureParameters parameters(params, n_params); - cap = new CvCapture_FFMPEG_proxy(filename, parameters); + cap = new CvCapture_FFMPEG_proxy(String(filename), parameters); + if (cap->isOpened()) + { + *handle = (CvPluginCapture)cap; + return CV_ERROR_OK; + } + } + catch (const std::exception& e) + { + CV_LOG_WARNING(NULL, "FFmpeg: Exception is raised: " << e.what()); + } + catch (...) + { + CV_LOG_WARNING(NULL, "FFmpeg: Unknown C++ exception is raised"); + } + if (cap) + delete cap; + return CV_ERROR_FAIL; +} + +static +CvResult CV_API_CALL cv_capture_open_buffer( + void* opaque, + long long(*read)(void* opaque, char* buffer, long long size), + long long(*seek)(void* opaque, long long offset, int way), + int* params, unsigned n_params, + CV_OUT CvPluginCapture* handle +) +{ + if (!handle) + return CV_ERROR_FAIL; + *handle = NULL; + CvCapture_FFMPEG_proxy *cap = 0; + try + { + cv::VideoCaptureParameters parameters(params, n_params); + cap = new CvCapture_FFMPEG_proxy(makePtr(opaque, read, seek), parameters); if (cap->isOpened()) { *handle = (CvPluginCapture)cap; @@ -609,6 +667,9 @@ static const OpenCV_VideoIO_Capture_Plugin_API capture_plugin_api = }, { /* 8*/cv_capture_open_with_params, + }, + { + /* 9*/cv_capture_open_buffer, } }; diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index f13cb8462b..d2359b066e 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -526,7 +526,7 @@ inline static std::string _opencv_ffmpeg_get_error_string(int error_code) struct CvCapture_FFMPEG { - bool open(const char* filename, const VideoCaptureParameters& params); + bool open(const char* filename, const Ptr& stream, const VideoCaptureParameters& params); void close(); double getProperty(int) const; @@ -563,6 +563,8 @@ struct CvCapture_FFMPEG int64_t pts_in_fps_time_base; int64_t dts_delay_in_fps_time_base; + AVIOContext * avio_context; + AVPacket packet; Image_FFMPEG frame; struct SwsContext *img_convert_ctx; @@ -580,6 +582,8 @@ struct CvCapture_FFMPEG */ char * filename; + Ptr readStream; + AVDictionary *dict; #if USE_AV_INTERRUPT_CALLBACK int open_timeout; @@ -628,11 +632,14 @@ void CvCapture_FFMPEG::init() avcodec = 0; context = 0; + avio_context = 0; frame_number = 0; eps_zero = 0.000025; rotation_angle = 0; + readStream.reset(); + dict = NULL; #if USE_AV_INTERRUPT_CALLBACK @@ -730,6 +737,13 @@ void CvCapture_FFMPEG::close() #endif } + if (avio_context) + { + av_free(avio_context->buffer); + av_freep(&avio_context); + } + readStream.reset(); + init(); } @@ -1019,7 +1033,7 @@ static bool isThreadSafe() { return threadSafe; } -bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& params) +bool CvCapture_FFMPEG::open(const char* _filename, const Ptr& stream, const VideoCaptureParameters& params) { const bool threadSafe = isThreadSafe(); InternalFFMpegRegister::init(threadSafe); @@ -1034,6 +1048,8 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& close(); + readStream = stream; + if (!params.empty()) { convertRGB = params.get(CAP_PROP_CONVERT_RGB, true); @@ -1145,6 +1161,56 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& input_format = av_find_input_format(entry->value); } + if (!_filename) + { + size_t avio_ctx_buffer_size = 4096; + uint8_t* avio_ctx_buffer = (uint8_t*)av_malloc(avio_ctx_buffer_size); + CV_Assert(avio_ctx_buffer); + avio_context = avio_alloc_context(avio_ctx_buffer, avio_ctx_buffer_size, 0, this, + [](void *opaque, uint8_t *buf, int buf_size) -> int { + try { + auto capture = reinterpret_cast(opaque); + auto is = capture->readStream; + int result = (int)is->read(reinterpret_cast(buf), buf_size); + + // https://github.com/FFmpeg/FFmpeg/commit/858db4b01fa2b55ee55056c033054ca54ac9b0fd#diff-863c87afc9bb02fe42d071015fc8218972c80b146d603239f20b483ad0988ae9R394 + // https://github.com/FFmpeg/FFmpeg/commit/a606f27f4c610708fa96e35eed7b7537d3d8f712 + // https://github.com/FFmpeg/FFmpeg/blob/n4.0/libavformat/version.h#L83C41-L83C73 +#if (LIBAVFORMAT_VERSION_MAJOR >= 58) && (LIBAVFORMAT_VERSION_MICRO >= 100) // FFmpeg n4.0+ + if (result == 0 && buf_size > 0) + { + result = AVERROR_EOF; + } +#endif + + CV_LOG_VERBOSE(NULL, 0, "FFMPEG: IStreamReader::read(" << buf_size << ") = " << result); + return result; + } catch (...) { + CV_LOG_WARNING(NULL, "FFMPEG: IStreamReader::read(" << buf_size << ") failed"); + return 0; + } + }, + NULL, + [](void *opaque, int64_t offset, int whence) -> int64_t { + try { + int64_t result = -1; + auto capture = reinterpret_cast(opaque); + auto is = capture->readStream; + int origin = whence & (~AVSEEK_FORCE); + if (origin == SEEK_SET || origin == SEEK_CUR || origin == SEEK_END) + { + result = is->seek(offset, origin); + } + CV_LOG_VERBOSE(NULL, 0, "FFMPEG: IStreamReader::seek(" << offset << ", whence=" << whence << ") = " << result); + return result; + } catch (...) { + CV_LOG_WARNING(NULL, "FFMPEG: IStreamReader::seek(" << offset << ", whence=" << whence << ") failed"); + return -1; + } + }); + CV_Assert(avio_context); + ic->pb = avio_context; + } int err = avformat_open_input(&ic, _filename, input_format, &dict); if (err < 0) @@ -3292,16 +3358,30 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, static CvCapture_FFMPEG* cvCreateFileCaptureWithParams_FFMPEG(const char* filename, const VideoCaptureParameters& params) { - // FIXIT: remove unsafe malloc() approach - CvCapture_FFMPEG* capture = (CvCapture_FFMPEG*)malloc(sizeof(*capture)); + CvCapture_FFMPEG* capture = new CvCapture_FFMPEG(); + if (!capture) + return 0; + capture->init(); + if (capture->open(filename, nullptr, params)) + return capture; + + capture->close(); + delete capture; + return 0; +} + +static +CvCapture_FFMPEG* cvCreateStreamCaptureWithParams_FFMPEG(const Ptr& stream, const VideoCaptureParameters& params) +{ + CvCapture_FFMPEG* capture = new CvCapture_FFMPEG(); if (!capture) return 0; capture->init(); - if (capture->open(filename, params)) + if (capture->open(nullptr, stream, params)) return capture; capture->close(); - free(capture); + delete capture; return 0; } @@ -3310,7 +3390,7 @@ void cvReleaseCapture_FFMPEG(CvCapture_FFMPEG** capture) if( capture && *capture ) { (*capture)->close(); - free(*capture); + delete *capture; *capture = 0; } } @@ -3344,14 +3424,14 @@ int cvRetrieveFrame2_FFMPEG(CvCapture_FFMPEG* capture, unsigned char** data, int static CvVideoWriter_FFMPEG* cvCreateVideoWriterWithParams_FFMPEG( const char* filename, int fourcc, double fps, int width, int height, const VideoWriterParameters& params ) { - CvVideoWriter_FFMPEG* writer = (CvVideoWriter_FFMPEG*)malloc(sizeof(*writer)); + CvVideoWriter_FFMPEG* writer = new CvVideoWriter_FFMPEG(); if (!writer) return 0; writer->init(); if( writer->open( filename, fourcc, fps, width, height, params )) return writer; writer->close(); - free(writer); + delete writer; return 0; } @@ -3368,7 +3448,7 @@ void cvReleaseVideoWriter_FFMPEG( CvVideoWriter_FFMPEG** writer ) if( writer && *writer ) { (*writer)->close(); - free(*writer); + delete *writer; *writer = 0; } } diff --git a/modules/videoio/src/cap_interface.hpp b/modules/videoio/src/cap_interface.hpp index a8e3e49dd0..624da9358a 100644 --- a/modules/videoio/src/cap_interface.hpp +++ b/modules/videoio/src/cap_interface.hpp @@ -9,6 +9,7 @@ #include "opencv2/core/core_c.h" #include "opencv2/videoio.hpp" #include "opencv2/videoio/videoio_c.h" +#include "opencv2/videoio/utils.private.hpp" //=================================================== @@ -326,6 +327,7 @@ protected: //================================================================================================== Ptr cvCreateFileCapture_FFMPEG_proxy(const std::string &filename, const VideoCaptureParameters& params); +Ptr cvCreateStreamCapture_FFMPEG_proxy(const Ptr& stream, const VideoCaptureParameters& params); Ptr cvCreateVideoWriter_FFMPEG_proxy(const std::string& filename, int fourcc, double fps, const Size& frameSize, const VideoWriterParameters& params); @@ -351,6 +353,7 @@ Ptr create_WRT_capture(int device); Ptr cvCreateCapture_MSMF(int index, const VideoCaptureParameters& params); Ptr cvCreateCapture_MSMF(const std::string& filename, const VideoCaptureParameters& params); +Ptr cvCreateCapture_MSMF(const Ptr& stream, const VideoCaptureParameters& params); Ptr cvCreateVideoWriter_MSMF(const std::string& filename, int fourcc, double fps, const Size& frameSize, const VideoWriterParameters& params); diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 138e26e87a..5c2121498d 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -746,7 +746,7 @@ public: virtual ~CvCapture_MSMF(); bool configureHW(const cv::VideoCaptureParameters& params); virtual bool open(int, const cv::VideoCaptureParameters* params); - virtual bool open(const cv::String&, const cv::VideoCaptureParameters* params); + virtual bool open(const cv::String&, const Ptr&, const cv::VideoCaptureParameters* params); virtual void close(); virtual double getProperty(int) const CV_OVERRIDE; virtual bool setProperty(int, double) CV_OVERRIDE; @@ -789,6 +789,7 @@ protected: _ComPtr D3DDev; _ComPtr D3DMgr; #endif + _ComPtr byteStream; _ComPtr videoFileSource; _ComPtr readCallback; // non-NULL for "live" streams (camera capture) std::vector dwStreamIndices; @@ -1034,7 +1035,7 @@ bool CvCapture_MSMF::configureHW(bool enable) } } // Reopen if needed - return reopen ? (prevcam >= 0 ? open(prevcam, NULL) : open(prevfile.c_str(), NULL)) : true; + return reopen ? (prevcam >= 0 ? open(prevcam, NULL) : open(prevfile.c_str(), nullptr, NULL)) : true; } D3DMgr.Release(); } @@ -1050,7 +1051,7 @@ bool CvCapture_MSMF::configureHW(bool enable) if (D3DDev) D3DDev.Release(); captureMode = MODE_SW; - return reopen ? (prevcam >= 0 ? open(prevcam, NULL) : open(prevfile.c_str(), NULL)) : true; + return reopen ? (prevcam >= 0 ? open(prevcam, NULL) : open(prevfile.c_str(), nullptr, NULL)) : true; } #else return !enable; @@ -1249,10 +1250,10 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) return isOpen; } -bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCaptureParameters* params) +bool CvCapture_MSMF::open(const cv::String& _filename, const Ptr& stream, const cv::VideoCaptureParameters* params) { close(); - if (_filename.empty()) + if (_filename.empty() && !stream) return false; if (params) @@ -1263,9 +1264,34 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar } // Set source reader parameters _ComPtr attr = getDefaultSourceConfig(); - cv::AutoBuffer unicodeFileName(_filename.length() + 1); - MultiByteToWideChar(CP_ACP, 0, _filename.c_str(), -1, unicodeFileName.data(), (int)_filename.length() + 1); - if (SUCCEEDED(MFCreateSourceReaderFromURL(unicodeFileName.data(), attr.Get(), &videoFileSource))) + bool succeeded = false; + if (!_filename.empty()) + { + cv::AutoBuffer unicodeFileName(_filename.length() + 1); + MultiByteToWideChar(CP_ACP, 0, _filename.c_str(), -1, unicodeFileName.data(), (int)_filename.length() + 1); + succeeded = SUCCEEDED(MFCreateSourceReaderFromURL(unicodeFileName.data(), attr.Get(), &videoFileSource)); + } + else if (stream) + { + // TODO: implement read by chunks + // FIXIT: save stream in field + std::vector data; + data.resize((size_t)stream->seek(0, SEEK_END)); + stream->seek(0, SEEK_SET); + stream->read(data.data(), data.size()); + IStream* s = SHCreateMemStream(reinterpret_cast(data.data()), static_cast(data.size())); + if (!s) + return false; + + succeeded = SUCCEEDED(MFCreateMFByteStreamOnStream(s, &byteStream)); + if (!succeeded) + return false; + if (!SUCCEEDED(MFStartup(MF_VERSION))) + return false; + succeeded = SUCCEEDED(MFCreateSourceReaderFromByteStream(byteStream.Get(), attr.Get(), &videoFileSource)); + } + + if (succeeded) { isOpen = true; usedVideoSampleTime = 0; @@ -2375,12 +2401,24 @@ cv::Ptr cv::cvCreateCapture_MSMF( int index, const cv::VideoC return cv::Ptr(); } -cv::Ptr cv::cvCreateCapture_MSMF (const cv::String& filename, const cv::VideoCaptureParameters& params) +cv::Ptr cv::cvCreateCapture_MSMF(const cv::String& filename, const cv::VideoCaptureParameters& params) { cv::Ptr capture = cv::makePtr(); if (capture) { - capture->open(filename, ¶ms); + capture->open(filename, nullptr, ¶ms); + if (capture->isOpened()) + return capture; + } + return cv::Ptr(); +} + +cv::Ptr cv::cvCreateCapture_MSMF(const Ptr& stream, const cv::VideoCaptureParameters& params) +{ + cv::Ptr capture = cv::makePtr(); + if (capture) + { + capture->open(std::string(), stream, ¶ms); if (capture->isOpened()) return capture; } @@ -2707,7 +2745,7 @@ cv::Ptr cv::cvCreateVideoWriter_MSMF( const std::string& filen #include "plugin_api.hpp" #else #define CAPTURE_ABI_VERSION 1 -#define CAPTURE_API_VERSION 1 +#define CAPTURE_API_VERSION 2 #include "plugin_capture_api.hpp" #define WRITER_ABI_VERSION 1 #define WRITER_API_VERSION 1 @@ -2736,7 +2774,9 @@ CvResult CV_API_CALL cv_capture_open_with_params( cap = new CaptureT(); bool res; if (filename) - res = cap->open(std::string(filename), ¶meters); + { + res = cap->open(std::string(filename), nullptr, ¶meters); + } else res = cap->open(camera_index, ¶meters); if (res) @@ -2758,6 +2798,44 @@ CvResult CV_API_CALL cv_capture_open_with_params( return CV_ERROR_FAIL; } +static +CvResult CV_API_CALL cv_capture_open_buffer( + void* opaque, + long long(*read)(void* opaque, char* buffer, long long size), + long long(*seek)(void* opaque, long long offset, int way), + int* params, unsigned n_params, + CV_OUT CvPluginCapture* handle +) +{ + if (!handle) + return CV_ERROR_FAIL; + + *handle = NULL; + CaptureT* cap = 0; + try + { + cv::VideoCaptureParameters parameters(params, n_params); + cap = new CaptureT(); + bool res = cap->open(std::string(), makePtr(opaque, read, seek), ¶meters); + if (res) + { + *handle = (CvPluginCapture)cap; + return CV_ERROR_OK; + } + } + catch (const std::exception& e) + { + CV_LOG_WARNING(NULL, "MSMF: Exception is raised: " << e.what()); + } + catch (...) + { + CV_LOG_WARNING(NULL, "MSMF: Unknown C++ exception is raised"); + } + if (cap) + delete cap; + return CV_ERROR_FAIL; +} + static CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_OUT CvPluginCapture* handle) { @@ -3027,6 +3105,9 @@ static const OpenCV_VideoIO_Capture_Plugin_API capture_plugin_api = }, { /* 8*/cv::cv_capture_open_with_params, + }, + { + /* 9*/cv::cv_capture_open_buffer, } }; diff --git a/modules/videoio/src/plugin_capture_api.hpp b/modules/videoio/src/plugin_capture_api.hpp index 8f33d40219..c616bbc928 100644 --- a/modules/videoio/src/plugin_capture_api.hpp +++ b/modules/videoio/src/plugin_capture_api.hpp @@ -13,7 +13,7 @@ /// increased for backward-compatible changes, e.g. add new function /// Caller API <= Plugin API -> plugin is fully compatible /// Caller API > Plugin API -> plugin is not fully compatible, caller should use extra checks to use plugins with older API -#define CAPTURE_API_VERSION 1 +#define CAPTURE_API_VERSION 2 /// increased for incompatible changes, e.g. remove function argument /// Caller ABI == Plugin ABI -> plugin is compatible @@ -121,6 +121,29 @@ struct OpenCV_VideoIO_Capture_Plugin_API_v1_1_api_entries CV_OUT CvPluginCapture* handle); }; // OpenCV_VideoIO_Capture_Plugin_API_v1_1_api_entries +struct OpenCV_VideoIO_Capture_Plugin_API_v1_2_api_entries +{ + /** @brief Open video capture from buffer with parameters + + @param opaque A pointer to user data + @param read A pointer to a function that is called to reads @p size bytes to allocated @p buffer. Returns a number of bytes that were actually read + @param seek A pointer to a function that is called to move starting position inside the stream buffer. + @p offset is a number of bytes and @p way is one of the markers SEEK_SET, SEEK_CUR, SEEK_END. + Function returns an absolute current position in bytes. + @param params pointer on 2*n_params array of 'key,value' pairs + @param n_params number of passed parameters + @param[out] handle pointer on Capture handle + + @note API-CALL 9, API-Version == 2 + */ + CvResult (CV_API_CALL *Capture_open_stream)( + void* opaque, + long long(*read)(void* opaque, char* buffer, long long size), + long long(*seek)(void* opaque, long long offset, int way), + int* params, unsigned n_params, + CV_OUT CvPluginCapture* handle); +}; // OpenCV_VideoIO_Capture_Plugin_API_v1_2_api_entries + typedef struct OpenCV_VideoIO_Capture_Plugin_API_v1_0 { OpenCV_API_Header api_header; @@ -134,7 +157,17 @@ typedef struct OpenCV_VideoIO_Capture_Plugin_API_v1_1 struct OpenCV_VideoIO_Capture_Plugin_API_v1_1_api_entries v1; } OpenCV_VideoIO_Capture_Plugin_API_v1_1; -#if CAPTURE_ABI_VERSION == 1 && CAPTURE_API_VERSION == 1 +typedef struct OpenCV_VideoIO_Capture_Plugin_API_v1_2 +{ + OpenCV_API_Header api_header; + struct OpenCV_VideoIO_Capture_Plugin_API_v1_0_api_entries v0; + struct OpenCV_VideoIO_Capture_Plugin_API_v1_1_api_entries v1; + struct OpenCV_VideoIO_Capture_Plugin_API_v1_2_api_entries v2; +} OpenCV_VideoIO_Capture_Plugin_API_v1_2; + +#if CAPTURE_ABI_VERSION == 1 && CAPTURE_API_VERSION == 2 +typedef struct OpenCV_VideoIO_Capture_Plugin_API_v1_2 OpenCV_VideoIO_Capture_Plugin_API; +#elif CAPTURE_ABI_VERSION == 1 && CAPTURE_API_VERSION == 1 typedef struct OpenCV_VideoIO_Capture_Plugin_API_v1_1 OpenCV_VideoIO_Capture_Plugin_API; #elif CAPTURE_ABI_VERSION == 1 && CAPTURE_API_VERSION == 0 typedef struct OpenCV_VideoIO_Capture_Plugin_API_v1_0 OpenCV_VideoIO_Capture_Plugin_API; diff --git a/modules/videoio/src/videoio_registry.cpp b/modules/videoio/src/videoio_registry.cpp index a3aee5abfe..a84258ad90 100644 --- a/modules/videoio/src/videoio_registry.cpp +++ b/modules/videoio/src/videoio_registry.cpp @@ -47,7 +47,12 @@ namespace { #define DECLARE_STATIC_BACKEND(cap, name, mode, createCaptureFile, createCaptureCamera, createWriter) \ { \ - cap, (BackendMode)(mode), 1000, name, createBackendFactory(createCaptureFile, createCaptureCamera, createWriter) \ + cap, (BackendMode)(mode), 1000, name, createBackendFactory(createCaptureFile, createCaptureCamera, 0, createWriter) \ +}, + +#define DECLARE_STATIC_BACKEND_WITH_STREAM_SUPPORT(cap, name, mode, createCaptureStream) \ +{ \ + cap, (BackendMode)(mode), 1000, name, createBackendFactory(0, 0, createCaptureStream, 0) \ }, /** Ordering guidelines: @@ -62,8 +67,9 @@ static const struct VideoBackendInfo builtin_backends[] = { #ifdef HAVE_FFMPEG DECLARE_STATIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, cvCreateFileCapture_FFMPEG_proxy, 0, cvCreateVideoWriter_FFMPEG_proxy) + DECLARE_STATIC_BACKEND_WITH_STREAM_SUPPORT(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_STREAM, cvCreateStreamCapture_FFMPEG_proxy) #elif defined(ENABLE_PLUGINS) || defined(HAVE_FFMPEG_WRAPPER) - DECLARE_DYNAMIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER) + DECLARE_DYNAMIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_CAPTURE_BY_STREAM | MODE_WRITER) #endif #ifdef HAVE_GSTREAMER @@ -90,8 +96,9 @@ static const struct VideoBackendInfo builtin_backends[] = #ifdef HAVE_MSMF DECLARE_STATIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER, cvCreateCapture_MSMF, cvCreateCapture_MSMF, cvCreateVideoWriter_MSMF) + DECLARE_STATIC_BACKEND_WITH_STREAM_SUPPORT(CAP_MSMF, "MSMF", MODE_CAPTURE_BY_STREAM, cvCreateCapture_MSMF) #elif defined(ENABLE_PLUGINS) && defined(_WIN32) - DECLARE_DYNAMIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER) + DECLARE_DYNAMIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_CAPTURE_BY_STREAM | MODE_WRITER) #endif #ifdef HAVE_DSHOW @@ -330,6 +337,17 @@ public: } return result; } + inline std::vector getAvailableBackends_CaptureByStream() const + { + std::vector result; + for (size_t i = 0; i < enabledBackends.size(); i++) + { + const VideoBackendInfo& info = enabledBackends[i]; + if (info.mode & MODE_CAPTURE_BY_STREAM) + result.push_back(info); + } + return result; + } inline std::vector getAvailableBackends_Writer() const { std::vector result; @@ -357,6 +375,11 @@ std::vector getAvailableBackends_CaptureByFilename() const std::vector result = VideoBackendRegistry::getInstance().getAvailableBackends_CaptureByFilename(); return result; } +std::vector getAvailableBackends_CaptureByStream() +{ + const std::vector result = VideoBackendRegistry::getInstance().getAvailableBackends_CaptureByStream(); + return result; +} std::vector getAvailableBackends_Writer() { const std::vector result = VideoBackendRegistry::getInstance().getAvailableBackends_Writer(); @@ -424,6 +447,15 @@ std::vector getStreamBackends() } +std::vector getStreamBufferedBackends() +{ + const std::vector backends = VideoBackendRegistry::getInstance().getAvailableBackends_CaptureByStream(); + std::vector result; + for (size_t i = 0; i < backends.size(); i++) + result.push_back((VideoCaptureAPIs)backends[i].id); + return result; +} + std::vector getWriterBackends() { const std::vector backends = VideoBackendRegistry::getInstance().getAvailableBackends_Writer(); @@ -501,6 +533,24 @@ std::string getStreamBackendPluginVersion(VideoCaptureAPIs api, CV_Error(Error::StsError, "Unknown or wrong backend ID"); } +std::string getStreamBufferedBackendPluginVersion(VideoCaptureAPIs api, + CV_OUT int& version_ABI, + CV_OUT int& version_API +) +{ + const std::vector backends = VideoBackendRegistry::getInstance().getAvailableBackends_CaptureByStream(); + for (size_t i = 0; i < backends.size(); i++) + { + const VideoBackendInfo& info = backends[i]; + if (api == info.id) + { + CV_Assert(!info.backendFactory.empty()); + CV_Assert(!info.backendFactory->isBuiltIn()); + return getCapturePluginVersion(info.backendFactory, version_ABI, version_API); + } + } + CV_Error(Error::StsError, "Unknown or wrong backend ID"); +} /** @brief Returns description and ABI/API version of videoio plugin's writer interface */ std::string getWriterBackendPluginVersion(VideoCaptureAPIs api, diff --git a/modules/videoio/src/videoio_registry.hpp b/modules/videoio/src/videoio_registry.hpp index 3a81df84b5..e2fe579a2f 100644 --- a/modules/videoio/src/videoio_registry.hpp +++ b/modules/videoio/src/videoio_registry.hpp @@ -14,6 +14,7 @@ namespace cv enum BackendMode { MODE_CAPTURE_BY_INDEX = 1 << 0, //!< device index MODE_CAPTURE_BY_FILENAME = 1 << 1, //!< filename or device path (v4l2) + MODE_CAPTURE_BY_STREAM = 1 << 2, //!< data stream MODE_WRITER = 1 << 4, //!< writer MODE_CAPTURE_ALL = MODE_CAPTURE_BY_INDEX + MODE_CAPTURE_BY_FILENAME, @@ -38,6 +39,7 @@ namespace videoio_registry { std::vector getAvailableBackends_CaptureByIndex(); std::vector getAvailableBackends_CaptureByFilename(); +std::vector getAvailableBackends_CaptureByStream(); std::vector getAvailableBackends_Writer(); bool checkDeprecatedBackend(int api); diff --git a/modules/videoio/test/test_video_io.cpp b/modules/videoio/test/test_video_io.cpp index 001035f35a..f3c0862999 100644 --- a/modules/videoio/test/test_video_io.cpp +++ b/modules/videoio/test/test_video_io.cpp @@ -3,6 +3,7 @@ // of this distribution and at http://opencv.org/license.html. #include "test_precomp.hpp" +#include "opencv2/core/utils/filesystem.hpp" namespace opencv_test { @@ -1024,4 +1025,133 @@ INSTANTIATE_TEST_CASE_P(videoio, videowriter_acceleration, testing::Combine( testing::ValuesIn(hw_use_umat) )); +class BufferStream : public cv::IStreamReader +{ +public: + BufferStream(const std::string& filename) + { + Ptr file = makePtr(); + file->open(filename.c_str(), std::ios::in | std::ios::binary); + stream = file; + } + + BufferStream(const Ptr& _stream) : stream(_stream) {} + + long long read(char* buffer, long long size) CV_OVERRIDE + { + auto result = stream->sgetn(buffer, size); + return result; + } + + long long seek(long long offset, int way) CV_OVERRIDE + { + auto result = stream->pubseekoff(offset, way == SEEK_SET ? std::ios_base::beg : (way == SEEK_END ? std::ios_base::end : std::ios_base::cur)); + return result; + } + +private: + Ptr stream; +}; + +typedef testing::TestWithParam> stream_capture; +TEST_P(stream_capture, read) +{ + std::string ext = get<0>(GetParam()); + VideoCaptureAPIs apiPref = get<1>(GetParam()); + std::vector supportedAPIs = videoio_registry::getStreamBufferedBackends(); + if (!videoio_registry::hasBackend(apiPref)) + throw SkipTestException(cv::String("Backend is not available/disabled: ") + cv::videoio_registry::getBackendName(apiPref)); + if (std::find(supportedAPIs.begin(), supportedAPIs.end(), apiPref) == supportedAPIs.end()) + throw SkipTestException(cv::String("Backend is not supported: ") + cv::videoio_registry::getBackendName(apiPref)); + if (cvtest::skipUnstableTests && apiPref == CAP_MSMF && (ext == "h264" || ext == "h265" || ext == "mpg")) + throw SkipTestException("Unstable MSMF test"); + + if (!videoio_registry::isBackendBuiltIn(apiPref)) + { + int pluginABI, pluginAPI; + videoio_registry::getStreamBufferedBackendPluginVersion(apiPref, pluginABI, pluginAPI); + if (pluginABI < 1 || (pluginABI == 1 && pluginAPI < 2)) + throw SkipTestException(format("Buffer capture supported since ABI/API = 1/2. %s plugin is %d/%d", + cv::videoio_registry::getBackendName(apiPref).c_str(), pluginABI, pluginAPI)); + } + + VideoCapture cap; + String video_file = BunnyParameters::getFilename(String(".") + ext); + ASSERT_TRUE(utils::fs::exists(video_file)); + EXPECT_NO_THROW(cap.open(makePtr(video_file), apiPref, {})); + ASSERT_TRUE(cap.isOpened()); + + const int numFrames = 10; + Mat frames[numFrames]; + Mat hardCopies[numFrames]; + for(int i = 0; i < numFrames; i++) + { + ASSERT_NO_THROW(cap >> frames[i]); + EXPECT_FALSE(frames[i].empty()); + hardCopies[i] = frames[i].clone(); + } + + for(int i = 0; i < numFrames; i++) + EXPECT_EQ(0, cv::norm(frames[i], hardCopies[i], NORM_INF)) << i; +} +INSTANTIATE_TEST_CASE_P(videoio, stream_capture, + testing::Combine( + testing::ValuesIn(bunny_params), + testing::ValuesIn(backend_params))); + +// This test for stream input for container format (See test_ffmpeg/videoio_container.read test) +typedef testing::TestWithParam stream_capture_ffmpeg; +TEST_P(stream_capture_ffmpeg, raw) +{ + std::string ext = GetParam(); + VideoCaptureAPIs apiPref = CAP_FFMPEG; + std::vector supportedAPIs = videoio_registry::getStreamBufferedBackends(); + if (!videoio_registry::hasBackend(apiPref)) + throw SkipTestException(cv::String("Backend is not available/disabled: ") + cv::videoio_registry::getBackendName(apiPref)); + if (std::find(supportedAPIs.begin(), supportedAPIs.end(), apiPref) == supportedAPIs.end()) + throw SkipTestException(cv::String("Backend is not supported: ") + cv::videoio_registry::getBackendName(apiPref)); + + if (!videoio_registry::isBackendBuiltIn(apiPref)) + { + int pluginABI, pluginAPI; + videoio_registry::getStreamBufferedBackendPluginVersion(apiPref, pluginABI, pluginAPI); + if (pluginABI < 1 || (pluginABI == 1 && pluginAPI < 2)) + throw SkipTestException(format("Buffer capture supported since ABI/API = 1/2. %s plugin is %d/%d", + cv::videoio_registry::getBackendName(apiPref).c_str(), pluginABI, pluginAPI)); + } + + VideoCapture container; + String video_file = BunnyParameters::getFilename(String(".") + ext); + ASSERT_TRUE(utils::fs::exists(video_file)); + EXPECT_NO_THROW(container.open(video_file, apiPref, {CAP_PROP_FORMAT, -1})); + ASSERT_TRUE(container.isOpened()); + ASSERT_EQ(-1.f, container.get(CAP_PROP_FORMAT)); + + auto stream = std::make_shared(); + Mat keyFrame; + while (true) + { + container >> keyFrame; + if (keyFrame.empty()) + break; + stream->sputn(keyFrame.ptr(), keyFrame.total()); + } + + VideoCapture capRef(video_file); + VideoCapture capStream; + EXPECT_NO_THROW(capStream.open(makePtr(stream), apiPref, {})); + ASSERT_TRUE(capStream.isOpened()); + + const int numFrames = 10; + Mat frameRef, frame; + for (int i = 0; i < numFrames; ++i) + { + capRef >> frameRef; + ASSERT_NO_THROW(capStream >> frame); + EXPECT_FALSE(frame.empty()); + EXPECT_EQ(0, cv::norm(frame, frameRef, NORM_INF)) << i; + } +} +INSTANTIATE_TEST_CASE_P(videoio, stream_capture_ffmpeg, testing::Values("h264", "h265", "mjpg.avi")); + } // namespace