From db4ab1c936d8dd1046c28741137510aab8f6537e Mon Sep 17 00:00:00 2001 From: Maxim Milashchenko <67949029+MaximMilashchenko@users.noreply.github.com> Date: Thu, 16 Dec 2021 22:43:02 +0300 Subject: [PATCH] Merge pull request #21264 from MaximMilashchenko:AudioGStreamer Audio GStreamer: added support .wav .flac audio formats * added support .wav, lossless compressed audio formats * fixed docs * fixes * videoio(gstreamer-audio): extra tests, improve error handling Co-authored-by: Alexander Alekhin --- modules/videoio/src/cap_gstreamer.cpp | 519 +++++++++++++++++++++----- modules/videoio/test/test_audio.cpp | 74 +++- 2 files changed, 479 insertions(+), 114 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index dd91bf1db4..adbc94fc95 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -59,6 +59,7 @@ #include #include #include +#include #include #include #include @@ -308,6 +309,8 @@ private: GSafePtr sample; GSafePtr caps; + gint videoStream; + gint audioStream; gint64 duration; gint width; gint height; @@ -315,6 +318,12 @@ private: bool isPosFramesSupported; bool isPosFramesEmulated; gint64 emulatedFrameNumber; + gint outputAudioFormat; + gint audioBaseIndex; + gint nAudioChannels; + gint audioSamplesPerSecond; + + Mat audioFrame; VideoAccelerationType va_type; int hw_device; @@ -323,6 +332,9 @@ public: virtual ~GStreamerCapture() CV_OVERRIDE; virtual bool grabFrame() CV_OVERRIDE; virtual bool retrieveFrame(int /*unused*/, OutputArray dst) CV_OVERRIDE; + bool grabAudioFrame(); + bool retrieveVideoFrame(int /*unused*/, OutputArray dst); + bool retrieveAudioFrame(int /*unused*/, OutputArray dst); virtual double getProperty(int propId) const CV_OVERRIDE; virtual bool setProperty(int propId, double value) CV_OVERRIDE; virtual bool isOpened() const CV_OVERRIDE { return (bool)pipeline; } @@ -330,6 +342,9 @@ public: bool open(int id, const cv::VideoCaptureParameters& params); bool open(const String &filename_, const cv::VideoCaptureParameters& params); static void newPad(GstElement * /*elem*/, GstPad *pad, gpointer data); + bool configureHW(const cv::VideoCaptureParameters&); + bool configureStreams(const cv::VideoCaptureParameters&); + bool setAudioProperties(const cv::VideoCaptureParameters&); protected: bool isPipelinePlaying(); @@ -341,10 +356,16 @@ protected: }; GStreamerCapture::GStreamerCapture() : + videoStream(0), + audioStream(-1), duration(-1), width(-1), height(-1), fps(-1), isPosFramesSupported(false), isPosFramesEmulated(false), - emulatedFrameNumber(-1) + emulatedFrameNumber(-1), + outputAudioFormat(CV_16S), + audioBaseIndex(1), + nAudioChannels(0), + audioSamplesPerSecond(44100) , va_type(VIDEO_ACCELERATION_NONE) , hw_device(-1) { @@ -365,6 +386,92 @@ GStreamerCapture::~GStreamerCapture() } } +bool GStreamerCapture::configureHW(const cv::VideoCaptureParameters& params) +{ + if (params.has(CAP_PROP_HW_ACCELERATION)) + { + va_type = params.get(CAP_PROP_HW_ACCELERATION); + } + if (params.has(CAP_PROP_HW_DEVICE)) + { + hw_device = params.get(CAP_PROP_HW_DEVICE); + if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE without requested H/W acceleration. Bailout"); + return false; + } + if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout"); + return false; + } + if (hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: CAP_PROP_HW_DEVICE is not supported. Specify -1 (auto) value. Bailout"); + return false; + } + } + return true; +} + +bool GStreamerCapture::configureStreams(const cv::VideoCaptureParameters& params) +{ + if (params.has(CAP_PROP_VIDEO_STREAM)) + { + double value = params.get(CAP_PROP_VIDEO_STREAM); + if (value == -1 || value == 0) + videoStream = static_cast(value); + else + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_VIDEO_STREAM parameter value is invalid/unsupported: " << value); + return false; + } + } + if (params.has(CAP_PROP_AUDIO_STREAM)) + { + double value = params.get(CAP_PROP_AUDIO_STREAM); + if (value == -1 || value > -1) + audioStream = static_cast(value); + else + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_STREAM parameter value is invalid/unsupported: " << value); + return false; + } + } + return true; +} + +bool GStreamerCapture::setAudioProperties(const cv::VideoCaptureParameters& params) +{ + if (params.has(CAP_PROP_AUDIO_DATA_DEPTH)) + { + gint value = static_cast(params.get(CAP_PROP_AUDIO_DATA_DEPTH)); + if (value != CV_8S && value != CV_16S && value != CV_32S && value != CV_32F) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_DATA_DEPTH parameter value is invalid/unsupported: " << value); + return false; + } + else + { + outputAudioFormat = value; + } + } + if (params.has(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + { + int value = static_cast(params.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)); + if (value < 0) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_SAMPLES_PER_SECOND parameter can't be negative: " << value); + return false; + } + else + { + audioSamplesPerSecond = value; + } + } + return true; +} + /*! * \brief CvCapture_GStreamer::grabFrame * \return @@ -391,21 +498,137 @@ bool GStreamerCapture::grabFrame() if (isPosFramesEmulated) emulatedFrameNumber++; + if (audioStream >= 0) + return grabAudioFrame(); + return true; } -/*! - * \brief CvCapture_GStreamer::retrieveFrame - * \return IplImage pointer. [Transfer Full] - * Retrieve the previously grabbed buffer, and wrap it in an IPLImage structure - */ -bool GStreamerCapture::retrieveFrame(int, OutputArray dst) +bool GStreamerCapture::grabAudioFrame() { - if (!sample) + GstCaps* frame_caps = gst_sample_get_caps(sample); // no lifetime transfer + if (!frame_caps) { + CV_LOG_ERROR(NULL, "GStreamer: gst_sample_get_caps() returns NULL"); return false; } + if (!GST_CAPS_IS_SIMPLE(frame_caps)) + { + // bail out in no caps + CV_LOG_ERROR(NULL, "GStreamer: GST_CAPS_IS_SIMPLE(frame_caps) check is failed"); + return false; + } + + GstAudioInfo info = {}; + gboolean audio_info_res = gst_audio_info_from_caps(&info, frame_caps); + if (!audio_info_res) + { + CV_Error(Error::StsError, "GStreamer: gst_audio_info_from_caps() is failed. Can't handle unknown layout"); + } + int bpf = GST_AUDIO_INFO_BPF(&info); + + GstStructure* structure = gst_caps_get_structure(frame_caps, 0); // no lifetime transfer + if (!structure) + { + CV_LOG_ERROR(NULL, "GStreamer: Can't query 'structure'-0 from GStreamer sample"); + return false; + } + + const gchar* name_ = gst_structure_get_name(structure); + if (!name_) + { + CV_LOG_ERROR(NULL, "GStreamer: Can't query 'name' from GStreamer sample"); + return false; + } + std::string name = toLowerCase(std::string(name_)); + + GstBuffer* buf = gst_sample_get_buffer(sample); + if (!buf) + return false; + GstMapInfo map_info = {}; + if (!gst_buffer_map(buf, &map_info, GST_MAP_READ)) + { + CV_LOG_ERROR(NULL, "GStreamer: Failed to map GStreamer buffer to system memory"); + return false; + } + ScopeGuardGstMapInfo map_guard(buf, &map_info); + if (name == "audio/x-raw") + { + const gchar* format_ = gst_structure_get_string(structure, "format"); + if (!format_) + { + CV_LOG_ERROR(NULL, "GStreamer: Can't query 'format' of 'video/x-raw'"); + return false; + } + std::string format = toUpperCase(std::string(format_)); + cv::Mat data; + if (format == "S8") + { + Mat(map_info.size/bpf, nAudioChannels, CV_8S, map_info.data).copyTo(audioFrame); + return true; + } + if (format == "S16LE") + { + Mat(map_info.size/bpf, nAudioChannels, CV_16S, map_info.data).copyTo(audioFrame); + return true; + } + if (format == "S32LE") + { + Mat(map_info.size/bpf, nAudioChannels, CV_32S, map_info.data).copyTo(audioFrame); + return true; + } + if (format == "F32LE") + { + Mat(map_info.size/bpf, nAudioChannels, CV_32F, map_info.data).copyTo(audioFrame); + return true; + } + CV_Error_(Error::StsNotImplemented, ("Unsupported GStreamer audio format: %s", format.c_str())); + } + + CV_Error_(Error::StsNotImplemented, ("Unsupported GStreamer layer type: %s", name.c_str())); +} + +bool GStreamerCapture::retrieveAudioFrame(int index, OutputArray dst) +{ + CV_Check(index, index >= audioBaseIndex && index < audioBaseIndex + nAudioChannels, ""); + index -= audioBaseIndex; + + CV_CheckType(outputAudioFormat, + outputAudioFormat == CV_8S || + outputAudioFormat == CV_16S || + outputAudioFormat == CV_32S || + outputAudioFormat == CV_32F, + ""); + + dst.create(1, audioFrame.rows, outputAudioFormat); + Mat data = dst.getMat(); + switch (outputAudioFormat) + { + case CV_8S: + for (int i = 0; i < audioFrame.rows; i++) + data.at(i) = audioFrame.at(i, index); + return true; + case CV_16S: + for (int i = 0; i < audioFrame.rows; i++) + data.at(i) = audioFrame.at(i, index); + return true; + case CV_32S: + for (int i = 0; i < audioFrame.rows; i++) + data.at(i) = audioFrame.at(i, index); + return true; + case CV_32F: + for (int i = 0; i < audioFrame.rows; i++) + data.at(i) = audioFrame.at(i, index); + return true; + } + + dst.release(); + return false; +} + +bool GStreamerCapture::retrieveVideoFrame(int, OutputArray dst) +{ GstCaps* frame_caps = gst_sample_get_caps(sample); // no lifetime transfer if (!frame_caps) { @@ -609,6 +832,28 @@ bool GStreamerCapture::retrieveFrame(int, OutputArray dst) CV_Error_(Error::StsNotImplemented, ("Unsupported GStreamer layer type: %s", name.c_str())); } +bool GStreamerCapture::retrieveFrame(int index, OutputArray dst) +{ + if (index < 0) + return false; + if (!sample) + return false; + + if (index == 0) + { + CV_CheckGE(videoStream, 0, "No video stream configured"); + return retrieveVideoFrame(index, dst); + } + else if (index >= audioBaseIndex) + { + CV_CheckGE(audioStream, 0, "No audio stream configured"); + return retrieveAudioFrame(index, dst); + } + + CV_LOG_ERROR(NULL, "GStreamer(retrive): unrecognized index=" << index); + return false; +} + bool GStreamerCapture::isPipelinePlaying() { if (!pipeline || !GST_IS_ELEMENT(pipeline.get())) @@ -810,28 +1055,33 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam { gst_initializer::init(); - if (params.has(CAP_PROP_HW_ACCELERATION)) + if (!configureHW(params)) { - va_type = params.get(CAP_PROP_HW_ACCELERATION); + CV_LOG_WARNING(NULL, "GStreamer: can't configure HW encoding/decoding support"); + return false; } - if (params.has(CAP_PROP_HW_DEVICE)) + + if (!configureStreams(params)) { - hw_device = params.get(CAP_PROP_HW_DEVICE); - if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1) - { - CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE without requested H/W acceleration. Bailout"); - return false; - } - if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1) - { - CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout"); - return false; - } - if (hw_device != -1) - { - CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: CAP_PROP_HW_DEVICE is not supported. Specify -1 (auto) value. Bailout"); - return false; - } + CV_LOG_WARNING(NULL, "GStreamer: can't configure streams"); + return false; + } + + if ((videoStream >= 0 && audioStream >= 0) || (videoStream < 0 && audioStream < 0)) + { + CV_LOG_ERROR(NULL, "GStreamer backend supports audio-only or video-only capturing. Only one of the properties CAP_PROP_AUDIO_STREAM=" << audioStream << " and CAP_PROP_VIDEO_STREAM=" << videoStream << " should be >= 0"); + return false; + } + if (audioStream > 0) + { + CV_LOG_ERROR(NULL, "GStreamer backend supports the first audio stream only. CAP_PROP_AUDIO_STREAM=" << audioStream); + return false; + } + + if (!setAudioProperties(params)) + { + CV_LOG_WARNING(NULL, "GStreamer: can't configure audio properties"); + return false; } const gchar* filename = filename_.c_str(); @@ -843,6 +1093,9 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam GSafePtr color; GstStateChangeReturn status; + GSafePtr convert; + GSafePtr resample; + // test if we have a valid uri. If so, open it with an uridecodebin // else, we might have a file or a manual pipeline. // if gstreamer cannot parse the manual pipeline, we assume we were given and @@ -884,20 +1137,29 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam bool element_from_uri = false; if (!uridecodebin) { - // At this writing, the v4l2 element (and maybe others too) does not support caps renegotiation. - // This means that we cannot use an uridecodebin when dealing with v4l2, since setting - // capture properties will not work. - // The solution (probably only until gstreamer 1.2) is to make an element from uri when dealing with v4l2. - GSafePtr protocol_; protocol_.attach(gst_uri_get_protocol(uri)); - CV_Assert(protocol_); - std::string protocol = toLowerCase(std::string(protocol_.get())); - if (protocol == "v4l2") + if (videoStream >= 0) { - uridecodebin.reset(gst_element_make_from_uri(GST_URI_SRC, uri.get(), "src", NULL)); - CV_Assert(uridecodebin); - element_from_uri = true; + // At this writing, the v4l2 element (and maybe others too) does not support caps renegotiation. + // This means that we cannot use an uridecodebin when dealing with v4l2, since setting + // capture properties will not work. + // The solution (probably only until gstreamer 1.2) is to make an element from uri when dealing with v4l2. + GSafePtr protocol_; protocol_.attach(gst_uri_get_protocol(uri)); + CV_Assert(protocol_); + std::string protocol = toLowerCase(std::string(protocol_.get())); + if (protocol == "v4l2") + { + uridecodebin.reset(gst_element_make_from_uri(GST_URI_SRC, uri.get(), "src", NULL)); + CV_Assert(uridecodebin); + element_from_uri = true; + } + else + { + uridecodebin.reset(gst_element_factory_make("uridecodebin", NULL)); + CV_Assert(uridecodebin); + g_object_set(G_OBJECT(uridecodebin.get()), "uri", uri.get(), NULL); + } } - else + else if (audioStream >= 0) { uridecodebin.reset(gst_element_factory_make("uridecodebin", NULL)); CV_Assert(uridecodebin); @@ -971,35 +1233,51 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam pipeline.reset(gst_pipeline_new(NULL)); CV_Assert(pipeline); - // videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert) - //automatically selects the correct colorspace conversion based on caps. - color.reset(gst_element_factory_make(COLOR_ELEM, NULL)); - CV_Assert(color); - sink.reset(gst_element_factory_make("appsink", NULL)); CV_Assert(sink); + if (videoStream >= 0) + { + // videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert) + //automatically selects the correct colorspace conversion based on caps. + color.reset(gst_element_factory_make(COLOR_ELEM, NULL)); + CV_Assert(color); - gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL); + gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL); - if (element_from_uri) - { - if(!gst_element_link(uridecodebin, color.get())) + if (element_from_uri) + { + if(!gst_element_link(uridecodebin, color.get())) + { + CV_WARN("GStreamer(video): cannot link color -> sink"); + pipeline.release(); + return false; + } + } + else + { + g_signal_connect(uridecodebin, "pad-added", G_CALLBACK(newPad), color.get()); + } + + if (!gst_element_link(color.get(), sink.get())) { - CV_WARN("cannot link color -> sink"); + CV_WARN("GStreamer(video): cannot link color -> sink"); pipeline.release(); return false; } } - else + else if (audioStream >= 0) { - g_signal_connect(uridecodebin, "pad-added", G_CALLBACK(newPad), color.get()); - } + convert.reset(gst_element_factory_make("audioconvert", NULL)); + resample.reset(gst_element_factory_make("audioresample", NULL)); - if (!gst_element_link(color.get(), sink.get())) - { - CV_WARN("GStreamer: cannot link color -> sink"); - pipeline.release(); - return false; + gst_bin_add_many (GST_BIN (pipeline.get()), uridecodebin.get(), convert.get(), resample.get(), sink.get(), NULL); + if (!gst_element_link_many (convert.get(), resample.get(), sink.get(), NULL)) + { + CV_WARN("GStreamer(audio): cannot link convert -> resample -> sink"); + pipeline.release(); + return false; + } + g_signal_connect (uridecodebin, "pad-added", G_CALLBACK (newPad), convert.get()); } } @@ -1008,17 +1286,41 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam //TODO: is 1 single buffer really high enough? gst_app_sink_set_max_buffers(GST_APP_SINK(sink.get()), 1); } - if (!manualpipeline) { gst_base_sink_set_sync(GST_BASE_SINK(sink.get()), FALSE); } - //do not emit signals: all calls will be synchronous and blocking gst_app_sink_set_emit_signals (GST_APP_SINK(sink.get()), FALSE); - - caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR, GRAY8}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; image/jpeg")); + if (videoStream >= 0) + { + caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR, GRAY8}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; image/jpeg")); + } + else if (audioStream >= 0) + { + std::string audioFormat; + switch (outputAudioFormat) + { + case CV_8S: + audioFormat = "S8"; + break; + case CV_16S: + audioFormat = "S16LE"; + break; + case CV_32S: + audioFormat = "S32LE"; + break; + case CV_32F: + audioFormat = "F32LE"; + break; + default: + audioFormat = "S16LE"; + break; + } + std::string stringCaps = "audio/x-raw, format=(string)" + audioFormat + ", rate=(int)" + std::to_string(audioSamplesPerSecond) + ", channels=(int){1, 2}, layout=(string)interleaved"; + caps.attach(gst_caps_from_string(stringCaps.c_str())); + } if (manualpipeline) { @@ -1054,58 +1356,73 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam return false; } - GstFormat format; - - format = GST_FORMAT_DEFAULT; - if(!gst_element_query_duration(sink, format, &duration)) - { - handleMessage(pipeline); - CV_WARN("unable to query duration of stream"); - duration = -1; - } - - handleMessage(pipeline); - GSafePtr pad; pad.attach(gst_element_get_static_pad(sink, "sink")); GSafePtr buffer_caps; buffer_caps.attach(gst_pad_get_current_caps(pad)); - const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer - if (!gst_structure_get_int (structure, "width", &width) || - !gst_structure_get_int (structure, "height", &height)) + if (videoStream >= 0) { - CV_WARN("cannot query video width/height"); - } + GstFormat format; - gint num = 0, denom=1; - if (!gst_structure_get_fraction(structure, "framerate", &num, &denom)) - { - CV_WARN("cannot query video fps"); - } + format = GST_FORMAT_DEFAULT; + if(!gst_element_query_duration(sink, format, &duration)) + { + handleMessage(pipeline); + CV_WARN("unable to query duration of stream"); + duration = -1; + } - fps = (double)num/(double)denom; + handleMessage(pipeline); - { - GstFormat format_; - gint64 value_ = -1; - gboolean status_; + const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer + if (!gst_structure_get_int (structure, "width", &width) || + !gst_structure_get_int (structure, "height", &height)) + { + CV_WARN("cannot query video width/height"); + } - format_ = GST_FORMAT_DEFAULT; + gint num = 0, denom=1; + if (!gst_structure_get_fraction(structure, "framerate", &num, &denom)) + { + CV_WARN("cannot query video fps"); + } + + fps = (double)num/(double)denom; + + { + GstFormat format_; + gint64 value_ = -1; + gboolean status_; + + format_ = GST_FORMAT_DEFAULT; - status_ = gst_element_query_position(sink, CV_GST_FORMAT(format_), &value_); - if (!status_ || value_ != 0 || duration < 0) + status_ = gst_element_query_position(sink, CV_GST_FORMAT(format_), &value_); + if (!status_ || value_ != 0 || duration < 0) + { + CV_WARN("Cannot query video position: status=" << status_ << ", value=" << value_ << ", duration=" << duration); + isPosFramesSupported = false; + isPosFramesEmulated = true; + emulatedFrameNumber = 0; + } + else + isPosFramesSupported = true; + } + } + else if (audioStream >= 0) + { + GstAudioInfo info = {}; + if (gst_audio_info_from_caps(&info, buffer_caps)) { - CV_WARN("Cannot query video position: status=" << status_ << ", value=" << value_ << ", duration=" << duration); - isPosFramesSupported = false; - isPosFramesEmulated = true; - emulatedFrameNumber = 0; + nAudioChannels = GST_AUDIO_INFO_CHANNELS(&info); + audioSamplesPerSecond = GST_AUDIO_INFO_RATE(&info); } else - isPosFramesSupported = true; + { + CV_WARN("cannot query audio nChannels and SamplesPerSecond"); + } } - GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline"); } @@ -1232,6 +1549,14 @@ double GStreamerCapture::getProperty(int propId) const return 0; } return gst_app_sink_get_max_buffers(GST_APP_SINK(sink.get())); + case CAP_PROP_AUDIO_TOTAL_CHANNELS: + return nAudioChannels; + case CAP_PROP_AUDIO_SAMPLES_PER_SECOND: + return audioSamplesPerSecond; + case CAP_PROP_AUDIO_DATA_DEPTH: + return outputAudioFormat; + case CAP_PROP_AUDIO_BASE_INDEX: + return audioBaseIndex; default: CV_WARN("unhandled property: " << propId); break; diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp index 7c66b83e34..b1eb0ed4b7 100644 --- a/modules/videoio/test/test_audio.cpp +++ b/modules/videoio/test/test_audio.cpp @@ -92,7 +92,7 @@ public: { for (int nCh = 0; nCh < numberOfChannels; nCh++) { - ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex)); + ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex + nCh)); ASSERT_EQ(CV_16SC1, audioFrame.type()) << audioData[nCh].size(); for (int i = 0; i < audioFrame.cols; i++) { @@ -111,10 +111,14 @@ public: const param audioParams[] = { +#ifdef _WIN32 param("test_audio.wav", 1, 132300, 0.0001, cv::CAP_MSMF), param("test_mono_audio.mp3", 1, 133104, 0.12, cv::CAP_MSMF), param("test_stereo_audio.mp3", 2, 133104, 0.12, cv::CAP_MSMF), - param("test_audio.mp4", 1, 133104, 0.15, cv::CAP_MSMF) + param("test_audio.mp4", 1, 133104, 0.15, cv::CAP_MSMF), +#endif + param("test_audio.wav", 1, 132300, 0.0001, cv::CAP_GSTREAMER), + param("test_audio.mp4", 1, 132522, 0.15, cv::CAP_GSTREAMER), }; class Audio : public AudioTestFixture{}; @@ -249,46 +253,82 @@ protected: const double psnrThreshold; }; +class Media : public MediaTestFixture{}; + +TEST_P(Media, audio) +{ + if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend))) + throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found"); + + doTest(); +} + +#ifdef _WIN32 const paramCombination mediaParams[] = { +#ifdef _WIN32 paramCombination("test_audio.mp4", 1, 0.15, CV_8UC3, 240, 320, 90, 131819, 30, 30., cv::CAP_MSMF) #if 0 // https://filesamples.com/samples/video/mp4/sample_960x400_ocean_with_audio.mp4 , paramCombination("sample_960x400_ocean_with_audio.mp4", 2, -1/*eplsilon*/, CV_8UC3, 400, 960, 1116, 2056588, 30, 30., cv::CAP_MSMF) #endif +#endif // _WIN32 }; -class Media : public MediaTestFixture{}; +INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams)); +#endif // _WIN32 -TEST_P(Media, audio) +TEST(AudioOpenCheck, bad_arg_invalid_audio_stream) { - if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend))) - throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found"); - - doTest(); + std::string fileName = "audio/test_audio.wav"; + std::vector params { + CAP_PROP_AUDIO_STREAM, 1, + CAP_PROP_VIDEO_STREAM, -1, // disabled + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S + }; + VideoCapture cap; + cap.open(findDataFile(fileName), cv::CAP_ANY, params); + ASSERT_FALSE(cap.isOpened()); } -INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams)); +TEST(AudioOpenCheck, bad_arg_invalid_audio_stream_video) +{ + std::string fileName = "audio/test_audio.mp4"; + std::vector params { + CAP_PROP_AUDIO_STREAM, 1, + CAP_PROP_VIDEO_STREAM, 0, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S + }; + VideoCapture cap; + cap.open(findDataFile(fileName), cv::CAP_ANY, params); + ASSERT_FALSE(cap.isOpened()); +} -TEST(AudioOpenCheck, bad_arg_invalid_audio_stream) +#ifdef _WIN32 +TEST(AudioOpenCheck, MSMF_bad_arg_invalid_audio_sample_per_second) { std::string fileName = "audio/test_audio.mp4"; - std::vector params { CAP_PROP_AUDIO_STREAM, 1, - CAP_PROP_VIDEO_STREAM, 0, - CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + std::vector params { + CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, // disabled + CAP_PROP_AUDIO_SAMPLES_PER_SECOND, (int)1e9 + }; VideoCapture cap; cap.open(findDataFile(fileName), cv::CAP_MSMF, params); ASSERT_FALSE(cap.isOpened()); } +#endif TEST(AudioOpenCheck, bad_arg_invalid_audio_sample_per_second) { std::string fileName = "audio/test_audio.mp4"; - std::vector params { CAP_PROP_AUDIO_STREAM, 0, - CAP_PROP_VIDEO_STREAM, -1, - CAP_PROP_AUDIO_SAMPLES_PER_SECOND, (int)1e9 }; + std::vector params { + CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, // disabled + CAP_PROP_AUDIO_SAMPLES_PER_SECOND, -1000 + }; VideoCapture cap; - cap.open(findDataFile(fileName), cv::CAP_MSMF, params); + cap.open(findDataFile(fileName), cv::CAP_ANY, params); ASSERT_FALSE(cap.isOpened()); }