Chiebot-Mirror
/
opencv
mirror of https://github.com/opencv/opencv.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #21264 from MaximMilashchenko:AudioGStreamer

Browse Source 
Audio GStreamer: added support .wav .flac audio formats

* added support .wav, lossless compressed audio formats

* fixed docs

* fixes

* videoio(gstreamer-audio): extra tests, improve error handling

Co-authored-by: Alexander Alekhin <alexander.a.alekhin@gmail.com>
pull/21281/head
Maxim Milashchenko 3 years ago committed by GitHub
parent 622b9d9276
commit db4ab1c936
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 479 additions and 114 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 519
      modules/videoio/src/cap_gstreamer.cpp
  2. 74
      modules/videoio/test/test_audio.cpp

519
modules/videoio/src/cap_gstreamer.cpp
Unescape View File

@ -59,6 +59,7 @@
#include <gst/gst.h>
#include <gst/gstbuffer.h>
#include <gst/video/video.h>
#include <gst/audio/audio.h>
#include <gst/app/gstappsink.h>
#include <gst/app/gstappsrc.h>
#include <gst/riff/riff-media.h>
@ -308,6 +309,8 @@ private:
GSafePtr<GstSample> sample;
GSafePtr<GstCaps> caps;
gint videoStream;
gint audioStream;
gint64 duration;
gint width;
gint height;
@ -315,6 +318,12 @@ private:
bool isPosFramesSupported;
bool isPosFramesEmulated;
gint64 emulatedFrameNumber;
gint outputAudioFormat;
gint audioBaseIndex;
gint nAudioChannels;
gint audioSamplesPerSecond;
Mat audioFrame;
VideoAccelerationType va_type;
int hw_device;
@ -323,6 +332,9 @@ public:
virtual ~GStreamerCapture() CV_OVERRIDE;
virtual bool grabFrame() CV_OVERRIDE;
virtual bool retrieveFrame(int /*unused*/, OutputArray dst) CV_OVERRIDE;
bool grabAudioFrame();
bool retrieveVideoFrame(int /*unused*/, OutputArray dst);
bool retrieveAudioFrame(int /*unused*/, OutputArray dst);
virtual double getProperty(int propId) const CV_OVERRIDE;
virtual bool setProperty(int propId, double value) CV_OVERRIDE;
virtual bool isOpened() const CV_OVERRIDE { return (bool)pipeline; }
@ -330,6 +342,9 @@ public:
bool open(int id, const cv::VideoCaptureParameters& params);
bool open(const String &filename_, const cv::VideoCaptureParameters& params);
static void newPad(GstElement * /*elem*/, GstPad *pad, gpointer data);
bool configureHW(const cv::VideoCaptureParameters&);
bool configureStreams(const cv::VideoCaptureParameters&);
bool setAudioProperties(const cv::VideoCaptureParameters&);
protected:
bool isPipelinePlaying();
@ -341,10 +356,16 @@ protected:
};
GStreamerCapture::GStreamerCapture() :
videoStream(0),
audioStream(-1),
duration(-1), width(-1), height(-1), fps(-1),
isPosFramesSupported(false),
isPosFramesEmulated(false),
emulatedFrameNumber(-1)
emulatedFrameNumber(-1),
outputAudioFormat(CV_16S),
audioBaseIndex(1),
nAudioChannels(0),
audioSamplesPerSecond(44100)
, va_type(VIDEO_ACCELERATION_NONE)
, hw_device(-1)
{
@ -365,6 +386,92 @@ GStreamerCapture::~GStreamerCapture()
}
}
bool GStreamerCapture::configureHW(const cv::VideoCaptureParameters& params)
{
if (params.has(CAP_PROP_HW_ACCELERATION))
{
va_type = params.get<VideoAccelerationType>(CAP_PROP_HW_ACCELERATION);
}
if (params.has(CAP_PROP_HW_DEVICE))
{
hw_device = params.get<int>(CAP_PROP_HW_DEVICE);
if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1)
{
CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE without requested H/W acceleration. Bailout");
return false;
}
if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1)
{
CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout");
return false;
}
if (hw_device != -1)
{
CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: CAP_PROP_HW_DEVICE is not supported. Specify -1 (auto) value. Bailout");
return false;
}
}
return true;
}
bool GStreamerCapture::configureStreams(const cv::VideoCaptureParameters& params)
{
if (params.has(CAP_PROP_VIDEO_STREAM))
{
double value = params.get<double>(CAP_PROP_VIDEO_STREAM);
if (value == -1 || value == 0)
videoStream = static_cast<gint>(value);
else
{
CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_VIDEO_STREAM parameter value is invalid/unsupported: " << value);
return false;
}
}
if (params.has(CAP_PROP_AUDIO_STREAM))
{
double value = params.get<double>(CAP_PROP_AUDIO_STREAM);
if (value == -1 || value > -1)
audioStream = static_cast<gint>(value);
else
{
CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_STREAM parameter value is invalid/unsupported: " << value);
return false;
}
}
return true;
}
bool GStreamerCapture::setAudioProperties(const cv::VideoCaptureParameters& params)
{
if (params.has(CAP_PROP_AUDIO_DATA_DEPTH))
{
gint value = static_cast<gint>(params.get<double>(CAP_PROP_AUDIO_DATA_DEPTH));
if (value != CV_8S && value != CV_16S && value != CV_32S && value != CV_32F)
{
CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_DATA_DEPTH parameter value is invalid/unsupported: " << value);
return false;
}
else
{
outputAudioFormat = value;
}
}
if (params.has(CAP_PROP_AUDIO_SAMPLES_PER_SECOND))
{
int value = static_cast<int>(params.get<double>(CAP_PROP_AUDIO_SAMPLES_PER_SECOND));
if (value < 0)
{
CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_SAMPLES_PER_SECOND parameter can't be negative: " << value);
return false;
}
else
{
audioSamplesPerSecond = value;
}
}
return true;
}
/*!
* \brief CvCapture_GStreamer::grabFrame
* \return
@ -391,21 +498,137 @@ bool GStreamerCapture::grabFrame()
if (isPosFramesEmulated)
emulatedFrameNumber++;
if (audioStream >= 0)
return grabAudioFrame();
return true;
}
/*!
* \brief CvCapture_GStreamer::retrieveFrame
* \return IplImage pointer. [Transfer Full]
* Retrieve the previously grabbed buffer, and wrap it in an IPLImage structure
*/
bool GStreamerCapture::retrieveFrame(int, OutputArray dst)
bool GStreamerCapture::grabAudioFrame()
{
if (!sample)
GstCaps* frame_caps = gst_sample_get_caps(sample); // no lifetime transfer
if (!frame_caps)
{
CV_LOG_ERROR(NULL, "GStreamer: gst_sample_get_caps() returns NULL");
return false;
}
if (!GST_CAPS_IS_SIMPLE(frame_caps))
{
// bail out in no caps
CV_LOG_ERROR(NULL, "GStreamer: GST_CAPS_IS_SIMPLE(frame_caps) check is failed");
return false;
}
GstAudioInfo info = {};
gboolean audio_info_res = gst_audio_info_from_caps(&info, frame_caps);
if (!audio_info_res)
{
CV_Error(Error::StsError, "GStreamer: gst_audio_info_from_caps() is failed. Can't handle unknown layout");
}
int bpf = GST_AUDIO_INFO_BPF(&info);
GstStructure* structure = gst_caps_get_structure(frame_caps, 0); // no lifetime transfer
if (!structure)
{
CV_LOG_ERROR(NULL, "GStreamer: Can't query 'structure'-0 from GStreamer sample");
return false;
}
const gchar* name_ = gst_structure_get_name(structure);
if (!name_)
{
CV_LOG_ERROR(NULL, "GStreamer: Can't query 'name' from GStreamer sample");
return false;
}
std::string name = toLowerCase(std::string(name_));
GstBuffer* buf = gst_sample_get_buffer(sample);
if (!buf)
return false;
GstMapInfo map_info = {};
if (!gst_buffer_map(buf, &map_info, GST_MAP_READ))
{
CV_LOG_ERROR(NULL, "GStreamer: Failed to map GStreamer buffer to system memory");
return false;
}
ScopeGuardGstMapInfo map_guard(buf, &map_info);
if (name == "audio/x-raw")
{
const gchar* format_ = gst_structure_get_string(structure, "format");
if (!format_)
{
CV_LOG_ERROR(NULL, "GStreamer: Can't query 'format' of 'video/x-raw'");
return false;
}
std::string format = toUpperCase(std::string(format_));
cv::Mat data;
if (format == "S8")
{
Mat(map_info.size/bpf, nAudioChannels, CV_8S, map_info.data).copyTo(audioFrame);
return true;
}
if (format == "S16LE")
{
Mat(map_info.size/bpf, nAudioChannels, CV_16S, map_info.data).copyTo(audioFrame);
return true;
}
if (format == "S32LE")
{
Mat(map_info.size/bpf, nAudioChannels, CV_32S, map_info.data).copyTo(audioFrame);
return true;
}
if (format == "F32LE")
{
Mat(map_info.size/bpf, nAudioChannels, CV_32F, map_info.data).copyTo(audioFrame);
return true;
}
CV_Error_(Error::StsNotImplemented, ("Unsupported GStreamer audio format: %s", format.c_str()));
}
CV_Error_(Error::StsNotImplemented, ("Unsupported GStreamer layer type: %s", name.c_str()));
}
bool GStreamerCapture::retrieveAudioFrame(int index, OutputArray dst)
{
CV_Check(index, index >= audioBaseIndex && index < audioBaseIndex + nAudioChannels, "");
index -= audioBaseIndex;
CV_CheckType(outputAudioFormat,
outputAudioFormat == CV_8S ||
outputAudioFormat == CV_16S ||
outputAudioFormat == CV_32S ||
outputAudioFormat == CV_32F,
"");
dst.create(1, audioFrame.rows, outputAudioFormat);
Mat data = dst.getMat();
switch (outputAudioFormat)
{
case CV_8S:
for (int i = 0; i < audioFrame.rows; i++)
data.at<char>(i) = audioFrame.at<char>(i, index);
return true;
case CV_16S:
for (int i = 0; i < audioFrame.rows; i++)
data.at<short>(i) = audioFrame.at<short>(i, index);
return true;
case CV_32S:
for (int i = 0; i < audioFrame.rows; i++)
data.at<int>(i) = audioFrame.at<int>(i, index);
return true;
case CV_32F:
for (int i = 0; i < audioFrame.rows; i++)
data.at<float>(i) = audioFrame.at<float>(i, index);
return true;
}
dst.release();
return false;
}
bool GStreamerCapture::retrieveVideoFrame(int, OutputArray dst)
{
GstCaps* frame_caps = gst_sample_get_caps(sample); // no lifetime transfer
if (!frame_caps)
{
@ -609,6 +832,28 @@ bool GStreamerCapture::retrieveFrame(int, OutputArray dst)
CV_Error_(Error::StsNotImplemented, ("Unsupported GStreamer layer type: %s", name.c_str()));
}
bool GStreamerCapture::retrieveFrame(int index, OutputArray dst)
{
if (index < 0)
return false;
if (!sample)
return false;
if (index == 0)
{
CV_CheckGE(videoStream, 0, "No video stream configured");
return retrieveVideoFrame(index, dst);
}
else if (index >= audioBaseIndex)
{
CV_CheckGE(audioStream, 0, "No audio stream configured");
return retrieveAudioFrame(index, dst);
}
CV_LOG_ERROR(NULL, "GStreamer(retrive): unrecognized index=" << index);
return false;
}
bool GStreamerCapture::isPipelinePlaying()
{
if (!pipeline || !GST_IS_ELEMENT(pipeline.get()))
@ -810,28 +1055,33 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
{
gst_initializer::init();
if (params.has(CAP_PROP_HW_ACCELERATION))
if (!configureHW(params))
{
va_type = params.get<VideoAccelerationType>(CAP_PROP_HW_ACCELERATION);
CV_LOG_WARNING(NULL, "GStreamer: can't configure HW encoding/decoding support");
return false;
}
if (params.has(CAP_PROP_HW_DEVICE))
if (!configureStreams(params))
{
hw_device = params.get<int>(CAP_PROP_HW_DEVICE);
if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1)
{
CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE without requested H/W acceleration. Bailout");
return false;
}
if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1)
{
CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout");
return false;
}
if (hw_device != -1)
{
CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: CAP_PROP_HW_DEVICE is not supported. Specify -1 (auto) value. Bailout");
return false;
}
CV_LOG_WARNING(NULL, "GStreamer: can't configure streams");
return false;
}
if ((videoStream >= 0 && audioStream >= 0) || (videoStream < 0 && audioStream < 0))
{
CV_LOG_ERROR(NULL, "GStreamer backend supports audio-only or video-only capturing. Only one of the properties CAP_PROP_AUDIO_STREAM=" << audioStream << " and CAP_PROP_VIDEO_STREAM=" << videoStream << " should be >= 0");
return false;
}
if (audioStream > 0)
{
CV_LOG_ERROR(NULL, "GStreamer backend supports the first audio stream only. CAP_PROP_AUDIO_STREAM=" << audioStream);
return false;
}
if (!setAudioProperties(params))
{
CV_LOG_WARNING(NULL, "GStreamer: can't configure audio properties");
return false;
}
const gchar* filename = filename_.c_str();
@ -843,6 +1093,9 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
GSafePtr<GstElement> color;
GstStateChangeReturn status;
GSafePtr<GstElement> convert;
GSafePtr<GstElement> resample;
// test if we have a valid uri. If so, open it with an uridecodebin
// else, we might have a file or a manual pipeline.
// if gstreamer cannot parse the manual pipeline, we assume we were given and
@ -884,20 +1137,29 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
bool element_from_uri = false;
if (!uridecodebin)
{
// At this writing, the v4l2 element (and maybe others too) does not support caps renegotiation.
// This means that we cannot use an uridecodebin when dealing with v4l2, since setting
// capture properties will not work.
// The solution (probably only until gstreamer 1.2) is to make an element from uri when dealing with v4l2.
GSafePtr<gchar> protocol_; protocol_.attach(gst_uri_get_protocol(uri));
CV_Assert(protocol_);
std::string protocol = toLowerCase(std::string(protocol_.get()));
if (protocol == "v4l2")
if (videoStream >= 0)
{
uridecodebin.reset(gst_element_make_from_uri(GST_URI_SRC, uri.get(), "src", NULL));
CV_Assert(uridecodebin);
element_from_uri = true;
// At this writing, the v4l2 element (and maybe others too) does not support caps renegotiation.
// This means that we cannot use an uridecodebin when dealing with v4l2, since setting
// capture properties will not work.
// The solution (probably only until gstreamer 1.2) is to make an element from uri when dealing with v4l2.
GSafePtr<gchar> protocol_; protocol_.attach(gst_uri_get_protocol(uri));
CV_Assert(protocol_);
std::string protocol = toLowerCase(std::string(protocol_.get()));
if (protocol == "v4l2")
{
uridecodebin.reset(gst_element_make_from_uri(GST_URI_SRC, uri.get(), "src", NULL));
CV_Assert(uridecodebin);
element_from_uri = true;
}
else
{
uridecodebin.reset(gst_element_factory_make("uridecodebin", NULL));
CV_Assert(uridecodebin);
g_object_set(G_OBJECT(uridecodebin.get()), "uri", uri.get(), NULL);
}
}
else
else if (audioStream >= 0)
{
uridecodebin.reset(gst_element_factory_make("uridecodebin", NULL));
CV_Assert(uridecodebin);
@ -971,35 +1233,51 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
pipeline.reset(gst_pipeline_new(NULL));
CV_Assert(pipeline);
// videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert)
//automatically selects the correct colorspace conversion based on caps.
color.reset(gst_element_factory_make(COLOR_ELEM, NULL));
CV_Assert(color);
sink.reset(gst_element_factory_make("appsink", NULL));
CV_Assert(sink);
if (videoStream >= 0)
{
// videoconvert (in 0.10: ffmpegcolorspace, in 1.x autovideoconvert)
//automatically selects the correct colorspace conversion based on caps.
color.reset(gst_element_factory_make(COLOR_ELEM, NULL));
CV_Assert(color);
gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL);
gst_bin_add_many(GST_BIN(pipeline.get()), uridecodebin.get(), color.get(), sink.get(), NULL);
if (element_from_uri)
{
if(!gst_element_link(uridecodebin, color.get()))
if (element_from_uri)
{
if(!gst_element_link(uridecodebin, color.get()))
{
CV_WARN("GStreamer(video): cannot link color -> sink");
pipeline.release();
return false;
}
}
else
{
g_signal_connect(uridecodebin, "pad-added", G_CALLBACK(newPad), color.get());
}
if (!gst_element_link(color.get(), sink.get()))
{
CV_WARN("cannot link color -> sink");
CV_WARN("GStreamer(video): cannot link color -> sink");
pipeline.release();
return false;
}
}
else
else if (audioStream >= 0)
{
g_signal_connect(uridecodebin, "pad-added", G_CALLBACK(newPad), color.get());
}
convert.reset(gst_element_factory_make("audioconvert", NULL));
resample.reset(gst_element_factory_make("audioresample", NULL));
if (!gst_element_link(color.get(), sink.get()))
{
CV_WARN("GStreamer: cannot link color -> sink");
pipeline.release();
return false;
gst_bin_add_many (GST_BIN (pipeline.get()), uridecodebin.get(), convert.get(), resample.get(), sink.get(), NULL);
if (!gst_element_link_many (convert.get(), resample.get(), sink.get(), NULL))
{
CV_WARN("GStreamer(audio): cannot link convert -> resample -> sink");
pipeline.release();
return false;
}
g_signal_connect (uridecodebin, "pad-added", G_CALLBACK (newPad), convert.get());
}
}
@ -1008,17 +1286,41 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
//TODO: is 1 single buffer really high enough?
gst_app_sink_set_max_buffers(GST_APP_SINK(sink.get()), 1);
}
if (!manualpipeline)
{
gst_base_sink_set_sync(GST_BASE_SINK(sink.get()), FALSE);
}
//do not emit signals: all calls will be synchronous and blocking
gst_app_sink_set_emit_signals (GST_APP_SINK(sink.get()), FALSE);
caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR, GRAY8}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; image/jpeg"));
if (videoStream >= 0)
{
caps.attach(gst_caps_from_string("video/x-raw, format=(string){BGR, GRAY8}; video/x-bayer,format=(string){rggb,bggr,grbg,gbrg}; image/jpeg"));
}
else if (audioStream >= 0)
{
std::string audioFormat;
switch (outputAudioFormat)
{
case CV_8S:
audioFormat = "S8";
break;
case CV_16S:
audioFormat = "S16LE";
break;
case CV_32S:
audioFormat = "S32LE";
break;
case CV_32F:
audioFormat = "F32LE";
break;
default:
audioFormat = "S16LE";
break;
}
std::string stringCaps = "audio/x-raw, format=(string)" + audioFormat + ", rate=(int)" + std::to_string(audioSamplesPerSecond) + ", channels=(int){1, 2}, layout=(string)interleaved";
caps.attach(gst_caps_from_string(stringCaps.c_str()));
}
if (manualpipeline)
{
@ -1054,58 +1356,73 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam
return false;
}
GstFormat format;
format = GST_FORMAT_DEFAULT;
if(!gst_element_query_duration(sink, format, &duration))
{
handleMessage(pipeline);
CV_WARN("unable to query duration of stream");
duration = -1;
}
handleMessage(pipeline);
GSafePtr<GstPad> pad;
pad.attach(gst_element_get_static_pad(sink, "sink"));
GSafePtr<GstCaps> buffer_caps;
buffer_caps.attach(gst_pad_get_current_caps(pad));
const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer
if (!gst_structure_get_int (structure, "width", &width) ||
!gst_structure_get_int (structure, "height", &height))
if (videoStream >= 0)
{
CV_WARN("cannot query video width/height");
}
GstFormat format;
gint num = 0, denom=1;
if (!gst_structure_get_fraction(structure, "framerate", &num, &denom))
{
CV_WARN("cannot query video fps");
}
format = GST_FORMAT_DEFAULT;
if(!gst_element_query_duration(sink, format, &duration))
{
handleMessage(pipeline);
CV_WARN("unable to query duration of stream");
duration = -1;
}
fps = (double)num/(double)denom;
handleMessage(pipeline);
{
GstFormat format_;
gint64 value_ = -1;
gboolean status_;
const GstStructure *structure = gst_caps_get_structure(buffer_caps, 0); // no lifetime transfer
if (!gst_structure_get_int (structure, "width", &width) ||
!gst_structure_get_int (structure, "height", &height))
{
CV_WARN("cannot query video width/height");
}
format_ = GST_FORMAT_DEFAULT;
gint num = 0, denom=1;
if (!gst_structure_get_fraction(structure, "framerate", &num, &denom))
{
CV_WARN("cannot query video fps");
}
fps = (double)num/(double)denom;
{
GstFormat format_;
gint64 value_ = -1;
gboolean status_;
format_ = GST_FORMAT_DEFAULT;
status_ = gst_element_query_position(sink, CV_GST_FORMAT(format_), &value_);
if (!status_ || value_ != 0 || duration < 0)
status_ = gst_element_query_position(sink, CV_GST_FORMAT(format_), &value_);
if (!status_ || value_ != 0 || duration < 0)
{
CV_WARN("Cannot query video position: status=" << status_ << ", value=" << value_ << ", duration=" << duration);
isPosFramesSupported = false;
isPosFramesEmulated = true;
emulatedFrameNumber = 0;
}
else
isPosFramesSupported = true;
}
}
else if (audioStream >= 0)
{
GstAudioInfo info = {};
if (gst_audio_info_from_caps(&info, buffer_caps))
{
CV_WARN("Cannot query video position: status=" << status_ << ", value=" << value_ << ", duration=" << duration);
isPosFramesSupported = false;
isPosFramesEmulated = true;
emulatedFrameNumber = 0;
nAudioChannels = GST_AUDIO_INFO_CHANNELS(&info);
audioSamplesPerSecond = GST_AUDIO_INFO_RATE(&info);
}
else
isPosFramesSupported = true;
{
CV_WARN("cannot query audio nChannels and SamplesPerSecond");
}
}
GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline");
}
@ -1232,6 +1549,14 @@ double GStreamerCapture::getProperty(int propId) const
return 0;
}
return gst_app_sink_get_max_buffers(GST_APP_SINK(sink.get()));
case CAP_PROP_AUDIO_TOTAL_CHANNELS:
return nAudioChannels;
case CAP_PROP_AUDIO_SAMPLES_PER_SECOND:
return audioSamplesPerSecond;
case CAP_PROP_AUDIO_DATA_DEPTH:
return outputAudioFormat;
case CAP_PROP_AUDIO_BASE_INDEX:
return audioBaseIndex;
default:
CV_WARN("unhandled property: " << propId);
break;

74
modules/videoio/test/test_audio.cpp
Unescape View File

@ -92,7 +92,7 @@ public:
{
for (int nCh = 0; nCh < numberOfChannels; nCh++)
{
ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex));
ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex + nCh));
ASSERT_EQ(CV_16SC1, audioFrame.type()) << audioData[nCh].size();
for (int i = 0; i < audioFrame.cols; i++)
{
@ -111,10 +111,14 @@ public:
const param audioParams[] =
{
#ifdef _WIN32
param("test_audio.wav", 1, 132300, 0.0001, cv::CAP_MSMF),
param("test_mono_audio.mp3", 1, 133104, 0.12, cv::CAP_MSMF),
param("test_stereo_audio.mp3", 2, 133104, 0.12, cv::CAP_MSMF),
param("test_audio.mp4", 1, 133104, 0.15, cv::CAP_MSMF)
param("test_audio.mp4", 1, 133104, 0.15, cv::CAP_MSMF),
#endif
param("test_audio.wav", 1, 132300, 0.0001, cv::CAP_GSTREAMER),
param("test_audio.mp4", 1, 132522, 0.15, cv::CAP_GSTREAMER),
};
class Audio : public AudioTestFixture{};
@ -249,46 +253,82 @@ protected:
const double psnrThreshold;
};
class Media : public MediaTestFixture{};
TEST_P(Media, audio)
{
if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend)))
throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found");
doTest();
}
#ifdef _WIN32
const paramCombination mediaParams[] =
{
#ifdef _WIN32
paramCombination("test_audio.mp4", 1, 0.15, CV_8UC3, 240, 320, 90, 131819, 30, 30., cv::CAP_MSMF)
#if 0
// https://filesamples.com/samples/video/mp4/sample_960x400_ocean_with_audio.mp4
, paramCombination("sample_960x400_ocean_with_audio.mp4", 2, -1/*eplsilon*/, CV_8UC3, 400, 960, 1116, 2056588, 30, 30., cv::CAP_MSMF)
#endif
#endif // _WIN32
};
class Media : public MediaTestFixture{};
INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams));
#endif // _WIN32
TEST_P(Media, audio)
TEST(AudioOpenCheck, bad_arg_invalid_audio_stream)
{
if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend)))
throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found");
doTest();
std::string fileName = "audio/test_audio.wav";
std::vector<int> params {
CAP_PROP_AUDIO_STREAM, 1,
CAP_PROP_VIDEO_STREAM, -1, // disabled
CAP_PROP_AUDIO_DATA_DEPTH, CV_16S
};
VideoCapture cap;
cap.open(findDataFile(fileName), cv::CAP_ANY, params);
ASSERT_FALSE(cap.isOpened());
}
INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams));
TEST(AudioOpenCheck, bad_arg_invalid_audio_stream_video)
{
std::string fileName = "audio/test_audio.mp4";
std::vector<int> params {
CAP_PROP_AUDIO_STREAM, 1,
CAP_PROP_VIDEO_STREAM, 0,
CAP_PROP_AUDIO_DATA_DEPTH, CV_16S
};
VideoCapture cap;
cap.open(findDataFile(fileName), cv::CAP_ANY, params);
ASSERT_FALSE(cap.isOpened());
}
TEST(AudioOpenCheck, bad_arg_invalid_audio_stream)
#ifdef _WIN32
TEST(AudioOpenCheck, MSMF_bad_arg_invalid_audio_sample_per_second)
{
std::string fileName = "audio/test_audio.mp4";
std::vector<int> params { CAP_PROP_AUDIO_STREAM, 1,
CAP_PROP_VIDEO_STREAM, 0,
CAP_PROP_AUDIO_DATA_DEPTH, CV_16S };
std::vector<int> params {
CAP_PROP_AUDIO_STREAM, 0,
CAP_PROP_VIDEO_STREAM, -1, // disabled
CAP_PROP_AUDIO_SAMPLES_PER_SECOND, (int)1e9
};
VideoCapture cap;
cap.open(findDataFile(fileName), cv::CAP_MSMF, params);
ASSERT_FALSE(cap.isOpened());
}
#endif
TEST(AudioOpenCheck, bad_arg_invalid_audio_sample_per_second)
{
std::string fileName = "audio/test_audio.mp4";
std::vector<int> params { CAP_PROP_AUDIO_STREAM, 0,
CAP_PROP_VIDEO_STREAM, -1,
CAP_PROP_AUDIO_SAMPLES_PER_SECOND, (int)1e9 };
std::vector<int> params {
CAP_PROP_AUDIO_STREAM, 0,
CAP_PROP_VIDEO_STREAM, -1, // disabled
CAP_PROP_AUDIO_SAMPLES_PER_SECOND, -1000
};
VideoCapture cap;
cap.open(findDataFile(fileName), cv::CAP_MSMF, params);
cap.open(findDataFile(fileName), cv::CAP_ANY, params);
ASSERT_FALSE(cap.isOpened());
}

Write Preview
Loading…
Cancel
Save