Chiebot-Mirror
/
opencv
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #19721 from MaximMilashchenko:Audio

Browse Source 
add audio support in cap_msmf

* audio msmf

* fixed warnings

* minor fix

* fixed SampleTime MSMF

* minor fix, fixed audio test, retrieveAudioFrame

* fixed warnings

* impelemented sync audio and video stream with start offset

* fixed error

* fixed docs

* fixed audio sample

* CAP_PROP_AUDIO_POS, minor fixed

* fixed warnings

* videoio(MSMF): update audio test checks, add debug logging

* fixed

* fixed desynchronization of time positions, warnings

* fixed warnings

* videoio(audio): tune tests checks

* videoio(audio): update properties description

* build warnings

Co-authored-by: Alexander Alekhin <alexander.a.alekhin@gmail.com>
pull/20921/head
MaximMilashchenko 3 years ago committed by GitHub
parent 824392a1c2
commit f36c268b9e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1360 additions and 136 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      modules/videoio/include/opencv2/videoio.hpp
  2. 956
      modules/videoio/src/cap_msmf.cpp
  3. 273
      modules/videoio/test/test_audio.cpp
  4. 41
      modules/videoio/test/test_microphone.cpp
  5. 59
      samples/cpp/videocapture_audio.cpp
  6. 69
      samples/cpp/videocapture_audio_combination.cpp
  7. 57
      samples/cpp/videocapture_microphone.cpp

11
modules/videoio/include/opencv2/videoio.hpp
Unescape View File

@ -189,6 +189,17 @@ enum VideoCaptureProperties {
CAP_PROP_OPEN_TIMEOUT_MSEC=53, //!< (**open-only**) timeout in milliseconds for opening a video capture (applicable for FFmpeg back-end only) CAP_PROP_OPEN_TIMEOUT_MSEC=53, //!< (**open-only**) timeout in milliseconds for opening a video capture (applicable for FFmpeg back-end only)
CAP_PROP_READ_TIMEOUT_MSEC=54, //!< (**open-only**) timeout in milliseconds for reading from a video capture (applicable for FFmpeg back-end only) CAP_PROP_READ_TIMEOUT_MSEC=54, //!< (**open-only**) timeout in milliseconds for reading from a video capture (applicable for FFmpeg back-end only)
CAP_PROP_STREAM_OPEN_TIME_USEC =55, //<! (read-only) time in microseconds since Jan 1 1970 when stream was opened. Applicable for FFmpeg backend only. Useful for RTSP and other live streams CAP_PROP_STREAM_OPEN_TIME_USEC =55, //<! (read-only) time in microseconds since Jan 1 1970 when stream was opened. Applicable for FFmpeg backend only. Useful for RTSP and other live streams
CAP_PROP_VIDEO_TOTAL_CHANNELS = 56, //!< (read-only) Number of video channels
CAP_PROP_VIDEO_STREAM = 57, //!< (**open-only**) Specify video stream, 0-based index. Use -1 to disable video stream from file or IP cameras. Default value is 0.
CAP_PROP_AUDIO_STREAM = 58, //!< (**open-only**) Specify stream in multi-language media files, -1 - disable audio processing or microphone. Default value is -1.
CAP_PROP_AUDIO_POS = 59, //!< (read-only) Audio position is measured in samples. Accurate audio sample timestamp of previous grabbed fragment. See CAP_PROP_AUDIO_SAMPLES_PER_SECOND and CAP_PROP_AUDIO_SHIFT_NSEC.
CAP_PROP_AUDIO_SHIFT_NSEC = 60, //!< (read only) Contains the time difference between the start of the audio stream and the video stream in nanoseconds. Positive value means that audio is started after the first video frame. Negative value means that audio is started before the first video frame.
CAP_PROP_AUDIO_DATA_DEPTH = 61, //!< (open, read) Alternative definition to bits-per-sample, but with clear handling of 32F / 32S
CAP_PROP_AUDIO_SAMPLES_PER_SECOND = 62, //!< (read-only) determined from file/codec input. If not specified, then selected audio sample rate is 44100
CAP_PROP_AUDIO_BASE_INDEX = 63, //!< (read-only) Index of the first audio channel for .retrieve() calls. That audio channel number continues enumeration after video channels.
CAP_PROP_AUDIO_TOTAL_CHANNELS = 64, //!< (read-only) Number of audio channels in the selected audio stream (mono, stereo, etc)
CAP_PROP_AUDIO_TOTAL_STREAMS = 65, //!< (read-only) Number of audio streams.
CAP_PROP_AUDIO_SYNCHRONIZE = 66, //!< (open, read) Enables audio synchronization.
#ifndef CV_DOXYGEN #ifndef CV_DOXYGEN
CV__CAP_PROP_LATEST CV__CAP_PROP_LATEST
#endif #endif

956
modules/videoio/src/cap_msmf.cpp
View File

File diff suppressed because it is too large Load Diff

273
modules/videoio/test/test_audio.cpp
Unescape View File

@ -0,0 +1,273 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "test_precomp.hpp"
namespace opencv_test { namespace {
//file name, number of audio channels, epsilon, video type, weight, height, number of frame, number of audio samples, fps, psnr Threshold, backend
typedef std::tuple<std::string, int, double, int, int, int, int, int, int, double, VideoCaptureAPIs> paramCombination;
//file name, number of audio channels, number of audio samples, epsilon, backend
typedef std::tuple<std::string, int, int, double, VideoCaptureAPIs> param;
class AudioBaseTest
{
protected:
AudioBaseTest(){};
void getValidAudioData()
{
const double step = 3.14/22050;
double value = 0;
validAudioData.resize(expectedNumAudioCh);
for (int nCh = 0; nCh < expectedNumAudioCh; nCh++)
{
value = 0;
for(unsigned int i = 0; i < numberOfSamples; i++)
{
if (i != 0 && i % 44100 == 0)
value = 0;
validAudioData[nCh].push_back(sin(value));
value += step;
}
}
}
void checkAudio()
{
getValidAudioData();
ASSERT_EQ(expectedNumAudioCh, (int)audioData.size());
for (unsigned int nCh = 0; nCh < audioData.size(); nCh++)
{
ASSERT_EQ(numberOfSamples, audioData[nCh].size()) << "nCh=" << nCh;
for (unsigned int i = 0; i < numberOfSamples; i++)
{
EXPECT_NEAR(validAudioData[nCh][i], audioData[nCh][i], epsilon) << "sample index=" << i << " nCh=" << nCh;
}
}
}
protected:
int expectedNumAudioCh;
unsigned int numberOfSamples;
double epsilon;
VideoCaptureAPIs backend;
std::string root;
std::string fileName;
std::vector<std::vector<double>> validAudioData;
std::vector<std::vector<double>> audioData;
std::vector<int> params;
Mat audioFrame;
VideoCapture cap;
};
class AudioTestFixture : public AudioBaseTest, public testing::TestWithParam <param>
{
public:
AudioTestFixture()
{
fileName = get<0>(GetParam());
expectedNumAudioCh = get<1>(GetParam());
numberOfSamples = get<2>(GetParam());
epsilon = get<3>(GetParam());
backend = get<4>(GetParam());
root = "audio/";
params = { CAP_PROP_AUDIO_STREAM, 0,
CAP_PROP_VIDEO_STREAM, -1,
CAP_PROP_AUDIO_DATA_DEPTH, CV_16S };
}
void doTest()
{
ASSERT_TRUE(cap.open(findDataFile(root + fileName), backend, params));
const int audioBaseIndex = static_cast<int>(cap.get(cv::CAP_PROP_AUDIO_BASE_INDEX));
const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS);
ASSERT_EQ(expectedNumAudioCh, numberOfChannels);
double f = 0;
audioData.resize(numberOfChannels);
for (;;)
{
if (cap.grab())
{
for (int nCh = 0; nCh < numberOfChannels; nCh++)
{
ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex));
ASSERT_EQ(CV_16SC1, audioFrame.type()) << audioData[nCh].size();
for (int i = 0; i < audioFrame.cols; i++)
{
f = ((double) audioFrame.at<signed short>(0,i)) / (double) 32768;
audioData[nCh].push_back(f);
}
}
}
else { break; }
}
ASSERT_FALSE(audioData.empty());
checkAudio();
}
};
const param audioParams[] =
{
param("test_audio.wav", 1, 132300, 0.0001, cv::CAP_MSMF),
param("test_mono_audio.mp3", 1, 133104, 0.12, cv::CAP_MSMF),
param("test_stereo_audio.mp3", 2, 133104, 0.12, cv::CAP_MSMF),
param("test_audio.mp4", 1, 133104, 0.15, cv::CAP_MSMF)
};
class Audio : public AudioTestFixture{};
TEST_P(Audio, audio)
{
if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend)))
throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found");
doTest();
}
INSTANTIATE_TEST_CASE_P(/**/, Audio, testing::ValuesIn(audioParams));
class MediaTestFixture : public AudioBaseTest, public testing::TestWithParam <paramCombination>
{
public:
MediaTestFixture():
videoType(get<3>(GetParam())),
height(get<4>(GetParam())),
width(get<5>(GetParam())),
numberOfFrames(get<6>(GetParam())),
fps(get<8>(GetParam())),
psnrThreshold(get<9>(GetParam()))
{
fileName = get<0>(GetParam());
expectedNumAudioCh = get<1>(GetParam());
numberOfSamples = get<7>(GetParam());
epsilon = get<2>(GetParam());
backend = get<10>(GetParam());
root = "audio/";
params = { CAP_PROP_AUDIO_STREAM, 0,
CAP_PROP_VIDEO_STREAM, 0,
CAP_PROP_AUDIO_DATA_DEPTH, CV_16S };
};
void doTest()
{
ASSERT_TRUE(cap.open(findDataFile(root + fileName), backend, params));
const int audioBaseIndex = static_cast<int>(cap.get(cv::CAP_PROP_AUDIO_BASE_INDEX));
const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS);
ASSERT_EQ(expectedNumAudioCh, numberOfChannels);
const int samplePerSecond = (int)cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND);
ASSERT_EQ(44100, samplePerSecond);
int samplesPerFrame = (int)(1./fps*samplePerSecond);
int audioSamplesTolerance = samplesPerFrame / 2;
double audio0_timestamp = 0;
Mat videoFrame;
Mat img(height, width, videoType);
audioData.resize(numberOfChannels);
for (int frame = 0; frame < numberOfFrames; frame++)
{
SCOPED_TRACE(cv::format("frame=%d", frame));
ASSERT_TRUE(cap.grab());
if (frame == 0)
{
double audio_shift = cap.get(CAP_PROP_AUDIO_SHIFT_NSEC);
double video0_timestamp = cap.get(CAP_PROP_POS_MSEC) * 1e-3;
audio0_timestamp = video0_timestamp + audio_shift * 1e-9;
std::cout << "video0 timestamp: " << video0_timestamp << " audio0 timestamp: " << audio0_timestamp << " (audio shift nanoseconds: " << audio_shift << " , seconds: " << audio_shift * 1e-9 << ")" << std::endl;
}
ASSERT_TRUE(cap.retrieve(videoFrame));
if (epsilon >= 0)
{
generateFrame(frame, numberOfFrames, img);
ASSERT_EQ(img.size, videoFrame.size);
double psnr = cvtest::PSNR(img, videoFrame);
EXPECT_GE(psnr, psnrThreshold);
}
int audioFrameCols = 0;
for (int nCh = 0; nCh < numberOfChannels; nCh++)
{
ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex+nCh));
if (audioFrame.empty())
continue;
ASSERT_EQ(CV_16SC1, audioFrame.type());
if (nCh == 0)
audioFrameCols = audioFrame.cols;
else
ASSERT_EQ(audioFrameCols, audioFrame.cols) << "channel "<< nCh;
for (int i = 0; i < audioFrame.cols; i++)
{
double f = audioFrame.at<signed short>(0,i) / 32768.0;
audioData[nCh].push_back(f);
}
}
if (frame < 5 || frame >= numberOfFrames-5)
std::cout << "frame=" << frame << ": audioFrameSize=" << audioFrameCols << " videoTimestamp=" << cap.get(CAP_PROP_POS_MSEC) << " ms" << std::endl;
else if (frame == 6)
std::cout << "frame..." << std::endl;
if (audioFrameCols == 0)
continue;
if (frame != 0 && frame != numberOfFrames-1)
{
// validate audio position
EXPECT_NEAR(
cap.get(CAP_PROP_AUDIO_POS) / samplePerSecond + audio0_timestamp,
cap.get(CAP_PROP_POS_MSEC) * 1e-3,
(1.0 / fps) * 0.3)
<< "CAP_PROP_AUDIO_POS=" << cap.get(CAP_PROP_AUDIO_POS) << " CAP_PROP_POS_MSEC=" << cap.get(CAP_PROP_POS_MSEC);
}
if (frame != 0 && frame != numberOfFrames-1 && audioData[0].size() != (size_t)numberOfSamples)
{
// validate audio frame size
EXPECT_NEAR(audioFrame.cols, samplesPerFrame, audioSamplesTolerance);
}
}
ASSERT_FALSE(cap.grab());
ASSERT_FALSE(audioData.empty());
std::cout << "Total audio samples=" << audioData[0].size() << std::endl;
if (epsilon >= 0)
checkAudio();
}
protected:
const int videoType;
const int height;
const int width;
const int numberOfFrames;
const int fps;
const double psnrThreshold;
};
const paramCombination mediaParams[] =
{
paramCombination("test_audio.mp4", 1, 0.15, CV_8UC3, 240, 320, 90, 131819, 30, 30., cv::CAP_MSMF)
#if 0
// https://filesamples.com/samples/video/mp4/sample_960x400_ocean_with_audio.mp4
, paramCombination("sample_960x400_ocean_with_audio.mp4", 2, -1/*eplsilon*/, CV_8UC3, 400, 960, 1116, 2056588, 30, 30., cv::CAP_MSMF)
#endif
};
class Media : public MediaTestFixture{};
TEST_P(Media, audio)
{
if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend)))
throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found");
doTest();
}
INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams));
}} //namespace

41
modules/videoio/test/test_microphone.cpp
Unescape View File

@ -0,0 +1,41 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Usage: opencv_test_videoio --gtest_also_run_disabled_tests
#include "test_precomp.hpp"
namespace opencv_test { namespace {
TEST(DISABLED_videoio_micro, basic)
{
int cursize = 0;
int validSize = 0;
Mat frame;
std::vector<int> params { CAP_PROP_AUDIO_STREAM, 0, CAP_PROP_VIDEO_STREAM, -1 };
VideoCapture cap(0, cv::CAP_MSMF, params);
ASSERT_TRUE(cap.isOpened());
int samplesPerSecond = (int)cap.get(cv::CAP_PROP_AUDIO_SAMPLES_PER_SECOND);
const int audio_base_index = (int)cap.get(cv::CAP_PROP_AUDIO_BASE_INDEX);
const double cvTickFreq = cv::getTickFrequency();
int64 sysTimePrev = cv::getTickCount();
int64 sysTimeCurr = cv::getTickCount();
cout << "Audio would be captured for the next 10 seconds" << endl;
while ((sysTimeCurr-sysTimePrev)/cvTickFreq < 10)
{
if (cap.grab())
{
ASSERT_TRUE(cap.retrieve(frame, audio_base_index));
sysTimeCurr = cv::getTickCount();
}
}
validSize = samplesPerSecond*(int)((sysTimeCurr-sysTimePrev)/cvTickFreq);
cursize = (int)cap.get(cv::CAP_PROP_AUDIO_POS);
ASSERT_LT(validSize - cursize, cursize*0.05);
}
}} // namespace

59
samples/cpp/videocapture_audio.cpp
Unescape View File

@ -0,0 +1,59 @@
#include <opencv2/core.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <iostream>
using namespace cv;
using namespace std;
int main(int argc, char** argv)
{
CommandLineParser parser(argc, argv, "{@audio||}");
string file = parser.get<string>("@audio");
if (file.empty())
{
return 1;
}
Mat frame;
vector<vector<Mat>> audioData;
VideoCapture cap;
vector<int> params { CAP_PROP_AUDIO_STREAM, 0,
CAP_PROP_VIDEO_STREAM, -1,
CAP_PROP_AUDIO_DATA_DEPTH, CV_16S };
cap.open(file, CAP_MSMF, params);
if (!cap.isOpened())
{
cerr << "ERROR! Can't to open file: " + file << endl;
return -1;
}
const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX);
const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS);
cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl;
cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl;
cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl;
cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl;
int numberOfSamples = 0;
audioData.resize(numberOfChannels);
for (;;)
{
if (cap.grab())
{
for (int nCh = 0; nCh < numberOfChannels; nCh++)
{
cap.retrieve(frame, audioBaseIndex+nCh);
audioData[nCh].push_back(frame);
numberOfSamples+=frame.cols;
}
}
else { break; }
}
cout << "Number of samples: " << numberOfSamples << endl;
return 0;
}

69
samples/cpp/videocapture_audio_combination.cpp
Unescape View File

@ -0,0 +1,69 @@
#include <opencv2/core.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <iostream>
using namespace cv;
using namespace std;
int main(int argc, char** argv)
{
cv::CommandLineParser parser(argc, argv, "{@audio||}");
string file = parser.get<string>("@audio");
if (file.empty())
{
return 1;
}
Mat videoFrame;
Mat audioFrame;
vector<vector<Mat>> audioData;
VideoCapture cap;
vector<int> params { CAP_PROP_AUDIO_STREAM, 0,
CAP_PROP_VIDEO_STREAM, 0,
CAP_PROP_AUDIO_DATA_DEPTH, CV_16S };
cap.open(file, CAP_MSMF, params);
if (!cap.isOpened())
{
cerr << "ERROR! Can't to open file: " + file << endl;
return -1;
}
const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX);
const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS);
cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl;
cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl;
cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS) << endl;
cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl;
int numberOfSamples = 0;
int numberOfFrames = 0;
audioData.resize(numberOfChannels);
for (;;)
{
if (cap.grab())
{
cap.retrieve(videoFrame);
for (int nCh = 0; nCh < numberOfChannels; nCh++)
{
cap.retrieve(audioFrame, audioBaseIndex+nCh);
if (!audioFrame.empty())
audioData[nCh].push_back(audioFrame);
numberOfSamples+=audioFrame.cols;
}
if (!videoFrame.empty())
{
numberOfFrames++;
imshow("Live", videoFrame);
if (waitKey(5) >= 0)
break;
}
} else { break; }
}
cout << "Number of audio samples: " << numberOfSamples << endl
<< "Number of video frames: " << numberOfFrames << endl;
return 0;
}

57
samples/cpp/videocapture_microphone.cpp
Unescape View File

@ -0,0 +1,57 @@
#include <opencv2/core.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <iostream>
using namespace cv;
using namespace std;
int main(int, char**)
{
Mat frame;
vector<Mat> audioData;
VideoCapture cap;
vector<int> params { CAP_PROP_AUDIO_STREAM, 0,
CAP_PROP_VIDEO_STREAM, -1 };
cap.open(0, CAP_MSMF, params);
if (!cap.isOpened())
{
cerr << "ERROR! Can't to open microphone" << endl;
return -1;
}
const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX);
const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS);
cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl;
cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl;
cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl;
cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl;
const double cvTickFreq = getTickFrequency();
int64 sysTimeCurr = getTickCount();
int64 sysTimePrev = sysTimeCurr;
while ((sysTimeCurr-sysTimePrev)/cvTickFreq < 10)
{
if (cap.grab())
{
for (int nCh = 0; nCh < numberOfChannels; nCh++)
{
cap.retrieve(frame, audioBaseIndex+nCh);
audioData.push_back(frame);
sysTimeCurr = getTickCount();
}
}
else
{
cerr << "Grab error" << endl;
break;
}
}
int numberOfSamles = 0;
for (auto item : audioData)
numberOfSamles+=item.cols;
cout << "Number of samples: " << numberOfSamles << endl;
return 0;
}
Write Preview
Loading…
Cancel
Save