From 0d1a0b126bf8b401e8e61c5616168fdf2ac7291d Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Wed, 20 Mar 2019 03:28:45 +0900 Subject: [PATCH] Merge pull request #14005 from komakai:android-video-cap * Add Android Media NDK video i/o file capture back-end * Fix failing test * Improve error handling/prevent resource leaks * Add license text * Modify default for WITH_ANDROID_MEDIANDK option * Fix spelling of deleter_AMediaExtractor --- CMakeLists.txt | 3 + modules/videoio/CMakeLists.txt | 6 + .../cmake/detect_android_mediandk.cmake | 8 + modules/videoio/cmake/init.cmake | 2 + modules/videoio/src/cap_android_mediandk.cpp | 248 ++++++++++++++++++ modules/videoio/src/cap_interface.hpp | 2 + modules/videoio/src/videoio_registry.cpp | 6 +- .../android/ndk-18-api-level-21.config.py | 6 + 8 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 modules/videoio/cmake/detect_android_mediandk.cmake create mode 100644 modules/videoio/src/cap_android_mediandk.cpp create mode 100644 platforms/android/ndk-18-api-level-21.config.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cbcdb938a..d7fbb1027b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -410,6 +410,9 @@ OCV_OPTION(WITH_IMGCODEC_PFM "Include PFM formats support" ON OCV_OPTION(WITH_QUIRC "Include library QR-code decoding" ON VISIBLE_IF TRUE VERIFY HAVE_QUIRC) +OCV_OPTION(WITH_ANDROID_MEDIANDK "Use Android Media NDK for Video I/O (Android)" (ANDROID_NATIVE_API_LEVEL GREATER 20) + VISIBLE_IF ANDROID + VERIFY HAVE_ANDROID_MEDIANDK) # OpenCV build components # =================================================== diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index c9034eb50f..38d59c225c 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -173,6 +173,12 @@ if(TARGET ocv.3rdparty.cap_ios) list(APPEND tgts ocv.3rdparty.cap_ios) endif() +if(TARGET ocv.3rdparty.android_mediandk) + list(APPEND videoio_srcs + ${CMAKE_CURRENT_LIST_DIR}/src/cap_android_mediandk.cpp) + list(APPEND tgts ocv.3rdparty.android_mediandk) +endif() + ocv_set_module_sources(HEADERS ${videoio_ext_hdrs} ${videoio_hdrs} SOURCES ${videoio_srcs}) ocv_module_include_directories() ocv_create_module() diff --git a/modules/videoio/cmake/detect_android_mediandk.cmake b/modules/videoio/cmake/detect_android_mediandk.cmake new file mode 100644 index 0000000000..edfb4bbbc5 --- /dev/null +++ b/modules/videoio/cmake/detect_android_mediandk.cmake @@ -0,0 +1,8 @@ +# if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER_EQUAL 21) <-- would be nicer but requires CMake 3.7 or later +if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER 20) + set(HAVE_ANDROID_MEDIANDK TRUE) + set(libs "-landroid -llog -lmediandk") + ocv_add_external_target(android_mediandk "" "${libs}" "HAVE_ANDROID_MEDIANDK") +endif() + +set(HAVE_ANDROID_MEDIANDK ${HAVE_ANDROID_MEDIANDK} PARENT_SCOPE) diff --git a/modules/videoio/cmake/init.cmake b/modules/videoio/cmake/init.cmake index 56ff4560c4..bbf57bc3b0 100644 --- a/modules/videoio/cmake/init.cmake +++ b/modules/videoio/cmake/init.cmake @@ -40,3 +40,5 @@ add_backend("ios" WITH_CAP_IOS) add_backend("dshow" WITH_DSHOW) add_backend("msmf" WITH_MSMF) + +add_backend("android_mediandk" WITH_ANDROID_MEDIANDK) diff --git a/modules/videoio/src/cap_android_mediandk.cpp b/modules/videoio/src/cap_android_mediandk.cpp new file mode 100644 index 0000000000..822024d53f --- /dev/null +++ b/modules/videoio/src/cap_android_mediandk.cpp @@ -0,0 +1,248 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "precomp.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "media/NdkMediaCodec.h" +#include "media/NdkMediaExtractor.h" + +#define INPUT_TIMEOUT_MS 2000 + +#define COLOR_FormatYUV420Planar 19 +#define COLOR_FormatYUV420SemiPlanar 21 + +using namespace cv; + +#define TAG "NativeCodec" +#define LOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) + + +static inline void deleter_AMediaExtractor(AMediaExtractor *extractor) { + AMediaExtractor_delete(extractor); +} + +static inline void deleter_AMediaCodec(AMediaCodec *codec) { + AMediaCodec_stop(codec); + AMediaCodec_delete(codec); +} + +static inline void deleter_AMediaFormat(AMediaFormat *format) { + AMediaFormat_delete(format); +} + +class AndroidMediaNdkCapture : public IVideoCapture +{ + +public: + AndroidMediaNdkCapture(): + sawInputEOS(false), sawOutputEOS(false), + frameWidth(0), frameHeight(0), colorFormat(0) {} + std::shared_ptr mediaExtractor; + std::shared_ptr mediaCodec; + bool sawInputEOS; + bool sawOutputEOS; + int32_t frameWidth; + int32_t frameHeight; + int32_t colorFormat; + std::vector buffer; + + ~AndroidMediaNdkCapture() { cleanUp(); } + + bool decodeFrame() { + while (!sawInputEOS || !sawOutputEOS) { + if (!sawInputEOS) { + auto bufferIndex = AMediaCodec_dequeueInputBuffer(mediaCodec.get(), INPUT_TIMEOUT_MS); + LOGV("input buffer %zd", bufferIndex); + if (bufferIndex >= 0) { + size_t bufferSize; + auto inputBuffer = AMediaCodec_getInputBuffer(mediaCodec.get(), bufferIndex, &bufferSize); + auto sampleSize = AMediaExtractor_readSampleData(mediaExtractor.get(), inputBuffer, bufferSize); + if (sampleSize < 0) { + sampleSize = 0; + sawInputEOS = true; + LOGV("EOS"); + } + auto presentationTimeUs = AMediaExtractor_getSampleTime(mediaExtractor.get()); + + AMediaCodec_queueInputBuffer(mediaCodec.get(), bufferIndex, 0, sampleSize, + presentationTimeUs, sawInputEOS ? AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM : 0); + AMediaExtractor_advance(mediaExtractor.get()); + } + } + + if (!sawOutputEOS) { + AMediaCodecBufferInfo info; + auto bufferIndex = AMediaCodec_dequeueOutputBuffer(mediaCodec.get(), &info, 0); + if (bufferIndex >= 0) { + size_t bufferSize = 0; + auto mediaFormat = std::shared_ptr(AMediaCodec_getOutputFormat(mediaCodec.get()), deleter_AMediaFormat); + AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_WIDTH, &frameWidth); + AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_HEIGHT, &frameHeight); + AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat); + uint8_t* codecBuffer = AMediaCodec_getOutputBuffer(mediaCodec.get(), bufferIndex, &bufferSize); + buffer = std::vector(codecBuffer + info.offset, codecBuffer + bufferSize); + LOGV("colorFormat: %d", colorFormat); + LOGV("buffer size: %zu", bufferSize); + LOGV("width (frame): %d", frameWidth); + LOGV("height (frame): %d", frameHeight); + if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) { + LOGV("output EOS"); + sawOutputEOS = true; + } + AMediaCodec_releaseOutputBuffer(mediaCodec.get(), bufferIndex, info.size != 0); + return true; + } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) { + LOGV("output buffers changed"); + } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) { + auto format = AMediaCodec_getOutputFormat(mediaCodec.get()); + LOGV("format changed to: %s", AMediaFormat_toString(format)); + AMediaFormat_delete(format); + } else if (bufferIndex == AMEDIACODEC_INFO_TRY_AGAIN_LATER) { + LOGV("no output buffer right now"); + } else { + LOGV("unexpected info code: %zd", bufferIndex); + } + } + } + return false; + } + + bool isOpened() const CV_OVERRIDE { return mediaCodec.get() != nullptr; } + + int getCaptureDomain() CV_OVERRIDE { return CAP_ANDROID; } + + bool grabFrame() CV_OVERRIDE + { + // clear the previous frame + buffer.clear(); + return decodeFrame(); + } + + bool retrieveFrame(int, OutputArray out) CV_OVERRIDE + { + if (buffer.empty()) { + return false; + } + Mat yuv(frameHeight + frameHeight/2, frameWidth, CV_8UC1, buffer.data()); + if (colorFormat == COLOR_FormatYUV420Planar) { + cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_YV12); + } else if (colorFormat == COLOR_FormatYUV420SemiPlanar) { + cv::cvtColor(yuv, out, cv::COLOR_YUV2BGR_NV21); + } else { + LOGE("Unsupported video format: %d", colorFormat); + return false; + } + return true; + } + + double getProperty(int property_id) const CV_OVERRIDE + { + switch (property_id) + { + case CV_CAP_PROP_FRAME_WIDTH: return frameWidth; + case CV_CAP_PROP_FRAME_HEIGHT: return frameHeight; + } + return 0; + } + + bool setProperty(int /* property_id */, double /* value */) CV_OVERRIDE + { + return false; + } + + bool initCapture(const char * filename) + { + struct stat statBuffer; + if (stat(filename, &statBuffer) != 0) { + LOGE("failed to stat file: %s (%s)", filename, strerror(errno)); + return false; + } + + int fd = open(filename, O_RDONLY); + + if (fd < 0) { + LOGE("failed to open file: %s %d (%s)", filename, fd, strerror(errno)); + return false; + } + + mediaExtractor = std::shared_ptr(AMediaExtractor_new(), deleter_AMediaExtractor); + if (!mediaExtractor) { + return false; + } + media_status_t err = AMediaExtractor_setDataSourceFd(mediaExtractor.get(), fd, 0, statBuffer.st_size); + close(fd); + if (err != AMEDIA_OK) { + LOGV("setDataSource error: %d", err); + return false; + } + + int numtracks = AMediaExtractor_getTrackCount(mediaExtractor.get()); + + LOGV("input has %d tracks", numtracks); + for (int i = 0; i < numtracks; i++) { + auto format = std::shared_ptr(AMediaExtractor_getTrackFormat(mediaExtractor.get(), i), deleter_AMediaFormat); + if (!format) { + continue; + } + const char *s = AMediaFormat_toString(format.get()); + LOGV("track %d format: %s", i, s); + const char *mime; + if (!AMediaFormat_getString(format.get(), AMEDIAFORMAT_KEY_MIME, &mime)) { + LOGV("no mime type"); + } else if (!strncmp(mime, "video/", 6)) { + int32_t trackWidth, trackHeight; + AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_WIDTH, &trackWidth); + AMediaFormat_getInt32(format.get(), AMEDIAFORMAT_KEY_HEIGHT, &trackHeight); + LOGV("width (track): %d", trackWidth); + LOGV("height (track): %d", trackHeight); + if (AMediaExtractor_selectTrack(mediaExtractor.get(), i) != AMEDIA_OK) { + continue; + } + mediaCodec = std::shared_ptr(AMediaCodec_createDecoderByType(mime), deleter_AMediaCodec); + if (!mediaCodec) { + continue; + } + if (AMediaCodec_configure(mediaCodec.get(), format.get(), NULL, NULL, 0) != AMEDIA_OK) { + continue; + } + sawInputEOS = false; + sawOutputEOS = false; + if (AMediaCodec_start(mediaCodec.get()) != AMEDIA_OK) { + continue; + } + return true; + } + } + + return false; + } + + void cleanUp() { + sawInputEOS = true; + sawOutputEOS = true; + frameWidth = 0; + frameHeight = 0; + colorFormat = 0; + } +}; + +/****************** Implementation of interface functions ********************/ + +Ptr cv::createAndroidCapture_file(const std::string &filename) { + Ptr res = makePtr(); + if (res && res->initCapture(filename.c_str())) + return res; + return Ptr(); +} diff --git a/modules/videoio/src/cap_interface.hpp b/modules/videoio/src/cap_interface.hpp index dc92039eb9..c303888828 100644 --- a/modules/videoio/src/cap_interface.hpp +++ b/modules/videoio/src/cap_interface.hpp @@ -206,6 +206,8 @@ Ptr createGPhoto2Capture(const std::string& deviceName); Ptr createXINECapture(const std::string &filename); +Ptr createAndroidCapture_file(const std::string &filename); + } // cv:: #endif // CAP_INTERFACE_HPP diff --git a/modules/videoio/src/videoio_registry.cpp b/modules/videoio/src/videoio_registry.cpp index 5b9641a6b5..cec435b75b 100644 --- a/modules/videoio/src/videoio_registry.cpp +++ b/modules/videoio/src/videoio_registry.cpp @@ -130,8 +130,10 @@ static const struct VideoBackendInfo builtin_backends[] = #ifdef HAVE_XINE DECLARE_STATIC_BACKEND(CAP_XINE, "XINE", MODE_CAPTURE_BY_FILENAME, createXINECapture, 0, 0), #endif - - // dropped backends: MIL, TYZX, Android +#ifdef HAVE_ANDROID_MEDIANDK + DECLARE_STATIC_BACKEND(CAP_ANDROID, "ANDROID_MEDIANDK", MODE_CAPTURE_BY_FILENAME, createAndroidCapture_file, 0, 0), +#endif + // dropped backends: MIL, TYZX }; bool sortByPriority(const VideoBackendInfo &lhs, const VideoBackendInfo &rhs) diff --git a/platforms/android/ndk-18-api-level-21.config.py b/platforms/android/ndk-18-api-level-21.config.py new file mode 100644 index 0000000000..2019c913ae --- /dev/null +++ b/platforms/android/ndk-18-api-level-21.config.py @@ -0,0 +1,6 @@ +ABIs = [ + ABI("2", "armeabi-v7a", None, 21, cmake_vars=dict(ANDROID_ABI='armeabi-v7a with NEON')), + ABI("3", "arm64-v8a", None, 21), + ABI("5", "x86_64", None, 21), + ABI("4", "x86", None, 21), +]