Merge pull request #19460 from mikhail-nikolskiy:videoio-hw
videoio: HW decode/encode in FFMPEG backend; new properties with support in FFMPEG/GST/MSMF * HW acceleration in FFMPEG backend * fixes on Windows, remove D3D9 * HW acceleration in FFMPEG backend * fixes on Windows, remove D3D9 * improve va test * Copyright * check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+ * CAP_MSMF test on .mp4 * .mp4 in test * improve va test * Copyright * check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+ * CAP_MSMF test on .mp4 * .mp4 in test * .avi for GStreamer test * revert changes around seek() * cv_writer_open_with_params * params.warnUnusedParameters * VideoCaptureParameters in GStreamer * open_with_params * params->getUnused * Reduce PSNR threshold 33->32 (other tests use 30) * require FFMPEG 4.0+; PSNR 30 as in other tests * GStreamer AVI-demux plugin not installed in Ubuntu test environment? * fix build on very old ffmpeg * fix build on very old ffmpeg * fix build issues * fix build issues (static_cast) * FFMPEG built on Windows without H264 encoder? * fix for write_nothing test on VAAPI * fix warnings * fix cv_writer_get_prop in plugins * use avcodec_get_hw_frames_parameters; more robust fallback to SW codecs * internal function hw_check_device() for device check/logging * two separate tests for HW read and write * image size 640x480 in encode test * WITH_VA=ON (only .h headers used in OpenCV, no linkage dependency) * exception on VP9 SW encoder? * rebase master; refine info message * videoio: fix FFmpeg standalone plugin build * videoio(ffmpeg): eliminate MSVC build warnings * address review comments * videoio(hw): update videocapture_acceleration.read test - remove parallel decoding by SW code path - check PSNR against the original generated image * videoio: minor fixes * videoio(test): disable unsupported MSMF cases (SW and HW) * videoio(test): update PSNR thresholds for HW acceleration read * videoio(test): update debug messages * "hw_acceleration" whitelisting parameter * little optimization in test * D3D11VA supports decoders, doesn't support encoders * videoio(test): adjust PSNR threshold in write_read_position tests * videoio(ffmpeg): fix rejecting on acceleration device name mismatch * videoio(ffmpeg): fix compilation USE_AV_HW_CODECS=0, add more debug logging * videoio: rework VideoAccelerationType behavior - enum is not a bitset - default value is backend specific - only '_NONE' and '_ANY' may fallback on software processing - specific H/W acceleration doesn't fallback on software processing. It fails if there is no support for specified H/W acceleration. * videoio(test): fix for current FFmpeg wrapper Co-authored-by: Alexander Alekhin <alexander.a.alekhin@gmail.com>pull/19606/head
Mikhail Nikolskii
4 years ago
committed by
GitHub
parent
f70e80a6ba
commit
7bcb51eded
13 changed files with 2211 additions and 218 deletions
@ -0,0 +1,555 @@ |
|||||||
|
// This file is part of OpenCV project.
|
||||||
|
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||||
|
// of this distribution and at http://opencv.org/license.html.
|
||||||
|
//
|
||||||
|
// Copyright (C) 2020-2021 Intel Corporation
|
||||||
|
|
||||||
|
#include "opencv2/videoio.hpp" |
||||||
|
#if defined(__OPENCV_BUILD) || defined(OPENCV_HAVE_CVCONFIG_H) // TODO Properly detect and add D3D11 / LIBVA dependencies for standalone plugins
|
||||||
|
#include "cvconfig.h" |
||||||
|
#endif |
||||||
|
#include <sstream> |
||||||
|
|
||||||
|
#ifdef HAVE_D3D11 |
||||||
|
#define D3D11_NO_HELPERS |
||||||
|
#include <d3d11.h> |
||||||
|
#include <codecvt> |
||||||
|
#endif |
||||||
|
|
||||||
|
#ifdef HAVE_VA |
||||||
|
#include <va/va_backend.h> |
||||||
|
#endif |
||||||
|
|
||||||
|
extern "C" { |
||||||
|
#include <libavcodec/avcodec.h> |
||||||
|
#include <libavutil/avutil.h> |
||||||
|
|
||||||
|
#include <libavutil/hwcontext.h> |
||||||
|
#ifdef HAVE_D3D11 |
||||||
|
#include <libavutil/hwcontext_d3d11va.h> |
||||||
|
#endif |
||||||
|
#ifdef HAVE_VA |
||||||
|
#include <libavutil/hwcontext_vaapi.h> |
||||||
|
#endif |
||||||
|
} |
||||||
|
|
||||||
|
static |
||||||
|
const char* getVideoAccelerationName(VideoAccelerationType va_type) |
||||||
|
{ |
||||||
|
switch (va_type) |
||||||
|
{ |
||||||
|
case VIDEO_ACCELERATION_NONE: return "none"; |
||||||
|
case VIDEO_ACCELERATION_ANY: return "any"; |
||||||
|
case VIDEO_ACCELERATION_D3D11: return "d3d11"; |
||||||
|
case VIDEO_ACCELERATION_VAAPI: return "vaapi"; |
||||||
|
case VIDEO_ACCELERATION_MFX: return "mfx"; |
||||||
|
} |
||||||
|
return "unknown"; |
||||||
|
} |
||||||
|
|
||||||
|
static |
||||||
|
std::string getDecoderConfiguration(VideoAccelerationType va_type, AVDictionary *dict) |
||||||
|
{ |
||||||
|
std::string va_name = getVideoAccelerationName(va_type); |
||||||
|
std::string key_name = std::string("hw_decoders_") + va_name; |
||||||
|
const char *hw_acceleration = NULL; |
||||||
|
if (dict) |
||||||
|
{ |
||||||
|
AVDictionaryEntry* entry = av_dict_get(dict, key_name.c_str(), NULL, 0); |
||||||
|
if (entry) |
||||||
|
hw_acceleration = entry->value; |
||||||
|
} |
||||||
|
if (hw_acceleration) |
||||||
|
return hw_acceleration; |
||||||
|
|
||||||
|
// some default values (FFMPEG_DECODE_ACCELERATION_TYPES)
|
||||||
|
#ifdef _WIN32 |
||||||
|
switch (va_type) |
||||||
|
{ |
||||||
|
case VIDEO_ACCELERATION_NONE: return ""; |
||||||
|
case VIDEO_ACCELERATION_ANY: return "d3d11va"; |
||||||
|
case VIDEO_ACCELERATION_D3D11: return "d3d11va"; |
||||||
|
case VIDEO_ACCELERATION_VAAPI: return ""; |
||||||
|
case VIDEO_ACCELERATION_MFX: return ""; |
||||||
|
} |
||||||
|
return ""; |
||||||
|
#else |
||||||
|
switch (va_type) |
||||||
|
{ |
||||||
|
case VIDEO_ACCELERATION_NONE: return ""; |
||||||
|
case VIDEO_ACCELERATION_ANY: return "vaapi.iHD"; |
||||||
|
case VIDEO_ACCELERATION_D3D11: return ""; |
||||||
|
case VIDEO_ACCELERATION_VAAPI: return "vaapi.iHD"; |
||||||
|
case VIDEO_ACCELERATION_MFX: return ""; |
||||||
|
} |
||||||
|
return ""; |
||||||
|
#endif |
||||||
|
} |
||||||
|
|
||||||
|
static |
||||||
|
std::string getEncoderConfiguration(VideoAccelerationType va_type, AVDictionary *dict) |
||||||
|
{ |
||||||
|
std::string va_name = getVideoAccelerationName(va_type); |
||||||
|
std::string key_name = std::string("hw_encoders_") + va_name; |
||||||
|
const char *hw_acceleration = NULL; |
||||||
|
if (dict) |
||||||
|
{ |
||||||
|
AVDictionaryEntry* entry = av_dict_get(dict, key_name.c_str(), NULL, 0); |
||||||
|
if (entry) |
||||||
|
hw_acceleration = entry->value; |
||||||
|
} |
||||||
|
if (hw_acceleration) |
||||||
|
return hw_acceleration; |
||||||
|
|
||||||
|
// some default values (FFMPEG_ENCODE_ACCELERATION_TYPES)
|
||||||
|
#ifdef _WIN32 |
||||||
|
switch (va_type) |
||||||
|
{ |
||||||
|
case VIDEO_ACCELERATION_NONE: return ""; |
||||||
|
case VIDEO_ACCELERATION_ANY: return "qsv"; |
||||||
|
case VIDEO_ACCELERATION_D3D11: return ""; |
||||||
|
case VIDEO_ACCELERATION_VAAPI: return ""; |
||||||
|
case VIDEO_ACCELERATION_MFX: return "qsv"; |
||||||
|
} |
||||||
|
return ""; |
||||||
|
#else |
||||||
|
switch (va_type) |
||||||
|
{ |
||||||
|
case VIDEO_ACCELERATION_NONE: return ""; |
||||||
|
case VIDEO_ACCELERATION_ANY: return "qsv.iHD,vaapi.iHD"; |
||||||
|
case VIDEO_ACCELERATION_D3D11: return ""; |
||||||
|
case VIDEO_ACCELERATION_VAAPI: return "vaapi.iHD"; |
||||||
|
case VIDEO_ACCELERATION_MFX: return "qsv.iHD"; |
||||||
|
} |
||||||
|
return "unknown"; |
||||||
|
#endif |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
static |
||||||
|
std::string getDecoderDisabledCodecs(AVDictionary *dict) |
||||||
|
{ |
||||||
|
std::string key_name = std::string("hw_disable_decoders"); |
||||||
|
const char *disabled_codecs = NULL; |
||||||
|
if (dict) |
||||||
|
{ |
||||||
|
AVDictionaryEntry* entry = av_dict_get(dict, key_name.c_str(), NULL, 0); |
||||||
|
if (entry) |
||||||
|
disabled_codecs = entry->value; |
||||||
|
} |
||||||
|
if (disabled_codecs) |
||||||
|
return disabled_codecs; |
||||||
|
|
||||||
|
// some default values (FFMPEG_DECODE_DISABLE_CODECS)
|
||||||
|
#ifdef _WIN32 |
||||||
|
return "none"; |
||||||
|
#else |
||||||
|
return "av1.vaapi,av1_qsv,vp8.vaapi,vp8_qsv"; // "vp9_qsv"
|
||||||
|
#endif |
||||||
|
} |
||||||
|
|
||||||
|
static |
||||||
|
std::string getEncoderDisabledCodecs(AVDictionary *dict) |
||||||
|
{ |
||||||
|
std::string key_name = std::string("hw_disabled_encoders"); |
||||||
|
const char *disabled_codecs = NULL; |
||||||
|
if (dict) |
||||||
|
{ |
||||||
|
AVDictionaryEntry* entry = av_dict_get(dict, key_name.c_str(), NULL, 0); |
||||||
|
if (entry) |
||||||
|
disabled_codecs = entry->value; |
||||||
|
} |
||||||
|
if (disabled_codecs) |
||||||
|
return disabled_codecs; |
||||||
|
|
||||||
|
// some default values (FFMPEG_ENCODE_DISABLE_CODECS)
|
||||||
|
#ifdef _WIN32 |
||||||
|
return "mjpeg_qsv"; |
||||||
|
#else |
||||||
|
return "mjpeg_vaapi,mjpeg_qsv,vp8_vaapi"; |
||||||
|
#endif |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
#define HW_DEFAULT_POOL_SIZE 32 |
||||||
|
#define HW_DEFAULT_SW_FORMAT AV_PIX_FMT_NV12 |
||||||
|
|
||||||
|
using namespace cv; |
||||||
|
|
||||||
|
static AVCodec *hw_find_codec(AVCodecID id, AVHWDeviceType hw_type, int (*check_category)(const AVCodec *), |
||||||
|
const char *disabled_codecs, AVPixelFormat *hw_pix_fmt); |
||||||
|
static AVBufferRef* hw_create_device(AVHWDeviceType hw_type, int hw_device, const std::string& device_subname); |
||||||
|
static AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device_ctx, int width, int height, AVPixelFormat hw_format); |
||||||
|
static AVPixelFormat hw_get_format_callback(struct AVCodecContext *ctx, const enum AVPixelFormat * fmt); |
||||||
|
static VideoAccelerationType hw_type_to_va_type(AVHWDeviceType hw_type); |
||||||
|
|
||||||
|
static |
||||||
|
bool hw_check_device(AVBufferRef* ctx, AVHWDeviceType hw_type, const std::string& device_subname) { |
||||||
|
if (!ctx) |
||||||
|
return false; |
||||||
|
AVHWDeviceContext* hw_device_ctx = (AVHWDeviceContext*)ctx->data; |
||||||
|
if (!hw_device_ctx->hwctx) |
||||||
|
return false; |
||||||
|
const char *hw_name = av_hwdevice_get_type_name(hw_type); |
||||||
|
if (hw_type == AV_HWDEVICE_TYPE_QSV) |
||||||
|
hw_name = "MFX"; |
||||||
|
bool ret = true; |
||||||
|
std::string device_name; |
||||||
|
#if defined(HAVE_D3D11) |
||||||
|
if (hw_device_ctx->type == AV_HWDEVICE_TYPE_D3D11VA) { |
||||||
|
ID3D11Device* device = ((AVD3D11VADeviceContext*)hw_device_ctx->hwctx)->device; |
||||||
|
IDXGIDevice* dxgiDevice = nullptr; |
||||||
|
if (device && SUCCEEDED(device->QueryInterface(__uuidof(IDXGIDevice), reinterpret_cast<void**>(&dxgiDevice)))) { |
||||||
|
IDXGIAdapter* adapter = nullptr; |
||||||
|
if (SUCCEEDED(dxgiDevice->GetAdapter(&adapter))) { |
||||||
|
DXGI_ADAPTER_DESC desc; |
||||||
|
if (SUCCEEDED(adapter->GetDesc(&desc))) { |
||||||
|
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> conv; |
||||||
|
device_name = conv.to_bytes(desc.Description); |
||||||
|
} |
||||||
|
adapter->Release(); |
||||||
|
} |
||||||
|
dxgiDevice->Release(); |
||||||
|
} |
||||||
|
} |
||||||
|
#endif |
||||||
|
if (hw_device_ctx->type == AV_HWDEVICE_TYPE_VAAPI) { |
||||||
|
#if defined(HAVE_VA) && (VA_MAJOR_VERSION >= 1) |
||||||
|
VADisplay display = ((AVVAAPIDeviceContext *) hw_device_ctx->hwctx)->display; |
||||||
|
if (display) { |
||||||
|
VADriverContext *va_ctx = ((VADisplayContext *) display)->pDriverContext; |
||||||
|
device_name = va_ctx->str_vendor; |
||||||
|
if (hw_type == AV_HWDEVICE_TYPE_QSV) { |
||||||
|
// Workaround for issue fixed in MediaSDK 21.x https://github.com/Intel-Media-SDK/MediaSDK/issues/2595
|
||||||
|
// Checks VAAPI driver for support of VideoProc operation required by MediaSDK
|
||||||
|
ret = false; |
||||||
|
int n_entrypoints = va_ctx->max_entrypoints; |
||||||
|
std::vector<VAEntrypoint> entrypoints(n_entrypoints); |
||||||
|
if (va_ctx->vtable->vaQueryConfigEntrypoints(va_ctx, VAProfileNone, entrypoints.data(), &n_entrypoints) == VA_STATUS_SUCCESS) { |
||||||
|
for (int i = 0; i < n_entrypoints; i++) { |
||||||
|
if (entrypoints[i] == VAEntrypointVideoProc) { |
||||||
|
ret = true; |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
if (!ret) |
||||||
|
CV_LOG_INFO(NULL, "FFMPEG: Skipping MFX video acceleration as entrypoint VideoProc not found in: " << device_name); |
||||||
|
} |
||||||
|
} |
||||||
|
#else |
||||||
|
ret = (hw_type != AV_HWDEVICE_TYPE_QSV); // disable MFX if we can't check VAAPI for VideoProc entrypoint
|
||||||
|
#endif |
||||||
|
} |
||||||
|
if (ret && !device_subname.empty() && device_name.find(device_subname) == std::string::npos) |
||||||
|
{ |
||||||
|
CV_LOG_INFO(NULL, "FFMPEG: Skipping '" << hw_name << |
||||||
|
"' video acceleration on the following device name as not matching substring '" << device_subname << "': " << device_name); |
||||||
|
ret = false; // reject configuration
|
||||||
|
} |
||||||
|
if (ret) |
||||||
|
{ |
||||||
|
if (!device_name.empty()) { |
||||||
|
CV_LOG_INFO(NULL, "FFMPEG: Using " << hw_name << " video acceleration on device: " << device_name); |
||||||
|
} else { |
||||||
|
CV_LOG_INFO(NULL, "FFMPEG: Using " << hw_name << " video acceleration"); |
||||||
|
} |
||||||
|
} |
||||||
|
return ret; |
||||||
|
} |
||||||
|
|
||||||
|
static |
||||||
|
AVBufferRef* hw_create_device(AVHWDeviceType hw_type, int hw_device, const std::string& device_subname) { |
||||||
|
if (AV_HWDEVICE_TYPE_NONE == hw_type) |
||||||
|
return NULL; |
||||||
|
|
||||||
|
AVHWDeviceType child_type = hw_type; |
||||||
|
if (hw_type == AV_HWDEVICE_TYPE_QSV) { |
||||||
|
#ifdef _WIN32 |
||||||
|
child_type = AV_HWDEVICE_TYPE_DXVA2; |
||||||
|
#else |
||||||
|
child_type = AV_HWDEVICE_TYPE_VAAPI; |
||||||
|
#endif |
||||||
|
} |
||||||
|
|
||||||
|
AVBufferRef* hw_device_ctx = NULL; |
||||||
|
char device[128] = ""; |
||||||
|
char* pdevice = NULL; |
||||||
|
if (hw_device >= 0 && hw_device < 100000) { |
||||||
|
if (child_type == AV_HWDEVICE_TYPE_VAAPI) { |
||||||
|
snprintf(device, sizeof(device), "/dev/dri/renderD%d", 128 + hw_device); |
||||||
|
} else { |
||||||
|
snprintf(device, sizeof(device), "%d", hw_device); |
||||||
|
} |
||||||
|
pdevice = device; |
||||||
|
} |
||||||
|
const char *hw_child_name = av_hwdevice_get_type_name(child_type); |
||||||
|
const char *device_name = pdevice ? pdevice : "'default'"; |
||||||
|
int err = av_hwdevice_ctx_create(&hw_device_ctx, child_type, pdevice, NULL, 0); |
||||||
|
if (hw_device_ctx && err >= 0) |
||||||
|
{ |
||||||
|
CV_LOG_DEBUG(NULL, "FFMPEG: Created video acceleration context (av_hwdevice_ctx_create) for " << hw_child_name << " on device " << device_name); |
||||||
|
if (!hw_check_device(hw_device_ctx, hw_type, device_subname)) { |
||||||
|
av_buffer_unref(&hw_device_ctx); |
||||||
|
return NULL; |
||||||
|
} |
||||||
|
if (hw_type != child_type) { |
||||||
|
AVBufferRef *derived_ctx = NULL; |
||||||
|
const char *hw_name = av_hwdevice_get_type_name(hw_type); |
||||||
|
err = av_hwdevice_ctx_create_derived(&derived_ctx, hw_type, hw_device_ctx, 0); |
||||||
|
if (!derived_ctx || err < 0) |
||||||
|
{ |
||||||
|
if (derived_ctx) |
||||||
|
av_buffer_unref(&derived_ctx); |
||||||
|
CV_LOG_INFO(NULL, "FFMPEG: Failed to create derived video acceleration (av_hwdevice_ctx_create_derived) for " << hw_name << ". Error=" << err); |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
CV_LOG_DEBUG(NULL, "FFMPEG: Created derived video acceleration context (av_hwdevice_ctx_create_derived) for " << hw_name); |
||||||
|
} |
||||||
|
av_buffer_unref(&hw_device_ctx); |
||||||
|
return derived_ctx; |
||||||
|
} else { |
||||||
|
return hw_device_ctx; |
||||||
|
} |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
const char *hw_name = hw_child_name; |
||||||
|
CV_LOG_INFO(NULL, "FFMPEG: Failed to create " << hw_name << " video acceleration (av_hwdevice_ctx_create) on device " << device_name); |
||||||
|
return NULL; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
static |
||||||
|
AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device_ctx, int width, int height, AVPixelFormat hw_format) |
||||||
|
{ |
||||||
|
AVBufferRef *hw_frames_ref = nullptr; |
||||||
|
if (ctx) |
||||||
|
{ |
||||||
|
int res = avcodec_get_hw_frames_parameters(ctx, hw_device_ctx, hw_format, &hw_frames_ref); |
||||||
|
if (res < 0) |
||||||
|
{ |
||||||
|
CV_LOG_DEBUG(NULL, "FFMPEG: avcodec_get_hw_frames_parameters() call failed: " << res) |
||||||
|
} |
||||||
|
} |
||||||
|
if (!hw_frames_ref) |
||||||
|
{ |
||||||
|
hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx); |
||||||
|
} |
||||||
|
if (!hw_frames_ref) |
||||||
|
{ |
||||||
|
CV_LOG_INFO(NULL, "FFMPEG: Failed to create HW frame context (av_hwframe_ctx_alloc)"); |
||||||
|
return NULL; |
||||||
|
} |
||||||
|
AVHWFramesContext *frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data); |
||||||
|
frames_ctx->width = width; |
||||||
|
frames_ctx->height = height; |
||||||
|
if (frames_ctx->format == AV_PIX_FMT_NONE) |
||||||
|
frames_ctx->format = hw_format; |
||||||
|
if (frames_ctx->sw_format == AV_PIX_FMT_NONE) |
||||||
|
frames_ctx->sw_format = HW_DEFAULT_SW_FORMAT; |
||||||
|
if (frames_ctx->initial_pool_size == 0) |
||||||
|
frames_ctx->initial_pool_size = HW_DEFAULT_POOL_SIZE; |
||||||
|
int res = av_hwframe_ctx_init(hw_frames_ref); |
||||||
|
if (res < 0) |
||||||
|
{ |
||||||
|
CV_LOG_INFO(NULL, "FFMPEG: Failed to initialize HW frame context (av_hwframe_ctx_init): " << res); |
||||||
|
av_buffer_unref(&hw_frames_ref); |
||||||
|
return NULL; |
||||||
|
} |
||||||
|
return hw_frames_ref; |
||||||
|
} |
||||||
|
|
||||||
|
static |
||||||
|
bool hw_check_codec(AVCodec* codec, AVHWDeviceType hw_type, const char *disabled_codecs) |
||||||
|
{ |
||||||
|
CV_Assert(disabled_codecs); |
||||||
|
std::string hw_name = std::string(".") + av_hwdevice_get_type_name(hw_type); |
||||||
|
std::stringstream s_stream(disabled_codecs); |
||||||
|
while (s_stream.good()) { |
||||||
|
std::string name; |
||||||
|
getline(s_stream, name, ','); |
||||||
|
if (name == codec->name || name == hw_name || name == codec->name + hw_name || name == "hw") { |
||||||
|
CV_LOG_INFO(NULL, "FFMPEG: skipping codec " << codec->name << hw_name); |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
static |
||||||
|
AVCodec *hw_find_codec(AVCodecID id, AVHWDeviceType hw_type, int (*check_category)(const AVCodec *), const char *disabled_codecs, AVPixelFormat *hw_pix_fmt) { |
||||||
|
AVCodec *c = 0; |
||||||
|
void *opaque = 0; |
||||||
|
|
||||||
|
while (NULL != (c = (AVCodec*)av_codec_iterate(&opaque))) |
||||||
|
{ |
||||||
|
if (!check_category(c)) |
||||||
|
continue; |
||||||
|
if (c->id != id) |
||||||
|
continue; |
||||||
|
if (c->capabilities & AV_CODEC_CAP_EXPERIMENTAL) |
||||||
|
continue; |
||||||
|
if (hw_type != AV_HWDEVICE_TYPE_NONE) { |
||||||
|
AVPixelFormat hw_native_fmt = AV_PIX_FMT_NONE; |
||||||
|
#if LIBAVUTIL_BUILD < AV_VERSION_INT(56, 51, 100) // VAAPI encoders support avcodec_get_hw_config() starting ffmpeg 4.3
|
||||||
|
if (hw_type == AV_HWDEVICE_TYPE_VAAPI) |
||||||
|
hw_native_fmt = AV_PIX_FMT_VAAPI_VLD; |
||||||
|
#endif |
||||||
|
if (hw_type == AV_HWDEVICE_TYPE_CUDA) // CUDA encoders don't support avcodec_get_hw_config()
|
||||||
|
hw_native_fmt = AV_PIX_FMT_CUDA; |
||||||
|
if (av_codec_is_encoder(c) && hw_native_fmt != AV_PIX_FMT_NONE && c->pix_fmts) { |
||||||
|
for (int i = 0; c->pix_fmts[i] != AV_PIX_FMT_NONE; i++) { |
||||||
|
if (c->pix_fmts[i] == hw_native_fmt) { |
||||||
|
*hw_pix_fmt = hw_native_fmt; |
||||||
|
if (hw_check_codec(c, hw_type, disabled_codecs)) |
||||||
|
return c; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
for (int i = 0;; i++) { |
||||||
|
const AVCodecHWConfig *hw_config = avcodec_get_hw_config(c, i); |
||||||
|
if (!hw_config) |
||||||
|
break; |
||||||
|
if (hw_config->device_type == hw_type) { |
||||||
|
*hw_pix_fmt = hw_config->pix_fmt; |
||||||
|
if (hw_check_codec(c, hw_type, disabled_codecs)) |
||||||
|
return c; |
||||||
|
} |
||||||
|
} |
||||||
|
} else { |
||||||
|
return c; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return NULL; |
||||||
|
} |
||||||
|
|
||||||
|
// Callback to select hardware pixel format (not software format) and allocate frame pool (hw_frames_ctx)
|
||||||
|
static |
||||||
|
AVPixelFormat hw_get_format_callback(struct AVCodecContext *ctx, const enum AVPixelFormat * fmt) { |
||||||
|
if (!ctx->hw_device_ctx) |
||||||
|
return fmt[0]; |
||||||
|
AVHWDeviceType hw_type = ((AVHWDeviceContext*)ctx->hw_device_ctx->data)->type; |
||||||
|
for (int j = 0;; j++) { |
||||||
|
const AVCodecHWConfig *hw_config = avcodec_get_hw_config(ctx->codec, j); |
||||||
|
if (!hw_config) |
||||||
|
break; |
||||||
|
if (hw_config->device_type == hw_type) { |
||||||
|
for (int i = 0; fmt[i] != AV_PIX_FMT_NONE; i++) { |
||||||
|
if (fmt[i] == hw_config->pix_fmt) { |
||||||
|
if (hw_config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) { |
||||||
|
ctx->sw_pix_fmt = HW_DEFAULT_SW_FORMAT; |
||||||
|
ctx->hw_frames_ctx = hw_create_frames(ctx, ctx->hw_device_ctx, ctx->width, ctx->height, fmt[i]); |
||||||
|
if (ctx->hw_frames_ctx) { |
||||||
|
//ctx->sw_pix_fmt = ((AVHWFramesContext *)(ctx->hw_frames_ctx->data))->sw_format;
|
||||||
|
return fmt[i]; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
CV_LOG_DEBUG(NULL, "FFMPEG: Can't select HW format in 'get_format()' callback, use default"); |
||||||
|
return fmt[0]; |
||||||
|
} |
||||||
|
|
||||||
|
static |
||||||
|
VideoAccelerationType hw_type_to_va_type(AVHWDeviceType hw_type) { |
||||||
|
struct HWTypeFFMPEG { |
||||||
|
AVHWDeviceType hw_type; |
||||||
|
VideoAccelerationType va_type; |
||||||
|
} known_hw_types[] = { |
||||||
|
{ AV_HWDEVICE_TYPE_D3D11VA, VIDEO_ACCELERATION_D3D11 }, |
||||||
|
{ AV_HWDEVICE_TYPE_VAAPI, VIDEO_ACCELERATION_VAAPI }, |
||||||
|
{ AV_HWDEVICE_TYPE_QSV, VIDEO_ACCELERATION_MFX }, |
||||||
|
{ AV_HWDEVICE_TYPE_CUDA, (VideoAccelerationType)(1 << 11) }, |
||||||
|
}; |
||||||
|
for (const HWTypeFFMPEG& hw : known_hw_types) { |
||||||
|
if (hw_type == hw.hw_type) |
||||||
|
return hw.va_type; |
||||||
|
} |
||||||
|
return VIDEO_ACCELERATION_NONE; |
||||||
|
} |
||||||
|
|
||||||
|
class HWAccelIterator { |
||||||
|
public: |
||||||
|
HWAccelIterator(VideoAccelerationType va_type, bool isEncoder, AVDictionary *dict) |
||||||
|
: hw_type_(AV_HWDEVICE_TYPE_NONE) |
||||||
|
{ |
||||||
|
std::string accel_list; |
||||||
|
if (va_type != VIDEO_ACCELERATION_NONE) |
||||||
|
{ |
||||||
|
updateAccelList_(accel_list, va_type, isEncoder, dict); |
||||||
|
} |
||||||
|
if (va_type == VIDEO_ACCELERATION_ANY) |
||||||
|
{ |
||||||
|
if (!accel_list.empty()) |
||||||
|
accel_list = ","; // add no-acceleration case to the end of the list
|
||||||
|
} |
||||||
|
CV_LOG_DEBUG(NULL, "FFMPEG: allowed acceleration types (" << getVideoAccelerationName(va_type) << "): '" << accel_list << "'"); |
||||||
|
|
||||||
|
if (accel_list.empty() && va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) |
||||||
|
{ |
||||||
|
// broke stream
|
||||||
|
std::string tmp; |
||||||
|
s_stream_ >> tmp; |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
s_stream_ = std::istringstream(accel_list); |
||||||
|
} |
||||||
|
|
||||||
|
if (va_type != VIDEO_ACCELERATION_NONE) |
||||||
|
{ |
||||||
|
disabled_codecs_ = isEncoder |
||||||
|
? getEncoderDisabledCodecs(dict) |
||||||
|
: getDecoderDisabledCodecs(dict); |
||||||
|
CV_LOG_DEBUG(NULL, "FFMPEG: disabled codecs: '" << disabled_codecs_ << "'"); |
||||||
|
} |
||||||
|
} |
||||||
|
bool good() const |
||||||
|
{ |
||||||
|
return s_stream_.good(); |
||||||
|
} |
||||||
|
void parse_next() |
||||||
|
{ |
||||||
|
getline(s_stream_, hw_type_device_string_, ','); |
||||||
|
size_t index = hw_type_device_string_.find('.'); |
||||||
|
if (index != std::string::npos) { |
||||||
|
device_subname_ = hw_type_device_string_.substr(index + 1); |
||||||
|
hw_type_string_ = hw_type_device_string_.substr(0, index); |
||||||
|
} else { |
||||||
|
device_subname_.clear(); |
||||||
|
hw_type_string_ = hw_type_device_string_; |
||||||
|
} |
||||||
|
hw_type_ = av_hwdevice_find_type_by_name(hw_type_string_.c_str()); |
||||||
|
} |
||||||
|
const std::string& hw_type_device_string() const { return hw_type_device_string_; } |
||||||
|
const std::string& hw_type_string() const { return hw_type_string_; } |
||||||
|
AVHWDeviceType hw_type() const { return hw_type_; } |
||||||
|
const std::string& device_subname() const { return device_subname_; } |
||||||
|
const std::string& disabled_codecs() const { return disabled_codecs_; } |
||||||
|
private: |
||||||
|
bool updateAccelList_(std::string& accel_list, VideoAccelerationType va_type, bool isEncoder, AVDictionary *dict) |
||||||
|
{ |
||||||
|
std::string new_accels = isEncoder |
||||||
|
? getEncoderConfiguration(va_type, dict) |
||||||
|
: getDecoderConfiguration(va_type, dict); |
||||||
|
if (new_accels.empty()) |
||||||
|
return false; |
||||||
|
if (accel_list.empty()) |
||||||
|
accel_list = new_accels; |
||||||
|
else |
||||||
|
accel_list = accel_list + "," + new_accels; |
||||||
|
return true; |
||||||
|
} |
||||||
|
std::istringstream s_stream_; |
||||||
|
std::string hw_type_device_string_; |
||||||
|
std::string hw_type_string_; |
||||||
|
AVHWDeviceType hw_type_; |
||||||
|
std::string device_subname_; |
||||||
|
|
||||||
|
std::string disabled_codecs_; |
||||||
|
}; |
||||||
@ -0,0 +1,211 @@ |
|||||||
|
#include <iostream> |
||||||
|
#include <chrono> |
||||||
|
#include "opencv2/core.hpp" |
||||||
|
#include "opencv2/core/ocl.hpp" |
||||||
|
#include "opencv2/core/utility.hpp" |
||||||
|
#include "opencv2/imgproc.hpp" |
||||||
|
#include "opencv2/videoio.hpp" |
||||||
|
#include "opencv2/highgui.hpp" |
||||||
|
|
||||||
|
using namespace cv; |
||||||
|
using namespace std; |
||||||
|
|
||||||
|
const char* keys = |
||||||
|
"{ i input | | input video file }" |
||||||
|
"{ o output | | output video file, or specify 'null' to measure decoding without rendering to screen}" |
||||||
|
"{ backend | any | VideoCapture and VideoWriter backend, valid values: 'any', 'ffmpeg', 'msmf', 'gstreamer' }" |
||||||
|
"{ accel | any | GPU Video Acceleration, valid values: 'none', 'any', 'd3d11', 'vaapi', 'mfx' }" |
||||||
|
"{ device | -1 | Video Acceleration device (GPU) index (-1 means default device) }" |
||||||
|
"{ out_w | | output width (resize by calling cv::resize) }" |
||||||
|
"{ out_h | | output height (resize by calling cv::resize) }" |
||||||
|
"{ bitwise_not| false | apply simple image processing - bitwise_not pixels by calling cv::bitwise_not }" |
||||||
|
"{ opencl | true | use OpenCL (inside VideoCapture/VideoWriter and for image processing) }" |
||||||
|
"{ codec | H264 | codec id (four characters string) of output file encoder }" |
||||||
|
"{ h help | | print help message }"; |
||||||
|
|
||||||
|
struct { |
||||||
|
cv::VideoCaptureAPIs backend; |
||||||
|
const char* str; |
||||||
|
} backend_strings[] = { |
||||||
|
{ cv::CAP_ANY, "any" }, |
||||||
|
{ cv::CAP_FFMPEG, "ffmpeg" }, |
||||||
|
{ cv::CAP_MSMF, "msmf" }, |
||||||
|
{ cv::CAP_GSTREAMER, "gstreamer" }, |
||||||
|
}; |
||||||
|
|
||||||
|
struct { |
||||||
|
VideoAccelerationType acceleration; |
||||||
|
const char* str; |
||||||
|
} acceleration_strings[] = { |
||||||
|
{ VIDEO_ACCELERATION_NONE, "none" }, |
||||||
|
{ VIDEO_ACCELERATION_ANY, "any" }, |
||||||
|
{ VIDEO_ACCELERATION_D3D11, "d3d11" }, |
||||||
|
{ VIDEO_ACCELERATION_VAAPI, "vaapi" }, |
||||||
|
{ VIDEO_ACCELERATION_MFX, "mfx" }, |
||||||
|
}; |
||||||
|
|
||||||
|
class FPSCounter { |
||||||
|
public: |
||||||
|
FPSCounter(double _interval) : interval(_interval) { |
||||||
|
} |
||||||
|
|
||||||
|
~FPSCounter() { |
||||||
|
NewFrame(true); |
||||||
|
} |
||||||
|
|
||||||
|
void NewFrame(bool last_frame = false) { |
||||||
|
num_frames++; |
||||||
|
auto now = std::chrono::high_resolution_clock::now(); |
||||||
|
if (!last_time.time_since_epoch().count()) { |
||||||
|
last_time = now; |
||||||
|
} |
||||||
|
|
||||||
|
double sec = std::chrono::duration_cast<std::chrono::duration<double>>(now - last_time).count(); |
||||||
|
if (sec >= interval || last_frame) { |
||||||
|
printf("FPS(last %.2f sec) = %.2f\n", sec, num_frames / sec); |
||||||
|
fflush(stdout); |
||||||
|
num_frames = 0; |
||||||
|
last_time = now; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
double interval = 1; |
||||||
|
std::chrono::time_point<std::chrono::high_resolution_clock> last_time; |
||||||
|
int num_frames = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
int main(int argc, char** argv) |
||||||
|
{ |
||||||
|
cv::CommandLineParser cmd(argc, argv, keys); |
||||||
|
if (cmd.has("help")) |
||||||
|
{ |
||||||
|
cout << "Usage : video_acceleration [options]" << endl; |
||||||
|
cout << "Available options:" << endl; |
||||||
|
cmd.printMessage(); |
||||||
|
return EXIT_SUCCESS; |
||||||
|
} |
||||||
|
|
||||||
|
string infile = cmd.get<string>("i"); |
||||||
|
string outfile = cmd.get<string>("o"); |
||||||
|
string codec = cmd.get<string>("codec"); |
||||||
|
int device = cmd.get<int>("device"); |
||||||
|
int out_w = cmd.get<int>("out_w"); |
||||||
|
int out_h = cmd.get<int>("out_h"); |
||||||
|
bool use_opencl = cmd.get<bool>("opencl"); |
||||||
|
bool bitwise_not = cmd.get<bool>("bitwise_not"); |
||||||
|
|
||||||
|
cv::VideoCaptureAPIs backend = cv::CAP_ANY; |
||||||
|
string backend_str = cmd.get<string>("backend"); |
||||||
|
for (size_t i = 0; i < sizeof(backend_strings)/sizeof(backend_strings[0]); i++) { |
||||||
|
if (backend_str == backend_strings[i].str) { |
||||||
|
backend = backend_strings[i].backend; |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
VideoAccelerationType accel = VIDEO_ACCELERATION_ANY; |
||||||
|
string accel_str = cmd.get<string>("accel"); |
||||||
|
for (size_t i = 0; i < sizeof(acceleration_strings) / sizeof(acceleration_strings[0]); i++) { |
||||||
|
if (accel_str == acceleration_strings[i].str) { |
||||||
|
accel = acceleration_strings[i].acceleration; |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
ocl::setUseOpenCL(use_opencl); |
||||||
|
|
||||||
|
VideoCapture capture(infile, backend, { |
||||||
|
CAP_PROP_HW_ACCELERATION, (int)accel, |
||||||
|
CAP_PROP_HW_DEVICE, device |
||||||
|
}); |
||||||
|
if (!capture.isOpened()) { |
||||||
|
cerr << "Failed to open VideoCapture" << endl; |
||||||
|
return 1; |
||||||
|
} |
||||||
|
cout << "VideoCapture backend = " << capture.getBackendName() << endl; |
||||||
|
VideoAccelerationType actual_accel = static_cast<VideoAccelerationType>(static_cast<int>(capture.get(CAP_PROP_HW_ACCELERATION))); |
||||||
|
for (size_t i = 0; i < sizeof(acceleration_strings) / sizeof(acceleration_strings[0]); i++) { |
||||||
|
if (actual_accel == acceleration_strings[i].acceleration) { |
||||||
|
cout << "VideoCapture acceleration = " << acceleration_strings[i].str << endl; |
||||||
|
cout << "VideoCapture acceleration device = " << (int)capture.get(CAP_PROP_HW_DEVICE) << endl; |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
VideoWriter writer; |
||||||
|
if (!outfile.empty() && outfile != "null") { |
||||||
|
const char* codec_str = codec.c_str(); |
||||||
|
int fourcc = VideoWriter::fourcc(codec_str[0], codec_str[1], codec_str[2], codec_str[3]); |
||||||
|
double fps = capture.get(CAP_PROP_FPS); |
||||||
|
Size frameSize = { out_w, out_h }; |
||||||
|
if (!out_w || !out_h) { |
||||||
|
frameSize = { (int)capture.get(CAP_PROP_FRAME_WIDTH), (int)capture.get(CAP_PROP_FRAME_HEIGHT) }; |
||||||
|
} |
||||||
|
writer = VideoWriter(outfile, backend, fourcc, fps, frameSize, { |
||||||
|
VIDEOWRITER_PROP_HW_ACCELERATION, (int)accel, |
||||||
|
VIDEOWRITER_PROP_HW_DEVICE, device |
||||||
|
}); |
||||||
|
if (!writer.isOpened()) { |
||||||
|
cerr << "Failed to open VideoWriter" << endl; |
||||||
|
return 1; |
||||||
|
} |
||||||
|
cout << "VideoWriter backend = " << writer.getBackendName() << endl; |
||||||
|
actual_accel = static_cast<VideoAccelerationType>(static_cast<int>(writer.get(CAP_PROP_HW_ACCELERATION))); |
||||||
|
for (size_t i = 0; i < sizeof(acceleration_strings) / sizeof(acceleration_strings[0]); i++) { |
||||||
|
if (actual_accel == acceleration_strings[i].acceleration) { |
||||||
|
cout << "VideoWriter acceleration = " << acceleration_strings[i].str << endl; |
||||||
|
cout << "VideoWriter acceleration device = " << (int)writer.get(CAP_PROP_HW_DEVICE) << endl; |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
cout << "\nStarting frame loop. Press ESC to exit\n"; |
||||||
|
|
||||||
|
FPSCounter fps_counter(0.5); // print FPS every 0.5 seconds
|
||||||
|
|
||||||
|
UMat frame, frame2, frame3; |
||||||
|
|
||||||
|
for (;;) |
||||||
|
{ |
||||||
|
capture.read(frame); |
||||||
|
if (frame.empty()) { |
||||||
|
cout << "End of stream" << endl; |
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
if (out_w && out_h) { |
||||||
|
cv::resize(frame, frame2, cv::Size(out_w, out_h)); |
||||||
|
//cv::cvtColor(frame, outframe, COLOR_BGRA2RGBA);
|
||||||
|
} |
||||||
|
else { |
||||||
|
frame2 = frame; |
||||||
|
} |
||||||
|
|
||||||
|
if (bitwise_not) { |
||||||
|
cv::bitwise_not(frame2, frame3); |
||||||
|
} |
||||||
|
else { |
||||||
|
frame3 = frame2; |
||||||
|
} |
||||||
|
|
||||||
|
if (writer.isOpened()) { |
||||||
|
writer.write(frame3); |
||||||
|
} |
||||||
|
|
||||||
|
if (outfile.empty()) { |
||||||
|
imshow("output", frame3); |
||||||
|
char key = (char) waitKey(1); |
||||||
|
if (key == 27) |
||||||
|
break; |
||||||
|
else if (key == 'm') { |
||||||
|
ocl::setUseOpenCL(!cv::ocl::useOpenCL()); |
||||||
|
cout << "Switched to " << (ocl::useOpenCL() ? "OpenCL enabled" : "CPU") << " mode\n"; |
||||||
|
} |
||||||
|
} |
||||||
|
fps_counter.NewFrame(); |
||||||
|
} |
||||||
|
|
||||||
|
return EXIT_SUCCESS; |
||||||
|
} |
||||||
Loading…
Reference in new issue