/* * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "config.h" #include #include #include #include "buffer.h" #include "buffer_internal.h" #include "common.h" #include "hwcontext.h" #include "hwcontext_internal.h" #include "hwcontext_videotoolbox.h" #include "mem.h" #include "pixfmt.h" #include "pixdesc.h" typedef struct VTFramesContext { CVPixelBufferPoolRef pool; } VTFramesContext; static const struct { uint32_t cv_fmt; bool full_range; enum AVPixelFormat pix_fmt; } cv_pix_fmts[] = { { kCVPixelFormatType_420YpCbCr8Planar, false, AV_PIX_FMT_YUV420P }, { kCVPixelFormatType_422YpCbCr8, false, AV_PIX_FMT_UYVY422 }, { kCVPixelFormatType_32BGRA, false, AV_PIX_FMT_BGRA }, #ifdef kCFCoreFoundationVersionNumber10_7 { kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, false, AV_PIX_FMT_NV12 }, { kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, true, AV_PIX_FMT_NV12 }, #endif #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P010 }, { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, true, AV_PIX_FMT_P010 }, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE { kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange, false, AV_PIX_FMT_NV16 }, { kCVPixelFormatType_422YpCbCr8BiPlanarFullRange, true, AV_PIX_FMT_NV16 }, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE { kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P210 }, { kCVPixelFormatType_422YpCbCr10BiPlanarFullRange, true, AV_PIX_FMT_P210 }, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR16BIPLANARVIDEORANGE { kCVPixelFormatType_422YpCbCr16BiPlanarVideoRange, false, AV_PIX_FMT_P216 }, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE { kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange, false, AV_PIX_FMT_NV24 }, { kCVPixelFormatType_444YpCbCr8BiPlanarFullRange, true, AV_PIX_FMT_NV24 }, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE { kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P410 }, { kCVPixelFormatType_444YpCbCr10BiPlanarFullRange, true, AV_PIX_FMT_P410 }, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE { kCVPixelFormatType_444YpCbCr16BiPlanarVideoRange, false, AV_PIX_FMT_P416 }, #endif }; static const enum AVPixelFormat supported_formats[] = { #ifdef kCFCoreFoundationVersionNumber10_7 AV_PIX_FMT_NV12, #endif AV_PIX_FMT_YUV420P, AV_PIX_FMT_UYVY422, #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE AV_PIX_FMT_P010, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE AV_PIX_FMT_NV16, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE AV_PIX_FMT_P210, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR16BIPLANARVIDEORANGE AV_PIX_FMT_P216, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE AV_PIX_FMT_NV24, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE AV_PIX_FMT_P410, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE AV_PIX_FMT_P416, #endif AV_PIX_FMT_BGRA, }; static int vt_frames_get_constraints(AVHWDeviceContext *ctx, const void *hwconfig, AVHWFramesConstraints *constraints) { int i; constraints->valid_sw_formats = av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1, sizeof(*constraints->valid_sw_formats)); if (!constraints->valid_sw_formats) return AVERROR(ENOMEM); for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) constraints->valid_sw_formats[i] = supported_formats[i]; constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] = AV_PIX_FMT_NONE; constraints->valid_hw_formats = av_malloc_array(2, sizeof(*constraints->valid_hw_formats)); if (!constraints->valid_hw_formats) return AVERROR(ENOMEM); constraints->valid_hw_formats[0] = AV_PIX_FMT_VIDEOTOOLBOX; constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE; return 0; } enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt) { int i; for (i = 0; i < FF_ARRAY_ELEMS(cv_pix_fmts); i++) { if (cv_pix_fmts[i].cv_fmt == cv_fmt) return cv_pix_fmts[i].pix_fmt; } return AV_PIX_FMT_NONE; } uint32_t av_map_videotoolbox_format_from_pixfmt(enum AVPixelFormat pix_fmt) { return av_map_videotoolbox_format_from_pixfmt2(pix_fmt, false); } uint32_t av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, bool full_range) { int i; for (i = 0; i < FF_ARRAY_ELEMS(cv_pix_fmts); i++) { if (cv_pix_fmts[i].pix_fmt == pix_fmt && cv_pix_fmts[i].full_range == full_range) return cv_pix_fmts[i].cv_fmt; } return 0; } static int vt_pool_alloc(AVHWFramesContext *ctx) { VTFramesContext *fctx = ctx->internal->priv; CVReturn err; CFNumberRef w, h, pixfmt; uint32_t cv_pixfmt; CFMutableDictionaryRef attributes, iosurface_properties; attributes = CFDictionaryCreateMutable( NULL, 2, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); cv_pixfmt = av_map_videotoolbox_format_from_pixfmt(ctx->sw_format); pixfmt = CFNumberCreate(NULL, kCFNumberSInt32Type, &cv_pixfmt); CFDictionarySetValue( attributes, kCVPixelBufferPixelFormatTypeKey, pixfmt); CFRelease(pixfmt); iosurface_properties = CFDictionaryCreateMutable( NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); CFDictionarySetValue(attributes, kCVPixelBufferIOSurfacePropertiesKey, iosurface_properties); CFRelease(iosurface_properties); w = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->width); h = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->height); CFDictionarySetValue(attributes, kCVPixelBufferWidthKey, w); CFDictionarySetValue(attributes, kCVPixelBufferHeightKey, h); CFRelease(w); CFRelease(h); err = CVPixelBufferPoolCreate( NULL, NULL, attributes, &fctx->pool); CFRelease(attributes); if (err == kCVReturnSuccess) return 0; av_log(ctx, AV_LOG_ERROR, "Error creating CVPixelBufferPool: %d\n", err); return AVERROR_EXTERNAL; } static AVBufferRef *vt_dummy_pool_alloc(void *opaque, size_t size) { return NULL; } static void vt_frames_uninit(AVHWFramesContext *ctx) { VTFramesContext *fctx = ctx->internal->priv; if (fctx->pool) { CVPixelBufferPoolRelease(fctx->pool); fctx->pool = NULL; } } static int vt_frames_init(AVHWFramesContext *ctx) { int i, ret; for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) { if (ctx->sw_format == supported_formats[i]) break; } if (i == FF_ARRAY_ELEMS(supported_formats)) { av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n", av_get_pix_fmt_name(ctx->sw_format)); return AVERROR(ENOSYS); } // create a dummy pool so av_hwframe_get_buffer doesn't EINVAL if (!ctx->pool) { ctx->internal->pool_internal = av_buffer_pool_init2(0, ctx, vt_dummy_pool_alloc, NULL); if (!ctx->internal->pool_internal) return AVERROR(ENOMEM); } ret = vt_pool_alloc(ctx); if (ret < 0) return ret; return 0; } static void videotoolbox_buffer_release(void *opaque, uint8_t *data) { CVPixelBufferRelease((CVPixelBufferRef)data); } static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame) { VTFramesContext *fctx = ctx->internal->priv; if (ctx->pool && ctx->pool->size != 0) { frame->buf[0] = av_buffer_pool_get(ctx->pool); if (!frame->buf[0]) return AVERROR(ENOMEM); } else { CVPixelBufferRef pixbuf; AVBufferRef *buf = NULL; CVReturn err; err = CVPixelBufferPoolCreatePixelBuffer( NULL, fctx->pool, &pixbuf ); if (err != kCVReturnSuccess) { av_log(ctx, AV_LOG_ERROR, "Failed to create pixel buffer from pool: %d\n", err); return AVERROR_EXTERNAL; } buf = av_buffer_create((uint8_t *)pixbuf, 1, videotoolbox_buffer_release, NULL, 0); if (!buf) { CVPixelBufferRelease(pixbuf); return AVERROR(ENOMEM); } frame->buf[0] = buf; } frame->data[3] = frame->buf[0]->data; frame->format = AV_PIX_FMT_VIDEOTOOLBOX; frame->width = ctx->width; frame->height = ctx->height; return 0; } static int vt_transfer_get_formats(AVHWFramesContext *ctx, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats) { enum AVPixelFormat *fmts = av_malloc_array(2, sizeof(*fmts)); if (!fmts) return AVERROR(ENOMEM); fmts[0] = ctx->sw_format; fmts[1] = AV_PIX_FMT_NONE; *formats = fmts; return 0; } static void vt_unmap(AVHWFramesContext *ctx, HWMapDescriptor *hwmap) { CVPixelBufferRef pixbuf = (CVPixelBufferRef)hwmap->source->data[3]; CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv); } static int vt_pixbuf_set_par(AVHWFramesContext *hwfc, CVPixelBufferRef pixbuf, const AVFrame *src) { CFMutableDictionaryRef par = NULL; CFNumberRef num = NULL, den = NULL; AVRational avpar = src->sample_aspect_ratio; if (avpar.num == 0) return 0; av_reduce(&avpar.num, &avpar.den, avpar.num, avpar.den, 0xFFFFFFFF); num = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &avpar.num); den = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &avpar.den); par = CFDictionaryCreateMutable(kCFAllocatorDefault, 2, &kCFCopyStringDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); if (!par || !num || !den) { if (par) CFRelease(par); if (num) CFRelease(num); if (den) CFRelease(den); return AVERROR(ENOMEM); } CFDictionarySetValue( par, kCVImageBufferPixelAspectRatioHorizontalSpacingKey, num); CFDictionarySetValue( par, kCVImageBufferPixelAspectRatioVerticalSpacingKey, den); CVBufferSetAttachment( pixbuf, kCVImageBufferPixelAspectRatioKey, par, kCVAttachmentMode_ShouldPropagate ); CFRelease(par); CFRelease(num); CFRelease(den); return 0; } static int vt_pixbuf_set_chromaloc(AVHWFramesContext *hwfc, CVPixelBufferRef pixbuf, const AVFrame *src) { CFStringRef loc = NULL; switch (src->chroma_location) { case AVCHROMA_LOC_LEFT: loc = kCVImageBufferChromaLocation_Left; break; case AVCHROMA_LOC_CENTER: loc = kCVImageBufferChromaLocation_Center; break; case AVCHROMA_LOC_TOP: loc = kCVImageBufferChromaLocation_Top; break; case AVCHROMA_LOC_BOTTOM: loc = kCVImageBufferChromaLocation_Bottom; break; case AVCHROMA_LOC_TOPLEFT: loc = kCVImageBufferChromaLocation_TopLeft; break; case AVCHROMA_LOC_BOTTOMLEFT: loc = kCVImageBufferChromaLocation_BottomLeft; break; } if (loc) { CVBufferSetAttachment( pixbuf, kCVImageBufferChromaLocationTopFieldKey, loc, kCVAttachmentMode_ShouldPropagate); } return 0; } static int vt_pixbuf_set_colorspace(AVHWFramesContext *hwfc, CVPixelBufferRef pixbuf, const AVFrame *src) { CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL; Float32 gamma = 0; switch (src->colorspace) { case AVCOL_SPC_BT2020_CL: case AVCOL_SPC_BT2020_NCL: #if HAVE_KCVIMAGEBUFFERYCBCRMATRIX_ITU_R_2020 colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_2020; #else colormatrix = CFSTR("ITU_R_2020"); #endif break; case AVCOL_SPC_BT470BG: case AVCOL_SPC_SMPTE170M: colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4; break; case AVCOL_SPC_BT709: colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2; break; case AVCOL_SPC_SMPTE240M: colormatrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995; break; case AVCOL_SPC_UNSPECIFIED: break; default: av_log(hwfc, AV_LOG_WARNING, "Color space %s is not supported.\n", av_color_space_name(src->colorspace)); } switch (src->color_primaries) { case AVCOL_PRI_BT2020: #if HAVE_KCVIMAGEBUFFERCOLORPRIMARIES_ITU_R_2020 colorpri = kCVImageBufferColorPrimaries_ITU_R_2020; #else colorpri = CFSTR("ITU_R_2020"); #endif break; case AVCOL_PRI_BT709: colorpri = kCVImageBufferColorPrimaries_ITU_R_709_2; break; case AVCOL_PRI_SMPTE170M: colorpri = kCVImageBufferColorPrimaries_SMPTE_C; break; case AVCOL_PRI_BT470BG: colorpri = kCVImageBufferColorPrimaries_EBU_3213; break; case AVCOL_PRI_UNSPECIFIED: break; default: av_log(hwfc, AV_LOG_WARNING, "Color primaries %s is not supported.\n", av_color_primaries_name(src->color_primaries)); } switch (src->color_trc) { case AVCOL_TRC_SMPTE2084: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_2084_PQ colortrc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ; #else colortrc = CFSTR("SMPTE_ST_2084_PQ"); #endif break; case AVCOL_TRC_BT2020_10: case AVCOL_TRC_BT2020_12: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2020 colortrc = kCVImageBufferTransferFunction_ITU_R_2020; #else colortrc = CFSTR("ITU_R_2020"); #endif break; case AVCOL_TRC_BT709: colortrc = kCVImageBufferTransferFunction_ITU_R_709_2; break; case AVCOL_TRC_SMPTE240M: colortrc = kCVImageBufferTransferFunction_SMPTE_240M_1995; break; case AVCOL_TRC_SMPTE428: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_428_1 colortrc = kCVImageBufferTransferFunction_SMPTE_ST_428_1; #else colortrc = CFSTR("SMPTE_ST_428_1"); #endif break; case AVCOL_TRC_ARIB_STD_B67: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG colortrc = kCVImageBufferTransferFunction_ITU_R_2100_HLG; #else colortrc = CFSTR("ITU_R_2100_HLG"); #endif break; case AVCOL_TRC_GAMMA22: gamma = 2.2; colortrc = kCVImageBufferTransferFunction_UseGamma; break; case AVCOL_TRC_GAMMA28: gamma = 2.8; colortrc = kCVImageBufferTransferFunction_UseGamma; break; case AVCOL_TRC_UNSPECIFIED: break; default: av_log(hwfc, AV_LOG_WARNING, "Color transfer function %s is not supported.\n", av_color_transfer_name(src->color_trc)); } if (colormatrix) { CVBufferSetAttachment( pixbuf, kCVImageBufferYCbCrMatrixKey, colormatrix, kCVAttachmentMode_ShouldPropagate); } if (colorpri) { CVBufferSetAttachment( pixbuf, kCVImageBufferColorPrimariesKey, colorpri, kCVAttachmentMode_ShouldPropagate); } if (colortrc) { CVBufferSetAttachment( pixbuf, kCVImageBufferTransferFunctionKey, colortrc, kCVAttachmentMode_ShouldPropagate); } if (gamma != 0) { CFNumberRef gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma); CVBufferSetAttachment( pixbuf, kCVImageBufferGammaLevelKey, gamma_level, kCVAttachmentMode_ShouldPropagate); CFRelease(gamma_level); } return 0; } static int vt_pixbuf_set_attachments(AVHWFramesContext *hwfc, CVPixelBufferRef pixbuf, const AVFrame *src) { int ret; ret = vt_pixbuf_set_par(hwfc, pixbuf, src); if (ret < 0) return ret; ret = vt_pixbuf_set_colorspace(hwfc, pixbuf, src); if (ret < 0) return ret; ret = vt_pixbuf_set_chromaloc(hwfc, pixbuf, src); if (ret < 0) return ret; return 0; } static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src, int flags) { CVPixelBufferRef pixbuf = (CVPixelBufferRef)src->data[3]; OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf); CVReturn err; uint32_t map_flags = 0; int ret; int i; enum AVPixelFormat format; format = av_map_videotoolbox_format_to_pixfmt(pixel_format); if (dst->format != format) { av_log(ctx, AV_LOG_ERROR, "Unsupported or mismatching pixel format: %s\n", av_fourcc2str(pixel_format)); return AVERROR_UNKNOWN; } if (CVPixelBufferGetWidth(pixbuf) != ctx->width || CVPixelBufferGetHeight(pixbuf) != ctx->height) { av_log(ctx, AV_LOG_ERROR, "Inconsistent frame dimensions.\n"); return AVERROR_UNKNOWN; } if (flags == AV_HWFRAME_MAP_READ) map_flags = kCVPixelBufferLock_ReadOnly; err = CVPixelBufferLockBaseAddress(pixbuf, map_flags); if (err != kCVReturnSuccess) { av_log(ctx, AV_LOG_ERROR, "Error locking the pixel buffer.\n"); return AVERROR_UNKNOWN; } if (CVPixelBufferIsPlanar(pixbuf)) { int planes = CVPixelBufferGetPlaneCount(pixbuf); for (i = 0; i < planes; i++) { dst->data[i] = CVPixelBufferGetBaseAddressOfPlane(pixbuf, i); dst->linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixbuf, i); } } else { dst->data[0] = CVPixelBufferGetBaseAddress(pixbuf); dst->linesize[0] = CVPixelBufferGetBytesPerRow(pixbuf); } ret = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, vt_unmap, (void *)(uintptr_t)map_flags); if (ret < 0) goto unlock; return 0; unlock: CVPixelBufferUnlockBaseAddress(pixbuf, map_flags); return ret; } static int vt_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src) { AVFrame *map; int err; if (dst->width > hwfc->width || dst->height > hwfc->height) return AVERROR(EINVAL); map = av_frame_alloc(); if (!map) return AVERROR(ENOMEM); map->format = dst->format; err = vt_map_frame(hwfc, map, src, AV_HWFRAME_MAP_READ); if (err) goto fail; map->width = dst->width; map->height = dst->height; err = av_frame_copy(dst, map); if (err) goto fail; err = 0; fail: av_frame_free(&map); return err; } static int vt_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src) { AVFrame *map; int err; if (src->width > hwfc->width || src->height > hwfc->height) return AVERROR(EINVAL); map = av_frame_alloc(); if (!map) return AVERROR(ENOMEM); map->format = src->format; err = vt_map_frame(hwfc, map, dst, AV_HWFRAME_MAP_WRITE | AV_HWFRAME_MAP_OVERWRITE); if (err) goto fail; map->width = src->width; map->height = src->height; err = av_frame_copy(map, src); if (err) goto fail; err = vt_pixbuf_set_attachments(hwfc, (CVPixelBufferRef)dst->data[3], src); if (err) goto fail; err = 0; fail: av_frame_free(&map); return err; } static int vt_device_create(AVHWDeviceContext *ctx, const char *device, AVDictionary *opts, int flags) { if (device && device[0]) { av_log(ctx, AV_LOG_ERROR, "Device selection unsupported.\n"); return AVERROR_UNKNOWN; } return 0; } const HWContextType ff_hwcontext_type_videotoolbox = { .type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX, .name = "videotoolbox", .frames_priv_size = sizeof(VTFramesContext), .device_create = vt_device_create, .frames_init = vt_frames_init, .frames_get_buffer = vt_get_buffer, .frames_get_constraints = vt_frames_get_constraints, .frames_uninit = vt_frames_uninit, .transfer_get_formats = vt_transfer_get_formats, .transfer_data_to = vt_transfer_data_to, .transfer_data_from = vt_transfer_data_from, .pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_VIDEOTOOLBOX, AV_PIX_FMT_NONE }, };