avcodec/videotoolbox: fix NULL pointer dereference

In the code path of av_videotoolbox_default_init/init2(),
avctx->internal->hwaccel_priv_data is NULL and passed to
decoder_cb.decompressionOutputRefCon. Then it will be dereferenced
inside videotoolbox_decoder_callback().

Delay videotoolbox_star() until ff_videotoolbox_common_init() to
fix the bug.

Signed-off-by: Zhao Zhili <zhilizhao@tencent.com>
release/6.0
Zhao Zhili 2 years ago
parent b8fca1f1f5
commit 79e9bdf749
  1. 5
      libavcodec/videotoolbox.c

@ -1181,9 +1181,8 @@ int ff_videotoolbox_common_init(AVCodecContext *avctx)
vtctx->logctx = avctx;
// Old API - do nothing.
if (avctx->hwaccel_context)
return 0;
return videotoolbox_start(avctx);
if (!avctx->hw_frames_ctx && !avctx->hw_device_ctx) {
av_log(avctx, AV_LOG_ERROR,
@ -1404,7 +1403,7 @@ int av_videotoolbox_default_init2(AVCodecContext *avctx, AVVideotoolboxContext *
avctx->hwaccel_context = vtctx ?: av_videotoolbox_alloc_context_with_pix_fmt(pix_fmt, full_range);
if (!avctx->hwaccel_context)
return AVERROR(ENOMEM);
return videotoolbox_start(avctx);
return 0;
}
void av_videotoolbox_default_free(AVCodecContext *avctx)

Loading…
Cancel
Save