avcodec/avcodec: Store whether AVCodec->close needs to be called

Right now all AVCodecContexts except those using frame-threaded decoding
call the codec's init function and expect its close function to be
called. In order to make sure that the close function is not called for
frame-threaded decoding ff_frame_thread_free() resets
AVCodecContext.codec (and because of this it has to free the private
AVOptions of the main AVCodecContext itself). This is not obvious and
potentially fragile. Instead add a field to AVCodecInternal that
indicates whether close should be called for this AVCodecContext.
It is always zero when using frame-threaded decoding, so that resetting
the codec is no longer necessary and has been removed.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

libavcodec/avcodec.c

@@ -135,7 +135,6 @@ static int64_t get_bit_rate(AVCodecContext *ctx)
 int attribute_align_arg avcodec_open2(AVCodecContext *avctx, const AVCodec *codec, AVDictionary **options)
 {
     int ret = 0;
-    int codec_init_ok = 0;
     AVDictionary *tmp = NULL;
     AVCodecInternal *avci;
 
@@ -320,14 +319,16 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, const AVCodec *code
     if (!HAVE_THREADS && !(codec->caps_internal & FF_CODEC_CAP_AUTO_THREADS))
         avctx->thread_count = 1;
 
-    if (   avctx->codec->init && (!(avctx->active_thread_type&FF_THREAD_FRAME)
-        || avci->frame_thread_encoder)) {
-        ret = avctx->codec->init(avctx);
-        if (ret < 0) {
-            codec_init_ok = -1;
-            goto free_and_end;
+    if (!(avctx->active_thread_type & FF_THREAD_FRAME) ||
+        avci->frame_thread_encoder) {
+        if (avctx->codec->init) {
+            ret = avctx->codec->init(avctx);
+            if (ret < 0) {
+                avci->needs_close = avctx->codec->caps_internal & FF_CODEC_CAP_INIT_CLEANUP;
+                goto free_and_end;
+            }
         }
-        codec_init_ok = 1;
+        avci->needs_close = 1;
     }
 
     ret=0;
@@ -378,9 +379,7 @@ end:
 
     return ret;
 free_and_end:
-    if (avctx->codec && avctx->codec->close &&
-        (codec_init_ok > 0 || (codec_init_ok < 0 &&
-         avctx->codec->caps_internal & FF_CODEC_CAP_INIT_CLEANUP)))
+    if (avci->needs_close && avctx->codec->close)
         avctx->codec->close(avctx);
 
     if (CONFIG_FRAME_THREAD_ENCODER && avci->frame_thread_encoder)
@@ -502,7 +501,7 @@ av_cold int avcodec_close(AVCodecContext *avctx)
         }
         if (HAVE_THREADS && avci->thread_ctx)
             ff_thread_free(avctx);
-        if (avctx->codec && avctx->codec->close)
+        if (avci->needs_close && avctx->codec->close)
             avctx->codec->close(avctx);
         avci->byte_buffer_size = 0;
         av_freep(&avci->byte_buffer);

libavcodec/internal.h

@@ -160,6 +160,12 @@ typedef struct AVCodecInternal {
 
     EncodeSimpleContext es;
 
+    /**
+     * If this is set, then AVCodec->close (if existing) needs to be called
+     * for the parent AVCodecContext.
+     */
+    int needs_close;
+
     /**
      * Number of audio samples to skip at the start of the next decoded frame
      */

libavcodec/pthread_frame.c

@@ -801,10 +801,6 @@ void ff_frame_thread_free(AVCodecContext *avctx, int thread_count)
     free_pthread(fctx, thread_ctx_offsets);
 
     av_freep(&avctx->internal->thread_ctx);
-
-    if (avctx->priv_data && avctx->codec && avctx->codec->priv_class)
-        av_opt_free(avctx->priv_data);
-    avctx->codec = NULL;
 }
 
 static av_cold int init_thread(PerThreadContext *p, int *threads_to_free,