diff --git a/libavcodec/pthread_frame.c b/libavcodec/pthread_frame.c
index 64121f5a9a..601f170447 100644
--- a/libavcodec/pthread_frame.c
+++ b/libavcodec/pthread_frame.c
@@ -310,7 +310,6 @@ static int update_context_from_thread(AVCodecContext *dst, AVCodecContext *src,
     }
 
     if (for_user) {
-        dst->delay       = src->thread_count - 1;
 #if FF_API_CODED_FRAME
 FF_DISABLE_DEPRECATION_WARNINGS
         dst->coded_frame = src->coded_frame;
@@ -790,6 +789,9 @@ int ff_frame_thread_init(AVCodecContext *avctx)
     fctx->async_lock = 1;
     fctx->delaying = 1;
 
+    if (codec->type == AVMEDIA_TYPE_VIDEO)
+        avctx->delay = src->thread_count - 1;
+
     for (i = 0; i < thread_count; i++) {
         AVCodecContext *copy = av_malloc(sizeof(AVCodecContext));
         PerThreadContext *p  = &fctx->threads[i];
@@ -827,6 +829,8 @@ int ff_frame_thread_init(AVCodecContext *avctx)
         copy->internal->thread_ctx = p;
         copy->internal->last_pkt_props = &p->avpkt;
 
+        copy->delay = avctx->delay;
+
         if (codec->priv_data_size) {
             copy->priv_data = av_mallocz(codec->priv_data_size);
             if (!copy->priv_data) {