proresenc: Properly account for alpha plane

The packet buffer allocation considers the alpha channel as DCT-coded,
while it is actually run-coded and thus requires a larger buffer.

CC: libav-stable@libav.org

Signed-off-by: Diego Biurrun <diego@biurrun.de>
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

libavcodec/proresenc.c

@@ -1246,8 +1246,6 @@ static av_cold int encode_init(AVCodecContext *avctx)
         ctx->bits_per_mb = ls * 8;
         if (ctx->chroma_factor == CFACTOR_Y444)
             ctx->bits_per_mb += ls * 4;
-        if (ctx->num_planes == 4)
-            ctx->bits_per_mb += ls * 4;
     }
 
     ctx->frame_size_upper_bound = ctx->pictures_per_frame *
@@ -1256,6 +1254,14 @@ static av_cold int encode_init(AVCodecContext *avctx)
                                    (mps * ctx->bits_per_mb) / 8)
                                   + 200;
 
+    if (ctx->alpha_bits) {
+         // The alpha plane is run-coded and might exceed the bit budget.
+         ctx->frame_size_upper_bound += ctx->pictures_per_frame *
+                                        ctx->slices_per_picture *
+         /* num pixels per slice */     (ctx->mbs_per_slice * 256 *
+         /* bits per pixel */            (1 + ctx->alpha_bits + 1) + 7 >> 3);
+    }
+
     avctx->codec_tag   = ctx->profile_info->tag;
 
     av_log(avctx, AV_LOG_DEBUG,