diff --git a/libavcodec/vda.c b/libavcodec/vda.c
index 61cb81e0ff..55012ceff5 100644
--- a/libavcodec/vda.c
+++ b/libavcodec/vda.c
@@ -168,6 +168,9 @@ int ff_vda_create_decoder(struct vda_context *vda_ctx,
     CFMutableDictionaryRef io_surface_properties;
     CFNumberRef cv_pix_fmt;
 
+    vda_ctx->bitstream = NULL;
+    vda_ctx->ref_size = 0;
+
     if (av_lockmgr_register(vda_lock_operation))
         return -1;
 
@@ -239,6 +242,9 @@ int ff_vda_destroy_decoder(struct vda_context *vda_ctx)
     if (vda_ctx->queue_mutex != NULL)
         vda_lock_operation(&vda_ctx->queue_mutex, AV_LOCK_DESTROY);
 
+    if (vda_ctx->bitstream)
+        av_freep(&vda_ctx->bitstream);
+
     if (kVDADecoderNoErr != status)
         return status;
 
diff --git a/libavcodec/vda.h b/libavcodec/vda.h
index 518bb9bfd6..ffdbea6f1d 100644
--- a/libavcodec/vda.h
+++ b/libavcodec/vda.h
@@ -129,6 +129,30 @@ struct vda_context {
     * - decoding: Set/Unset by user.
     */
     OSType              cv_pix_fmt_type;
+
+    /**
+    * The current bitstream buffer.
+    *
+    * - encoding: unused
+    * - decoding: Set/Unset by libavcodec.
+    */
+    uint8_t             *bitstream;
+
+    /**
+    * The current size of the bitstream.
+    *
+    * - encoding: unused
+    * - decoding: Set/Unset by libavcodec.
+    */
+    int                 bitstream_size;
+
+    /**
+    * The reference size used for fast reallocation.
+    *
+    * - encoding: unused
+    * - decoding: Set/Unset by libavcodec.
+    */
+    int                 ref_size;
 };
 
 /** Creates the video decoder. */
diff --git a/libavcodec/vda_h264.c b/libavcodec/vda_h264.c
index f4791a168c..3fa5614eac 100644
--- a/libavcodec/vda_h264.c
+++ b/libavcodec/vda_h264.c
@@ -22,25 +22,16 @@
 
 #include "vda_internal.h"
 
-/* This structure is used to store the bitstream of the current frame. */
-struct vda_picture_context {
-    uint8_t         *bitstream;
-    int              bitstream_size;
-};
-
 static int start_frame(AVCodecContext *avctx,
                        av_unused const uint8_t *buffer,
                        av_unused uint32_t size)
 {
-    const H264Context *h = avctx->priv_data;
     struct vda_context *vda_ctx = avctx->hwaccel_context;
-    struct vda_picture_context *pic_ctx = h->s.current_picture_ptr->f.hwaccel_picture_private;
 
     if (!vda_ctx->decoder)
         return -1;
 
-    pic_ctx->bitstream = NULL;
-    pic_ctx->bitstream_size = 0;
+    vda_ctx->bitstream_size = 0;
 
     return 0;
 }
@@ -49,24 +40,22 @@ static int decode_slice(AVCodecContext *avctx,
                         const uint8_t *buffer,
                         uint32_t size)
 {
-    H264Context *h = avctx->priv_data;
     struct vda_context *vda_ctx = avctx->hwaccel_context;
-    struct vda_picture_context *pic_ctx = h->s.current_picture_ptr->f.hwaccel_picture_private;
     void *tmp;
 
     if (!vda_ctx->decoder)
         return -1;
 
-    tmp = av_realloc(pic_ctx->bitstream, pic_ctx->bitstream_size+size+4);
+    tmp = av_fast_realloc(vda_ctx->bitstream, &vda_ctx->ref_size, vda_ctx->bitstream_size+size+4);
     if (!tmp)
         return AVERROR(ENOMEM);
 
-    pic_ctx->bitstream = tmp;
+    vda_ctx->bitstream = tmp;
 
-    AV_WB32(pic_ctx->bitstream+pic_ctx->bitstream_size, size);
-    memcpy(pic_ctx->bitstream+pic_ctx->bitstream_size+4, buffer, size);
+    AV_WB32(vda_ctx->bitstream+vda_ctx->bitstream_size, size);
+    memcpy(vda_ctx->bitstream+vda_ctx->bitstream_size+4, buffer, size);
 
-    pic_ctx->bitstream_size += size + 4;
+    vda_ctx->bitstream_size += size + 4;
 
     return 0;
 }
@@ -75,22 +64,19 @@ static int end_frame(AVCodecContext *avctx)
 {
     H264Context *h = avctx->priv_data;
     struct vda_context *vda_ctx = avctx->hwaccel_context;
-    struct vda_picture_context *pic_ctx = h->s.current_picture_ptr->f.hwaccel_picture_private;
     AVFrame *frame = (AVFrame*)h->s.current_picture_ptr;
     int status;
 
-    if (!vda_ctx->decoder || !pic_ctx->bitstream)
+    if (!vda_ctx->decoder || !vda_ctx->bitstream)
         return -1;
 
-    status = ff_vda_decoder_decode(vda_ctx, pic_ctx->bitstream,
-                                   pic_ctx->bitstream_size,
+    status = ff_vda_decoder_decode(vda_ctx, vda_ctx->bitstream,
+                                   vda_ctx->bitstream_size,
                                    frame->reordered_opaque);
 
     if (status)
         av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
 
-    av_freep(&pic_ctx->bitstream);
-
     return status;
 }
 
@@ -103,5 +89,5 @@ AVHWAccel ff_h264_vda_hwaccel = {
     .start_frame    = start_frame,
     .decode_slice   = decode_slice,
     .end_frame      = end_frame,
-    .priv_data_size = sizeof(struct vda_picture_context),
+    .priv_data_size = 0,
 };