diff --git a/libavcodec/h263dec.c b/libavcodec/h263dec.c
index eea3d13c77..d90de7dd47 100644
--- a/libavcodec/h263dec.c
+++ b/libavcodec/h263dec.c
@@ -185,7 +185,7 @@ static int decode_slice(MpegEncContext *s)
         const int qscale = s->qscale;
 
         if (CONFIG_MPEG4_DECODER && s->codec_id == AV_CODEC_ID_MPEG4)
-            if ((ret = ff_mpeg4_decode_partitions(s)) < 0)
+            if ((ret = ff_mpeg4_decode_partitions(s->avctx->priv_data)) < 0)
                 return ret;
 
         /* restore variables which were modified */
diff --git a/libavcodec/mpeg4video.h b/libavcodec/mpeg4video.h
index eacf0e0963..bda44e6453 100644
--- a/libavcodec/mpeg4video.h
+++ b/libavcodec/mpeg4video.h
@@ -65,6 +65,7 @@ typedef struct Mpeg4DecContext {
     ///< number of bits to represent the fractional part of time
     int time_increment_bits;
     int shape;
+    int vol_sprite_usage;
 } Mpeg4DecContext;
 
 /* dc encoding for mpeg4 */
@@ -109,7 +110,7 @@ void ff_mpeg4_stuffing(PutBitContext *pbc);
 void ff_mpeg4_init_partitions(MpegEncContext *s);
 void ff_mpeg4_merge_partitions(MpegEncContext *s);
 void ff_clean_mpeg4_qscales(MpegEncContext *s);
-int ff_mpeg4_decode_partitions(MpegEncContext *s);
+int ff_mpeg4_decode_partitions(Mpeg4DecContext *ctx);
 int ff_mpeg4_get_video_packet_prefix_length(MpegEncContext *s);
 int ff_mpeg4_decode_video_packet_header(Mpeg4DecContext *ctx);
 void ff_mpeg4_init_direct_mv(MpegEncContext *s);
diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c
index 38d3d2af57..b6aeecc5c6 100644
--- a/libavcodec/mpeg4videodec.c
+++ b/libavcodec/mpeg4videodec.c
@@ -442,7 +442,7 @@ int ff_mpeg4_decode_video_packet_header(Mpeg4DecContext *ctx)
             skip_bits(&s->gb, 3); /* intra dc vlc threshold */
             // FIXME don't just ignore everything
             if (s->pict_type == AV_PICTURE_TYPE_S &&
-                s->vol_sprite_usage == GMC_SPRITE) {
+                ctx->vol_sprite_usage == GMC_SPRITE) {
                 if (mpeg4_decode_sprite_trajectory(s, &s->gb) < 0)
                     return AVERROR_INVALIDDATA;
                 av_log(s->avctx, AV_LOG_ERROR, "untested\n");
@@ -573,8 +573,9 @@ static inline int mpeg4_decode_dc(MpegEncContext *s, int n, int *dir_ptr)
  * Decode first partition.
  * @return number of MBs decoded or <0 if an error occurred
  */
-static int mpeg4_decode_partition_a(MpegEncContext *s)
+static int mpeg4_decode_partition_a(Mpeg4DecContext *ctx)
 {
+    MpegEncContext *s = &ctx->m;
     int mb_num = 0;
     static const int8_t quant_tab[4] = { -1, -2, 1, 2 };
 
@@ -644,7 +645,7 @@ try_again:
                 if (bits & 0x10000) {
                     /* skip mb */
                     if (s->pict_type == AV_PICTURE_TYPE_S &&
-                        s->vol_sprite_usage == GMC_SPRITE) {
+                        ctx->vol_sprite_usage == GMC_SPRITE) {
                         s->current_picture.mb_type[xy] = MB_TYPE_SKIP  |
                                                          MB_TYPE_16x16 |
                                                          MB_TYPE_GMC   |
@@ -700,7 +701,7 @@ try_again:
                         ff_clean_intra_table_entries(s);
 
                     if (s->pict_type == AV_PICTURE_TYPE_S &&
-                        s->vol_sprite_usage == GMC_SPRITE &&
+                        ctx->vol_sprite_usage == GMC_SPRITE &&
                         (cbpc & 16) == 0)
                         s->mcsel = get_bits1(&s->gb);
                     else
@@ -859,13 +860,14 @@ static int mpeg4_decode_partition_b(MpegEncContext *s, int mb_count)
  * Decode the first and second partition.
  * @return <0 if error (and sets error type in the error_status_table)
  */
-int ff_mpeg4_decode_partitions(MpegEncContext *s)
+int ff_mpeg4_decode_partitions(Mpeg4DecContext *ctx)
 {
+    MpegEncContext *s = &ctx->m;
     int mb_num;
     const int part_a_error = s->pict_type == AV_PICTURE_TYPE_I ? (ER_DC_ERROR | ER_MV_ERROR) : ER_MV_ERROR;
     const int part_a_end   = s->pict_type == AV_PICTURE_TYPE_I ? (ER_DC_END   | ER_MV_END)   : ER_MV_END;
 
-    mb_num = mpeg4_decode_partition_a(s);
+    mb_num = mpeg4_decode_partition_a(ctx);
     if (mb_num < 0) {
         ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y,
                         s->mb_x, s->mb_y, part_a_error);
@@ -1167,6 +1169,7 @@ not_coded:
  */
 static int mpeg4_decode_partitioned_mb(MpegEncContext *s, int16_t block[6][64])
 {
+    Mpeg4DecContext *ctx = (Mpeg4DecContext *)s;
     int cbp, mb_type;
     const int xy = s->mb_x + s->mb_y * s->mb_stride;
 
@@ -1194,7 +1197,7 @@ static int mpeg4_decode_partitioned_mb(MpegEncContext *s, int16_t block[6][64])
             s->mv_dir  = MV_DIR_FORWARD;
             s->mv_type = MV_TYPE_16X16;
             if (s->pict_type == AV_PICTURE_TYPE_S
-                && s->vol_sprite_usage == GMC_SPRITE) {
+                && ctx->vol_sprite_usage == GMC_SPRITE) {
                 s->mcsel      = 1;
                 s->mb_skipped = 0;
             } else {
@@ -1251,6 +1254,7 @@ static int mpeg4_decode_partitioned_mb(MpegEncContext *s, int16_t block[6][64])
 
 static int mpeg4_decode_mb(MpegEncContext *s, int16_t block[6][64])
 {
+    Mpeg4DecContext *ctx = (Mpeg4DecContext *)s;
     int cbpc, cbpy, i, cbp, pred_x, pred_y, mx, my, dquant;
     int16_t *mot_val;
     static int8_t quant_tab[4] = { -1, -2, 1, 2 };
@@ -1269,7 +1273,7 @@ static int mpeg4_decode_mb(MpegEncContext *s, int16_t block[6][64])
                 s->mv_dir  = MV_DIR_FORWARD;
                 s->mv_type = MV_TYPE_16X16;
                 if (s->pict_type == AV_PICTURE_TYPE_S &&
-                    s->vol_sprite_usage == GMC_SPRITE) {
+                    ctx->vol_sprite_usage == GMC_SPRITE) {
                     s->current_picture.mb_type[xy] = MB_TYPE_SKIP  |
                                                      MB_TYPE_GMC   |
                                                      MB_TYPE_16x16 |
@@ -1304,7 +1308,7 @@ static int mpeg4_decode_mb(MpegEncContext *s, int16_t block[6][64])
             goto intra;
 
         if (s->pict_type == AV_PICTURE_TYPE_S &&
-            s->vol_sprite_usage == GMC_SPRITE && (cbpc & 16) == 0)
+            ctx->vol_sprite_usage == GMC_SPRITE && (cbpc & 16) == 0)
             s->mcsel = get_bits1(&s->gb);
         else
             s->mcsel = 0;
@@ -1756,15 +1760,15 @@ static int decode_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
             av_log(s->avctx, AV_LOG_INFO,           /* OBMC Disable */
                    "MPEG4 OBMC not supported (very likely buggy encoder)\n");
         if (vo_ver_id == 1)
-            s->vol_sprite_usage = get_bits1(gb);    /* vol_sprite_usage */
+            ctx->vol_sprite_usage = get_bits1(gb);    /* vol_sprite_usage */
         else
-            s->vol_sprite_usage = get_bits(gb, 2);  /* vol_sprite_usage */
+            ctx->vol_sprite_usage = get_bits(gb, 2);  /* vol_sprite_usage */
 
-        if (s->vol_sprite_usage == STATIC_SPRITE)
+        if (ctx->vol_sprite_usage == STATIC_SPRITE)
             av_log(s->avctx, AV_LOG_ERROR, "Static Sprites not supported\n");
-        if (s->vol_sprite_usage == STATIC_SPRITE ||
-            s->vol_sprite_usage == GMC_SPRITE) {
-            if (s->vol_sprite_usage == STATIC_SPRITE) {
+        if (ctx->vol_sprite_usage == STATIC_SPRITE ||
+            ctx->vol_sprite_usage == GMC_SPRITE) {
+            if (ctx->vol_sprite_usage == STATIC_SPRITE) {
                 s->sprite_width = get_bits(gb, 13);
                 skip_bits1(gb); /* marker */
                 s->sprite_height = get_bits(gb, 13);
@@ -1784,7 +1788,7 @@ static int decode_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
             }
             s->sprite_warping_accuracy  = get_bits(gb, 2);
             s->sprite_brightness_change = get_bits1(gb);
-            if (s->vol_sprite_usage == STATIC_SPRITE)
+            if (ctx->vol_sprite_usage == STATIC_SPRITE)
                 s->low_latency_sprite = get_bits1(gb);
         }
         // FIXME sadct disable bit if verid!=1 && shape not rect
@@ -2075,7 +2079,7 @@ static int decode_vop_header(Mpeg4DecContext *ctx, GetBitContext *gb)
              ctx->time_increment_bits++) {
             if (s->pict_type == AV_PICTURE_TYPE_P ||
                 (s->pict_type == AV_PICTURE_TYPE_S &&
-                 s->vol_sprite_usage == GMC_SPRITE)) {
+                 ctx->vol_sprite_usage == GMC_SPRITE)) {
                 if ((show_bits(gb, ctx->time_increment_bits + 6) & 0x37) == 0x30)
                     break;
             } else if ((show_bits(gb, ctx->time_increment_bits + 5) & 0x1F) == 0x18)
@@ -2141,7 +2145,7 @@ static int decode_vop_header(Mpeg4DecContext *ctx, GetBitContext *gb)
     if (ctx->shape != BIN_ONLY_SHAPE &&
                     (s->pict_type == AV_PICTURE_TYPE_P ||
                      (s->pict_type == AV_PICTURE_TYPE_S &&
-                      s->vol_sprite_usage == GMC_SPRITE))) {
+                      ctx->vol_sprite_usage == GMC_SPRITE))) {
         /* rounding type for motion estimation */
         s->no_rounding = get_bits1(gb);
     } else {
@@ -2150,7 +2154,7 @@ static int decode_vop_header(Mpeg4DecContext *ctx, GetBitContext *gb)
     // FIXME reduced res stuff
 
     if (ctx->shape != RECT_SHAPE) {
-        if (s->vol_sprite_usage != 1 || s->pict_type != AV_PICTURE_TYPE_I) {
+        if (ctx->vol_sprite_usage != 1 || s->pict_type != AV_PICTURE_TYPE_I) {
             skip_bits(gb, 13);  /* width */
             skip_bits1(gb);     /* marker */
             skip_bits(gb, 13);  /* height */
@@ -2195,14 +2199,14 @@ static int decode_vop_header(Mpeg4DecContext *ctx, GetBitContext *gb)
     }
 
     if (s->pict_type == AV_PICTURE_TYPE_S &&
-        (s->vol_sprite_usage == STATIC_SPRITE ||
-         s->vol_sprite_usage == GMC_SPRITE)) {
+        (ctx->vol_sprite_usage == STATIC_SPRITE ||
+         ctx->vol_sprite_usage == GMC_SPRITE)) {
         if (mpeg4_decode_sprite_trajectory(s, gb) < 0)
             return AVERROR_INVALIDDATA;
         if (s->sprite_brightness_change)
             av_log(s->avctx, AV_LOG_ERROR,
                    "sprite_brightness_change not supported\n");
-        if (s->vol_sprite_usage == STATIC_SPRITE)
+        if (ctx->vol_sprite_usage == STATIC_SPRITE)
             av_log(s->avctx, AV_LOG_ERROR, "static sprite not supported\n");
     }
 
diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index dc8f527ece..8154600726 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -579,7 +579,6 @@ typedef struct MpegEncContext {
     uint16_t pb_time;               ///< time distance between the last b and p,s,i frame
     uint16_t pp_field_time;
     uint16_t pb_field_time;         ///< like above, just for interlaced
-    int vol_sprite_usage;
     int sprite_width;
     int sprite_height;
     int sprite_left;
diff --git a/libavcodec/vaapi_mpeg4.c b/libavcodec/vaapi_mpeg4.c
index a0f2966621..7973a6491b 100644
--- a/libavcodec/vaapi_mpeg4.c
+++ b/libavcodec/vaapi_mpeg4.c
@@ -22,6 +22,7 @@
 
 #include "vaapi_internal.h"
 #include "h263.h"
+#include "mpeg4video.h"
 
 /** Reconstruct bitstream intra_dc_vlc_thr */
 static int mpeg4_get_intra_dc_vlc_thr(MpegEncContext *s)
@@ -41,7 +42,8 @@ static int mpeg4_get_intra_dc_vlc_thr(MpegEncContext *s)
 
 static int vaapi_mpeg4_start_frame(AVCodecContext *avctx, av_unused const uint8_t *buffer, av_unused uint32_t size)
 {
-    MpegEncContext * const s = avctx->priv_data;
+    Mpeg4DecContext *ctx = avctx->priv_data;
+    MpegEncContext * const s = &ctx->m;
     struct vaapi_context * const vactx = avctx->hwaccel_context;
     VAPictureParameterBufferMPEG4 *pic_param;
     VAIQMatrixBufferMPEG4 *iq_matrix;
@@ -64,7 +66,7 @@ static int vaapi_mpeg4_start_frame(AVCodecContext *avctx, av_unused const uint8_
     pic_param->vol_fields.bits.chroma_format            = CHROMA_420;
     pic_param->vol_fields.bits.interlaced               = !s->progressive_sequence;
     pic_param->vol_fields.bits.obmc_disable             = 1;
-    pic_param->vol_fields.bits.sprite_enable            = s->vol_sprite_usage;
+    pic_param->vol_fields.bits.sprite_enable            = ctx->vol_sprite_usage;
     pic_param->vol_fields.bits.sprite_warping_accuracy  = s->sprite_warping_accuracy;
     pic_param->vol_fields.bits.quant_type               = s->mpeg_quant;
     pic_param->vol_fields.bits.quarter_sample           = s->quarter_sample;