diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index c7dc5da0f9..8fa83c72b2 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -132,7 +132,7 @@ OBJS-$(CONFIG_MPEGAUDIODSP)            += mpegaudiodsp.o                \
                                           mpegaudiodsp_float.o
 OBJS-$(CONFIG_MPEGAUDIOHEADER)         += mpegaudiodecheader.o mpegaudiotabs.o
 OBJS-$(CONFIG_MPEG4AUDIO)              += mpeg4audio.o mpeg4audio_sample_rates.o
-OBJS-$(CONFIG_MPEGVIDEO)               += mpegvideo.o mpegvideodsp.o rl.o \
+OBJS-$(CONFIG_MPEGVIDEO)               += mpegvideo.o rl.o \
                                           mpegvideo_motion.o \
                                           mpegvideodata.o mpegpicture.o  \
                                           to_upper4.o
@@ -523,7 +523,7 @@ OBJS-$(CONFIG_MPEG2_CUVID_DECODER)     += cuviddec.o
 OBJS-$(CONFIG_MPEG2_MEDIACODEC_DECODER) += mediacodecdec.o
 OBJS-$(CONFIG_MPEG2_VAAPI_ENCODER)     += vaapi_encode_mpeg2.o
 OBJS-$(CONFIG_MPEG2_V4L2M2M_DECODER)   += v4l2_m2m_dec.o
-OBJS-$(CONFIG_MPEG4_DECODER)           += xvididct.o
+OBJS-$(CONFIG_MPEG4_DECODER)           += mpeg4videodsp.o xvididct.o
 OBJS-$(CONFIG_MPEG4_ENCODER)           += mpeg4videoenc.o
 OBJS-$(CONFIG_MPEG4_CUVID_DECODER)     += cuviddec.o
 OBJS-$(CONFIG_MPEG4_MEDIACODEC_DECODER) += mediacodecdec.o
diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c
index 58a8ac9027..1638664e00 100644
--- a/libavcodec/mpeg4videodec.c
+++ b/libavcodec/mpeg4videodec.c
@@ -72,7 +72,7 @@ static const int mb_type_b_map[4] = {
     MB_TYPE_L0      | MB_TYPE_16x16,
 };
 
-static void gmc1_motion(MpegEncContext *s,
+static void gmc1_motion(MpegEncContext *s, const Mpeg4DecContext *ctx,
                         uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,
                         uint8_t *const *ref_picture)
 {
@@ -110,10 +110,10 @@ static void gmc1_motion(MpegEncContext *s,
     }
 
     if ((motion_x | motion_y) & 7) {
-        s->mdsp.gmc1(dest_y, ptr, linesize, 16,
-                     motion_x & 15, motion_y & 15, 128 - s->no_rounding);
-        s->mdsp.gmc1(dest_y + 8, ptr + 8, linesize, 16,
-                     motion_x & 15, motion_y & 15, 128 - s->no_rounding);
+        ctx->mdsp.gmc1(dest_y, ptr, linesize, 16,
+                       motion_x & 15, motion_y & 15, 128 - s->no_rounding);
+        ctx->mdsp.gmc1(dest_y + 8, ptr + 8, linesize, 16,
+                       motion_x & 15, motion_y & 15, 128 - s->no_rounding);
     } else {
         int dxy;
 
@@ -153,8 +153,8 @@ static void gmc1_motion(MpegEncContext *s,
         ptr = s->sc.edge_emu_buffer;
         emu = 1;
     }
-    s->mdsp.gmc1(dest_cb, ptr, uvlinesize, 8,
-                 motion_x & 15, motion_y & 15, 128 - s->no_rounding);
+    ctx->mdsp.gmc1(dest_cb, ptr, uvlinesize, 8,
+                   motion_x & 15, motion_y & 15, 128 - s->no_rounding);
 
     ptr = ref_picture[2] + offset;
     if (emu) {
@@ -165,11 +165,11 @@ static void gmc1_motion(MpegEncContext *s,
                                  s->h_edge_pos >> 1, s->v_edge_pos >> 1);
         ptr = s->sc.edge_emu_buffer;
     }
-    s->mdsp.gmc1(dest_cr, ptr, uvlinesize, 8,
-                 motion_x & 15, motion_y & 15, 128 - s->no_rounding);
+    ctx->mdsp.gmc1(dest_cr, ptr, uvlinesize, 8,
+                   motion_x & 15, motion_y & 15, 128 - s->no_rounding);
 }
 
-static void gmc_motion(MpegEncContext *s,
+static void gmc_motion(MpegEncContext *s, const Mpeg4DecContext *ctx,
                        uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,
                        uint8_t *const *ref_picture)
 {
@@ -188,13 +188,13 @@ static void gmc_motion(MpegEncContext *s,
     oy = s->sprite_offset[0][1] + s->sprite_delta[1][0] * s->mb_x * 16 +
          s->sprite_delta[1][1] * s->mb_y * 16;
 
-    s->mdsp.gmc(dest_y, ptr, linesize, 16,
-                ox, oy,
+    ctx->mdsp.gmc(dest_y, ptr, linesize, 16,
+                  ox, oy,
                 s->sprite_delta[0][0], s->sprite_delta[0][1],
                 s->sprite_delta[1][0], s->sprite_delta[1][1],
-                a + 1, (1 << (2 * a + 1)) - s->no_rounding,
-                s->h_edge_pos, s->v_edge_pos);
-    s->mdsp.gmc(dest_y + 8, ptr, linesize, 16,
+                  a + 1, (1 << (2 * a + 1)) - s->no_rounding,
+                  s->h_edge_pos, s->v_edge_pos);
+    ctx->mdsp.gmc(dest_y + 8, ptr, linesize, 16,
                 ox + s->sprite_delta[0][0] * 8,
                 oy + s->sprite_delta[1][0] * 8,
                 s->sprite_delta[0][0], s->sprite_delta[0][1],
@@ -211,31 +211,33 @@ static void gmc_motion(MpegEncContext *s,
          s->sprite_delta[1][1] * s->mb_y * 8;
 
     ptr = ref_picture[1];
-    s->mdsp.gmc(dest_cb, ptr, uvlinesize, 8,
-                ox, oy,
+    ctx->mdsp.gmc(dest_cb, ptr, uvlinesize, 8,
+                  ox, oy,
                 s->sprite_delta[0][0], s->sprite_delta[0][1],
                 s->sprite_delta[1][0], s->sprite_delta[1][1],
-                a + 1, (1 << (2 * a + 1)) - s->no_rounding,
-                (s->h_edge_pos + 1) >> 1, (s->v_edge_pos + 1) >> 1);
+                  a + 1, (1 << (2 * a + 1)) - s->no_rounding,
+                  (s->h_edge_pos + 1) >> 1, (s->v_edge_pos + 1) >> 1);
 
     ptr = ref_picture[2];
-    s->mdsp.gmc(dest_cr, ptr, uvlinesize, 8,
-                ox, oy,
+    ctx->mdsp.gmc(dest_cr, ptr, uvlinesize, 8,
+                  ox, oy,
                 s->sprite_delta[0][0], s->sprite_delta[0][1],
                 s->sprite_delta[1][0], s->sprite_delta[1][1],
-                a + 1, (1 << (2 * a + 1)) - s->no_rounding,
-                (s->h_edge_pos + 1) >> 1, (s->v_edge_pos + 1) >> 1);
+                  a + 1, (1 << (2 * a + 1)) - s->no_rounding,
+                  (s->h_edge_pos + 1) >> 1, (s->v_edge_pos + 1) >> 1);
 }
 
 void ff_mpeg4_mcsel_motion(MpegEncContext *s,
                            uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,
                            uint8_t *const *ref_picture)
 {
+    const Mpeg4DecContext *const ctx = (Mpeg4DecContext*)s;
+
     if (s->real_sprite_warping_points == 1) {
-        gmc1_motion(s, dest_y, dest_cb, dest_cr,
+        gmc1_motion(s, ctx, dest_y, dest_cb, dest_cr,
                     ref_picture);
     } else {
-        gmc_motion(s, dest_y, dest_cb, dest_cr,
+        gmc_motion(s, ctx, dest_y, dest_cb, dest_cr,
                     ref_picture);
     }
 }
@@ -3684,6 +3686,7 @@ int ff_mpeg4_frame_end(AVCodecContext *avctx, const uint8_t *buf, int buf_size)
     return 0;
 }
 
+#if CONFIG_MPEG4_DECODER
 #if HAVE_THREADS
 static int mpeg4_update_thread_context(AVCodecContext *dst,
                                        const AVCodecContext *src)
@@ -3726,7 +3729,7 @@ static int mpeg4_update_thread_context(AVCodecContext *dst,
     memcpy(s->sprite_shift, s1->sprite_shift, sizeof(s1->sprite_shift));
     memcpy(s->sprite_traj,  s1->sprite_traj,  sizeof(s1->sprite_traj));
 
-    if (CONFIG_MPEG4_DECODER && !init && s1->xvid_build >= 0)
+    if (!init && s1->xvid_build >= 0)
         ff_xvid_idct_init(&s->m.idsp, dst);
 
     return 0;
@@ -3814,6 +3817,8 @@ static av_cold int decode_init(AVCodecContext *avctx)
 
     avctx->chroma_sample_location = AVCHROMA_LOC_LEFT;
 
+    ff_mpeg4videodsp_init(&ctx->mdsp);
+
     ff_thread_once(&init_static_once, mpeg4_init_static);
 
     return 0;
@@ -3873,3 +3878,4 @@ const FFCodec ff_mpeg4_decoder = {
                                NULL
                            },
 };
+#endif /* CONFIG_MPEG4_DECODER */
diff --git a/libavcodec/mpeg4videodec.h b/libavcodec/mpeg4videodec.h
index 8d1e121b67..302c5c38da 100644
--- a/libavcodec/mpeg4videodec.h
+++ b/libavcodec/mpeg4videodec.h
@@ -27,6 +27,7 @@
 
 #include "get_bits.h"
 #include "mpegvideo.h"
+#include "mpeg4videodsp.h"
 
 
 typedef struct Mpeg4DecContext {
@@ -76,6 +77,8 @@ typedef struct Mpeg4DecContext {
 
     int rgb;
 
+    Mpeg4VideoDSPContext mdsp;
+
     int32_t block32[12][64];
     // 0 = DCT, 1 = DPCM top to bottom scan, -1 = DPCM bottom to top scan
     int dpcm_direction;
diff --git a/libavcodec/mpegvideodsp.c b/libavcodec/mpeg4videodsp.c
similarity index 96%
rename from libavcodec/mpegvideodsp.c
rename to libavcodec/mpeg4videodsp.c
index 05893d0e01..1c5661a076 100644
--- a/libavcodec/mpegvideodsp.c
+++ b/libavcodec/mpeg4videodsp.c
@@ -19,7 +19,7 @@
 #include "config.h"
 #include "libavutil/attributes.h"
 #include "libavutil/common.h"
-#include "mpegvideodsp.h"
+#include "mpeg4videodsp.h"
 
 static void gmc1_c(uint8_t *dst, const uint8_t *src, int stride, int h,
                    int x16, int y16, int rounder)
@@ -107,14 +107,14 @@ void ff_gmc_c(uint8_t *dst, const uint8_t *src, int stride, int h, int ox, int o
     }
 }
 
-av_cold void ff_mpegvideodsp_init(MpegVideoDSPContext *c)
+av_cold void ff_mpeg4videodsp_init(Mpeg4VideoDSPContext *c)
 {
     c->gmc1 = gmc1_c;
     c->gmc  = ff_gmc_c;
 
 #if ARCH_PPC
-    ff_mpegvideodsp_init_ppc(c);
+    ff_mpeg4videodsp_init_ppc(c);
 #elif ARCH_X86
-    ff_mpegvideodsp_init_x86(c);
+    ff_mpeg4videodsp_init_x86(c);
 #endif
 }
diff --git a/libavcodec/mpegvideodsp.h b/libavcodec/mpeg4videodsp.h
similarity index 81%
rename from libavcodec/mpegvideodsp.h
rename to libavcodec/mpeg4videodsp.h
index 69e6053c68..e1ccb71ce9 100644
--- a/libavcodec/mpegvideodsp.h
+++ b/libavcodec/mpeg4videodsp.h
@@ -16,8 +16,8 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
-#ifndef AVCODEC_MPEGVIDEODSP_H
-#define AVCODEC_MPEGVIDEODSP_H
+#ifndef AVCODEC_MPEG4VIDEODSP_H
+#define AVCODEC_MPEG4VIDEODSP_H
 
 #include <stdint.h>
 
@@ -25,7 +25,7 @@ void ff_gmc_c(uint8_t *dst, const uint8_t *src, int stride, int h, int ox, int o
               int dxx, int dxy, int dyx, int dyy, int shift, int r,
               int width, int height);
 
-typedef struct MpegVideoDSPContext {
+typedef struct Mpeg4VideoDSPContext {
     /**
      * translational global motion compensation.
      */
@@ -38,10 +38,10 @@ typedef struct MpegVideoDSPContext {
                 int stride, int h, int ox, int oy,
                 int dxx, int dxy, int dyx, int dyy,
                 int shift, int r, int width, int height);
-} MpegVideoDSPContext;
+} Mpeg4VideoDSPContext;
 
-void ff_mpegvideodsp_init(MpegVideoDSPContext *c);
-void ff_mpegvideodsp_init_ppc(MpegVideoDSPContext *c);
-void ff_mpegvideodsp_init_x86(MpegVideoDSPContext *c);
+void ff_mpeg4videodsp_init(Mpeg4VideoDSPContext *c);
+void ff_mpeg4videodsp_init_ppc(Mpeg4VideoDSPContext *c);
+void ff_mpeg4videodsp_init_x86(Mpeg4VideoDSPContext *c);
 
-#endif /* AVCODEC_MPEGVIDEODSP_H */
+#endif /* AVCODEC_MPEG4VIDEODSP_H */
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 5a8ddcb5df..fbe9884b4c 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -277,7 +277,6 @@ static av_cold int dct_init(MpegEncContext *s)
     ff_blockdsp_init(&s->bdsp);
     ff_h264chroma_init(&s->h264chroma, 8); //for lowres
     ff_hpeldsp_init(&s->hdsp, s->avctx->flags);
-    ff_mpegvideodsp_init(&s->mdsp);
     ff_videodsp_init(&s->vdsp, s->avctx->bits_per_raw_sample);
 
     if (s->avctx->debug & FF_DEBUG_NOMC) {
diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index 9d5ca38eb7..929ce25dad 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -40,7 +40,6 @@
 #include "me_cmp.h"
 #include "motion_est.h"
 #include "mpegpicture.h"
-#include "mpegvideodsp.h"
 #include "mpegvideoencdsp.h"
 #include "pixblockdsp.h"
 #include "put_bits.h"
@@ -209,7 +208,6 @@ typedef struct MpegEncContext {
     HpelDSPContext hdsp;
     IDCTDSPContext idsp;
     MECmpContext mecc;
-    MpegVideoDSPContext mdsp;
     MpegvideoEncDSPContext mpvencdsp;
     PixblockDSPContext pdsp;
     QpelDSPContext qdsp;
diff --git a/libavcodec/ppc/Makefile b/libavcodec/ppc/Makefile
index 03e5b42d33..bc13d8a0ce 100644
--- a/libavcodec/ppc/Makefile
+++ b/libavcodec/ppc/Makefile
@@ -14,8 +14,7 @@ OBJS-$(CONFIG_IDCTDSP)                 += ppc/idctdsp.o
 OBJS-$(CONFIG_LLVIDDSP)                += ppc/lossless_videodsp_altivec.o
 OBJS-$(CONFIG_ME_CMP)                  += ppc/me_cmp.o
 OBJS-$(CONFIG_MPEGAUDIODSP)            += ppc/mpegaudiodsp_altivec.o
-OBJS-$(CONFIG_MPEGVIDEO)               += ppc/mpegvideo_altivec.o      \
-                                          ppc/mpegvideodsp.o
+OBJS-$(CONFIG_MPEGVIDEO)               += ppc/mpegvideo_altivec.o
 OBJS-$(CONFIG_MPEGVIDEOENC)            += ppc/mpegvideoencdsp.o
 OBJS-$(CONFIG_PIXBLOCKDSP)             += ppc/pixblockdsp.o
 OBJS-$(CONFIG_VC1DSP)                  += ppc/vc1dsp_altivec.o
@@ -26,6 +25,7 @@ OBJS-$(CONFIG_VP8DSP)                  += ppc/vp8dsp_altivec.o
 # decoders/encoders
 OBJS-$(CONFIG_HEVC_DECODER)            += ppc/hevcdsp.o
 OBJS-$(CONFIG_LLAUDDSP)                += ppc/lossless_audiodsp_altivec.o
+OBJS-$(CONFIG_MPEG4_DECODER)           += ppc/mpeg4videodsp.o
 OBJS-$(CONFIG_SVQ1_ENCODER)            += ppc/svq1enc_altivec.o
 OBJS-$(CONFIG_VORBIS_DECODER)          += ppc/vorbisdsp_altivec.o
 OBJS-$(CONFIG_VP7_DECODER)             += ppc/vp8dsp_altivec.o
diff --git a/libavcodec/ppc/mpegvideodsp.c b/libavcodec/ppc/mpeg4videodsp.c
similarity index 98%
rename from libavcodec/ppc/mpegvideodsp.c
rename to libavcodec/ppc/mpeg4videodsp.c
index 3e99e089ea..8b30af4258 100644
--- a/libavcodec/ppc/mpegvideodsp.c
+++ b/libavcodec/ppc/mpeg4videodsp.c
@@ -26,7 +26,7 @@
 #include "libavutil/ppc/cpu.h"
 #include "libavutil/ppc/util_altivec.h"
 
-#include "libavcodec/mpegvideodsp.h"
+#include "libavcodec/mpeg4videodsp.h"
 
 #if HAVE_ALTIVEC
 /* AltiVec-enhanced gmc1. ATM this code assumes stride is a multiple of 8
@@ -128,7 +128,7 @@ static void gmc1_altivec(uint8_t *dst /* align 8 */, const uint8_t *src /* align
 }
 #endif /* HAVE_ALTIVEC */
 
-av_cold void ff_mpegvideodsp_init_ppc(MpegVideoDSPContext *c)
+av_cold void ff_mpeg4videodsp_init_ppc(Mpeg4VideoDSPContext *c)
 {
 #if HAVE_ALTIVEC
     if (!PPC_ALTIVEC(av_get_cpu_flags()))
diff --git a/libavcodec/x86/Makefile b/libavcodec/x86/Makefile
index e1120b7e15..ec6adcd8b0 100644
--- a/libavcodec/x86/Makefile
+++ b/libavcodec/x86/Makefile
@@ -27,8 +27,7 @@ OBJS-$(CONFIG_LPC)                     += x86/lpc_init.o
 OBJS-$(CONFIG_MDCT15)                  += x86/mdct15_init.o
 OBJS-$(CONFIG_ME_CMP)                  += x86/me_cmp_init.o
 OBJS-$(CONFIG_MPEGAUDIODSP)            += x86/mpegaudiodsp.o
-OBJS-$(CONFIG_MPEGVIDEO)               += x86/mpegvideo.o              \
-                                          x86/mpegvideodsp.o
+OBJS-$(CONFIG_MPEGVIDEO)               += x86/mpegvideo.o
 OBJS-$(CONFIG_MPEGVIDEOENC)            += x86/mpegvideoenc.o           \
                                           x86/mpegvideoencdsp_init.o
 OBJS-$(CONFIG_PIXBLOCKDSP)             += x86/pixblockdsp_init.o
@@ -62,7 +61,7 @@ OBJS-$(CONFIG_HEVC_DECODER)            += x86/hevcdsp_init.o
 OBJS-$(CONFIG_JPEG2000_DECODER)        += x86/jpeg2000dsp_init.o
 OBJS-$(CONFIG_LSCR_DECODER)            += x86/pngdsp_init.o
 OBJS-$(CONFIG_MLP_DECODER)             += x86/mlpdsp_init.o
-OBJS-$(CONFIG_MPEG4_DECODER)           += x86/xvididct_init.o
+OBJS-$(CONFIG_MPEG4_DECODER)           += x86/mpeg4videodsp.o x86/xvididct_init.o
 OBJS-$(CONFIG_PNG_DECODER)             += x86/pngdsp_init.o
 OBJS-$(CONFIG_PRORES_DECODER)          += x86/proresdsp_init.o
 OBJS-$(CONFIG_PRORES_LGPL_DECODER)     += x86/proresdsp_init.o
diff --git a/libavcodec/x86/mpegvideodsp.c b/libavcodec/x86/mpeg4videodsp.c
similarity index 98%
rename from libavcodec/x86/mpegvideodsp.c
rename to libavcodec/x86/mpeg4videodsp.c
index ea1d941fba..6a1c6c5064 100644
--- a/libavcodec/x86/mpegvideodsp.c
+++ b/libavcodec/x86/mpeg4videodsp.c
@@ -20,7 +20,7 @@
 #include "libavutil/attributes.h"
 #include "libavutil/cpu.h"
 #include "libavutil/x86/cpu.h"
-#include "libavcodec/mpegvideodsp.h"
+#include "libavcodec/mpeg4videodsp.h"
 #include "libavcodec/videodsp.h"
 
 #if HAVE_INLINE_ASM
@@ -150,7 +150,7 @@ static void gmc_mmx(uint8_t *dst, const uint8_t *src,
 
 #endif /* HAVE_INLINE_ASM */
 
-av_cold void ff_mpegvideodsp_init_x86(MpegVideoDSPContext *c)
+av_cold void ff_mpeg4videodsp_init_x86(Mpeg4VideoDSPContext *c)
 {
 #if HAVE_INLINE_ASM
     int cpu_flags = av_get_cpu_flags();