diff --git a/libavcodec/error_resilience.c b/libavcodec/error_resilience.c
index b1ca2d9919..0120109448 100644
--- a/libavcodec/error_resilience.c
+++ b/libavcodec/error_resilience.c
@@ -30,6 +30,7 @@
 #include "libavutil/internal.h"
 #include "avcodec.h"
 #include "error_resilience.h"
+#include "me_cmp.h"
 #include "mpegutils.h"
 #include "mpegvideo.h"
 #include "rectangle.h"
@@ -713,11 +714,11 @@ FF_ENABLE_DEPRECATION_WARNINGS
                 } else {
                     ff_thread_await_progress(s->last_pic.tf, mb_y, 0);
                 }
-                is_intra_likely += s->mecc->sad[0](NULL, last_mb_ptr, mb_ptr,
-                                                   linesize[0], 16);
-                is_intra_likely -= s->mecc->sad[0](NULL, last_mb_ptr,
-                                                   last_mb_ptr + linesize[0] * 16,
-                                                   linesize[0], 16);
+                is_intra_likely += s->mecc.sad[0](NULL, last_mb_ptr, mb_ptr,
+                                                  linesize[0], 16);
+                is_intra_likely -= s->mecc.sad[0](NULL, last_mb_ptr,
+                                                  last_mb_ptr + linesize[0] * 16,
+                                                  linesize[0], 16);
             } else {
                 if (IS_INTRA(s->cur_pic.mb_type[mb_xy]))
                    is_intra_likely++;
@@ -734,6 +735,11 @@ void ff_er_frame_start(ERContext *s)
     if (!s->avctx->error_concealment)
         return;
 
+    if (!s->mecc_inited) {
+        ff_me_cmp_init(&s->mecc, s->avctx);
+        s->mecc_inited = 1;
+    }
+
     memset(s->error_status_table, ER_MB_ERROR | VP_START | ER_MB_END,
            s->mb_stride * s->mb_height * sizeof(uint8_t));
     s->error_count    = 3 * s->mb_num;
diff --git a/libavcodec/error_resilience.h b/libavcodec/error_resilience.h
index 7b9ec1918e..611b529311 100644
--- a/libavcodec/error_resilience.h
+++ b/libavcodec/error_resilience.h
@@ -52,7 +52,8 @@ typedef struct ERPicture {
 
 typedef struct ERContext {
     AVCodecContext *avctx;
-    MECmpContext *mecc;
+    MECmpContext mecc;
+    int mecc_inited;
 
     int *mb_index2xy;
     int mb_num;
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 684c78fcdc..f9667517d7 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -490,7 +490,6 @@ int ff_h264_context_init(H264Context *h)
     if (CONFIG_ERROR_RESILIENCE) {
         /* init ER */
         er->avctx          = h->avctx;
-        er->mecc           = &h->mecc;
         er->decode_mb      = h264_er_decode_mb;
         er->opaque         = h;
         er->quarter_sample = 1;
@@ -619,8 +618,6 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx)
     h->dequant_coeff_pps = -1;
 
     /* needed so that IDCT permutation is known early */
-    if (CONFIG_ERROR_RESILIENCE)
-        ff_me_cmp_init(&h->mecc, h->avctx);
     ff_videodsp_init(&h->vdsp, 8);
 
     memset(h->pps.scaling_matrix4, 16, 6 * 16 * sizeof(uint8_t));
@@ -1231,8 +1228,6 @@ int ff_h264_set_parameter_from_sps(H264Context *h)
             ff_h264qpel_init(&h->h264qpel, h->sps.bit_depth_luma);
             ff_h264_pred_init(&h->hpc, h->avctx->codec_id, h->sps.bit_depth_luma,
                               h->sps.chroma_format_idc);
-            if (CONFIG_ERROR_RESILIENCE)
-                ff_me_cmp_init(&h->mecc, h->avctx);
             ff_videodsp_init(&h->vdsp, h->sps.bit_depth_luma);
         } else {
             av_log(h->avctx, AV_LOG_ERROR, "Unsupported bit depth %d\n",
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 047937aea0..fa6e19ebc5 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -36,7 +36,6 @@
 #include "h264dsp.h"
 #include "h264pred.h"
 #include "h264qpel.h"
-#include "me_cmp.h"
 #include "mpegutils.h"
 #include "parser.h"
 #include "qpeldsp.h"
@@ -302,7 +301,6 @@ typedef struct H264Picture {
  */
 typedef struct H264Context {
     AVCodecContext *avctx;
-    MECmpContext mecc;
     VideoDSPContext vdsp;
     H264DSPContext h264dsp;
     H264ChromaContext h264chroma;
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index edce30f3a5..e291aab6d6 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -1112,7 +1112,6 @@ static int h264_slice_header_init(H264Context *h, int reinit)
             if (!c)
                 return AVERROR(ENOMEM);
             c->avctx             = h->avctx;
-            c->mecc              = h->mecc;
             c->vdsp              = h->vdsp;
             c->h264dsp           = h->h264dsp;
             c->h264qpel          = h->h264qpel;
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 44cf8c53bd..876ec51fc1 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -381,7 +381,6 @@ static av_cold int dct_init(MpegEncContext *s)
 {
     ff_blockdsp_init(&s->bdsp, s->avctx);
     ff_hpeldsp_init(&s->hdsp, s->avctx->flags);
-    ff_me_cmp_init(&s->mecc, s->avctx);
     ff_mpegvideodsp_init(&s->mdsp);
     ff_videodsp_init(&s->vdsp, s->avctx->bits_per_raw_sample);
 
@@ -1068,7 +1067,6 @@ static int init_er(MpegEncContext *s)
     int i;
 
     er->avctx       = s->avctx;
-    er->mecc        = &s->mecc;
 
     er->mb_index2xy = s->mb_index2xy;
     er->mb_num      = s->mb_num;