diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index d38026174b..a09591346b 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -68,7 +68,7 @@ static void h264_er_decode_mb(void *opaque, int ref, int mv_dir, int mv_type,
 
     h->mb_x  = mb_x;
     h->mb_y  = mb_y;
-    h->mb_xy = mb_x + mb_y * h->mb_stride;
+    sl->mb_xy = mb_x + mb_y * h->mb_stride;
     memset(sl->non_zero_count_cache, 0, sizeof(sl->non_zero_count_cache));
     av_assert1(ref >= 0);
     /* FIXME: It is possible albeit uncommon that slice references
@@ -85,7 +85,7 @@ static void h264_er_decode_mb(void *opaque, int ref, int mv_dir, int mv_type,
         av_log(h->avctx, AV_LOG_DEBUG, "Reference invalid\n");
         return;
     }
-    fill_rectangle(&h->cur_pic.ref_index[0][4 * h->mb_xy],
+    fill_rectangle(&h->cur_pic.ref_index[0][4 * sl->mb_xy],
                    2, 2, 2, ref, 1);
     fill_rectangle(&sl->ref_cache[0][scan8[0]], 4, 4, 8, ref, 1);
     fill_rectangle(sl->mv_cache[0][scan8[0]], 4, 4, 8,
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 04ddde06c3..40afb11689 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -393,6 +393,7 @@ typedef struct H264SliceContext {
     ptrdiff_t mb_linesize;  ///< may be equal to s->linesize or s->linesize * 2, for mbaff
     ptrdiff_t mb_uvlinesize;
 
+    int mb_xy;
     int mb_skip_run;
     int is_complex;
 
@@ -574,7 +575,6 @@ typedef struct H264Context {
     int mb_height, mb_width;
     int mb_stride;
     int mb_num;
-    int mb_xy;
 
     // =============================================================
     // Things below are not used in the MB or more inner code
@@ -1020,7 +1020,7 @@ static av_always_inline int pred_intra_mode(H264Context *h,
 static av_always_inline void write_back_intra_pred_mode(H264Context *h,
                                                         H264SliceContext *sl)
 {
-    int8_t *i4x4       = sl->intra4x4_pred_mode + h->mb2br_xy[h->mb_xy];
+    int8_t *i4x4       = sl->intra4x4_pred_mode + h->mb2br_xy[sl->mb_xy];
     int8_t *i4x4_cache = sl->intra4x4_pred_mode_cache;
 
     AV_COPY32(i4x4, i4x4_cache + 4 + 8 * 4);
@@ -1032,7 +1032,7 @@ static av_always_inline void write_back_intra_pred_mode(H264Context *h,
 static av_always_inline void write_back_non_zero_count(H264Context *h,
                                                        H264SliceContext *sl)
 {
-    const int mb_xy    = h->mb_xy;
+    const int mb_xy    = sl->mb_xy;
     uint8_t *nnz       = h->non_zero_count[mb_xy];
     uint8_t *nnz_cache = sl->non_zero_count_cache;
 
@@ -1066,8 +1066,8 @@ static av_always_inline void write_back_motion_list(H264Context *h,
     AV_COPY128(mv_dst + 2 * b_stride, mv_src + 8 * 2);
     AV_COPY128(mv_dst + 3 * b_stride, mv_src + 8 * 3);
     if (CABAC(h)) {
-        uint8_t (*mvd_dst)[2] = &sl->mvd_table[list][FMO ? 8 * h->mb_xy
-                                                        : h->mb2br_xy[h->mb_xy]];
+        uint8_t (*mvd_dst)[2] = &sl->mvd_table[list][FMO ? 8 * sl->mb_xy
+                                                        : h->mb2br_xy[sl->mb_xy]];
         uint8_t(*mvd_src)[2]  = &sl->mvd_cache[list][scan8[0]];
         if (IS_SKIP(mb_type)) {
             AV_ZERO128(mvd_dst);
@@ -1095,7 +1095,7 @@ static av_always_inline void write_back_motion(H264Context *h,
 {
     const int b_stride      = h->b_stride;
     const int b_xy  = 4 * h->mb_x + 4 * h->mb_y * h->b_stride; // try mb2b(8)_xy
-    const int b8_xy = 4 * h->mb_xy;
+    const int b8_xy = 4 * sl->mb_xy;
 
     if (USES_LIST(mb_type, 0)) {
         write_back_motion_list(h, sl, b_stride, b_xy, b8_xy, mb_type, 0);
@@ -1108,7 +1108,7 @@ static av_always_inline void write_back_motion(H264Context *h,
 
     if (sl->slice_type_nos == AV_PICTURE_TYPE_B && CABAC(h)) {
         if (IS_8X8(mb_type)) {
-            uint8_t *direct_table = &h->direct_table[4 * h->mb_xy];
+            uint8_t *direct_table = &h->direct_table[4 * sl->mb_xy];
             direct_table[1] = sl->sub_mb_type[1] >> 1;
             direct_table[2] = sl->sub_mb_type[2] >> 1;
             direct_table[3] = sl->sub_mb_type[3] >> 1;
diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c
index 74c4b8d0aa..c9318facec 100644
--- a/libavcodec/h264_cabac.c
+++ b/libavcodec/h264_cabac.c
@@ -1284,7 +1284,7 @@ void ff_h264_init_cabac_states(H264Context *h, H264SliceContext *sl)
 
 static int decode_cabac_field_decoding_flag(H264Context *h, H264SliceContext *sl)
 {
-    const int mbb_xy = h->mb_xy - 2*h->mb_stride;
+    const int mbb_xy = sl->mb_xy - 2*h->mb_stride;
 
     unsigned long ctx = 0;
 
@@ -1348,7 +1348,7 @@ static int decode_cabac_mb_skip(H264Context *h, H264SliceContext *sl,
         }else
             mbb_xy = mb_x + (mb_y-1)*h->mb_stride;
     }else{
-        int mb_xy = h->mb_xy;
+        int mb_xy = sl->mb_xy;
         mba_xy = mb_xy - 1;
         mbb_xy = mb_xy - (h->mb_stride << FIELD_PICTURE(h));
     }
@@ -1695,9 +1695,9 @@ decode_cabac_residual_internal(H264Context *h, H264SliceContext *sl,
 
     if( is_dc ) {
         if( cat == 3 )
-            h->cbp_table[h->mb_xy] |= 0x40 << (n - CHROMA_DC_BLOCK_INDEX);
+            h->cbp_table[sl->mb_xy] |= 0x40 << (n - CHROMA_DC_BLOCK_INDEX);
         else
-            h->cbp_table[h->mb_xy] |= 0x100 << (n - LUMA_DC_BLOCK_INDEX);
+            h->cbp_table[sl->mb_xy] |= 0x100 << (n - LUMA_DC_BLOCK_INDEX);
         sl->non_zero_count_cache[scan8[n]] = coeff_count;
     } else {
         if( max_coeff == 64 )
@@ -1917,7 +1917,7 @@ int ff_h264_decode_mb_cabac(H264Context *h, H264SliceContext *sl)
     const int pixel_shift = h->pixel_shift;
     unsigned local_ref_count[2];
 
-    mb_xy = h->mb_xy = h->mb_x + h->mb_y*h->mb_stride;
+    mb_xy = sl->mb_xy = h->mb_x + h->mb_y*h->mb_stride;
 
     tprintf(h->avctx, "pic:%d mb:%d/%d\n", h->frame_num, h->mb_x, h->mb_y);
     if (sl->slice_type_nos != AV_PICTURE_TYPE_I) {
diff --git a/libavcodec/h264_cavlc.c b/libavcodec/h264_cavlc.c
index a0e3a725ba..6d77d21f75 100644
--- a/libavcodec/h264_cavlc.c
+++ b/libavcodec/h264_cavlc.c
@@ -711,7 +711,7 @@ int ff_h264_decode_mb_cavlc(H264Context *h, H264SliceContext *sl)
     const int pixel_shift = h->pixel_shift;
     unsigned local_ref_count[2];
 
-    mb_xy = h->mb_xy = h->mb_x + h->mb_y*h->mb_stride;
+    mb_xy = sl->mb_xy = h->mb_x + h->mb_y*h->mb_stride;
 
     tprintf(h->avctx, "pic:%d mb:%d/%d\n", h->frame_num, h->mb_x, h->mb_y);
     cbp = 0; /* avoid warning. FIXME: find a solution without slowing
diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index cee83cd6cc..891a24bf35 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -179,7 +179,7 @@ static void pred_spatial_direct_motion(H264Context *const h, H264SliceContext *s
 {
     int b8_stride = 2;
     int b4_stride = h->b_stride;
-    int mb_xy = h->mb_xy, mb_y = h->mb_y;
+    int mb_xy = sl->mb_xy, mb_y = h->mb_y;
     int mb_type_col[2];
     const int16_t (*l1mv0)[2], (*l1mv1)[2];
     const int8_t *l1ref0, *l1ref1;
@@ -466,7 +466,7 @@ static void pred_temp_direct_motion(H264Context *const h, H264SliceContext *sl,
 {
     int b8_stride = 2;
     int b4_stride = h->b_stride;
-    int mb_xy = h->mb_xy, mb_y = h->mb_y;
+    int mb_xy = sl->mb_xy, mb_y = h->mb_y;
     int mb_type_col[2];
     const int16_t (*l1mv0)[2], (*l1mv1)[2];
     const int8_t *l1ref0, *l1ref1;
diff --git a/libavcodec/h264_loopfilter.c b/libavcodec/h264_loopfilter.c
index e81751a204..f92cfc0beb 100644
--- a/libavcodec/h264_loopfilter.c
+++ b/libavcodec/h264_loopfilter.c
@@ -246,7 +246,7 @@ static av_always_inline void h264_filter_mb_fast_internal(H264Context *h,
     int chroma444 = CHROMA444(h);
     int chroma422 = CHROMA422(h);
 
-    int mb_xy = h->mb_xy;
+    int mb_xy = sl->mb_xy;
     int left_type = sl->left_type[LTOP];
     int top_type  = sl->top_type;
 
diff --git a/libavcodec/h264_mb.c b/libavcodec/h264_mb.c
index b50902fc41..9216d213fa 100644
--- a/libavcodec/h264_mb.c
+++ b/libavcodec/h264_mb.c
@@ -96,7 +96,7 @@ static inline void get_lowest_part_y(H264Context *h, H264SliceContext *sl,
  */
 static void await_references(H264Context *h, H264SliceContext *sl)
 {
-    const int mb_xy   = h->mb_xy;
+    const int mb_xy   = sl->mb_xy;
     const int mb_type = h->cur_pic.mb_type[mb_xy];
     int16_t refs[2][48];
     int nrefs[2] = { 0 };
@@ -528,7 +528,7 @@ static av_always_inline void xchg_mb_border(H264Context *h, H264SliceContext *sl
     }
 
     if (sl->deblocking_filter == 2) {
-        deblock_topleft = h->slice_table[h->mb_xy - 1 - h->mb_stride] == sl->slice_num;
+        deblock_topleft = h->slice_table[sl->mb_xy - 1 - h->mb_stride] == sl->slice_num;
         deblock_top     = sl->top_type;
     } else {
         deblock_topleft = (h->mb_x > 0);
@@ -817,7 +817,7 @@ static av_always_inline void hl_decode_mb_idct_luma(H264Context *h, H264SliceCon
 
 void ff_h264_hl_decode_mb(H264Context *h, H264SliceContext *sl)
 {
-    const int mb_xy   = h->mb_xy;
+    const int mb_xy   = sl->mb_xy;
     const int mb_type = h->cur_pic.mb_type[mb_xy];
     int is_complex    = CONFIG_SMALL || sl->is_complex ||
                         IS_INTRA_PCM(mb_type) || sl->qscale == 0;
diff --git a/libavcodec/h264_mb_template.c b/libavcodec/h264_mb_template.c
index a9671ae265..3259232a4b 100644
--- a/libavcodec/h264_mb_template.c
+++ b/libavcodec/h264_mb_template.c
@@ -44,7 +44,7 @@ static av_noinline void FUNC(hl_decode_mb)(H264Context *h, H264SliceContext *sl)
 {
     const int mb_x    = h->mb_x;
     const int mb_y    = h->mb_y;
-    const int mb_xy   = h->mb_xy;
+    const int mb_xy   = sl->mb_xy;
     const int mb_type = h->cur_pic.mb_type[mb_xy];
     uint8_t *dest_y, *dest_cb, *dest_cr;
     int linesize, uvlinesize /*dct_offset*/;
@@ -273,7 +273,7 @@ static av_noinline void FUNC(hl_decode_mb_444)(H264Context *h, H264SliceContext
 {
     const int mb_x    = h->mb_x;
     const int mb_y    = h->mb_y;
-    const int mb_xy   = h->mb_xy;
+    const int mb_xy   = sl->mb_xy;
     const int mb_type = h->cur_pic.mb_type[mb_xy];
     uint8_t *dest[3];
     int linesize;
diff --git a/libavcodec/h264_mc_template.c b/libavcodec/h264_mc_template.c
index d20ccc58f7..2ca82a321d 100644
--- a/libavcodec/h264_mc_template.c
+++ b/libavcodec/h264_mc_template.c
@@ -71,7 +71,7 @@ static void MCFUNC(hl_motion)(H264Context *h, H264SliceContext *sl,
                               h264_weight_func *weight_op,
                               h264_biweight_func *weight_avg)
 {
-    const int mb_xy   = h->mb_xy;
+    const int mb_xy   = sl->mb_xy;
     const int mb_type = h->cur_pic.mb_type[mb_xy];
 
     av_assert2(IS_INTER(mb_type));
diff --git a/libavcodec/h264_mvpred.h b/libavcodec/h264_mvpred.h
index 241e90565c..81ce4dc5d7 100644
--- a/libavcodec/h264_mvpred.h
+++ b/libavcodec/h264_mvpred.h
@@ -355,7 +355,7 @@ zeromv:
 
 static void fill_decode_neighbors(H264Context *h, H264SliceContext *sl, int mb_type)
 {
-    const int mb_xy = h->mb_xy;
+    const int mb_xy = sl->mb_xy;
     int topleft_xy, top_xy, topright_xy, left_xy[LEFT_MBS];
     static const uint8_t left_block_options[4][32] = {
         { 0, 1, 2, 3, 7, 10, 8, 11, 3 + 0 * 4, 3 + 1 * 4, 3 + 2 * 4, 3 + 3 * 4, 1 + 4 * 4, 1 + 8 * 4, 1 + 5 * 4, 1 + 9 * 4 },
@@ -802,7 +802,7 @@ static void fill_decode_caches(H264Context *h, H264SliceContext *sl, int mb_type
  */
 static void av_unused decode_mb_skip(H264Context *h, H264SliceContext *sl)
 {
-    const int mb_xy = h->mb_xy;
+    const int mb_xy = sl->mb_xy;
     int mb_type     = 0;
 
     memset(h->non_zero_count[mb_xy], 0, 48);
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index d0f8dbc686..94422435f9 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -2127,7 +2127,7 @@ static av_always_inline void fill_filter_caches_inter(H264Context *h,
  */
 static int fill_filter_caches(H264Context *h, H264SliceContext *sl, int mb_type)
 {
-    const int mb_xy = h->mb_xy;
+    const int mb_xy = sl->mb_xy;
     int top_xy, left_xy[LEFT_MBS];
     int top_type, left_type[LEFT_MBS];
     uint8_t *nnz;
@@ -2283,7 +2283,7 @@ static void loop_filter(H264Context *h, H264SliceContext *sl, int start_x, int e
         for (mb_x = start_x; mb_x < end_x; mb_x++)
             for (mb_y = end_mb_y - FRAME_MBAFF(h); mb_y <= end_mb_y; mb_y++) {
                 int mb_xy, mb_type;
-                mb_xy         = h->mb_xy = mb_x + mb_y * h->mb_stride;
+                mb_xy         = sl->mb_xy = mb_x + mb_y * h->mb_stride;
                 sl->slice_num = h->slice_table[mb_xy];
                 mb_type       = h->cur_pic.mb_type[mb_xy];
                 sl->list_count = h->list_counts[mb_xy];
diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c
index 077eccb0bb..32a347b6c7 100644
--- a/libavcodec/svq3.c
+++ b/libavcodec/svq3.c
@@ -494,7 +494,7 @@ static int svq3_decode_mb(SVQ3Context *s, unsigned int mb_type)
     int cbp = 0;
     uint32_t vlc;
     int8_t *top, *left;
-    const int mb_xy         = h->mb_xy;
+    const int mb_xy         = sl->mb_xy;
     const int b_xy          = 4 * h->mb_x + 4 * h->mb_y * h->b_stride;
 
     sl->top_samples_available      = (h->mb_y == 0) ? 0x33FF : 0xFFFF;
@@ -782,7 +782,7 @@ static int svq3_decode_slice_header(AVCodecContext *avctx)
     SVQ3Context *s = avctx->priv_data;
     H264Context *h    = &s->h;
     H264SliceContext *sl = &h->slice_ctx[0];
-    const int mb_xy   = h->mb_xy;
+    const int mb_xy   = sl->mb_xy;
     int i, header;
     unsigned slice_id;
 
@@ -1153,7 +1153,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
         return 0;
     }
 
-    h->mb_x = h->mb_y = h->mb_xy = 0;
+    h->mb_x = h->mb_y = sl->mb_xy = 0;
 
     if (s->watermark_key) {
         av_fast_padded_malloc(&s->buf, &s->buf_size, buf_size);
@@ -1283,7 +1283,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
     for (h->mb_y = 0; h->mb_y < h->mb_height; h->mb_y++) {
         for (h->mb_x = 0; h->mb_x < h->mb_width; h->mb_x++) {
             unsigned mb_type;
-            h->mb_xy = h->mb_x + h->mb_y * h->mb_stride;
+            sl->mb_xy = h->mb_x + h->mb_y * h->mb_stride;
 
             if ((get_bits_count(&h->gb) + 7) >= h->gb.size_in_bits &&
                 ((get_bits_count(&h->gb) & 7) == 0 ||