diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 29c9746813..3440cca74b 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -393,6 +393,7 @@ typedef struct H264SliceContext {
     DECLARE_ALIGNED(16, int16_t, mv_cache)[2][5 * 8][2];
     DECLARE_ALIGNED(8,  int8_t, ref_cache)[2][5 * 8];
     DECLARE_ALIGNED(16, uint8_t, mvd_cache)[2][5 * 8][2];
+    uint8_t direct_cache[5 * 8];
 
     DECLARE_ALIGNED(8, uint16_t, sub_mb_type)[4];
 
@@ -497,7 +498,6 @@ typedef struct H264Context {
     uint8_t *chroma_pred_mode_table;
     uint8_t (*mvd_table[2])[2];
     uint8_t *direct_table;
-    uint8_t direct_cache[5 * 8];
 
     uint8_t zigzag_scan[16];
     uint8_t zigzag_scan8x8[64];
diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c
index cc3db5dc7d..2dbd7ced80 100644
--- a/libavcodec/h264_cabac.c
+++ b/libavcodec/h264_cabac.c
@@ -1475,9 +1475,9 @@ static int decode_cabac_mb_ref(H264Context *h, H264SliceContext *sl, int list, i
     int ctx  = 0;
 
     if (sl->slice_type_nos == AV_PICTURE_TYPE_B) {
-        if( refa > 0 && !(h->direct_cache[scan8[n] - 1]&(MB_TYPE_DIRECT2>>1)) )
+        if( refa > 0 && !(sl->direct_cache[scan8[n] - 1]&(MB_TYPE_DIRECT2>>1)) )
             ctx++;
-        if( refb > 0 && !(h->direct_cache[scan8[n] - 8]&(MB_TYPE_DIRECT2>>1)) )
+        if( refb > 0 && !(sl->direct_cache[scan8[n] - 8]&(MB_TYPE_DIRECT2>>1)) )
             ctx += 2;
     } else {
         if( refa > 0 )
@@ -2108,7 +2108,7 @@ decode_intra_mb:
                 sl->ref_cache[0][scan8[12]] =
                 sl->ref_cache[1][scan8[12]] = PART_NOT_AVAILABLE;
                     for( i = 0; i < 4; i++ )
-                        fill_rectangle(&h->direct_cache[scan8[4*i]], 2, 2, 8, (sl->sub_mb_type[i] >> 1) & 0xFF, 1);
+                        fill_rectangle(&sl->direct_cache[scan8[4*i]], 2, 2, 8, (sl->sub_mb_type[i] >> 1) & 0xFF, 1);
             }
         } else {
             for( i = 0; i < 4; i++ ) {
diff --git a/libavcodec/h264_mvpred.h b/libavcodec/h264_mvpred.h
index d325b73db9..e81c6e9d34 100644
--- a/libavcodec/h264_mvpred.h
+++ b/libavcodec/h264_mvpred.h
@@ -721,7 +721,7 @@ static void fill_decode_caches(H264Context *h, H264SliceContext *sl, int mb_type
                     AV_ZERO16(mvd_cache[2 + 8 * 0]);
                     AV_ZERO16(mvd_cache[2 + 8 * 2]);
                     if (sl->slice_type_nos == AV_PICTURE_TYPE_B) {
-                        uint8_t *direct_cache = &h->direct_cache[scan8[0]];
+                        uint8_t *direct_cache = &sl->direct_cache[scan8[0]];
                         uint8_t *direct_table = h->direct_table;
                         fill_rectangle(direct_cache, 4, 4, 8, MB_TYPE_16x16 >> 1, 1);