diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index d0f555313f..0135348d66 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -947,7 +947,7 @@ static void fill_decode_caches(H264Context *h, int mb_type){
     }
 
 #if 1
-    if(IS_INTER(mb_type) || IS_DIRECT(mb_type)){
+    if(IS_INTER(mb_type) || (IS_DIRECT(mb_type) && h->direct_spatial_mv_pred)){
         int list;
         for(list=0; list<h->list_count; list++){
             if(!USES_LIST(mb_type, list) && !IS_DIRECT(mb_type)){
@@ -958,6 +958,8 @@ static void fill_decode_caches(H264Context *h, int mb_type){
                 }*/
                 continue;
             }
+            assert(!(IS_DIRECT(mb_type) && !h->direct_spatial_mv_pred));
+
             h->mv_cache_clean[list]= 0;
 
             if(USES_LIST(top_type, list)){
@@ -990,9 +992,6 @@ static void fill_decode_caches(H264Context *h, int mb_type){
                 }
             }
 
-            if((IS_DIRECT(mb_type) && !h->direct_spatial_mv_pred))
-                continue;
-
             if(USES_LIST(topleft_type, list)){
                 const int b_xy = h->mb2b_xy[topleft_xy] + 3 + h->b_stride + (topleft_partition & 2*h->b_stride);
                 const int b8_xy= h->mb2b8_xy[topleft_xy] + 1 + (topleft_partition & h->b8_stride);