diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c
index 5c7e93e5e6..530b890de1 100644
--- a/libavcodec/vp56.c
+++ b/libavcodec/vp56.c
@@ -401,6 +401,8 @@ static void vp56_decode_mb(VP56Context *s, int row, int col, int is_alpha)
 
     frame_current = s->framep[VP56_FRAME_CURRENT];
     frame_ref = s->framep[ref_frame];
+    if (mb_type != VP56_MB_INTRA && !frame_ref->data[0])
+        return;
 
     ab = 6*is_alpha;
     b_max = 6 - 2*is_alpha;