diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 84183108dd..89fc2227ae 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1704,7 +1704,7 @@ int ff_h264_frame_start(H264Context *h)
 
     if ((ret = alloc_picture(h, pic)) < 0)
         return ret;
-    if(!h->sync)
+    if(!h->sync && !h->avctx->hwaccel)
         avpriv_color_frame(&pic->f, c);
 
     h->cur_pic_ptr = pic;