avcodec/h264: Avoid using gray gap frames as references

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/h264_refs.c

@@ -410,6 +410,17 @@ int ff_h264_build_ref_list(H264Context *h, H264SliceContext *sl)
                 else
                     return -1;
             }
+            if (h->noref_gray>0 && sl->ref_list[list][index].parent->gray && h->non_gray) {
+                for (int j=0; j<sl->list_count; j++) {
+                    int list2 = (list+j)&1;
+                    if (h->default_ref[list2].parent && !h->default_ref[list2].parent->gray
+                        && !(!FIELD_PICTURE(h) && (h->default_ref[list2].reference&3) != 3)) {
+                        sl->ref_list[list][index] = h->default_ref[list2];
+                        av_log(h, AV_LOG_DEBUG, "replacement of gray gap frame\n");
+                        break;
+                    }
+                }
+            }
             av_assert0(av_buffer_get_ref_count(sl->ref_list[list][index].parent->f->buf[0]) > 0);
         }
     }

libavcodec/h264dec.c

@@ -1091,6 +1091,7 @@ static const AVOption h264_options[] = {
     { "nal_length_size", "nal_length_size", OFFSET(nal_length_size), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 4, VDX },
     { "enable_er", "Enable error resilience on damaged frames (unsafe)", OFFSET(enable_er), AV_OPT_TYPE_BOOL, { .i64 = -1 }, -1, 1, VD },
     { "x264_build", "Assume this x264 version if no x264 version found in any SEI", OFFSET(x264_build), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, VD },
+    { "noref_gray", "Avoid using gray gap frames as references", OFFSET(noref_gray), AV_OPT_TYPE_INT, {.i64 = 1}, -1, 1, VD },
     { NULL },
 };
 

libavcodec/h264dec.h

@@ -571,6 +571,7 @@ typedef struct H264Context {
     int ref2frm[MAX_SLICES][2][64];     ///< reference to frame number lists, used in the loop filter, the first 2 are for -2,-1
 
     int non_gray;                       ///< Did we encounter a intra frame after a gray gap frame
+    int noref_gray;
 } H264Context;
 
 extern const uint16_t ff_h264_mb_sizes[4];