From 29f6c9b04d37305ec7b19cefdd4e4c71b1bdfe78 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Tue, 17 Oct 2023 00:54:30 +0200
Subject: [PATCH] avcodec/h264: keep track of which frames used gray references

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/h264_picture.c |  1 +
 libavcodec/h264_slice.c   | 22 ++++++++++++++++++++--
 libavcodec/h264dec.c      |  1 +
 libavcodec/h264dec.h      |  4 ++++
 4 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/libavcodec/h264_picture.c b/libavcodec/h264_picture.c
index 691c61eea2..3234141dbd 100644
--- a/libavcodec/h264_picture.c
+++ b/libavcodec/h264_picture.c
@@ -96,6 +96,7 @@ static void h264_copy_picture_params(H264Picture *dst, const H264Picture *src)
     dst->field_picture = src->field_picture;
     dst->reference     = src->reference;
     dst->recovered     = src->recovered;
+    dst->gray          = src->gray;
     dst->invalid_gap   = src->invalid_gap;
     dst->sei_recovery_frame_cnt = src->sei_recovery_frame_cnt;
     dst->mb_width      = src->mb_width;
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 4861a2cabb..3b6c6c3592 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -457,6 +457,7 @@ int ff_h264_update_thread_context(AVCodecContext *dst,
     h->poc.prev_frame_num        = h->poc.frame_num;
 
     h->recovery_frame        = h1->recovery_frame;
+    h->non_gray              = h1->non_gray;
 
     return err;
 }
@@ -1544,11 +1545,15 @@ static int h264_field_start(H264Context *h, const H264SliceContext *sl,
                 if (ret < 0)
                     return ret;
                 h->short_ref[0]->poc = prev->poc + 2U;
+                h->short_ref[0]->gray = prev->gray;
                 ff_thread_report_progress(&h->short_ref[0]->tf, INT_MAX, 0);
                 if (h->short_ref[0]->field_picture)
                     ff_thread_report_progress(&h->short_ref[0]->tf, INT_MAX, 1);
-            } else if (!h->frame_recovered && !h->avctx->hwaccel)
-                color_frame(h->short_ref[0]->f, c);
+            } else if (!h->frame_recovered) {
+                if (!h->avctx->hwaccel)
+                    color_frame(h->short_ref[0]->f, c);
+                h->short_ref[0]->gray = 1;
+            }
             h->short_ref[0]->frame_num = h->poc.prev_frame_num;
         }
     }
@@ -2007,6 +2012,19 @@ static int h264_slice_init(H264Context *h, H264SliceContext *sl,
                              (sl->ref_list[j][i].reference & 3);
     }
 
+    if (sl->slice_type_nos == AV_PICTURE_TYPE_I) {
+        h->cur_pic_ptr->gray = 0;
+        h->non_gray = 1;
+    } else {
+        int gray = 0;
+        for (j = 0; j < sl->list_count; j++) {
+            for (i = 0; i < sl->ref_count[j]; i++) {
+                gray |= sl->ref_list[j][i].parent->gray;
+            }
+        }
+        h->cur_pic_ptr->gray = gray;
+    }
+
     if (h->avctx->debug & FF_DEBUG_PICT_INFO) {
         av_log(h->avctx, AV_LOG_DEBUG,
                "slice:%d %c mb:%d %c%s%s frame:%d poc:%d/%d ref:%d/%d qp:%d loop:%d:%d:%d weight:%d%s %s\n",
diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c
index ef9e60bbf3..7ea55db75d 100644
--- a/libavcodec/h264dec.c
+++ b/libavcodec/h264dec.c
@@ -492,6 +492,7 @@ static void h264_decode_flush(AVCodecContext *avctx)
     ff_h264_unref_picture(&h->cur_pic);
 
     h->mb_y = 0;
+    h->non_gray = 0;
 
     ff_h264_free_tables(h);
     h->context_initialized = 0;
diff --git a/libavcodec/h264dec.h b/libavcodec/h264dec.h
index ede5195172..366626c056 100644
--- a/libavcodec/h264dec.h
+++ b/libavcodec/h264dec.h
@@ -154,6 +154,8 @@ typedef struct H264Picture {
 
     /// RefStruct reference; its pointee is shared between decoding threads.
     atomic_int *decode_error_flags;
+
+    int gray;
 } H264Picture;
 
 typedef struct H264Ref {
@@ -567,6 +569,8 @@ typedef struct H264Context {
     struct FFRefStructPool *ref_index_pool;
     struct FFRefStructPool *decode_error_flags_pool;
     int ref2frm[MAX_SLICES][2][64];     ///< reference to frame number lists, used in the loop filter, the first 2 are for -2,-1
+
+    int non_gray;                       ///< Did we encounter a intra frame after a gray gap frame
 } H264Context;
 
 extern const uint16_t ff_h264_mb_sizes[4];