diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 08b089187d..f1f9c7304b 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1005,14 +1005,9 @@ int ff_set_ref_count(H264Context *h, H264SliceContext *sl)
         return AVERROR_INVALIDDATA;
     }
 
-    if (list_count   != sl->list_count   ||
-        ref_count[0] != sl->ref_count[0] ||
-        ref_count[1] != sl->ref_count[1]) {
-        sl->ref_count[0] = ref_count[0];
-        sl->ref_count[1] = ref_count[1];
-        sl->list_count   = list_count;
-        return 1;
-    }
+    sl->ref_count[0] = ref_count[0];
+    sl->ref_count[1] = ref_count[1];
+    sl->list_count   = list_count;
 
     return 0;
 }