diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 4d08642779..d4858d02ef 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -3715,7 +3715,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){
             if((err = decode_slice_header(hx, h)))
                break;
 
-            if (h->sei_recovery_frame_cnt >= 0) {
+            if (h->sei_recovery_frame_cnt >= 0 && h->recovery_frame < 0) {
                 h->recovery_frame = (h->frame_num + h->sei_recovery_frame_cnt) %
                                     (1 << h->sps.log2_max_frame_num);
             }