avcodec/libx264: add a flush callback

Signed-off-by: James Almer <jamrial@gmail.com>

libavcodec/libx264.c

@@ -151,7 +151,7 @@ static int encode_nals(AVCodecContext *ctx, AVPacket *pkt,
 {
     X264Context *x4 = ctx->priv_data;
     uint8_t *p;
-    uint64_t size = x4->sei_size;
+    uint64_t size = FFMAX(x4->sei_size, 0);
     int ret;
 
     if (!nnal)
@@ -178,8 +178,8 @@ static int encode_nals(AVCodecContext *ctx, AVPacket *pkt,
         memcpy(p, x4->sei, x4->sei_size);
         p += x4->sei_size;
         size -= x4->sei_size;
-        x4->sei_size = 0;
-        av_freep(&x4->sei);
+        /* Keep the value around in case of flush */
+        x4->sei_size = -x4->sei_size;
     }
 
     /* x264 guarantees the payloads of the NALs
@@ -648,6 +648,24 @@ FF_ENABLE_DEPRECATION_WARNINGS
     return 0;
 }
 
+static void X264_flush(AVCodecContext *avctx)
+{
+    X264Context *x4 = avctx->priv_data;
+    x264_nal_t *nal;
+    int nnal, ret;
+    x264_picture_t pic_out = {0};
+
+    do {
+        ret = x264_encoder_encode(x4->enc, &nal, &nnal, NULL, &pic_out);
+    } while (ret > 0 && x264_encoder_delayed_frames(x4->enc));
+
+    for (int i = 0; i < x4->nb_reordered_opaque; i++)
+        opaque_uninit(&x4->reordered_opaque[i]);
+
+    if (x4->sei_size < 0)
+        x4->sei_size = -x4->sei_size;
+}
+
 static av_cold int X264_close(AVCodecContext *avctx)
 {
     X264Context *x4 = avctx->priv_data;
@@ -1335,12 +1353,14 @@ FFCodec ff_libx264_encoder = {
     .p.capabilities   = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY |
                         AV_CODEC_CAP_OTHER_THREADS |
                         AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE |
+                        AV_CODEC_CAP_ENCODER_FLUSH |
                         AV_CODEC_CAP_ENCODER_RECON_FRAME,
     .p.priv_class     = &x264_class,
     .p.wrapper_name   = "libx264",
     .priv_data_size   = sizeof(X264Context),
     .init             = X264_init,
     FF_CODEC_ENCODE_CB(X264_frame),
+    .flush            = X264_flush,
     .close            = X264_close,
     .defaults         = x264_defaults,
 #if X264_BUILD < 153