avformat/mux: Fix leak when adding packet to interleavement queue fails

When an error happened in ff_interleave_add_packet() when adding
a packet to the packet queue, said packet would not be unreferenced
in ff_interleave_add_packet(), but would be zeroed in
av_interleaved_write_frame(), which results in a memleak.

This has been fixed: ff_interleave_add_packet() now always unreferences
the input packet on error; as a result, it always returns blank packets
which has been documented. Relying on this a call to av_packet_unref()
in ff_audio_rechunk_interleave() can be removed.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>

libavformat/audiointerleave.c

@@ -136,10 +136,8 @@ int ff_audio_rechunk_interleave(AVFormatContext *s, AVPacket *out, AVPacket *pkt
         if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
             AVPacket new_pkt;
             while ((ret = interleave_new_audio_packet(s, &new_pkt, i, flush)) > 0) {
-                if ((ret = ff_interleave_add_packet(s, &new_pkt, compare_ts)) < 0) {
-                    av_packet_unref(&new_pkt);
+                if ((ret = ff_interleave_add_packet(s, &new_pkt, compare_ts)) < 0)
                     return ret;
-                }
             }
             if (ret < 0)
                 return ret;

libavformat/internal.h

@@ -230,9 +230,9 @@ char *ff_data_to_hex(char *buf, const uint8_t *src, int size, int lowercase);
 int ff_hex_to_data(uint8_t *data, const char *p);
 
 /**
- * Add packet to AVFormatContext->packet_buffer list, determining its
+ * Add packet to an AVFormatContext's packet_buffer list, determining its
  * interleaved position using compare() function argument.
- * @return 0, or < 0 on error
+ * @return 0 on success, < 0 on error. pkt will always be blank on return.
  */
 int ff_interleave_add_packet(AVFormatContext *s, AVPacket *pkt,
                              int (*compare)(AVFormatContext *, const AVPacket *, const AVPacket *));

libavformat/mux.c

@@ -917,10 +917,13 @@ int ff_interleave_add_packet(AVFormatContext *s, AVPacket *pkt,
     int chunked  = s->max_chunk_size || s->max_chunk_duration;
 
     this_pktl    = av_malloc(sizeof(AVPacketList));
-    if (!this_pktl)
+    if (!this_pktl) {
+        av_packet_unref(pkt);
         return AVERROR(ENOMEM);
+    }
     if ((ret = av_packet_make_refcounted(pkt)) < 0) {
         av_free(this_pktl);
+        av_packet_unref(pkt);
         return ret;
     }
 
@@ -1215,7 +1218,7 @@ int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
             av_init_packet(pkt);
             pkt = NULL;
         }
-        if (ret <= 0) //FIXME cleanup needed for ret<0 ?
+        if (ret <= 0)
             return ret;
 
         ret = write_packet(s, &opkt);