avfilter/ff_merge_formats: only merge if doing so does not loose chroma or alpha

Fixes Ticket1280

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavfilter/formats.c

@@ -92,10 +92,29 @@ do {
 AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b)
 {
     AVFilterFormats *ret = NULL;
+    int i, j;
+    int alpha1=0, alpha2=0;
+    int chroma1=0, chroma2=0;
 
     if (a == b)
         return a;
 
+    for (i = 0; i < a->format_count; i++)
+        for (j = 0; j < b->format_count; j++) {
+            const AVPixFmtDescriptor *adesc = av_pix_fmt_desc_get(a->formats[i]);
+            const AVPixFmtDescriptor *bdesc = av_pix_fmt_desc_get(b->formats[j]);
+            alpha2 |= adesc->flags & bdesc->flags & PIX_FMT_ALPHA;
+            chroma2|= adesc->nb_components > 1 && bdesc->nb_components > 1;
+            if (a->formats[i] == b->formats[j]) {
+                alpha1 |= adesc->flags & PIX_FMT_ALPHA;
+                chroma1|= adesc->nb_components > 1;
+            }
+        }
+
+    // If chroma or alpha can be lost through merging then do not merge
+    if (alpha2 > alpha1 || chroma2 > chroma1)
+        return NULL;
+
     MERGE_FORMATS(ret, a, b, formats, format_count, AVFilterFormats, fail);
 
     return ret;