Adds a new pix fmt function that selects the best dest fmt from among 2 formats to convert to, given a certain src format.

Similar to avcodec_find_best_pix_fmt(), but instead only compares two destination
pix fmts and selects one of these fmts as the best dest format when converting from
a given source format. Also, as opposed to avcodec_find_best_pix_fmt() which supports
only the first 64 defined pixel formats, this supports as input any of the defined
pixel formats.

Also replaces the old function to rely on the new function.

Signed-off-by: Matthew Einhorn <moiein2000@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/avcodec.h

@@ -3531,6 +3531,39 @@ int avcodec_get_pix_fmt_loss(enum PixelFormat dst_pix_fmt, enum PixelFormat src_
 enum PixelFormat avcodec_find_best_pix_fmt(int64_t pix_fmt_mask, enum PixelFormat src_pix_fmt,
                               int has_alpha, int *loss_ptr);
 
+/**
+ * Find the best pixel format to convert to given a certain source pixel
+ * format and a selection of two destination pixel formats. When converting from
+ * one pixel format to another, information loss may occur.  For example, when converting
+ * from RGB24 to GRAY, the color information will be lost. Similarly, other losses occur when
+ * converting from some formats to other formats. avcodec_find_best_pix_fmt2() selects which of
+ * the given pixel formats should be used to suffer the least amount of loss.
+ *
+ * If one of the destination formats is PIX_FMT_NONE the other pixel format (if valid) will be
+ * returned.
+ *
+ * @code
+ * src_pix_fmt = PIX_FMT_YUV420P;
+ * dst_pix_fmt1= PIX_FMT_RGB24;
+ * dst_pix_fmt2= PIX_FMT_GRAY8;
+ * dst_pix_fmt3= PIX_FMT_RGB8;
+ * loss= FF_LOSS_CHROMA; // don't care about chroma loss, so chroma loss will be ignored.
+ * dst_pix_fmt = avcodec_find_best_pix_fmt2(dst_pix_fmt1, dst_pix_fmt2, src_pix_fmt, alpha, &loss);
+ * dst_pix_fmt = avcodec_find_best_pix_fmt2(dst_pix_fmt, dst_pix_fmt3, src_pix_fmt, alpha, &loss);
+ * @endcode
+ *
+ * @param[in] dst_pix_fmt1 One of the two destination pixel formats to choose from
+ * @param[in] dst_pix_fmt2 The other of the two destination pixel formats to choose from
+ * @param[in] src_pix_fmt Source pixel format
+ * @param[in] has_alpha Whether the source pixel format alpha channel is used.
+ * @param[in, out] loss_ptr Combination of loss flags. In: selects which of the losses to ignore, i.e.
+ *                               NULL or value of zero means we care about all losses. Out: the loss
+ *                               that occurs when converting from src to selected dst pixel format.
+ * @return The best pixel format to convert to or -1 if none was found.
+ */
+enum PixelFormat avcodec_find_best_pix_fmt2(enum PixelFormat dst_pix_fmt1, enum PixelFormat dst_pix_fmt2,
+                                            enum PixelFormat src_pix_fmt, int has_alpha, int *loss_ptr);
+
 #define FF_ALPHA_TRANSP       0x0001 /* image has some totally transparent pixels */
 #define FF_ALPHA_SEMI_TRANSP  0x0002 /* image has some transparent pixels */
 

libavcodec/imgconvert.c

@@ -461,7 +461,24 @@ enum PixelFormat avcodec_find_best_pix_fmt(int64_t pix_fmt_mask, enum PixelForma
                                             int has_alpha, int *loss_ptr)
 {
     enum PixelFormat dst_pix_fmt;
-    int loss_mask, i;
+    int i;
+
+    if (loss_ptr) /* all losses count (for backward compatibility) */
+        *loss_ptr = 0;
+
+    dst_pix_fmt = PIX_FMT_NONE; /* so first iteration doesn't have to be treated special */
+    for(i = 0; i< FFMIN(PIX_FMT_NB, 64); i++){
+        if (pix_fmt_mask & (1ULL << i))
+            dst_pix_fmt = avcodec_find_best_pix_fmt2(dst_pix_fmt, i, src_pix_fmt, has_alpha, loss_ptr);
+    }
+    return dst_pix_fmt;
+}
+
+enum PixelFormat avcodec_find_best_pix_fmt2(enum PixelFormat dst_pix_fmt1, enum PixelFormat dst_pix_fmt2,
+                                            enum PixelFormat src_pix_fmt, int has_alpha, int *loss_ptr)
+{
+    enum PixelFormat dst_pix_fmt;
+    int loss1, loss2, loss_order1, loss_order2, i, loss_mask;
     static const int loss_mask_order[] = {
         ~0, /* no loss first */
         ~FF_LOSS_ALPHA,
@@ -469,22 +486,28 @@ enum PixelFormat avcodec_find_best_pix_fmt(int64_t pix_fmt_mask, enum PixelForma
         ~(FF_LOSS_COLORSPACE | FF_LOSS_RESOLUTION),
         ~FF_LOSS_COLORQUANT,
         ~FF_LOSS_DEPTH,
+        ~(FF_LOSS_RESOLUTION | FF_LOSS_DEPTH | FF_LOSS_COLORSPACE | FF_LOSS_ALPHA |
+          FF_LOSS_COLORQUANT | FF_LOSS_CHROMA),
         0,
     };
 
+    loss_mask= loss_ptr?~*loss_ptr:~0; /* use loss mask if provided */
+    dst_pix_fmt = PIX_FMT_NONE;
+    loss1 = avcodec_get_pix_fmt_loss(dst_pix_fmt1, src_pix_fmt, has_alpha) & loss_mask;
+    loss2 = avcodec_get_pix_fmt_loss(dst_pix_fmt2, src_pix_fmt, has_alpha) & loss_mask;
+
     /* try with successive loss */
-    i = 0;
-    for(;;) {
-        loss_mask = loss_mask_order[i++];
-        dst_pix_fmt = avcodec_find_best_pix_fmt1(pix_fmt_mask, src_pix_fmt,
-                                                 has_alpha, loss_mask);
-        if (dst_pix_fmt >= 0)
-            goto found;
-        if (loss_mask == 0)
-            break;
+    for(i = 0;loss_mask_order[i] != 0 && dst_pix_fmt == PIX_FMT_NONE;i++) {
+        loss_order1 = loss1 & loss_mask_order[i];
+        loss_order2 = loss2 & loss_mask_order[i];
+
+        if (loss_order1 == 0 && loss_order2 == 0){ /* use format with smallest depth */
+            dst_pix_fmt = avg_bits_per_pixel(dst_pix_fmt2) < avg_bits_per_pixel(dst_pix_fmt1) ? dst_pix_fmt2 : dst_pix_fmt1;
+        } else if (loss_order1 == 0 || loss_order2 == 0) { /* use format with no loss */
+            dst_pix_fmt = loss_order2 ? dst_pix_fmt1 : dst_pix_fmt2;
         }
-    return PIX_FMT_NONE;
- found:
+    }
+
     if (loss_ptr)
         *loss_ptr = avcodec_get_pix_fmt_loss(dst_pix_fmt, src_pix_fmt, has_alpha);
     return dst_pix_fmt;