swscale: improve dither checks

Bssed on patch by Øyvind Kolås <pippin@gimp.org>

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libswscale/swscale_unscaled.c

@@ -1180,7 +1180,7 @@ void ff_get_unscaled_swscale(SwsContext *c)
     /* yuv2bgr */
     if ((srcFormat == AV_PIX_FMT_YUV420P || srcFormat == AV_PIX_FMT_YUV422P ||
          srcFormat == AV_PIX_FMT_YUVA420P) && isAnyRGB(dstFormat) &&
-        !(flags & SWS_ACCURATE_RND) && c->dither != SWS_DITHER_ED && !(dstH & 1)) {
+        !(flags & SWS_ACCURATE_RND) && (c->dither == SWS_DITHER_BAYER || c->dither == SWS_DITHER_AUTO) && !(dstH & 1)) {
         c->swScale = ff_yuv2rgb_get_func_ptr(c);
     }
 

libswscale/utils.c

@@ -1205,18 +1205,22 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
        dstFormat == AV_PIX_FMT_RGB8) {
         if (c->dither == SWS_DITHER_AUTO)
             c->dither = (flags & SWS_FULL_CHR_H_INT) ? SWS_DITHER_ED : SWS_DITHER_BAYER;
-        if (c->dither == SWS_DITHER_ED && !(flags & SWS_FULL_CHR_H_INT)) {
-            av_log(c, AV_LOG_DEBUG,
-                "Error diffusion dither is only supported in full chroma interpolation for destination format '%s'\n",
-                av_get_pix_fmt_name(dstFormat));
-            flags   |= SWS_FULL_CHR_H_INT;
-            c->flags = flags;
+        if (!(flags & SWS_FULL_CHR_H_INT)) {
+            if (c->dither == SWS_DITHER_ED) {
+                av_log(c, AV_LOG_DEBUG,
+                    "Desired dithering only supported in full chroma interpolation for destination format '%s'\n",
+                    av_get_pix_fmt_name(dstFormat));
+                flags   |= SWS_FULL_CHR_H_INT;
+                c->flags = flags;
+            }
         }
-        if (c->dither != SWS_DITHER_ED && (flags & SWS_FULL_CHR_H_INT)) {
-            av_log(c, AV_LOG_DEBUG,
-                "Ordered dither is not supported in full chroma interpolation for destination format '%s'\n",
-                av_get_pix_fmt_name(dstFormat));
-            c->dither = SWS_DITHER_ED;
+        if (flags & SWS_FULL_CHR_H_INT) {
+            if (c->dither == SWS_DITHER_BAYER) {
+                av_log(c, AV_LOG_DEBUG,
+                    "Ordered dither is not supported in full chroma interpolation for destination format '%s'\n",
+                    av_get_pix_fmt_name(dstFormat));
+                c->dither = SWS_DITHER_ED;
+            }
         }
     }
     if (isPlanarRGB(dstFormat)) {