diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index bf0c13013a..8227ff50b2 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -910,7 +910,7 @@ int attribute_align_arg sws_scale(struct SwsContext *c,
         src2[0] = base;
     }
 
-    if (!srcSliceY && (c->flags & SWS_BITEXACT) && (c->flags & SWS_ERROR_DIFFUSION))
+    if (!srcSliceY && (c->flags & SWS_BITEXACT) && (c->flags & SWS_ERROR_DIFFUSION) && c->dither_error[0])
         for (i = 0; i < 4; i++)
             memset(c->dither_error[i], 0, sizeof(c->dither_error[0][0]) * (c->dstW+2));