swscale/rgb2xyz: add explicit width parameter

This fixes an 11-year-old bug in the rgb2xyz functions, when used with a
negative stride. The current loop bounds turned it into a no-op.

Additionally, this increases performance on highly cropped images, whose
stride may be substantially higher than the effective width.

Sponsored-by: Sovereign Tech Fund
Signed-off-by: Niklas Haas <git@haasn.dev>

libswscale/swscale.c

@@ -638,12 +638,12 @@ static int check_image_pointers(const uint8_t * const data[4], enum AVPixelForma
 }
 
 static void xyz12Torgb48(struct SwsContext *c, uint16_t *dst,
-                         const uint16_t *src, int stride, int h)
+                         const uint16_t *src, int stride, int w, int h)
 {
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(c->srcFormat);
 
     for (int yp = 0; yp < h; yp++) {
-        for (int xp = 0; xp + 2 < stride; xp += 3) {
+        for (int xp = 0; xp < 3 * w; xp += 3) {
             int x, y, z, r, g, b;
 
             if (desc->flags & AV_PIX_FMT_FLAG_BE) {
@@ -693,12 +693,12 @@ static void xyz12Torgb48(struct SwsContext *c, uint16_t *dst,
 }
 
 static void rgb48Toxyz12(struct SwsContext *c, uint16_t *dst,
-                         const uint16_t *src, int stride, int h)
+                         const uint16_t *src, int stride, int w, int h)
 {
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(c->dstFormat);
 
     for (int yp = 0; yp < h; yp++) {
-        for (int xp = 0; xp + 2 < stride; xp += 3) {
+        for (int xp = 0; xp < 3 * w; xp += 3) {
             int x, y, z, r, g, b;
 
             if (desc->flags & AV_PIX_FMT_FLAG_BE) {
@@ -993,7 +993,8 @@ static int scale_internal(SwsContext *c,
         base = srcStride[0] < 0 ? c->xyz_scratch - srcStride[0] * (srcSliceH-1) :
                                   c->xyz_scratch;
 
-        xyz12Torgb48(c, (uint16_t*)base, (const uint16_t*)src2[0], srcStride[0]/2, srcSliceH);
+        xyz12Torgb48(c, (uint16_t*)base, (const uint16_t*)src2[0], srcStride[0]/2,
+                     c->srcW, srcSliceH);
         src2[0] = base;
     }
 
@@ -1065,7 +1066,7 @@ static int scale_internal(SwsContext *c,
         }
 
         /* replace on the same data */
-        rgb48Toxyz12(c, dst16, dst16, dstStride2[0]/2, ret);
+        rgb48Toxyz12(c, dst16, dst16, dstStride2[0]/2, c->dstW, ret);
     }
 
     /* reset slice direction at end of frame */

tests/ref/fate/filter-pixfmts-vflip

@@ -103,8 +103,8 @@ x2bgr10le           795b66a5fc83cd2cf300aae51c230f80
 x2rgb10le           262c502230cf3724f8e2cf4737f18a42
 xv30le              7e29ee107a1fabf3c7251f337d4b9fe5
 xv36le              bf1cbef0745f90881e15f5c5db3c5949
-xyz12be             810644e008deb231850d779aaa27cc7e
-xyz12le             829701db461b43533cf9241e0743bc61
+xyz12be             23fa9fb36d49dce61e284d41b83e0e6b
+xyz12le             ef73e6d1f932a9a355df1eedd628394f
 y210le              9544c81f8e1fc95e9fa4009dbecfea25
 y212le              c801725ae31e3b8f5be269359d49f191
 ya16be              55b1dbbe4d56ed0d22461685ce85520d