avfilter/vf_rotate: fix several of by 1 errors

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavfilter/vf_rotate.c

@@ -328,8 +328,8 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int job, int nb_jobs)
     int i, j, x, y;
 
     for (j = start; j < end; j++) {
-        x = xprime + xi + FIXP*inw/2;
-        y = yprime + yi + FIXP*inh/2;
+        x = xprime + xi + FIXP*(inw-1)/2;
+        y = yprime + yi + FIXP*(inh-1)/2;
 
         for (i = 0; i < outw; i++) {
             int32_t v;
@@ -421,9 +421,9 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
                           .inw  = FF_CEIL_RSHIFT(inlink->w, hsub),
                           .inh  = FF_CEIL_RSHIFT(inlink->h, vsub),
                           .outh = outh, .outw = outw,
-                          .xi = -outw/2 * c, .yi =  outw/2 * s,
-                          .xprime = -outh/2 * s,
-                          .yprime = -outh/2 * c,
+                          .xi = -(outw-1) * c / 2, .yi =  (outw-1) * s / 2,
+                          .xprime = -(outh-1) * s / 2,
+                          .yprime = -(outh-1) * c / 2,
                           .plane = plane, .c = c, .s = s };
 
 

tests/ref/fate/filter-pixfmts-rotate

@@ -1,20 +1,20 @@
-0bgr                979522e1115cd253cb531e26216084ee
-0rgb                1eadac4ec11fdcc2fe205f1b443f5360
-abgr                cff4da9c08268792efadcdb3192cab4d
-argb                cb9db4de21026dffd63716660b77e104
-bgr0                aa85111028cfcf59d22b21b764e767e3
-bgr24               aad1662159f1b4a7258270a175dcc3f3
-bgra                9eeaa89344ed4266bc6c9bd5ffc03b16
-gbrap               3aab798f7ff4e641bbd710cfea84f5b4
-gbrp                f5f2c992b6bbe1a5a11ce361eed95308
-gray                28371cc84367c01807d7ac374343aa37
-rgb0                f843a29d4dd63f27203f05a9475af657
-rgb24               f06c68509dd7a4b3a5ca39f0783edba0
-rgba                029074e91fbcdd43ba2e6e48a0bdbbd5
-yuv410p             b98886adb9fc00a8be5e58a515278a48
-yuv420p             f3ddf03e0802a292b4cfecdc9f64e522
-yuv444p             1c8f2bd43afbb1de8ec3078ca4f94117
-yuva420p            761944cf562e2a968e8275ce6d750fd7
-yuva444p            dbe62d6e155c8b16e7a708efe3967208
-yuvj420p            e3750224275c04421e7b9a35af7b8677
-yuvj444p            107d01d31b86dd8381d6c28e5a090281
+0bgr                cdd2136dc74f315020290ebf8b95fb75
+0rgb                55a14614aaa4144928b4e795c3990da8
+abgr                6138620129451d433ed169600476aaad
+argb                64ade9d0e67fea30e3e119995df3ffa0
+bgr0                c5cca289d918de99885968ef7be34039
+bgr24               f0132dd813a7d99ca6a412759e55db76
+bgra                9c1e00c6b6baa25ed54660c6ca4f2914
+gbrap               f26cdfccf9f1f5da750127a9406005c8
+gbrp                50e955e81205ea20cfd5170830f91e0b
+gray                f0ef450ff90edc58dc48e790f6c10fcc
+rgb0                01e16160f78fa5a171d794b3594bbf5b
+rgb24               e445aeb43f0a03c03af571df1dd98914
+rgba                39b3bd5b8af36873d6723b17f16e5fa7
+yuv410p             0b702cab08cb79d07cbd54ad1bfd08ba
+yuv420p             4a8c4786b9bd2b0156d719870dc98fbf
+yuv444p             85a0a7d0a28d2b9f5239cd2ee06dc0e4
+yuva420p            bf368d93ae54ca84235cb482b948229b
+yuva444p            6d528668eb4ee9f06aa56b0bd9a8bf34
+yuvj420p            715334a97fcf8b0a911174c45966d90e
+yuvj444p            f41bd0d4b8f238c518178159613745b4