avcodec/jpeg2000dec: Merge i_stepsize/2 decoder special case into dequant

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/jpeg2000.c

@@ -330,13 +330,14 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
                     band->f_stepsize *= pow(F_LFTG_K, 2*(codsty->nreslevels2decode - reslevelno) + lband - 2);
                 }
             }
+
+            band->i_stepsize = band->f_stepsize * (1 << 15);
+
             /* FIXME: In openjepg code stespize = stepsize * 0.5. Why?
              * If not set output of entropic decoder is not correct. */
             if (!av_codec_is_encoder(avctx->codec))
                 band->f_stepsize *= 0.5;
 
-            band->i_stepsize = band->f_stepsize * (1 << 15);
-
             /* computation of tbx_0, tbx_1, tby_0, tby_1
              * see ISO/IEC 15444-1:2002 B.5 eq. B-15 and tbl B.1
              * codeblock width and height is computed for

libavcodec/jpeg2000dec.c

@@ -1559,13 +1559,13 @@ static void dequantization_int(int x, int y, Jpeg2000Cblk *cblk,
     for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j) {
         int32_t *datap = &comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * (y + j) + x];
         int *src = t1->data + j*t1->stride;
-        if (band->i_stepsize == 16384) {
+        if (band->i_stepsize == 32768) {
             for (i = 0; i < w; ++i)
                 datap[i] = src[i] / 2;
         } else {
             // This should be VERY uncommon
             for (i = 0; i < w; ++i)
-                datap[i] = (src[i] * (int64_t)band->i_stepsize) / 32768;
+                datap[i] = (src[i] * (int64_t)band->i_stepsize) / 65536;
         }
     }
 }
@@ -1580,7 +1580,7 @@ static void dequantization_int_97(int x, int y, Jpeg2000Cblk *cblk,
         int32_t *datap = &comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * (y + j) + x];
         int *src = t1->data + j*t1->stride;
         for (i = 0; i < w; ++i)
-            datap[i] = (src[i] * (int64_t)band->i_stepsize + (1<<14)) >> 15;
+            datap[i] = (src[i] * (int64_t)band->i_stepsize + (1<<15)) >> 16;
     }
 }
 

tests/ref/fate/jpeg2000-dcinema

@@ -1,3 +1,3 @@
 #tb 0: 1/24
-0,          0,          0,        1, 12441600, 0xda6b6cde
-0,          1,          1,        1, 12441600, 0xb0994664
+0,          0,          0,        1, 12441600, 0xfcf6a127
+0,          1,          1,        1, 12441600, 0x577b6a64

tests/ref/vsynth/vsynth1-jpeg2000-97

@@ -1,4 +1,4 @@
 a19cc0e1a1c1bf76ff5a0b63a0bdfbd1 *tests/data/fate/vsynth1-jpeg2000-97.avi
 3654420 tests/data/fate/vsynth1-jpeg2000-97.avi
-3b71c0f8aebf45122da77d892a6ebf00 *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
+5073771a78e1f5366a7eb0df341662fc *tests/data/fate/vsynth1-jpeg2000-97.out.rawvideo
 stddev:    4.23 PSNR: 35.59 MAXDIFF:   53 bytes:  7603200/  7603200

tests/ref/vsynth/vsynth2-jpeg2000-97

@@ -1,4 +1,4 @@
 b86217f0bcbd84a9368ad3f98af32157 *tests/data/fate/vsynth2-jpeg2000-97.avi
 2448506 tests/data/fate/vsynth2-jpeg2000-97.avi
-4d9d9db91075a1eca2a6b9f152e4defc *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
+a6e2453118a0de135836a868b2ca0e60 *tests/data/fate/vsynth2-jpeg2000-97.out.rawvideo
 stddev:    3.23 PSNR: 37.94 MAXDIFF:   29 bytes:  7603200/  7603200

tests/ref/vsynth/vsynth3-jpeg2000-97

@@ -1,4 +1,4 @@
 5e17fdaae1a22f3eef8c82b512e4b1b9 *tests/data/fate/vsynth3-jpeg2000-97.avi
 83670 tests/data/fate/vsynth3-jpeg2000-97.avi
-8ec04513b2e6645c9ea340e3fe9fe8f2 *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
+0cd707bfb1bbe5312b00c094f695b1fa *tests/data/fate/vsynth3-jpeg2000-97.out.rawvideo
 stddev:    4.52 PSNR: 35.02 MAXDIFF:   47 bytes:    86700/    86700

tests/ref/vsynth/vsynth_lena-jpeg2000-97

@@ -1,4 +1,4 @@
 ca78db12e1af7cbf44fdce165aaa5130 *tests/data/fate/vsynth_lena-jpeg2000-97.avi
 1918756 tests/data/fate/vsynth_lena-jpeg2000-97.avi
-5fd8a2e35503b48af302b3ef5e317683 *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
+93a4ba0c230f2430a813df594676e58a *tests/data/fate/vsynth_lena-jpeg2000-97.out.rawvideo
 stddev:    2.84 PSNR: 39.04 MAXDIFF:   28 bytes:  7603200/  7603200