avcodec/mpegvideo: Change mpeg2 unquant to work with higher precission qscale

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/mips/mpegvideo_init_mips.c

@@ -26,6 +26,7 @@ static av_cold void dct_unquantize_init_msa(MpegEncContext *s)
 {
     s->dct_unquantize_h263_intra = ff_dct_unquantize_h263_intra_msa;
     s->dct_unquantize_h263_inter = ff_dct_unquantize_h263_inter_msa;
+    if (!s->q_scale_type)
         s->dct_unquantize_mpeg2_inter = ff_dct_unquantize_mpeg2_inter_msa;
 }
 #endif  // #if HAVE_MSA
@@ -39,6 +40,7 @@ static av_cold void dct_unquantize_init_mmi(MpegEncContext *s)
     s->dct_unquantize_mpeg1_inter = ff_dct_unquantize_mpeg1_inter_mmi;
 
     if (!(s->avctx->flags & AV_CODEC_FLAG_BITEXACT))
+        if (!s->q_scale_type)
             s->dct_unquantize_mpeg2_intra = ff_dct_unquantize_mpeg2_intra_mmi;
 
     s->denoise_dct= ff_denoise_dct_mmi;

libavcodec/mpegvideo.c

@@ -114,6 +114,8 @@ static void dct_unquantize_mpeg2_intra_c(MpegEncContext *s,
     int i, level, nCoeffs;
     const uint16_t *quant_matrix;
 
+    qscale <<= 1;
+
     if(s->alternate_scan) nCoeffs= 63;
     else nCoeffs= s->block_last_index[n];
 
@@ -125,10 +127,10 @@ static void dct_unquantize_mpeg2_intra_c(MpegEncContext *s,
         if (level) {
             if (level < 0) {
                 level = -level;
-                level = (int)(level * qscale * quant_matrix[j]) >> 3;
+                level = (int)(level * qscale * quant_matrix[j]) >> 4;
                 level = -level;
             } else {
-                level = (int)(level * qscale * quant_matrix[j]) >> 3;
+                level = (int)(level * qscale * quant_matrix[j]) >> 4;
             }
             block[j] = level;
         }
@@ -142,6 +144,8 @@ static void dct_unquantize_mpeg2_intra_bitexact(MpegEncContext *s,
     const uint16_t *quant_matrix;
     int sum=-1;
 
+    qscale <<= 1;
+
     if(s->alternate_scan) nCoeffs= 63;
     else nCoeffs= s->block_last_index[n];
 
@@ -154,10 +158,10 @@ static void dct_unquantize_mpeg2_intra_bitexact(MpegEncContext *s,
         if (level) {
             if (level < 0) {
                 level = -level;
-                level = (int)(level * qscale * quant_matrix[j]) >> 3;
+                level = (int)(level * qscale * quant_matrix[j]) >> 4;
                 level = -level;
             } else {
-                level = (int)(level * qscale * quant_matrix[j]) >> 3;
+                level = (int)(level * qscale * quant_matrix[j]) >> 4;
             }
             block[j] = level;
             sum+=level;
@@ -173,6 +177,8 @@ static void dct_unquantize_mpeg2_inter_c(MpegEncContext *s,
     const uint16_t *quant_matrix;
     int sum=-1;
 
+    qscale <<= 1;
+
     if(s->alternate_scan) nCoeffs= 63;
     else nCoeffs= s->block_last_index[n];
 
@@ -184,11 +190,11 @@ static void dct_unquantize_mpeg2_inter_c(MpegEncContext *s,
             if (level < 0) {
                 level = -level;
                 level = (((level << 1) + 1) * qscale *
-                         ((int) (quant_matrix[j]))) >> 4;
+                         ((int) (quant_matrix[j]))) >> 5;
                 level = -level;
             } else {
                 level = (((level << 1) + 1) * qscale *
-                         ((int) (quant_matrix[j]))) >> 4;
+                         ((int) (quant_matrix[j]))) >> 5;
             }
             block[j] = level;
             sum+=level;

libavcodec/x86/mpegvideo.c

@@ -308,6 +308,8 @@ static void dct_unquantize_mpeg2_intra_mmx(MpegEncContext *s,
 
     av_assert2(s->block_last_index[n]>=0);
 
+    qscale <<= 1;
+
     if(s->alternate_scan) nCoeffs= 63; //FIXME
     else nCoeffs= s->intra_scantable.raster_end[ s->block_last_index[n] ];
 
@@ -345,8 +347,8 @@ __asm__ volatile(
                 "pxor %%mm5, %%mm5              \n\t" // FIXME slow
                 "pcmpeqw (%0, %%"REG_a"), %%mm4 \n\t" // block[i] == 0 ? -1 : 0
                 "pcmpeqw 8(%0, %%"REG_a"), %%mm5\n\t" // block[i] == 0 ? -1 : 0
-                "psraw $3, %%mm0                \n\t"
-                "psraw $3, %%mm1                \n\t"
+                "psraw $4, %%mm0                \n\t"
+                "psraw $4, %%mm1                \n\t"
                 "pxor %%mm2, %%mm0              \n\t"
                 "pxor %%mm3, %%mm1              \n\t"
                 "psubw %%mm2, %%mm0             \n\t"
@@ -373,6 +375,8 @@ static void dct_unquantize_mpeg2_inter_mmx(MpegEncContext *s,
 
     av_assert2(s->block_last_index[n]>=0);
 
+    qscale <<= 1;
+
     if(s->alternate_scan) nCoeffs= 63; //FIXME
     else nCoeffs= s->intra_scantable.raster_end[ s->block_last_index[n] ];
 
@@ -410,8 +414,8 @@ __asm__ volatile(
                 "pxor %%mm5, %%mm5              \n\t" // FIXME slow
                 "pcmpeqw (%0, %%"REG_a"), %%mm4 \n\t" // block[i] == 0 ? -1 : 0
                 "pcmpeqw 8(%0, %%"REG_a"), %%mm5\n\t" // block[i] == 0 ? -1 : 0
-                "psrlw $4, %%mm0                \n\t"
-                "psrlw $4, %%mm1                \n\t"
+                "psrlw $5, %%mm0                \n\t"
+                "psrlw $5, %%mm1                \n\t"
                 "pxor %%mm2, %%mm0              \n\t"
                 "pxor %%mm3, %%mm1              \n\t"
                 "psubw %%mm2, %%mm0             \n\t"