eac3dec: make GAQ dequantization 24-bit

Originally committed as revision 18888 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/ac3dec_data.c

@@ -87,19 +87,18 @@ const int16_t ff_eac3_gaq_remap_2_4_a[9][2] = {
 /**
  * Table E3.6, Gk=2 & Gk=4, B
  * Large mantissa inverse quantization, negative mantissa remapping offsets
- * Table values from the spec are right-shifted by 8 to simplify calculations.
  * ff_eac3_gaq_remap_3_4_b[hebap-8][Gk=2,4]
  */
-const int8_t ff_eac3_gaq_remap_2_4_b[9][2] = {
-    { -22,  -5 },
-    { -46, -20 },
-    { -56, -26 },
-    { -60, -29 },
-    { -62, -31 },
-    { -63, -32 },
-    { -64, -32 },
-    { -64, -32 },
-    { -64, -32 },
+const int16_t ff_eac3_gaq_remap_2_4_b[9][2] = {
+    {  -5461, -1170 },
+    { -11703, -4915 },
+    { -14199, -6606 },
+    { -15327, -7412 },
+    { -15864, -7805 },
+    { -16126, -7999 },
+    { -16255, -8096 },
+    { -16320, -8144 },
+    { -16352, -8168 }
 };
 
 static const int16_t vq_hebap1[4][6] = {

libavcodec/ac3dec_data.h

@@ -29,7 +29,7 @@ extern const uint8_t ff_eac3_hebap_tab[64];
 extern const uint8_t ff_eac3_bits_vs_hebap[20];
 extern const int16_t ff_eac3_gaq_remap_1[12];
 extern const int16_t ff_eac3_gaq_remap_2_4_a[9][2];
-extern const int8_t  ff_eac3_gaq_remap_2_4_b[9][2];
+extern const int16_t ff_eac3_gaq_remap_2_4_b[9][2];
 
 extern const int16_t (* const ff_eac3_mantissa_vq[8])[6];
 extern const uint8_t ff_eac3_frm_expstr[32][6];

libavcodec/eac3dec.c

@@ -178,19 +178,21 @@ void ff_eac3_decode_transform_coeffs_aht_ch(AC3DecodeContext *s, int ch)
                 if (mant == -(1 << (gbits-1))) {
                     /* large mantissa */
                     int b;
-                    mant = get_sbits(gbc, bits-2+log_gain) << (26-log_gain-bits);
+                    int mbits = bits - (2 - log_gain);
+                    mant = get_sbits(gbc, mbits);
+                    mant <<= (23 - (mbits - 1));
                     /* remap mantissa value to correct for asymmetric quantization */
                     if (mant >= 0)
-                        b = 32768 >> (log_gain+8);
+                        b = 1 << (23 - (mbits - 1));
                     else
-                        b = ff_eac3_gaq_remap_2_4_b[hebap-8][log_gain-1];
-                    mant += (ff_eac3_gaq_remap_2_4_a[hebap-8][log_gain-1] * (mant>>8) + b) >> 7;
+                        b = ff_eac3_gaq_remap_2_4_b[hebap-8][log_gain-1] << 8;
+                    mant += (((ff_eac3_gaq_remap_2_4_a[hebap-8][log_gain-1] << 8) * (int64_t)mant) >> 23) + b;
                 } else {
                     /* small mantissa, no GAQ, or Gk=1 */
                     mant <<= 24 - bits;
                     if (!log_gain) {
                         /* remap mantissa value for no GAQ or Gk=1 */
-                        mant += (ff_eac3_gaq_remap_1[hebap-8] * (mant>>8)) >> 7;
+                        mant += ((ff_eac3_gaq_remap_1[hebap-8] << 8) * (int64_t)mant) >> 23;
                     }
                 }
                 s->pre_mantissa[ch][bin][blk] = mant;