Make Smacker audio decoder output audio in original bit depth

Patch by Daniel Verkamp
($firstname) at (three-letter file extension for drivers in Win 3.1) dot (nu)

Thread: [PATCH] Smacker: Output audio in original bit depth

Originally committed as revision 18111 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/smacker.c

@@ -558,7 +558,6 @@ static av_cold int decode_end(AVCodecContext *avctx)
 
 static av_cold int smka_decode_init(AVCodecContext *avctx)
 {
-    avctx->sample_fmt = SAMPLE_FMT_S16;
     avctx->channel_layout = (avctx->channels==2) ? CH_LAYOUT_STEREO : CH_LAYOUT_MONO;
     return 0;
 }
@@ -572,6 +571,7 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     HuffContext h[4];
     VLC vlc[4];
     int16_t *samples = data;
+    int8_t *samples8 = data;
     int val;
     int i, res;
     int unp_size;
@@ -589,7 +589,7 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     }
     stereo = get_bits1(&gb);
     bits = get_bits1(&gb);
-    if (unp_size & 0xC0000000 || (unp_size << !bits) > *data_size) {
+    if (unp_size & 0xC0000000 || unp_size > *data_size) {
         av_log(avctx, AV_LOG_ERROR, "Frame is too large to fit in buffer\n");
         return -1;
     }
@@ -655,7 +655,7 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         for(i = stereo; i >= 0; i--)
             pred[i] = get_bits(&gb, 8);
         for(i = 0; i < stereo; i++)
-            *samples++ = (pred[i] - 0x80) << 8;
+            *samples8++ = pred[i];
         for(i = 0; i < unp_size; i++) {
             if(i & stereo){
                 if(vlc[1].table)
@@ -663,17 +663,16 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
                 else
                     res = 0;
                 pred[1] += (int8_t)h[1].values[res];
-                *samples++ = (pred[1] - 0x80) << 8;
+                *samples8++ = pred[1];
             } else {
                 if(vlc[0].table)
                     res = get_vlc2(&gb, vlc[0].table, SMKTREE_BITS, 3);
                 else
                     res = 0;
                 pred[0] += (int8_t)h[0].values[res];
-                *samples++ = (pred[0] - 0x80) << 8;
+                *samples8++ = pred[0];
             }
         }
-        unp_size *= 2;
     }
 
     for(i = 0; i < 4; i++) {

libavformat/smacker.c

@@ -183,6 +183,7 @@ static int smacker_read_header(AVFormatContext *s, AVFormatParameters *ap)
             ast[i]->codec->bits_per_coded_sample = (smk->rates[i] & SMK_AUD_16BITS) ? 16 : 8;
             if(ast[i]->codec->bits_per_coded_sample == 16 && ast[i]->codec->codec_id == CODEC_ID_PCM_U8)
                 ast[i]->codec->codec_id = CODEC_ID_PCM_S16LE;
+            ast[i]->codec->sample_fmt = ast[i]->codec->bits_per_coded_sample == 8 ? SAMPLE_FMT_U8 : SAMPLE_FMT_S16;
             av_set_pts_info(ast[i], 64, 1, ast[i]->codec->sample_rate
                     * ast[i]->codec->channels * ast[i]->codec->bits_per_coded_sample / 8);
         }