From d7eabd50425a61b31e90c763a0c3e4316a725404 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sat, 10 Mar 2012 14:28:08 -0800 Subject: [PATCH 1/3] mpc: pad mpc_CC/SCF[] tables to allow for negative indices. MPC8 allows indices of mpc_CC up to -1, and mpc_SCF up to -6, thus pad the tables by that much on the left end. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libavcodec/mpc.c | 6 +++--- libavcodec/mpc7.c | 4 ++-- libavcodec/mpcdata.h | 10 +++++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/libavcodec/mpc.c b/libavcodec/mpc.c index 4573860525..6b15a33e5a 100644 --- a/libavcodec/mpc.c +++ b/libavcodec/mpc.c @@ -78,13 +78,13 @@ void ff_mpc_dequantize_and_synth(MPCContext * c, int maxband, void *data, int ch for(ch = 0; ch < 2; ch++){ if(bands[i].res[ch]){ j = 0; - mul = mpc_CC[bands[i].res[ch]] * mpc_SCF[bands[i].scf_idx[ch][0]]; + mul = mpc_CC[bands[i].res[ch] + 1] * mpc_SCF[bands[i].scf_idx[ch][0]+6]; for(; j < 12; j++) c->sb_samples[ch][j][i] = mul * c->Q[ch][j + off]; - mul = mpc_CC[bands[i].res[ch]] * mpc_SCF[bands[i].scf_idx[ch][1]]; + mul = mpc_CC[bands[i].res[ch] + 1] * mpc_SCF[bands[i].scf_idx[ch][1]+6]; for(; j < 24; j++) c->sb_samples[ch][j][i] = mul * c->Q[ch][j + off]; - mul = mpc_CC[bands[i].res[ch]] * mpc_SCF[bands[i].scf_idx[ch][2]]; + mul = mpc_CC[bands[i].res[ch] + 1] * mpc_SCF[bands[i].scf_idx[ch][2]+6]; for(; j < 36; j++) c->sb_samples[ch][j][i] = mul * c->Q[ch][j + off]; } diff --git a/libavcodec/mpc7.c b/libavcodec/mpc7.c index 2e8271a120..4e75841008 100644 --- a/libavcodec/mpc7.c +++ b/libavcodec/mpc7.c @@ -193,7 +193,7 @@ static int get_scale_idx(GetBitContext *gb, int ref) int t = get_vlc2(gb, dscf_vlc.table, MPC7_DSCF_BITS, 1) - 7; if (t == 8) return get_bits(gb, 6); - return ref + t; + return av_clip_uintp2(ref + t, 7); } static int mpc7_decode_frame(AVCodecContext * avctx, void *data, @@ -247,7 +247,7 @@ static int mpc7_decode_frame(AVCodecContext * avctx, void *data, int t = 4; if(i) t = get_vlc2(&gb, hdr_vlc.table, MPC7_HDR_BITS, 1) - 5; if(t == 4) bands[i].res[ch] = get_bits(&gb, 4); - else bands[i].res[ch] = bands[i-1].res[ch] + t; + else bands[i].res[ch] = av_clip(bands[i-1].res[ch] + t, 0, 17); } if(bands[i].res[0] || bands[i].res[1]){ diff --git a/libavcodec/mpcdata.h b/libavcodec/mpcdata.h index 397dad59d8..15724f3b74 100644 --- a/libavcodec/mpcdata.h +++ b/libavcodec/mpcdata.h @@ -22,13 +22,17 @@ #ifndef AVCODEC_MPCDATA_H #define AVCODEC_MPCDATA_H -static const float mpc_CC[18] = { - 65536.0000, 21845.3333, 13107.2000, 9362.2857, 7281.7778, 4369.0667, 2114.0645, +static const float mpc_CC[18+1] = { + 111.285962475327f, // 32768/2/255*sqrt(3) + 65536.0000 /* this value is never used */, + 21845.3333, 13107.2000, 9362.2857, 7281.7778, 4369.0667, 2114.0645, 1040.2539, 516.0315, 257.0039, 128.2505, 64.0626, 32.0156, 16.0039, 8.0010, 4.0002, 2.0001, 1.0000 }; -static const float mpc_SCF[128] = { +static const float mpc_SCF[128+6] = { + 920.016296386718750000, 766.355773925781250000, 638.359558105468750000, + 531.741149902343750000, 442.930114746093750000, 368.952209472656250000, 307.330047607421875000, 255.999984741210937500, 213.243041992187500000, 177.627334594726562500, 147.960128784179687500, 123.247924804687500000, 102.663139343261718750, 85.516410827636718750, 71.233520507812500000, 59.336143493652343750, 49.425861358642578125, 41.170787811279296875, From 97e48b2f541396ef6e8816a555bac1bb993d7a6a Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sat, 10 Mar 2012 17:51:28 -0800 Subject: [PATCH 2/3] cook: error out on quant_index values outside [-63, 63] range. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libavcodec/cook.c | 50 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/libavcodec/cook.c b/libavcodec/cook.c index 41ce0e528e..74378521b3 100644 --- a/libavcodec/cook.c +++ b/libavcodec/cook.c @@ -366,8 +366,8 @@ static void decode_gain_info(GetBitContext *gb, int *gaininfo) * @param q pointer to the COOKContext * @param quant_index_table pointer to the array */ -static void decode_envelope(COOKContext *q, COOKSubpacket *p, - int *quant_index_table) +static int decode_envelope(COOKContext *q, COOKSubpacket *p, + int *quant_index_table) { int i, j, vlc_index; @@ -388,7 +388,15 @@ static void decode_envelope(COOKContext *q, COOKSubpacket *p, j = get_vlc2(&q->gb, q->envelope_quant_index[vlc_index - 1].table, q->envelope_quant_index[vlc_index - 1].bits, 2); quant_index_table[i] = quant_index_table[i - 1] + j - 12; // differential encoding + if (quant_index_table[i] > 63 || quant_index_table[i] < -63) { + av_log(q->avctx, AV_LOG_ERROR, + "Invalid quantizer %d at position %d, outside [-63, 63] range\n", + quant_index_table[i], i); + return AVERROR_INVALIDDATA; + } } + + return 0; } /** @@ -639,20 +647,24 @@ static void decode_vectors(COOKContext *q, COOKSubpacket *p, int *category, * @param q pointer to the COOKContext * @param mlt_buffer pointer to mlt coefficients */ -static void mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer) +static int mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer) { int category_index[128]; int quant_index_table[102]; int category[128]; + int res; memset(&category, 0, sizeof(category)); memset(&category_index, 0, sizeof(category_index)); - decode_envelope(q, p, quant_index_table); + if ((res = decode_envelope(q, p, quant_index_table)) < 0) + return res; q->num_vectors = get_bits(&q->gb, p->log2_numvector_size); categorize(q, p, quant_index_table, category, category_index); expand_category(q, category, category_index); decode_vectors(q, p, category, quant_index_table, mlt_buffer); + + return 0; } @@ -802,10 +814,10 @@ static void decouple_float(COOKContext *q, * @param mlt_buffer1 pointer to left channel mlt coefficients * @param mlt_buffer2 pointer to right channel mlt coefficients */ -static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1, - float *mlt_buffer2) +static int joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1, + float *mlt_buffer2) { - int i, j; + int i, j, res; int decouple_tab[SUBBAND_SIZE]; float *decode_buffer = q->decode_buffer_0; int idx, cpl_tmp; @@ -819,7 +831,8 @@ static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1, memset(mlt_buffer1, 0, 1024 * sizeof(*mlt_buffer1)); memset(mlt_buffer2, 0, 1024 * sizeof(*mlt_buffer2)); decouple_info(q, p, decouple_tab); - mono_decode(q, p, decode_buffer); + if ((res = mono_decode(q, p, decode_buffer)) < 0) + return res; /* The two channels are stored interleaved in decode_buffer. */ for (i = 0; i < p->js_subband_start; i++) { @@ -841,6 +854,8 @@ static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1, q->decouple(q, p, i, f1, f2, decode_buffer, mlt_buffer1, mlt_buffer2); idx = (1 << p->js_vlc_bits) - 1; } + + return 0; } /** @@ -913,10 +928,11 @@ static inline void mlt_compensate_output(COOKContext *q, float *decode_buffer, * @param inbuffer pointer to the inbuffer * @param outbuffer pointer to the outbuffer */ -static void decode_subpacket(COOKContext *q, COOKSubpacket *p, - const uint8_t *inbuffer, float *outbuffer) +static int decode_subpacket(COOKContext *q, COOKSubpacket *p, + const uint8_t *inbuffer, float *outbuffer) { int sub_packet_size = p->size; + int res; /* packet dump */ // for (i = 0; i < sub_packet_size ; i++) // av_log(q->avctx, AV_LOG_ERROR, "%02x", inbuffer[i]); @@ -925,13 +941,16 @@ static void decode_subpacket(COOKContext *q, COOKSubpacket *p, decode_bytes_and_gain(q, p, inbuffer, &p->gains1); if (p->joint_stereo) { - joint_decode(q, p, q->decode_buffer_1, q->decode_buffer_2); + if ((res = joint_decode(q, p, q->decode_buffer_1, q->decode_buffer_2)) < 0) + return res; } else { - mono_decode(q, p, q->decode_buffer_1); + if ((res = mono_decode(q, p, q->decode_buffer_1)) < 0) + return res; if (p->num_channels == 2) { decode_bytes_and_gain(q, p, inbuffer + sub_packet_size / 2, &p->gains2); - mono_decode(q, p, q->decode_buffer_2); + if ((res = mono_decode(q, p, q->decode_buffer_2)) < 0) + return res; } } @@ -945,6 +964,8 @@ static void decode_subpacket(COOKContext *q, COOKSubpacket *p, else mlt_compensate_output(q, q->decode_buffer_2, &p->gains2, p->mono_previous_buffer2, outbuffer, p->ch_idx + 1); + + return 0; } @@ -1000,7 +1021,8 @@ static int cook_decode_frame(AVCodecContext *avctx, void *data, i, q->subpacket[i].size, q->subpacket[i].joint_stereo, offset, avctx->block_align); - decode_subpacket(q, &q->subpacket[i], buf + offset, samples); + if ((ret = decode_subpacket(q, &q->subpacket[i], buf + offset, samples)) < 0) + return ret; offset += q->subpacket[i].size; chidx += q->subpacket[i].num_channels; av_log(avctx, AV_LOG_DEBUG, "subpacket[%i] %i %i\n", From c95fefa0420be9cc0f09a95041acf11114aaacd0 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sun, 11 Mar 2012 07:28:54 -0700 Subject: [PATCH 3/3] dsicinvideo: validate buffer offset before copying pixels. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libavcodec/dsicinav.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/libavcodec/dsicinav.c b/libavcodec/dsicinav.c index 37d39f5405..a379531613 100644 --- a/libavcodec/dsicinav.c +++ b/libavcodec/dsicinav.c @@ -146,11 +146,11 @@ static int cin_decode_huffman(const unsigned char *src, int src_size, unsigned c return dst_cur - dst; } -static void cin_decode_lzss(const unsigned char *src, int src_size, unsigned char *dst, int dst_size) +static int cin_decode_lzss(const unsigned char *src, int src_size, unsigned char *dst, int dst_size) { uint16_t cmd; int i, sz, offset, code; - unsigned char *dst_end = dst + dst_size; + unsigned char *dst_end = dst + dst_size, *dst_start = dst; const unsigned char *src_end = src + src_size; while (src < src_end && dst < dst_end) { @@ -161,6 +161,8 @@ static void cin_decode_lzss(const unsigned char *src, int src_size, unsigned cha } else { cmd = AV_RL16(src); src += 2; offset = cmd >> 4; + if ((int) (dst - dst_start) < offset + 1) + return AVERROR_INVALIDDATA; sz = (cmd & 0xF) + 2; /* don't use memcpy/memmove here as the decoding routine (ab)uses */ /* buffer overlappings to repeat bytes in the destination */ @@ -172,6 +174,8 @@ static void cin_decode_lzss(const unsigned char *src, int src_size, unsigned cha } } } + + return 0; } static void cin_decode_rle(const unsigned char *src, int src_size, unsigned char *dst, int dst_size) @@ -201,13 +205,7 @@ static int cinvideo_decode_frame(AVCodecContext *avctx, const uint8_t *buf = avpkt->data; int buf_size = avpkt->size; CinVideoContext *cin = avctx->priv_data; - int i, y, palette_type, palette_colors_count, bitmap_frame_type, bitmap_frame_size; - - cin->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; - if (avctx->reget_buffer(avctx, &cin->frame)) { - av_log(cin->avctx, AV_LOG_ERROR, "delphinecinvideo: reget_buffer() failed to allocate a frame\n"); - return -1; - } + int i, y, palette_type, palette_colors_count, bitmap_frame_type, bitmap_frame_size, res = 0; palette_type = buf[0]; palette_colors_count = AV_RL16(buf+1); @@ -233,8 +231,6 @@ static int cinvideo_decode_frame(AVCodecContext *avctx, bitmap_frame_size -= 4; } } - memcpy(cin->frame.data[1], cin->palette, sizeof(cin->palette)); - cin->frame.palette_has_changed = 1; /* note: the decoding routines below assumes that surface.width = surface.pitch */ switch (bitmap_frame_type) { @@ -267,17 +263,31 @@ static int cinvideo_decode_frame(AVCodecContext *avctx, cin->bitmap_table[CIN_CUR_BMP], cin->bitmap_size); break; case 38: - cin_decode_lzss(buf, bitmap_frame_size, - cin->bitmap_table[CIN_CUR_BMP], cin->bitmap_size); + res = cin_decode_lzss(buf, bitmap_frame_size, + cin->bitmap_table[CIN_CUR_BMP], + cin->bitmap_size); + if (res < 0) + return res; break; case 39: - cin_decode_lzss(buf, bitmap_frame_size, - cin->bitmap_table[CIN_CUR_BMP], cin->bitmap_size); + res = cin_decode_lzss(buf, bitmap_frame_size, + cin->bitmap_table[CIN_CUR_BMP], + cin->bitmap_size); + if (res < 0) + return res; cin_apply_delta_data(cin->bitmap_table[CIN_PRE_BMP], cin->bitmap_table[CIN_CUR_BMP], cin->bitmap_size); break; } + cin->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; + if (avctx->reget_buffer(avctx, &cin->frame)) { + av_log(cin->avctx, AV_LOG_ERROR, "delphinecinvideo: reget_buffer() failed to allocate a frame\n"); + return -1; + } + + memcpy(cin->frame.data[1], cin->palette, sizeof(cin->palette)); + cin->frame.palette_has_changed = 1; for (y = 0; y < cin->avctx->height; ++y) memcpy(cin->frame.data[0] + (cin->avctx->height - 1 - y) * cin->frame.linesize[0], cin->bitmap_table[CIN_CUR_BMP] + y * cin->avctx->width,