Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

huffyuv: Check and propagate function return values

Browse Source 
Bug-Id: CVE-2013-0868

inspired by a patch from Michael Niedermayer <michaelni@gmx.at>
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Diego Biurrun <diego@biurrun.de>

CC: libav-stable@libav.org
pull/80/head
Diego Biurrun 11 years ago
parent 6234058148
commit d0393d79bc
1 changed files with 61 additions and 44 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 105
      libavcodec/huffyuvdec.c

105
libavcodec/huffyuvdec.c
Unescape View File

@ -106,11 +106,13 @@ static int read_len_table(uint8_t *dst, GetBitContext *gb)
return 0; return 0;
} }
static void generate_joint_tables(HYuvContext *s) static int generate_joint_tables(HYuvContext *s)
{ {
uint16_t symbols[1 << VLC_BITS]; uint16_t symbols[1 << VLC_BITS];
uint16_t bits[1 << VLC_BITS]; uint16_t bits[1 << VLC_BITS];
uint8_t len[1 << VLC_BITS]; uint8_t len[1 << VLC_BITS];
int ret;
if (s->bitstream_bpp < 24) { if (s->bitstream_bpp < 24) {
int p, i, y, u; int p, i, y, u;
for (p = 0; p < 3; p++) { for (p = 0; p < 3; p++) {
@ -131,8 +133,9 @@ static void generate_joint_tables(HYuvContext *s)
} }
} }
ff_free_vlc(&s->vlc[3 + p]); ff_free_vlc(&s->vlc[3 + p]);
ff_init_vlc_sparse(&s->vlc[3 + p], VLC_BITS, i, len, 1, 1, if ((ret = ff_init_vlc_sparse(&s->vlc[3 + p], VLC_BITS, i, len, 1, 1,
bits, 2, 2, symbols, 2, 2, 0); bits, 2, 2, symbols, 2, 2, 0)) < 0)
return ret;
} }
} else { } else {
uint8_t (*map)[4] = (uint8_t(*)[4]) s->pix_bgr_map; uint8_t (*map)[4] = (uint8_t(*)[4]) s->pix_bgr_map;
@ -173,28 +176,34 @@ static void generate_joint_tables(HYuvContext *s)
} }
} }
ff_free_vlc(&s->vlc[3]); ff_free_vlc(&s->vlc[3]);
init_vlc(&s->vlc[3], VLC_BITS, i, len, 1, 1, bits, 2, 2, 0); if ((ret = init_vlc(&s->vlc[3], VLC_BITS, i, len, 1, 1,
bits, 2, 2, 0)) < 0)
return ret;
} }
return 0;
} }
static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length) static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length)
{ {
GetBitContext gb; GetBitContext gb;
int i; int i, ret;
init_get_bits(&gb, src, length * 8); if ((ret = init_get_bits(&gb, src, length * 8)) < 0)
return ret;
for (i = 0; i < 3; i++) { for (i = 0; i < 3; i++) {
if (read_len_table(s->len[i], &gb) < 0) if ((ret = read_len_table(s->len[i], &gb)) < 0)
return -1; return ret;
if (ff_huffyuv_generate_bits_table(s->bits[i], s->len[i]) < 0) if ((ret = ff_huffyuv_generate_bits_table(s->bits[i], s->len[i])) < 0)
return -1; return ret;
ff_free_vlc(&s->vlc[i]); ff_free_vlc(&s->vlc[i]);
init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
s->bits[i], 4, 4, 0); s->bits[i], 4, 4, 0)) < 0)
return ret;
} }
generate_joint_tables(s); if ((ret = generate_joint_tables(s)) < 0)
return ret;
return (get_bits_count(&gb) + 7) / 8; return (get_bits_count(&gb) + 7) / 8;
} }
@ -202,17 +211,19 @@ static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length)
static int read_old_huffman_tables(HYuvContext *s) static int read_old_huffman_tables(HYuvContext *s)
{ {
GetBitContext gb; GetBitContext gb;
int i; int i, ret;
init_get_bits(&gb, classic_shift_luma, if ((ret = init_get_bits(&gb, classic_shift_luma,
classic_shift_luma_table_size * 8); classic_shift_luma_table_size * 8)) < 0)
if (read_len_table(s->len[0], &gb) < 0) return ret;
return -1; if ((ret = read_len_table(s->len[0], &gb)) < 0)
return ret;
init_get_bits(&gb, classic_shift_chroma, if ((ret = init_get_bits(&gb, classic_shift_chroma,
classic_shift_chroma_table_size * 8); classic_shift_chroma_table_size * 8)) < 0)
if (read_len_table(s->len[1], &gb) < 0) return ret;
return -1; if ((ret = read_len_table(s->len[1], &gb)) < 0)
return ret;
for (i = 0; i < 256; i++) for (i = 0; i < 256; i++)
s->bits[0][i] = classic_add_luma[i]; s->bits[0][i] = classic_add_luma[i];
@ -228,11 +239,13 @@ static int read_old_huffman_tables(HYuvContext *s)
for (i = 0; i < 3; i++) { for (i = 0; i < 3; i++) {
ff_free_vlc(&s->vlc[i]); ff_free_vlc(&s->vlc[i]);
init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
s->bits[i], 4, 4, 0); s->bits[i], 4, 4, 0)) < 0)
return ret;
} }
generate_joint_tables(s); if ((ret = generate_joint_tables(s)) < 0)
return ret;
return 0; return 0;
} }
@ -240,6 +253,7 @@ static int read_old_huffman_tables(HYuvContext *s)
static av_cold int decode_init(AVCodecContext *avctx) static av_cold int decode_init(AVCodecContext *avctx)
{ {
HYuvContext *s = avctx->priv_data; HYuvContext *s = avctx->priv_data;
int ret;
ff_huffyuv_common_init(avctx); ff_huffyuv_common_init(avctx);
ff_huffyuvdsp_init(&s->hdsp); ff_huffyuvdsp_init(&s->hdsp);
@ -273,9 +287,9 @@ static av_cold int decode_init(AVCodecContext *avctx)
s->interlaced = (interlace == 1) ? 1 : (interlace == 2) ? 0 : s->interlaced; s->interlaced = (interlace == 1) ? 1 : (interlace == 2) ? 0 : s->interlaced;
s->context = avctx->extradata[2] & 0x40 ? 1 : 0; s->context = avctx->extradata[2] & 0x40 ? 1 : 0;
if (read_huffman_tables(s, avctx->extradata + 4, if ((ret = read_huffman_tables(s, avctx->extradata + 4,
avctx->extradata_size - 4) < 0) avctx->extradata_size - 4)) < 0)
return -1; return ret;
} else { } else {
switch (avctx->bits_per_coded_sample & 7) { switch (avctx->bits_per_coded_sample & 7) {
case 1: case 1:
@ -302,8 +316,8 @@ static av_cold int decode_init(AVCodecContext *avctx)
s->bitstream_bpp = avctx->bits_per_coded_sample & ~7; s->bitstream_bpp = avctx->bits_per_coded_sample & ~7;
s->context = 0; s->context = 0;
if (read_old_huffman_tables(s) < 0) if ((ret = read_old_huffman_tables(s)) < 0)
return -1; return ret;
} }
switch (s->bitstream_bpp) { switch (s->bitstream_bpp) {
@ -327,7 +341,8 @@ static av_cold int decode_init(AVCodecContext *avctx)
return AVERROR_INVALIDDATA; return AVERROR_INVALIDDATA;
} }
ff_huffyuv_alloc_temp(s); if ((ret = ff_huffyuv_alloc_temp(s)) < 0)
return ret;
return 0; return 0;
} }
@ -335,20 +350,21 @@ static av_cold int decode_init(AVCodecContext *avctx)
static av_cold int decode_init_thread_copy(AVCodecContext *avctx) static av_cold int decode_init_thread_copy(AVCodecContext *avctx)
{ {
HYuvContext *s = avctx->priv_data; HYuvContext *s = avctx->priv_data;
int i; int i, ret;
ff_huffyuv_alloc_temp(s); if ((ret = ff_huffyuv_alloc_temp(s)) < 0)
return ret;
for (i = 0; i < 6; i++) for (i = 0; i < 6; i++)
s->vlc[i].table = NULL; s->vlc[i].table = NULL;
if (s->version == 2) { if (s->version == 2) {
if (read_huffman_tables(s, avctx->extradata + 4, if ((ret = read_huffman_tables(s, avctx->extradata + 4,
avctx->extradata_size) < 0) avctx->extradata_size)) < 0)
return -1; return ret;
} else { } else {
if (read_old_huffman_tables(s) < 0) if ((ret = read_old_huffman_tables(s)) < 0)
return -1; return ret;
} }
return 0; return 0;
@ -484,7 +500,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
int fake_ystride, fake_ustride, fake_vstride; int fake_ystride, fake_ustride, fake_vstride;
ThreadFrame frame = { .f = data }; ThreadFrame frame = { .f = data };
AVFrame *const p = data; AVFrame *const p = data;
int table_size = 0; int table_size = 0, ret;
av_fast_malloc(&s->bitstream_buffer, av_fast_malloc(&s->bitstream_buffer,
&s->bitstream_buffer_size, &s->bitstream_buffer_size,
@ -496,22 +512,23 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
s->bdsp.bswap_buf((uint32_t *) s->bitstream_buffer, s->bdsp.bswap_buf((uint32_t *) s->bitstream_buffer,
(const uint32_t *) buf, buf_size / 4); (const uint32_t *) buf, buf_size / 4);
if (ff_thread_get_buffer(avctx, &frame, 0) < 0) { if ((ret = ff_thread_get_buffer(avctx, &frame, 0)) < 0) {
av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
return -1; return ret;
} }
if (s->context) { if (s->context) {
table_size = read_huffman_tables(s, s->bitstream_buffer, buf_size); table_size = read_huffman_tables(s, s->bitstream_buffer, buf_size);
if (table_size < 0) if (table_size < 0)
return -1; return table_size;
} }
if ((unsigned) (buf_size - table_size) >= INT_MAX / 8) if ((unsigned) (buf_size - table_size) >= INT_MAX / 8)
return AVERROR_INVALIDDATA; return AVERROR_INVALIDDATA;
init_get_bits(&s->gb, s->bitstream_buffer + table_size, if ((ret = init_get_bits(&s->gb, s->bitstream_buffer + table_size,
(buf_size - table_size) * 8); (buf_size - table_size) * 8)) < 0)
return ret;
fake_ystride = s->interlaced ? p->linesize[0] * 2 : p->linesize[0]; fake_ystride = s->interlaced ? p->linesize[0] * 2 : p->linesize[0];
fake_ustride = s->interlaced ? p->linesize[1] * 2 : p->linesize[1]; fake_ustride = s->interlaced ? p->linesize[1] * 2 : p->linesize[1];

Write Preview
Loading…
Cancel
Save