avcodec/dstdec: Check for overflow in build_filter()

Fixes: signed integer overflow: 1917019860 + 265558963 cannot be represented in type 'int'
Fixes: 48798/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_DST_fuzzer-4833165046317056

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
pull/388/head
Michael Niedermayer 2 years ago
parent aa441ac105
commit 8008940da5
No known key found for this signature in database
GPG Key ID: B18E8928B3948D64
  1. 11
      libavcodec/dstdec.c

@ -215,7 +215,7 @@ static uint8_t prob_dst_x_bit(int c)
return (ff_reverse[c & 127] >> 1) + 1;
}
static void build_filter(int16_t table[DST_MAX_ELEMENTS][16][256], const Table *fsets)
static int build_filter(int16_t table[DST_MAX_ELEMENTS][16][256], const Table *fsets)
{
int i, j, k, l;
@ -226,14 +226,17 @@ static void build_filter(int16_t table[DST_MAX_ELEMENTS][16][256], const Table *
int total = av_clip(length - j * 8, 0, 8);
for (k = 0; k < 256; k++) {
int v = 0;
int64_t v = 0;
for (l = 0; l < total; l++)
v += (((k >> l) & 1) * 2 - 1) * fsets->coeff[i][j * 8 + l];
if ((int16_t)v != v)
return AVERROR_INVALIDDATA;
table[i][j][k] = v;
}
}
}
return 0;
}
static int decode_frame(AVCodecContext *avctx, AVFrame *frame,
@ -328,7 +331,9 @@ static int decode_frame(AVCodecContext *avctx, AVFrame *frame,
return AVERROR_INVALIDDATA;
ac_init(ac, gb);
build_filter(s->filter, &s->fsets);
ret = build_filter(s->filter, &s->fsets);
if (ret < 0)
return ret;
memset(s->status, 0xAA, sizeof(s->status));
memset(dsd, 0, frame->nb_samples * 4 * channels);

Loading…
Cancel
Save