Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

avcodec/vp9dsp_template: Fix integer overflows in idct16_1d()

Browse Source 
Fixes: signed integer overflow: -190760 * 11585 cannot be represented in type 'int'
Fixes: 25471/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VP9_fuzzer-5743354917421056

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
pull/355/head
Michael Niedermayer 4 years ago
parent 949f0a6be9
commit 394e8bb385
1 changed files with 26 additions and 26 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 52
      libavcodec/vp9dsp_template.c

52
libavcodec/vp9dsp_template.c
Unescape View File

@ -1290,22 +1290,22 @@ static av_always_inline void idct16_1d(const dctcoef *in, ptrdiff_t stride,
dctint t0a, t1a, t2a, t3a, t4a, t5a, t6a, t7a;
dctint t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a;
t0a = ((IN(0) + IN(8)) * 11585 + (1 << 13)) >> 14;
t1a = ((IN(0) - IN(8)) * 11585 + (1 << 13)) >> 14;
t2a = (IN(4) * 6270 - IN(12) * 15137 + (1 << 13)) >> 14;
t3a = (IN(4) * 15137 + IN(12) * 6270 + (1 << 13)) >> 14;
t4a = (IN(2) * 3196 - IN(14) * 16069 + (1 << 13)) >> 14;
t7a = (IN(2) * 16069 + IN(14) * 3196 + (1 << 13)) >> 14;
t5a = (IN(10) * 13623 - IN(6) * 9102 + (1 << 13)) >> 14;
t6a = (IN(10) * 9102 + IN(6) * 13623 + (1 << 13)) >> 14;
t8a = (IN(1) * 1606 - IN(15) * 16305 + (1 << 13)) >> 14;
t15a = (IN(1) * 16305 + IN(15) * 1606 + (1 << 13)) >> 14;
t9a = (IN(9) * 12665 - IN(7) * 10394 + (1 << 13)) >> 14;
t14a = (IN(9) * 10394 + IN(7) * 12665 + (1 << 13)) >> 14;
t10a = (IN(5) * 7723 - IN(11) * 14449 + (1 << 13)) >> 14;
t13a = (IN(5) * 14449 + IN(11) * 7723 + (1 << 13)) >> 14;
t11a = (IN(13) * 15679 - IN(3) * 4756 + (1 << 13)) >> 14;
t12a = (IN(13) * 4756 + IN(3) * 15679 + (1 << 13)) >> 14;
t0a = (dctint)((IN(0) + IN(8)) * 11585U + (1 << 13)) >> 14;
t1a = (dctint)((IN(0) - IN(8)) * 11585U + (1 << 13)) >> 14;
t2a = (dctint)(IN(4) * 6270U - IN(12) * 15137U + (1 << 13)) >> 14;
t3a = (dctint)(IN(4) * 15137U + IN(12) * 6270U + (1 << 13)) >> 14;
t4a = (dctint)(IN(2) * 3196U - IN(14) * 16069U + (1 << 13)) >> 14;
t7a = (dctint)(IN(2) * 16069U + IN(14) * 3196U + (1 << 13)) >> 14;
t5a = (dctint)(IN(10) * 13623U - IN(6) * 9102U + (1 << 13)) >> 14;
t6a = (dctint)(IN(10) * 9102U + IN(6) * 13623U + (1 << 13)) >> 14;
t8a = (dctint)(IN(1) * 1606U - IN(15) * 16305U + (1 << 13)) >> 14;
t15a = (dctint)(IN(1) * 16305U + IN(15) * 1606U + (1 << 13)) >> 14;
t9a = (dctint)(IN(9) * 12665U - IN(7) * 10394U + (1 << 13)) >> 14;
t14a = (dctint)(IN(9) * 10394U + IN(7) * 12665U + (1 << 13)) >> 14;
t10a = (dctint)(IN(5) * 7723U - IN(11) * 14449U + (1 << 13)) >> 14;
t13a = (dctint)(IN(5) * 14449U + IN(11) * 7723U + (1 << 13)) >> 14;
t11a = (dctint)(IN(13) * 15679U - IN(3) * 4756U + (1 << 13)) >> 14;
t12a = (dctint)(IN(13) * 4756U + IN(3) * 15679U + (1 << 13)) >> 14;
t0 = t0a + t3a;
t1 = t1a + t2a;
@ -1324,12 +1324,12 @@ static av_always_inline void idct16_1d(const dctcoef *in, ptrdiff_t stride,
t14 = t15a - t14a;
t15 = t15a + t14a;
t5a = ((t6 - t5) * 11585 + (1 << 13)) >> 14;
t6a = ((t6 + t5) * 11585 + (1 << 13)) >> 14;
t9a = ( t14 * 6270 - t9 * 15137 + (1 << 13)) >> 14;
t14a = ( t14 * 15137 + t9 * 6270 + (1 << 13)) >> 14;
t10a = (-(t13 * 15137 + t10 * 6270) + (1 << 13)) >> 14;
t13a = ( t13 * 6270 - t10 * 15137 + (1 << 13)) >> 14;
t5a = (dctint)((t6 - t5) * 11585U + (1 << 13)) >> 14;
t6a = (dctint)((t6 + t5) * 11585U + (1 << 13)) >> 14;
t9a = (dctint)( t14 * 6270U - t9 * 15137U + (1 << 13)) >> 14;
t14a = (dctint)( t14 * 15137U + t9 * 6270U + (1 << 13)) >> 14;
t10a = (dctint)(-(t13 * 15137U + t10 * 6270U) + (1 << 13)) >> 14;
t13a = (dctint)( t13 * 6270U - t10 * 15137U + (1 << 13)) >> 14;
t0a = t0 + t7;
t1a = t1 + t6a;
@ -1348,10 +1348,10 @@ static av_always_inline void idct16_1d(const dctcoef *in, ptrdiff_t stride,
t14 = t14a + t13a;
t15a = t15 + t12;
t10a = ((t13 - t10) * 11585 + (1 << 13)) >> 14;
t13a = ((t13 + t10) * 11585 + (1 << 13)) >> 14;
t11 = ((t12a - t11a) * 11585 + (1 << 13)) >> 14;
t12 = ((t12a + t11a) * 11585 + (1 << 13)) >> 14;
t10a = (dctint)((t13 - t10) * 11585U + (1 << 13)) >> 14;
t13a = (dctint)((t13 + t10) * 11585U + (1 << 13)) >> 14;
t11 = (dctint)((t12a - t11a) * 11585U + (1 << 13)) >> 14;
t12 = (dctint)((t12a + t11a) * 11585U + (1 << 13)) >> 14;
out[ 0] = t0a + t15a;
out[ 1] = t1a + t14;

Write Preview
Loading…
Cancel
Save