Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

avcodec/vp9dsp_template: Fix integer overflow(s) in iadst16_1d()

Browse Source 
Fixes: signed integer overflow: 1080285923 - -1130879337 cannot be represented in type 'int'
Fixes: 22002/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VP9_fuzzer-6260237310099456

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
pull/357/head
Michael Niedermayer 5 years ago
parent bafaf95116
commit 071e293723
1 changed files with 76 additions and 76 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 152
      libavcodec/vp9dsp_template.c

152
libavcodec/vp9dsp_template.c
Unescape View File

@ -1378,48 +1378,48 @@ static av_always_inline void iadst16_1d(const dctcoef *in, ptrdiff_t stride,
dctint t0a, t1a, t2a, t3a, t4a, t5a, t6a, t7a;
dctint t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a;
t0 = IN(15) * 16364 + IN(0) * 804;
t1 = IN(15) * 804 - IN(0) * 16364;
t2 = IN(13) * 15893 + IN(2) * 3981;
t3 = IN(13) * 3981 - IN(2) * 15893;
t4 = IN(11) * 14811 + IN(4) * 7005;
t5 = IN(11) * 7005 - IN(4) * 14811;
t6 = IN(9) * 13160 + IN(6) * 9760;
t7 = IN(9) * 9760 - IN(6) * 13160;
t8 = IN(7) * 11003 + IN(8) * 12140;
t9 = IN(7) * 12140 - IN(8) * 11003;
t10 = IN(5) * 8423 + IN(10) * 14053;
t11 = IN(5) * 14053 - IN(10) * 8423;
t12 = IN(3) * 5520 + IN(12) * 15426;
t13 = IN(3) * 15426 - IN(12) * 5520;
t14 = IN(1) * 2404 + IN(14) * 16207;
t15 = IN(1) * 16207 - IN(14) * 2404;
t0a = (t0 + t8 + (1 << 13)) >> 14;
t1a = (t1 + t9 + (1 << 13)) >> 14;
t2a = (t2 + t10 + (1 << 13)) >> 14;
t3a = (t3 + t11 + (1 << 13)) >> 14;
t4a = (t4 + t12 + (1 << 13)) >> 14;
t5a = (t5 + t13 + (1 << 13)) >> 14;
t6a = (t6 + t14 + (1 << 13)) >> 14;
t7a = (t7 + t15 + (1 << 13)) >> 14;
t8a = (t0 - t8 + (1 << 13)) >> 14;
t9a = (t1 - t9 + (1 << 13)) >> 14;
t10a = (t2 - t10 + (1 << 13)) >> 14;
t11a = (t3 - t11 + (1 << 13)) >> 14;
t12a = (t4 - t12 + (1 << 13)) >> 14;
t13a = (t5 - t13 + (1 << 13)) >> 14;
t14a = (t6 - t14 + (1 << 13)) >> 14;
t15a = (t7 - t15 + (1 << 13)) >> 14;
t8 = t8a * 16069 + t9a * 3196;
t9 = t8a * 3196 - t9a * 16069;
t10 = t10a * 9102 + t11a * 13623;
t11 = t10a * 13623 - t11a * 9102;
t12 = t13a * 16069 - t12a * 3196;
t13 = t13a * 3196 + t12a * 16069;
t14 = t15a * 9102 - t14a * 13623;
t15 = t15a * 13623 + t14a * 9102;
t0 = IN(15) * 16364U + IN(0) * 804U;
t1 = IN(15) * 804U - IN(0) * 16364U;
t2 = IN(13) * 15893U + IN(2) * 3981U;
t3 = IN(13) * 3981U - IN(2) * 15893U;
t4 = IN(11) * 14811U + IN(4) * 7005U;
t5 = IN(11) * 7005U - IN(4) * 14811U;
t6 = IN(9) * 13160U + IN(6) * 9760U;
t7 = IN(9) * 9760U - IN(6) * 13160U;
t8 = IN(7) * 11003U + IN(8) * 12140U;
t9 = IN(7) * 12140U - IN(8) * 11003U;
t10 = IN(5) * 8423U + IN(10) * 14053U;
t11 = IN(5) * 14053U - IN(10) * 8423U;
t12 = IN(3) * 5520U + IN(12) * 15426U;
t13 = IN(3) * 15426U - IN(12) * 5520U;
t14 = IN(1) * 2404U + IN(14) * 16207U;
t15 = IN(1) * 16207U - IN(14) * 2404U;
t0a = (dctint)((1U << 13) + t0 + t8 ) >> 14;
t1a = (dctint)((1U << 13) + t1 + t9 ) >> 14;
t2a = (dctint)((1U << 13) + t2 + t10) >> 14;
t3a = (dctint)((1U << 13) + t3 + t11) >> 14;
t4a = (dctint)((1U << 13) + t4 + t12) >> 14;
t5a = (dctint)((1U << 13) + t5 + t13) >> 14;
t6a = (dctint)((1U << 13) + t6 + t14) >> 14;
t7a = (dctint)((1U << 13) + t7 + t15) >> 14;
t8a = (dctint)((1U << 13) + t0 - t8 ) >> 14;
t9a = (dctint)((1U << 13) + t1 - t9 ) >> 14;
t10a = (dctint)((1U << 13) + t2 - t10) >> 14;
t11a = (dctint)((1U << 13) + t3 - t11) >> 14;
t12a = (dctint)((1U << 13) + t4 - t12) >> 14;
t13a = (dctint)((1U << 13) + t5 - t13) >> 14;
t14a = (dctint)((1U << 13) + t6 - t14) >> 14;
t15a = (dctint)((1U << 13) + t7 - t15) >> 14;
t8 = t8a * 16069U + t9a * 3196U;
t9 = t8a * 3196U - t9a * 16069U;
t10 = t10a * 9102U + t11a * 13623U;
t11 = t10a * 13623U - t11a * 9102U;
t12 = t13a * 16069U - t12a * 3196U;
t13 = t13a * 3196U + t12a * 16069U;
t14 = t15a * 9102U - t14a * 13623U;
t15 = t15a * 13623U + t14a * 9102U;
t0 = t0a + t4a;
t1 = t1a + t5a;
@ -1429,49 +1429,49 @@ static av_always_inline void iadst16_1d(const dctcoef *in, ptrdiff_t stride,
t5 = t1a - t5a;
t6 = t2a - t6a;
t7 = t3a - t7a;
t8a = (t8 + t12 + (1 << 13)) >> 14;
t9a = (t9 + t13 + (1 << 13)) >> 14;
t10a = (t10 + t14 + (1 << 13)) >> 14;
t11a = (t11 + t15 + (1 << 13)) >> 14;
t12a = (t8 - t12 + (1 << 13)) >> 14;
t13a = (t9 - t13 + (1 << 13)) >> 14;
t14a = (t10 - t14 + (1 << 13)) >> 14;
t15a = (t11 - t15 + (1 << 13)) >> 14;
t4a = t4 * 15137 + t5 * 6270;
t5a = t4 * 6270 - t5 * 15137;
t6a = t7 * 15137 - t6 * 6270;
t7a = t7 * 6270 + t6 * 15137;
t12 = t12a * 15137 + t13a * 6270;
t13 = t12a * 6270 - t13a * 15137;
t14 = t15a * 15137 - t14a * 6270;
t15 = t15a * 6270 + t14a * 15137;
t8a = (dctint)((1U << 13) + t8 + t12) >> 14;
t9a = (dctint)((1U << 13) + t9 + t13) >> 14;
t10a = (dctint)((1U << 13) + t10 + t14) >> 14;
t11a = (dctint)((1U << 13) + t11 + t15) >> 14;
t12a = (dctint)((1U << 13) + t8 - t12) >> 14;
t13a = (dctint)((1U << 13) + t9 - t13) >> 14;
t14a = (dctint)((1U << 13) + t10 - t14) >> 14;
t15a = (dctint)((1U << 13) + t11 - t15) >> 14;
t4a = t4 * 15137U + t5 * 6270U;
t5a = t4 * 6270U - t5 * 15137U;
t6a = t7 * 15137U - t6 * 6270U;
t7a = t7 * 6270U + t6 * 15137U;
t12 = t12a * 15137U + t13a * 6270U;
t13 = t12a * 6270U - t13a * 15137U;
t14 = t15a * 15137U - t14a * 6270U;
t15 = t15a * 6270U + t14a * 15137U;
out[ 0] = t0 + t2;
out[15] = -(t1 + t3);
t2a = t0 - t2;
t3a = t1 - t3;
out[ 3] = -((t4a + t6a + (1 << 13)) >> 14);
out[12] = (t5a + t7a + (1 << 13)) >> 14;
t6 = (t4a - t6a + (1 << 13)) >> 14;
t7 = (t5a - t7a + (1 << 13)) >> 14;
out[ 3] = -((dctint)((1U << 13) + t4a + t6a) >> 14);
out[12] = (dctint)((1U << 13) + t5a + t7a) >> 14;
t6 = (dctint)((1U << 13) + t4a - t6a) >> 14;
t7 = (dctint)((1U << 13) + t5a - t7a) >> 14;
out[ 1] = -(t8a + t10a);
out[14] = t9a + t11a;
t10 = t8a - t10a;
t11 = t9a - t11a;
out[ 2] = (t12 + t14 + (1 << 13)) >> 14;
out[13] = -((t13 + t15 + (1 << 13)) >> 14);
t14a = (t12 - t14 + (1 << 13)) >> 14;
t15a = (t13 - t15 + (1 << 13)) >> 14;
out[ 7] = ((t2a + t3a) * -11585 + (1 << 13)) >> 14;
out[ 8] = ((t2a - t3a) * 11585 + (1 << 13)) >> 14;
out[ 4] = ((t7 + t6) * 11585 + (1 << 13)) >> 14;
out[11] = ((t7 - t6) * 11585 + (1 << 13)) >> 14;
out[ 6] = ((t11 + t10) * 11585 + (1 << 13)) >> 14;
out[ 9] = ((t11 - t10) * 11585 + (1 << 13)) >> 14;
out[ 5] = ((t14a + t15a) * -11585 + (1 << 13)) >> 14;
out[10] = ((t14a - t15a) * 11585 + (1 << 13)) >> 14;
out[ 2] = (dctint)((1U << 13) + t12 + t14) >> 14;
out[13] = -((dctint)((1U << 13) + t13 + t15) >> 14);
t14a = (dctint)((1U << 13) + t12 - t14) >> 14;
t15a = (dctint)((1U << 13) + t13 - t15) >> 14;
out[ 7] = (dctint)(-(t2a + t3a) * 11585U + (1 << 13)) >> 14;
out[ 8] = (dctint)( (t2a - t3a) * 11585U + (1 << 13)) >> 14;
out[ 4] = (dctint)( (t7 + t6) * 11585U + (1 << 13)) >> 14;
out[11] = (dctint)( (t7 - t6) * 11585U + (1 << 13)) >> 14;
out[ 6] = (dctint)( (t11 + t10) * 11585U + (1 << 13)) >> 14;
out[ 9] = (dctint)( (t11 - t10) * 11585U + (1 << 13)) >> 14;
out[ 5] = (dctint)(-(t14a + t15a) * 11585U + (1 << 13)) >> 14;
out[10] = (dctint)( (t14a - t15a) * 11585U + (1 << 13)) >> 14;
}
itxfm_wrap(16, 6)

Write Preview
Loading…
Cancel
Save