Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

avcodec/xvididct: Fix integer overflow in idct_row()

Browse Source 
Fixes: signed integer overflow: -1238335488 + -1003634688 cannot be represented in type 'int'
Fixes: 21649/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MJPEG_fuzzer-5112005765890048

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
pull/338/head
Michael Niedermayer 5 years ago
parent e9a4c4fe99
commit 620236e4d2
1 changed files with 18 additions and 18 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 36
      libavcodec/xvididct.c

36
libavcodec/xvididct.c
Unescape View File

@ -115,24 +115,24 @@ static int idct_row(short *in, const int *const tab, int rnd)
in[6] = a1; in[6] = a1;
} else { } else {
const int k = c4 * in[0] + rnd; const int k = c4 * in[0] + rnd;
const int a0 = k + c2 * in[2] + c4 * in[4] + c6 * in[6]; const unsigned int a0 = k + c2 * in[2] + c4 * in[4] + c6 * in[6];
const int a1 = k + c6 * in[2] - c4 * in[4] - c2 * in[6]; const unsigned int a1 = k + c6 * in[2] - c4 * in[4] - c2 * in[6];
const int a2 = k - c6 * in[2] - c4 * in[4] + c2 * in[6]; const unsigned int a2 = k - c6 * in[2] - c4 * in[4] + c2 * in[6];
const int a3 = k - c2 * in[2] + c4 * in[4] - c6 * in[6]; const unsigned int a3 = k - c2 * in[2] + c4 * in[4] - c6 * in[6];
const int b0 = c1 * in[1] + c3 * in[3] + c5 * in[5] + c7 * in[7]; const unsigned int b0 = c1 * in[1] + c3 * in[3] + c5 * in[5] + c7 * in[7];
const int b1 = c3 * in[1] - c7 * in[3] - c1 * in[5] - c5 * in[7]; const unsigned int b1 = c3 * in[1] - c7 * in[3] - c1 * in[5] - c5 * in[7];
const int b2 = c5 * in[1] - c1 * in[3] + c7 * in[5] + c3 * in[7]; const unsigned int b2 = c5 * in[1] - c1 * in[3] + c7 * in[5] + c3 * in[7];
const int b3 = c7 * in[1] - c5 * in[3] + c3 * in[5] - c1 * in[7]; const unsigned int b3 = c7 * in[1] - c5 * in[3] + c3 * in[5] - c1 * in[7];
in[0] = (a0 + b0) >> ROW_SHIFT; in[0] = (int)(a0 + b0) >> ROW_SHIFT;
in[1] = (a1 + b1) >> ROW_SHIFT; in[1] = (int)(a1 + b1) >> ROW_SHIFT;
in[2] = (a2 + b2) >> ROW_SHIFT; in[2] = (int)(a2 + b2) >> ROW_SHIFT;
in[3] = (a3 + b3) >> ROW_SHIFT; in[3] = (int)(a3 + b3) >> ROW_SHIFT;
in[4] = (a3 - b3) >> ROW_SHIFT; in[4] = (int)(a3 - b3) >> ROW_SHIFT;
in[5] = (a2 - b2) >> ROW_SHIFT; in[5] = (int)(a2 - b2) >> ROW_SHIFT;
in[6] = (a1 - b1) >> ROW_SHIFT; in[6] = (int)(a1 - b1) >> ROW_SHIFT;
in[7] = (a0 - b0) >> ROW_SHIFT; in[7] = (int)(a0 - b0) >> ROW_SHIFT;
} }
return 1; return 1;
} }

Write Preview
Loading…
Cancel
Save