swscale/output: Fix undefined integer overflow in yuv2rgba64_2_c_template()

Fixes: signed integer overflow: -1082982400 + -1083218484 cannot be represented in type 'int'
Fixes: 70657/clusterfuzz-testcase-minimized-ffmpeg_SWS_fuzzer-6707819712675840

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
release/7.1
Michael Niedermayer 3 months ago
parent 44c5641ae8
commit bd80c97391
No known key found for this signature in database
GPG Key ID: B18E8928B3948D64
  1. 22
      libswscale/output.c

@ -1150,8 +1150,8 @@ yuv2rgba64_2_c_template(SwsContext *c, const int32_t *buf[2],
av_assert2(uvalpha <= 4096U);
for (i = 0; i < ((dstW + 1) >> 1); i++) {
int Y1 = (buf0[i * 2] * yalpha1 + buf1[i * 2] * yalpha) >> 14;
int Y2 = (buf0[i * 2 + 1] * yalpha1 + buf1[i * 2 + 1] * yalpha) >> 14;
unsigned Y1 = (buf0[i * 2] * yalpha1 + buf1[i * 2] * yalpha) >> 14;
unsigned Y2 = (buf0[i * 2 + 1] * yalpha1 + buf1[i * 2 + 1] * yalpha) >> 14;
int U = (ubuf0[i] * uvalpha1 + ubuf1[i] * uvalpha - (128 << 23)) >> 14;
int V = (vbuf0[i] * uvalpha1 + vbuf1[i] * uvalpha - (128 << 23)) >> 14;
int R, G, B;
@ -1175,20 +1175,20 @@ yuv2rgba64_2_c_template(SwsContext *c, const int32_t *buf[2],
A2 += 1 << 13;
}
output_pixel(&dest[0], av_clip_uintp2(((R_B + Y1) >> 14) + (1<<15), 16));
output_pixel(&dest[1], av_clip_uintp2((( G + Y1) >> 14) + (1<<15), 16));
output_pixel(&dest[2], av_clip_uintp2(((B_R + Y1) >> 14) + (1<<15), 16));
output_pixel(&dest[0], av_clip_uintp2(((int)(R_B + Y1) >> 14) + (1<<15), 16));
output_pixel(&dest[1], av_clip_uintp2(((int)( G + Y1) >> 14) + (1<<15), 16));
output_pixel(&dest[2], av_clip_uintp2(((int)(B_R + Y1) >> 14) + (1<<15), 16));
if (eightbytes) {
output_pixel(&dest[3], av_clip_uintp2(A1 , 30) >> 14);
output_pixel(&dest[4], av_clip_uintp2(((R_B + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[5], av_clip_uintp2((( G + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[6], av_clip_uintp2(((B_R + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[4], av_clip_uintp2(((int)(R_B + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[5], av_clip_uintp2(((int)( G + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[6], av_clip_uintp2(((int)(B_R + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[7], av_clip_uintp2(A2 , 30) >> 14);
dest += 8;
} else {
output_pixel(&dest[3], av_clip_uintp2(((R_B + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[4], av_clip_uintp2((( G + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[5], av_clip_uintp2(((B_R + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[3], av_clip_uintp2(((int)(R_B + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[4], av_clip_uintp2(((int)( G + Y2) >> 14) + (1<<15), 16));
output_pixel(&dest[5], av_clip_uintp2(((int)(B_R + Y2) >> 14) + (1<<15), 16));
dest += 6;
}
}

Loading…
Cancel
Save