From bd80c97391969f9dbb312d6c498211ad85bb67cb Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 12 Sep 2024 22:51:53 +0200 Subject: [PATCH] swscale/output: Fix undefined integer overflow in yuv2rgba64_2_c_template() Fixes: signed integer overflow: -1082982400 + -1083218484 cannot be represented in type 'int' Fixes: 70657/clusterfuzz-testcase-minimized-ffmpeg_SWS_fuzzer-6707819712675840 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libswscale/output.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/libswscale/output.c b/libswscale/output.c index abfb0fd1ce..31921a3cce 100644 --- a/libswscale/output.c +++ b/libswscale/output.c @@ -1150,8 +1150,8 @@ yuv2rgba64_2_c_template(SwsContext *c, const int32_t *buf[2], av_assert2(uvalpha <= 4096U); for (i = 0; i < ((dstW + 1) >> 1); i++) { - int Y1 = (buf0[i * 2] * yalpha1 + buf1[i * 2] * yalpha) >> 14; - int Y2 = (buf0[i * 2 + 1] * yalpha1 + buf1[i * 2 + 1] * yalpha) >> 14; + unsigned Y1 = (buf0[i * 2] * yalpha1 + buf1[i * 2] * yalpha) >> 14; + unsigned Y2 = (buf0[i * 2 + 1] * yalpha1 + buf1[i * 2 + 1] * yalpha) >> 14; int U = (ubuf0[i] * uvalpha1 + ubuf1[i] * uvalpha - (128 << 23)) >> 14; int V = (vbuf0[i] * uvalpha1 + vbuf1[i] * uvalpha - (128 << 23)) >> 14; int R, G, B; @@ -1175,20 +1175,20 @@ yuv2rgba64_2_c_template(SwsContext *c, const int32_t *buf[2], A2 += 1 << 13; } - output_pixel(&dest[0], av_clip_uintp2(((R_B + Y1) >> 14) + (1<<15), 16)); - output_pixel(&dest[1], av_clip_uintp2((( G + Y1) >> 14) + (1<<15), 16)); - output_pixel(&dest[2], av_clip_uintp2(((B_R + Y1) >> 14) + (1<<15), 16)); + output_pixel(&dest[0], av_clip_uintp2(((int)(R_B + Y1) >> 14) + (1<<15), 16)); + output_pixel(&dest[1], av_clip_uintp2(((int)( G + Y1) >> 14) + (1<<15), 16)); + output_pixel(&dest[2], av_clip_uintp2(((int)(B_R + Y1) >> 14) + (1<<15), 16)); if (eightbytes) { output_pixel(&dest[3], av_clip_uintp2(A1 , 30) >> 14); - output_pixel(&dest[4], av_clip_uintp2(((R_B + Y2) >> 14) + (1<<15), 16)); - output_pixel(&dest[5], av_clip_uintp2((( G + Y2) >> 14) + (1<<15), 16)); - output_pixel(&dest[6], av_clip_uintp2(((B_R + Y2) >> 14) + (1<<15), 16)); + output_pixel(&dest[4], av_clip_uintp2(((int)(R_B + Y2) >> 14) + (1<<15), 16)); + output_pixel(&dest[5], av_clip_uintp2(((int)( G + Y2) >> 14) + (1<<15), 16)); + output_pixel(&dest[6], av_clip_uintp2(((int)(B_R + Y2) >> 14) + (1<<15), 16)); output_pixel(&dest[7], av_clip_uintp2(A2 , 30) >> 14); dest += 8; } else { - output_pixel(&dest[3], av_clip_uintp2(((R_B + Y2) >> 14) + (1<<15), 16)); - output_pixel(&dest[4], av_clip_uintp2((( G + Y2) >> 14) + (1<<15), 16)); - output_pixel(&dest[5], av_clip_uintp2(((B_R + Y2) >> 14) + (1<<15), 16)); + output_pixel(&dest[3], av_clip_uintp2(((int)(R_B + Y2) >> 14) + (1<<15), 16)); + output_pixel(&dest[4], av_clip_uintp2(((int)( G + Y2) >> 14) + (1<<15), 16)); + output_pixel(&dest[5], av_clip_uintp2(((int)(B_R + Y2) >> 14) + (1<<15), 16)); dest += 6; } }