avcodec/dxtory: Fix get_raw_size() for YUV

Fixes: out of array read
Fixes: 25455/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_DXTORY_fuzzer-6327985731534848

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
pull/353/head
Michael Niedermayer 4 years ago
parent cc25ae5d8a
commit 497debb171
  1. 4
      libavcodec/dxtory.c

@ -44,9 +44,9 @@ static int64_t get_raw_size(enum AVPixelFormat fmt, int width, int height)
case AV_PIX_FMT_YUV444P:
return width * height * 3LL;
case AV_PIX_FMT_YUV420P:
return (int64_t)(width * height) + AV_CEIL_RSHIFT(width, 1) * AV_CEIL_RSHIFT(height, 1);
return (int64_t)(width * height) + 2 * AV_CEIL_RSHIFT(width, 1) * AV_CEIL_RSHIFT(height, 1);
case AV_PIX_FMT_YUV410P:
return (int64_t)(width * height) + AV_CEIL_RSHIFT(width, 2) * AV_CEIL_RSHIFT(height, 2);
return (int64_t)(width * height) + 2 * AV_CEIL_RSHIFT(width, 2) * AV_CEIL_RSHIFT(height, 2);
}
return 0;

Loading…
Cancel
Save