From addd5cfee3a677f1cdf22319a892ecd1ff3457a8 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Fri, 15 Sep 2023 00:49:41 +0200 Subject: [PATCH] avcodec/osq: avoid using too large numbers for shifts and integers in update_residue_parameter() Fixes: 2.96539e+09 is outside the range of representable values of type 'int' Fixes: Assertion n>=0 && n<=32 failed at libavcodec/get_bits.h:423 Fixes: 62241/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_OSQ_fuzzer-4525761925873664 Fixes: 70406/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_OSQ_fuzzer-6545326804434944 Signed-off-by: Michael Niedermayer (cherry picked from commit 56c334d732dbbce43b0c8fc0809ec545b7946832) Signed-off-by: Michael Niedermayer --- libavcodec/osq.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/libavcodec/osq.c b/libavcodec/osq.c index fa4aeee35e..333ca506ea 100644 --- a/libavcodec/osq.c +++ b/libavcodec/osq.c @@ -160,11 +160,15 @@ static int update_residue_parameter(OSQChannel *cb) sum = cb->sum; x = sum / cb->count; - rice_k = av_ceil_log2(x); + rice_k = ceil(log2(x)); if (rice_k >= 30) { - rice_k = floor(sum / 1.4426952 + 0.5); - if (rice_k < 1) + double f = floor(sum / 1.4426952 + 0.5); + if (f <= 1) { rice_k = 1; + } else if (f >= 31) { + rice_k = 31; + } else + rice_k = f; } return rice_k;