avcodec/aacsbr_fixed: Fix overflows in rounding in sbr_hf_assemble()

Fixes: runtime error: signed integer overflow: 2052929346 + 204817098 cannot be represented in type 'int'
Fixes: 5275/clusterfuzz-testcase-minimized-5367635958038528

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
pull/277/head
Michael Niedermayer 7 years ago
parent 0a2560a977
commit b1bef755f6
  1. 9
      libavcodec/aacsbr_fixed.c

@ -568,7 +568,8 @@ static void sbr_hf_assemble(int Y1[38][64][2],
int A = (1-((indexsine+(kx & 1))&2)); int A = (1-((indexsine+(kx & 1))&2));
int B = (A^(-idx)) + idx; int B = (A^(-idx)) + idx;
int *out = &Y1[i][kx][idx]; int *out = &Y1[i][kx][idx];
int shift, round; int shift;
unsigned round;
SoftFloat *in = sbr->s_m[e]; SoftFloat *in = sbr->s_m[e];
for (m = 0; m+1 < m_max; m+=2) { for (m = 0; m+1 < m_max; m+=2) {
@ -581,12 +582,12 @@ static void sbr_hf_assemble(int Y1[38][64][2],
} }
if (shift < 32) { if (shift < 32) {
round = 1 << (shift-1); round = 1 << (shift-1);
out[2*m ] += (in[m ].mant * A + round) >> shift; out[2*m ] += (int)(in[m ].mant * A + round) >> shift;
} }
if (shift2 < 32) { if (shift2 < 32) {
round = 1 << (shift2-1); round = 1 << (shift2-1);
out[2*m+2] += (in[m+1].mant * B + round) >> shift2; out[2*m+2] += (int)(in[m+1].mant * B + round) >> shift2;
} }
} }
if(m_max&1) if(m_max&1)
@ -597,7 +598,7 @@ static void sbr_hf_assemble(int Y1[38][64][2],
return; return;
} else if (shift < 32) { } else if (shift < 32) {
round = 1 << (shift-1); round = 1 << (shift-1);
out[2*m ] += (in[m ].mant * A + round) >> shift; out[2*m ] += (int)(in[m ].mant * A + round) >> shift;
} }
} }
} }

Loading…
Cancel
Save