avcodec/h264dsp_template: Fix undefined shifts

Fixes: asan_heap-oob_17212bc_2243_cov_594210248_h264_TTA.mkv

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
pull/267/head
Michael Niedermayer 10 years ago
parent ce81e47c91
commit 06a0d5ef5c
  1. 4
      libavcodec/h264dsp_template.c

@ -110,7 +110,7 @@ static av_always_inline av_flatten void FUNCC(h264_loop_filter_luma)(uint8_t *p_
alpha <<= BIT_DEPTH - 8;
beta <<= BIT_DEPTH - 8;
for( i = 0; i < 4; i++ ) {
const int tc_orig = tc0[i] << (BIT_DEPTH - 8);
const int tc_orig = tc0[i] * (1 << (BIT_DEPTH - 8));
if( tc_orig < 0 ) {
pix += inner_iters*ystride;
continue;
@ -141,7 +141,7 @@ static av_always_inline av_flatten void FUNCC(h264_loop_filter_luma)(uint8_t *p_
tc++;
}
i_delta = av_clip( (((q0 - p0 ) << 2) + (p1 - q1) + 4) >> 3, -tc, tc );
i_delta = av_clip( (((q0 - p0 ) * 4) + (p1 - q1) + 4) >> 3, -tc, tc );
pix[-xstride] = av_clip_pixel( p0 + i_delta ); /* p0' */
pix[0] = av_clip_pixel( q0 - i_delta ); /* q0' */
}

Loading…
Cancel
Save