avcodec/motion_est: Fix some undefined shifts

Fixes: asan_heap-oob_1dd60fd_1049_cov_4200102444_P4250048.MOV

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
pull/267/head
Michael Niedermayer 10 years ago
parent f91126643a
commit ac78014f0b
  1. 10
      libavcodec/motion_est.c

@ -426,13 +426,13 @@ static int sad_hpel_motion_search(MpegEncContext * s,
my > ymin && my < ymax) {
int dx=0, dy=0;
int d, pen_x, pen_y;
const int index= (my<<ME_MAP_SHIFT) + mx;
const int index= my*(1<<ME_MAP_SHIFT) + mx;
const int t= score_map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)];
const int l= score_map[(index- 1 )&(ME_MAP_SIZE-1)];
const int r= score_map[(index+ 1 )&(ME_MAP_SIZE-1)];
const int b= score_map[(index+(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)];
mx<<=1;
my<<=1;
mx += mx;
my += my;
pen_x= pred_x + mx;
@ -490,8 +490,8 @@ static int sad_hpel_motion_search(MpegEncContext * s,
my+=dy;
}else{
mx<<=1;
my<<=1;
mx += mx;
my += my;
}
*mx_ptr = mx;

Loading…
Cancel
Save