Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

avcodec/motion_est: Fix mv_penalty table size

Browse Source 
Fixes out of array read

Found-by: Tyson Smith <twsmith@mozilla.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
pull/172/head
Michael Niedermayer 9 years ago
parent e7271e6d07
commit 5b4da8a38a
6 changed files with 18 additions and 17 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      libavcodec/ituh263enc.c
  2. 16
      libavcodec/motion_est.c
  3. 3
      libavcodec/motion_est.h
  4. 6
      libavcodec/mpeg12enc.c
  5. 2
      libavcodec/mpegvideo_enc.c
  6. 2
      libavcodec/snowenc.c

6
libavcodec/ituh263enc.c
Unescape View File

@ -45,7 +45,7 @@
/**
* Table of number of bits a motion vector component needs.
*/
static uint8_t mv_penalty[MAX_FCODE+1][MAX_MV*2+1];
static uint8_t mv_penalty[MAX_FCODE+1][MAX_DMV*2+1];
/**
* Minimal fcode that a motion vector component would need.
@ -678,7 +678,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s)
int mv;
for(f_code=1; f_code<=MAX_FCODE; f_code++){
for(mv=-MAX_MV; mv<=MAX_MV; mv++){
for(mv=-MAX_DMV; mv<=MAX_DMV; mv++){
int len;
if(mv==0) len= ff_mvtab[0][1];
@ -699,7 +699,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s)
}
}
mv_penalty[f_code][mv+MAX_MV]= len;
mv_penalty[f_code][mv+MAX_DMV]= len;
}
}

16
libavcodec/motion_est.c
Unescape View File

@ -923,7 +923,7 @@ void ff_estimate_p_frame_motion(MpegEncContext * s,
c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
get_limits(s, 16*mb_x, 16*mb_y);
c->skip=0;
@ -1090,7 +1090,7 @@ int ff_pre_estimate_p_frame_motion(MpegEncContext * s,
av_assert0(s->quarter_sample==0 || s->quarter_sample==1);
c->pre_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_pre_cmp);
c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
get_limits(s, 16*mb_x, 16*mb_y);
c->skip=0;
@ -1139,7 +1139,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y,
const int shift= 1+s->quarter_sample;
const int mot_stride = s->mb_stride;
const int mot_xy = mb_y*mot_stride + mb_x;
uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_MV;
uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_DMV;
int mv_scale;
c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
@ -1203,8 +1203,8 @@ static inline int check_bidir_mv(MpegEncContext * s,
//FIXME better f_code prediction (max mv & distance)
//FIXME pointers
MotionEstContext * const c= &s->me;
uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_MV; // f_code of the prev frame
uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_MV; // f_code of the prev frame
uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_DMV; // f_code of the prev frame
uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_DMV; // f_code of the prev frame
int stride= c->stride;
uint8_t *dest_y = c->scratchpad;
uint8_t *ptr;
@ -1417,7 +1417,7 @@ static inline int direct_search(MpegEncContext * s, int mb_x, int mb_y)
int mx, my, xmin, xmax, ymin, ymax;
int16_t (*mv_table)[2]= s->b_direct_mv_table;
c->current_mv_penalty= c->mv_penalty[1] + MAX_MV;
c->current_mv_penalty= c->mv_penalty[1] + MAX_DMV;
ymin= xmin=(-32)>>shift;
ymax= xmax= 31>>shift;
@ -1553,11 +1553,11 @@ void ff_estimate_b_frame_motion(MpegEncContext * s,
if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
//FIXME mb type penalty
c->skip=0;
c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
fimin= interlaced_search(s, 0,
s->b_field_mv_table[0], s->b_field_select_table[0],
s->b_forw_mv_table[xy][0], s->b_forw_mv_table[xy][1], 0);
c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_MV;
c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_DMV;
bimin= interlaced_search(s, 2,
s->b_field_mv_table[1], s->b_field_select_table[1],
s->b_back_mv_table[xy][0], s->b_back_mv_table[xy][1], 0);

3
libavcodec/motion_est.h
Unescape View File

@ -30,6 +30,7 @@
struct MpegEncContext;
#define MAX_MV 4096
#define MAX_DMV (2*MAX_MV)
#define ME_MAP_SIZE 64
#define FF_ME_ZERO 0
@ -85,7 +86,7 @@ typedef struct MotionEstContext {
op_pixels_func(*hpel_avg)[4];
qpel_mc_func(*qpel_put)[16];
qpel_mc_func(*qpel_avg)[16];
uint8_t (*mv_penalty)[MAX_MV * 2 + 1]; ///< bit amount needed to encode a MV
uint8_t (*mv_penalty)[MAX_DMV * 2 + 1]; ///< bit amount needed to encode a MV
uint8_t *current_mv_penalty;
int (*sub_motion_search)(struct MpegEncContext *s,
int *mx_ptr, int *my_ptr, int dmin,

6
libavcodec/mpeg12enc.c
Unescape View File

@ -47,7 +47,7 @@ static const uint8_t svcd_scan_offset_placeholder[] = {
0x81, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
};
static uint8_t mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
static uint8_t mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
static uint8_t fcode_tab[MAX_MV * 2 + 1];
static uint8_t uni_mpeg1_ac_vlc_len[64 * 64 * 2];
@ -1041,7 +1041,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s)
}
for (f_code = 1; f_code <= MAX_FCODE; f_code++)
for (mv = -MAX_MV; mv <= MAX_MV; mv++) {
for (mv = -MAX_DMV; mv <= MAX_DMV; mv++) {
int len;
if (mv == 0) {
@ -1064,7 +1064,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s)
2 + bit_size;
}
mv_penalty[f_code][mv + MAX_MV] = len;
mv_penalty[f_code][mv + MAX_DMV] = len;
}

2
libavcodec/mpegvideo_enc.c
Unescape View File

@ -78,7 +78,7 @@ static int sse_mb(MpegEncContext *s);
static void denoise_dct_c(MpegEncContext *s, int16_t *block);
static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
const AVOption ff_mpv_generic_options[] = {

2
libavcodec/snowenc.c
Unescape View File

@ -292,7 +292,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_MV;
c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_DMV;
c->xmin = - x*block_w - 16+3;
c->ymin = - y*block_w - 16+3;

Write Preview
Loading…
Cancel
Save