snow: remove a VLA used for edge emulation

Signed-off-by: Martin Storsjö <martin@martin.st>

libavcodec/snow.c

@@ -395,6 +395,7 @@ av_cold int ff_snow_common_init(AVCodecContext *avctx){
     SnowContext *s = avctx->priv_data;
     int width, height;
     int i, j, ret;
+    int emu_buf_size;
 
     s->avctx= avctx;
     s->max_ref_frames=1; //just make sure its not an invalid value in case of no initial keyframe
@@ -462,6 +463,8 @@ av_cold int ff_snow_common_init(AVCodecContext *avctx){
         return ret;
     }
     FF_ALLOC_OR_GOTO(avctx, s->scratchbuf, s->mconly_picture.linesize[0]*7*MB_SIZE, fail);
+    emu_buf_size = s->mconly_picture.linesize[0] * (2 * MB_SIZE + HTAPS_MAX - 1);
+    FF_ALLOC_OR_GOTO(avctx, s->emu_edge_buffer, emu_buf_size, fail);
 
     return 0;
 fail:
@@ -648,6 +651,7 @@ av_cold void ff_snow_common_end(SnowContext *s)
 
     av_freep(&s->block);
     av_freep(&s->scratchbuf);
+    av_freep(&s->emu_edge_buffer);
 
     for(i=0; i<MAX_REF_FRAMES; i++){
         av_freep(&s->ref_mvs[i]);

libavcodec/snow.h

@@ -165,6 +165,7 @@ typedef struct SnowContext{
     MpegEncContext m; // needed for motion estimation, should not be used for anything else, the idea is to eventually make the motion estimation independent of MpegEncContext, so this will be removed then (FIXME/XXX)
 
     uint8_t *scratchbuf;
+    uint8_t *emu_edge_buffer;
 }SnowContext;
 
 /* Tables */

libavcodec/snowenc.c

@@ -675,7 +675,7 @@ static int get_block_rd(SnowContext *s, int mb_x, int mb_y, int plane_index, uin
     uint8_t *src= s->  input_picture.data[plane_index];
     IDWTELEM *pred= (IDWTELEM*)s->m.obmc_scratchpad + plane_index*block_size*block_size*4;
     uint8_t *cur = s->scratchbuf;
-    uint8_t tmp[ref_stride*(2*MB_SIZE+HTAPS_MAX-1)];
+    uint8_t *tmp = s->emu_edge_buffer;
     const int b_stride = s->b_width << s->block_max_depth;
     const int b_height = s->b_height<< s->block_max_depth;
     const int w= p->width;