handle direct rendering buffer allocation failure

Originally committed as revision 1075 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/avcodec.h

@@ -5,8 +5,8 @@
 
 #define LIBAVCODEC_VERSION_INT 0x000406
 #define LIBAVCODEC_VERSION     "0.4.6"
-#define LIBAVCODEC_BUILD       4631
-#define LIBAVCODEC_BUILD_STR   "4631"
+#define LIBAVCODEC_BUILD       4632
+#define LIBAVCODEC_BUILD_STR   "4632"
 
 enum CodecID {
     CODEC_ID_NONE, 
@@ -503,7 +503,7 @@ typedef struct AVCodecContext {
      * encoding: unused
      * decoding: set by user
      */
-    void (*get_buffer_callback)(struct AVCodecContext *c, int width, int height, int pict_type);
+    int (*get_buffer_callback)(struct AVCodecContext *c, int width, int height, int pict_type);
 
     /**
      * is 1 if the decoded stream contains b frames, 0 otherwise

libavcodec/h263dec.c

@@ -457,7 +457,8 @@ retry:
             s->next_p_frame_damaged=0;
     }
 
-    MPV_frame_start(s, avctx);
+    if(MPV_frame_start(s, avctx) < 0)
+        return -1;
 
 #ifdef DEBUG
     printf("qscale=%d\n", s->qscale);

libavcodec/mpeg12.c

@@ -1581,7 +1581,8 @@ static int mpeg_decode_slice(AVCodecContext *avctx,
     /* start frame decoding */
     if (s->first_slice) {
         s->first_slice = 0;
-        MPV_frame_start(s, avctx);
+        if(MPV_frame_start(s, avctx) < 0)
+            return -1;
     }
 
     init_get_bits(&s->gb, buf, buf_size);

libavcodec/mpegvideo.c

@@ -791,7 +791,7 @@ static void draw_edges_c(UINT8 *buf, int wrap, int width, int height, int w)
 }
 
 /* generic function for encode/decode called before a frame is coded/decoded */
-void MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx)
+int MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx)
 {
     int i;
     UINT8 *tmp;
@@ -800,7 +800,10 @@ void MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx)
     avctx->mbskip_table= s->mbskip_table;
 
     if(avctx->flags&CODEC_FLAG_DR1){
-        avctx->get_buffer_callback(avctx, s->width, s->height, s->pict_type);
+        if(avctx->get_buffer_callback(avctx, s->width, s->height, s->pict_type) < 0){
+            fprintf(stderr, "get_buffer() failed\n");
+            return -1;
+        }
 
         s->linesize  = avctx->dr_stride;
         s->uvlinesize= avctx->dr_uvstride;
@@ -854,6 +857,8 @@ void MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx)
             s->dct_unquantize = s->dct_unquantize_h263;
     }else 
         s->dct_unquantize = s->dct_unquantize_mpeg1;
+
+    return 0;
 }
 
 /* generic function for encode/decode called after a frame has been coded/decoded */

libavcodec/mpegvideo.h

@@ -518,7 +518,7 @@ int DCT_common_init(MpegEncContext *s);
 int MPV_common_init(MpegEncContext *s);
 void MPV_common_end(MpegEncContext *s);
 void MPV_decode_mb(MpegEncContext *s, DCTELEM block[6][64]);
-void MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx);
+int MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx);
 void MPV_frame_end(MpegEncContext *s);
 #ifdef HAVE_MMX
 void MPV_common_init_mmx(MpegEncContext *s);

libavcodec/rv10.c

@@ -416,7 +416,8 @@ static int rv10_decode_packet(AVCodecContext *avctx,
     }
 
     if (s->mb_x == 0 && s->mb_y == 0) {
-        MPV_frame_start(s, avctx);
+        if(MPV_frame_start(s, avctx) < 0)
+            return -1;
     }
 
 #ifdef DEBUG

libavcodec/svq1.c

@@ -1085,7 +1085,8 @@ static int svq1_decode_frame(AVCodecContext *avctx,
 
   result = svq1_decode_frame_header (&s->gb, s);
   
-  MPV_frame_start(s, avctx);
+  if(MPV_frame_start(s, avctx) < 0)
+      return -1;
 
   if (result != 0)
   {