Fix design of the pad filter.

Previously the pad filter just drawed borders in the surrounding of the input
without checking if this area was allocated or writeable. Now we check and
allocate a new buffer if the input is unsuitable.

Originally committed as revision 26315 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavfilter/vf_pad.c

@@ -27,6 +27,7 @@
 #include "avfilter.h"
 #include "libavutil/pixdesc.h"
 #include "libavutil/colorspace.h"
+#include "libavutil/avassert.h"
 #include "libavcore/imgutils.h"
 #include "libavcore/parseutils.h"
 
@@ -100,6 +101,24 @@ static void draw_rectangle(AVFilterBufferRef *outpic, uint8_t *line[4], int line
     }
 }
 
+static void copy_rectangle(AVFilterBufferRef *outpic,uint8_t *line[4], int line_step[4], int linesize[4],
+                           int hsub, int vsub, int x, int y, int y2, int w, int h)
+{
+    int i, plane;
+    uint8_t *p;
+
+    for (plane = 0; plane < 4 && outpic->data[plane]; plane++) {
+        int hsub1 = plane == 1 || plane == 2 ? hsub : 0;
+        int vsub1 = plane == 1 || plane == 2 ? vsub : 0;
+
+        p = outpic->data[plane] + (y >> vsub1) * outpic->linesize[plane];
+        for (i = 0; i < (h >> vsub1); i++) {
+            memcpy(p + (x >> hsub1) * line_step[plane], line[plane] + linesize[plane]*(i+(y2>>vsub1)), (w >> hsub1) * line_step[plane]);
+            p += outpic->linesize[plane];
+        }
+    }
+}
+
 static int query_formats(AVFilterContext *ctx)
 {
     static const enum PixelFormat pix_fmts[] = {
@@ -132,6 +151,7 @@ typedef struct {
     uint8_t *line[4];
     int      line_step[4];
     int hsub, vsub;         ///< chroma subsampling values
+    int needs_copy;
 } PadContext;
 
 static av_cold int init(AVFilterContext *ctx, const char *args, void *opaque)
@@ -243,21 +263,65 @@ static AVFilterBufferRef *get_video_buffer(AVFilterLink *inlink, int perms, int
     return picref;
 }
 
+static int does_clip(PadContext *pad, AVFilterBufferRef *outpicref, int plane, int hsub, int vsub, int x, int y)
+{
+    int64_t x_in_buf, y_in_buf;
+
+    x_in_buf =  outpicref->data[plane] - outpicref->buf->data[plane]
+             +  (x >> hsub) * pad      ->line_step[plane]
+             +  (y >> vsub) * outpicref->linesize [plane];
+
+    if(x_in_buf < 0 || x_in_buf % pad->line_step[plane])
+        return 1;
+    x_in_buf /= pad->line_step[plane];
+
+    av_assert0(outpicref->buf->linesize[plane]>0); //while reference can use negative linesize the main buffer should not
+
+    y_in_buf = x_in_buf / outpicref->buf->linesize[plane];
+    x_in_buf %= outpicref->buf->linesize[plane];
+
+    if(   y_in_buf<<vsub >= outpicref->buf->h
+       || x_in_buf<<hsub >= outpicref->buf->w)
+        return 1;
+    return 0;
+}
+
 static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
 {
     PadContext *pad = inlink->dst->priv;
     AVFilterBufferRef *outpicref = avfilter_ref_buffer(inpicref, ~0);
     int plane;
 
-    inlink->dst->outputs[0]->out_buf = outpicref;
-
     for (plane = 0; plane < 4 && outpicref->data[plane]; plane++) {
         int hsub = (plane == 1 || plane == 2) ? pad->hsub : 0;
         int vsub = (plane == 1 || plane == 2) ? pad->vsub : 0;
 
-        outpicref->data[plane] -= (pad->x >> hsub) * pad->line_step[plane] +
-            (pad->y >> vsub) * outpicref->linesize[plane];
+        av_assert0(outpicref->buf->w>0 && outpicref->buf->h>0);
+
+        if(outpicref->format != outpicref->buf->format) //unsupported currently
+            break;
+
+        outpicref->data[plane] -=   (pad->x  >> hsub) * pad      ->line_step[plane]
+                                  + (pad->y  >> vsub) * outpicref->linesize [plane];
+
+        if(   does_clip(pad, outpicref, plane, hsub, vsub, 0, 0)
+           || does_clip(pad, outpicref, plane, hsub, vsub, 0, pad->h-1)
+           || does_clip(pad, outpicref, plane, hsub, vsub, pad->w-1, 0)
+           || does_clip(pad, outpicref, plane, hsub, vsub, pad->w-1, pad->h-1)
+          )
+            break;
     }
+    pad->needs_copy= plane < 4 && outpicref->data[plane];
+    if(pad->needs_copy){
+        av_log(inlink->dst, AV_LOG_DEBUG, "Direct padding impossible allocating new frame\n");
+        avfilter_unref_buffer(outpicref);
+        outpicref = avfilter_get_video_buffer(inlink->dst->outputs[0], AV_PERM_WRITE | AV_PERM_NEG_LINESIZES,
+                                                       FFMAX(inlink->w, pad->w),
+                                                       FFMAX(inlink->h, pad->h));
+        avfilter_copy_buffer_ref_props(outpicref, inpicref);
+    }
+
+    inlink->dst->outputs[0]->out_buf = outpicref;
 
     outpicref->video->w = pad->w;
     outpicref->video->h = pad->h;
@@ -298,6 +362,7 @@ static void draw_slice(AVFilterLink *link, int y, int h, int slice_dir)
 {
     PadContext *pad = link->dst->priv;
     AVFilterBufferRef *outpic = link->dst->outputs[0]->out_buf;
+    AVFilterBufferRef *inpic = link->cur_buf;
 
     y += pad->y;
 
@@ -311,6 +376,13 @@ static void draw_slice(AVFilterLink *link, int y, int h, int slice_dir)
     /* left border */
     draw_rectangle(outpic, pad->line, pad->line_step, pad->hsub, pad->vsub,
                    0, y, pad->x, h);
+
+    if(pad->needs_copy){
+        copy_rectangle(outpic,
+                       inpic->data, pad->line_step, inpic->linesize, pad->hsub, pad->vsub,
+                       pad->x, y, y-pad->y, inpic->video->w, h);
+    }
+
     /* right border */
     draw_rectangle(outpic, pad->line, pad->line_step, pad->hsub, pad->vsub,
                    pad->x + pad->in_w, y, pad->w - pad->x - pad->in_w, h);