Add new vf_tiltandshift filter

Signed-off-by: Vittorio Giovara <vittorio.giovara@gmail.com>

Changelog

@@ -12,6 +12,7 @@ version <next>:
 - enable gdigrab device to grab a window using the hwnd=HANDLER syntax
 - IAMF raw demuxer and muxer
 - D3D12VA hardware accelerated H264, HEVC, VP9, AV1, MPEG-2 and VC1 decoding
+- tiltandshift filter
 
 version 6.1:
 - libaribcaption decoder

doc/filters.texi

@@ -23186,6 +23186,69 @@ tile=3x2:nb_frames=5:padding=7:margin=2
 @end example
 @end itemize
 
+@section tiltandshift
+
+What happens when you invert time and space?
+
+Normally a video is composed of several frames that represent a different
+instant of time and shows a scence that evolves in the space captured by the
+frame. This filter is the antipode of that concept, taking inspiration by
+tilt and shift photography.
+
+A filtered frame contains the whole timeline of events composing the sequence,
+and this is obtained by placing a slice of pixels from each frame into a single
+one. However, since there are no infinite-width frames, this is done up the
+width of the input frame, and a video is recomposed by shifting away one
+column for each subsequent frame. In order to map space to time, the filter
+tilts each input frame as well, so that motion is preseved. This is accomplished
+by progressively selecting a different column from each input frame.
+
+The end result is a sort of inverted parralax, so that far away objects move
+much faster that the ones in the front. The ideal conditions for this video
+effect are when there is either very little motion and the backgroud is static,
+or when there is a lot of motion and a very wide depth of field (eg. wide
+panorama, while moving on a train).
+
+The filter accepts the following parameters:
+
+@table @option
+
+@item tilt
+Tilt video while shifting (default). When unset, video will be sliding a
+static image, composed of the first column of each frame.
+
+@item start
+What to do at the start of filtering (see below).
+
+@item end
+What to do at the end of filtering (see below).
+
+@item hold
+How many columns should pass through before start of filtering.
+
+@item pad
+How many columns should be inserted before end of filtering.
+
+@end table
+
+Normally the filter shifts and tils from the very first frame, and stops when
+the last one is received. However, before filtering starts, normal video may
+be preseved, so that the effect is slowly shifted in its place. Similarly,
+the last video frame may be reconstructed at the end. Alternatively it is
+possible to just start and end with black.
+
+@table @samp
+@item none
+Filtering is starts immediately and ends when the last frame is received.
+
+@item frame
+The first frames or the very last frame are kept intact during processing.
+
+@item black
+Black is padded at the beginning or at the end of filtering.
+
+@end table
+
 @section tinterlace
 
 Perform various types of temporal field interlacing.

libavfilter/Makefile

@@ -514,6 +514,7 @@ OBJS-$(CONFIG_THUMBNAIL_FILTER)              += vf_thumbnail.o
 OBJS-$(CONFIG_THUMBNAIL_CUDA_FILTER)         += vf_thumbnail_cuda.o vf_thumbnail_cuda.ptx.o \
                                                 cuda/load_helper.o
 OBJS-$(CONFIG_TILE_FILTER)                   += vf_tile.o
+OBJS-$(CONFIG_TILTANDSHIFT_FILTER)           += vf_tiltandshift.o
 OBJS-$(CONFIG_TINTERLACE_FILTER)             += vf_tinterlace.o
 OBJS-$(CONFIG_TLUT2_FILTER)                  += vf_lut2.o framesync.o
 OBJS-$(CONFIG_TMEDIAN_FILTER)                += vf_xmedian.o framesync.o

libavfilter/allfilters.c

@@ -483,6 +483,7 @@ extern const AVFilter ff_vf_threshold;
 extern const AVFilter ff_vf_thumbnail;
 extern const AVFilter ff_vf_thumbnail_cuda;
 extern const AVFilter ff_vf_tile;
+extern const AVFilter ff_vf_tiltandshift;
 extern const AVFilter ff_vf_tinterlace;
 extern const AVFilter ff_vf_tlut2;
 extern const AVFilter ff_vf_tmedian;

libavfilter/version.h

@@ -32,7 +32,7 @@
 #include "version_major.h"
 
 #define LIBAVFILTER_VERSION_MINOR  14
-#define LIBAVFILTER_VERSION_MICRO 100
+#define LIBAVFILTER_VERSION_MICRO 101
 
 
 #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \

libavfilter/vf_tiltandshift.c

@@ -0,0 +1,367 @@
+/*
+ * Copyright (c) 2014 Vittorio Giovara
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file vf_tiltandshift.c
+ * Simple time and space inverter.
+ */
+
+#include <string.h>
+
+#include "libavutil/common.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
+#include "libavutil/rational.h"
+
+#include "avfilter.h"
+#include "formats.h"
+#include "internal.h"
+#include "video.h"
+
+enum PaddingOption {
+    TILT_NONE,
+    TILT_FRAME,
+    TILT_BLACK,
+    TILT_OPT_MAX,
+};
+
+typedef struct TiltandshiftContext {
+    const AVClass *class;
+
+    /* set when all input frames have been processed and we have to
+     * empty buffers, pad and then return */
+    int eof_recv;
+
+    /* live or static sliding */
+    int tilt;
+
+    /* initial or final actions to perform (pad/hold a frame/black/nothing) */
+    enum PaddingOption start;
+    enum PaddingOption end;
+
+    /* columns to hold or pad at the beginning or at the end (respectively) */
+    int hold;
+    int pad;
+
+    /* buffers for black columns */
+    uint8_t *black_buffers[4];
+    int black_linesizes[4];
+
+    /* list containing all input frames */
+    size_t input_size;
+    AVFrame *input;
+    AVFrame *prev;
+
+    const AVPixFmtDescriptor *desc;
+} TiltandshiftContext;
+
+static int list_add_frame(TiltandshiftContext *s, AVFrame *frame)
+{
+    if (s->input == NULL) {
+        s->input = frame;
+    } else {
+        AVFrame *head = s->input;
+        while (head->opaque)
+            head = head->opaque;
+        head->opaque = frame;
+    }
+    s->input_size++;
+    return 0;
+}
+
+static void list_remove_head(TiltandshiftContext *s)
+{
+    AVFrame *head = s->input;
+    if (head) {
+        s->input = head->opaque;
+        av_frame_free(&head);
+    }
+    s->input_size--;
+}
+
+static const enum AVPixelFormat pix_fmts[] = {
+    AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_YUV410P,
+    AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P,
+    AV_PIX_FMT_YUVJ440P,
+    AV_PIX_FMT_NONE
+};
+
+static av_cold void uninit(AVFilterContext *ctx)
+{
+    TiltandshiftContext *s = ctx->priv;
+    while (s->input)
+        list_remove_head(s);
+    av_freep(&s->black_buffers);
+}
+
+static int config_props(AVFilterLink *outlink)
+{
+    AVFilterContext *ctx = outlink->src;
+    TiltandshiftContext *s = ctx->priv;
+
+    outlink->w = ctx->inputs[0]->w;
+    outlink->h = ctx->inputs[0]->h;
+    outlink->format = ctx->inputs[0]->format;
+
+    // when we have to pad black or a frame at the start, skip navigating
+    // the list and use either the frame or black for the requested value
+    if (s->start != TILT_NONE && !s->hold)
+        s->hold = outlink->w;
+
+    // Init black buffers if we pad with black at the start or at the end.
+    // For the end, we always have to init on NONE and BLACK because we never
+    // know if there are going to be enough input frames to fill an output one.
+    if (s->start == TILT_BLACK || s->end != TILT_FRAME) {
+        int i, j, ret;
+        uint8_t black_data[] = { 0x10, 0x80, 0x80, 0x10 };
+        const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(outlink->format);
+        if (!desc)
+            return AVERROR_BUG;
+
+        if (outlink->format == AV_PIX_FMT_YUVJ420P ||
+            outlink->format == AV_PIX_FMT_YUVJ422P ||
+            outlink->format == AV_PIX_FMT_YUVJ444P ||
+            outlink->format == AV_PIX_FMT_YUVJ440P)
+            black_data[0] = black_data[3] = 0;
+
+        ret = av_image_alloc(s->black_buffers, s->black_linesizes, 1,
+                             outlink->h, outlink->format, 1);
+        if (ret < 0)
+            return ret;
+
+        for (i = 0; i < FFMIN(desc->nb_components, 4); i++)
+            for (j = 0; j < (!i ? outlink->h
+                                : -((-outlink->h) >> desc->log2_chroma_h)); j++)
+                memset(s->black_buffers[i] + j * s->black_linesizes[i],
+                       black_data[i], 1);
+
+        av_log(ctx, AV_LOG_VERBOSE, "Padding buffers initialized.\n");
+    }
+
+    s->desc = av_pix_fmt_desc_get(outlink->format);
+    if (!s->desc)
+        return AVERROR_BUG;
+
+    return 0;
+}
+
+
+static void copy_column(AVFilterLink *outlink,
+                        uint8_t *dst_data[4], int dst_linesizes[4],
+                        const uint8_t *src_data[4], const int src_linesizes[4],
+                        int ncol, int tilt)
+{
+    AVFilterContext *ctx = outlink->src;
+    TiltandshiftContext *s = ctx->priv;
+    uint8_t *dst[4];
+    const uint8_t *src[4];
+
+    dst[0] = dst_data[0] + ncol;
+    dst[1] = dst_data[1] + (ncol >> s->desc->log2_chroma_h);
+    dst[2] = dst_data[2] + (ncol >> s->desc->log2_chroma_h);
+
+    if (!tilt)
+        ncol = 0;
+    src[0] = src_data[0] + ncol;
+    src[1] = src_data[1] + (ncol >> s->desc->log2_chroma_h);
+    src[2] = src_data[2] + (ncol >> s->desc->log2_chroma_h);
+
+    av_image_copy(dst, dst_linesizes, src, src_linesizes, outlink->format, 1, outlink->h);
+}
+
+static int output_frame(AVFilterLink *outlink)
+{
+    TiltandshiftContext *s = outlink->src->priv;
+    AVFrame *head;
+    int ret;
+
+    int ncol = 0;
+    AVFrame *dst = ff_get_video_buffer(outlink, outlink->w, outlink->h);
+    if (!dst)
+        return AVERROR(ENOMEM);
+
+    // in case we have to do any initial black padding
+    if (s->start == TILT_BLACK) {
+        for ( ; ncol < s->hold; ncol++)
+            copy_column(outlink, dst->data, dst->linesize,
+                        (const uint8_t **)s->black_buffers, s->black_linesizes,
+                        ncol, 0);
+    }
+
+    head = s->input;
+    // copy a column from each input frame
+    for ( ; ncol < s->input_size; ncol++) {
+        AVFrame *src = head;
+
+        copy_column(outlink, dst->data, dst->linesize,
+                    (const uint8_t **)src->data, src->linesize,
+                    ncol, s->tilt);
+
+        // keep track of the last known frame in case we need it below
+        s->prev = head;
+        // advance to the next frame unless we have to hold it
+        if (s->hold <= ncol)
+            head = head->opaque;
+    }
+
+    // pad any remaining space with black or last frame
+    if (s->end == TILT_FRAME) {
+        for ( ; ncol < outlink->w; ncol++)
+            copy_column(outlink, dst->data, dst->linesize,
+                        (const uint8_t **)s->prev->data,
+                        s->prev->linesize, ncol, 1);
+    } else { // TILT_BLACK and TILT_NONE
+        for ( ; ncol < outlink->w; ncol++)
+            copy_column(outlink, dst->data, dst->linesize,
+                        (const uint8_t **)s->black_buffers, s->black_linesizes,
+                        ncol, 0);
+    }
+
+    // set correct timestamps and props as long as there is proper input
+    ret = av_frame_copy_props(dst, s->input);
+    if (ret < 0)
+        return ret;
+
+    // discard frame at the top of the list since it has been fully processed
+    list_remove_head(s);
+    // and it is safe to reduce the hold value (even if unused)
+    s->hold--;
+
+    // output
+    return ff_filter_frame(outlink, dst);
+}
+
+// This function just polls for new frames and queues them on a list
+static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
+{
+    AVFilterLink *outlink = inlink->dst->outputs[0];
+    AVFilterContext *ctx = outlink->src;
+    TiltandshiftContext *s = inlink->dst->priv;
+
+    int ret = list_add_frame(s, frame);
+    if (ret < 0) {
+        return ret;
+    }
+
+    // load up enough frames to fill a frame and keep the queue filled on subsequent
+    // calls, until we receive EOF, and then we either pad or end
+    if (!s->eof_recv && s->input_size < outlink->w - s->pad) {
+        av_log(ctx, AV_LOG_DEBUG, "Not enough frames in the list (%zu/%d), waiting for more.\n", s->input_size, outlink->w - s->pad);
+        return 0;
+    }
+
+    return output_frame(outlink);
+}
+
+static int request_frame(AVFilterLink *outlink)
+{
+    AVFilterContext *ctx = outlink->src;
+    TiltandshiftContext *s = ctx->priv;
+    int ret;
+
+    // signal job finished when list is empty or when padding is either
+    // limited or disabled and eof was received
+    if ((s->input_size <= 0 || s->input_size == outlink->w - s->pad || s->end == TILT_NONE) && s->eof_recv) {
+        return AVERROR_EOF;
+    }
+
+    ret = ff_request_frame(ctx->inputs[0]);
+    if (ret == AVERROR_EOF) {
+        s->eof_recv = 1;
+    } else if (ret < 0) {
+        return ret;
+    }
+
+    if (s->eof_recv) {
+        while (s->input_size) {
+            av_log(ctx, AV_LOG_DEBUG, "Emptying buffers (%zu/%d).\n", s->input_size, outlink->w - s->pad);
+            ret = output_frame(outlink);
+            if (ret < 0) {
+                return ret;
+            }
+        }
+    }
+
+    return 0;
+}
+
+#define OFFSET(x) offsetof(TiltandshiftContext, x)
+#define V AV_OPT_FLAG_VIDEO_PARAM
+static const AVOption tiltandshift_options[] = {
+    { "tilt", "Tilt the video horizontally while shifting", OFFSET(tilt), AV_OPT_TYPE_INT,
+        { .i64 = 1 }, 0, 1, .flags = V, .unit = "tilt" },
+
+    { "start", "Action at the start of input", OFFSET(start), AV_OPT_TYPE_INT,
+        { .i64 = TILT_NONE }, 0, TILT_OPT_MAX, .flags = V, .unit = "start" },
+    { "none", "Start immediately (default)", 0, AV_OPT_TYPE_CONST,
+        { .i64 = TILT_NONE }, INT_MIN, INT_MAX, .flags = V, .unit = "start" },
+    { "frame", "Use the first frames", 0, AV_OPT_TYPE_CONST,
+        { .i64 = TILT_FRAME }, INT_MIN, INT_MAX, .flags = V, .unit = "start" },
+    { "black", "Fill with black", 0, AV_OPT_TYPE_CONST,
+        { .i64 = TILT_BLACK }, INT_MIN, INT_MAX, .flags = V, .unit = "start" },
+
+    { "end", "Action at the end of input", OFFSET(end), AV_OPT_TYPE_INT,
+        { .i64 = TILT_NONE }, 0, TILT_OPT_MAX, .flags = V, .unit = "end" },
+    { "none", "Do not pad at the end (default)", 0, AV_OPT_TYPE_CONST,
+        { .i64 = TILT_NONE }, INT_MIN, INT_MAX, .flags = V, .unit = "end" },
+    { "frame", "Use the last frame", 0, AV_OPT_TYPE_CONST,
+        { .i64 = TILT_FRAME }, INT_MIN, INT_MAX, .flags = V, .unit = "end" },
+    { "black", "Fill with black", 0, AV_OPT_TYPE_CONST,
+        { .i64 = TILT_BLACK }, INT_MIN, INT_MAX, .flags = V, .unit = "end" },
+
+    { "hold", "Number of columns to hold at the start of the video", OFFSET(hold), AV_OPT_TYPE_INT,
+        { .i64 = 0 }, 0, INT_MAX, .flags = V, .unit = "hold" },
+    { "pad", "Number of columns to pad at the end of the video", OFFSET(pad), AV_OPT_TYPE_INT,
+        { .i64 = 0 }, 0, INT_MAX, .flags = V, .unit = "pad" },
+
+    { NULL },
+};
+
+AVFILTER_DEFINE_CLASS(tiltandshift);
+
+static const AVFilterPad tiltandshift_inputs[] = {
+    {
+        .name         = "in",
+        .type         = AVMEDIA_TYPE_VIDEO,
+        .filter_frame = filter_frame,
+    },
+};
+
+static const AVFilterPad tiltandshift_outputs[] = {
+    {
+        .name          = "out",
+        .type          = AVMEDIA_TYPE_VIDEO,
+        .config_props  = config_props,
+        .request_frame = request_frame,
+    },
+};
+
+const AVFilter ff_vf_tiltandshift = {
+    .name          = "tiltandshift",
+    .description   = NULL_IF_CONFIG_SMALL("Generate a tilt-and-shift'd video."),
+    .priv_size     = sizeof(TiltandshiftContext),
+    .priv_class    = &tiltandshift_class,
+    .uninit        = uninit,
+    FILTER_INPUTS(tiltandshift_inputs),
+    FILTER_OUTPUTS(tiltandshift_outputs),
+    FILTER_PIXFMTS_ARRAY(pix_fmts),
+};

tests/fate/filter-video.mak

@@ -544,6 +544,8 @@ fate-filter-pixelize-min: CMD = video_filter "pixelize=mode=min"
 FATE_FILTER_VSYNTH_VIDEO_FILTER-$(CONFIG_PIXELIZE_FILTER) += fate-filter-pixelize-max
 fate-filter-pixelize-max: CMD = video_filter "pixelize=mode=max"
 
+FATE_FILTER_VSYNTH_VIDEO_FILTER-$(CONFIG_TILTANDSHIFT_FILTER) += fate-filter-tiltandshift
+fate-filter-tiltandshift: CMD = framecrc -c:v pgmyuv -i $(SRC) -flags +bitexact -vf tiltandshift
 
 tests/pixfmts.mak: TAG = GEN
 tests/pixfmts.mak: ffmpeg$(PROGSSUF)$(EXESUF) | tests

tests/ref/fate/filter-tiltandshift

@@ -0,0 +1,55 @@
+#tb 0: 1/25
+#media_type 0: video
+#codec_id 0: rawvideo
+#dimensions 0: 352x288
+#sar 0: 0/1
+0,          0,          0,        1,   152064, 0x25250b4b
+0,          1,          1,        1,   152064, 0x8d6ada7a
+0,          2,          2,        1,   152064, 0xc033c3d7
+0,          3,          3,        1,   152064, 0x035b5d08
+0,          4,          4,        1,   152064, 0x5841e969
+0,          5,          5,        1,   152064, 0x76b809b2
+0,          6,          6,        1,   152064, 0x1cfcd9c3
+0,          7,          7,        1,   152064, 0x663a42ec
+0,          8,          8,        1,   152064, 0x8230b191
+0,          9,          9,        1,   152064, 0xb9d627a3
+0,         10,         10,        1,   152064, 0xb3749370
+0,         11,         11,        1,   152064, 0x549f2eab
+0,         12,         12,        1,   152064, 0x257d1476
+0,         13,         13,        1,   152064, 0xd4c76aea
+0,         14,         14,        1,   152064, 0x315c9f3d
+0,         15,         15,        1,   152064, 0x80dff6df
+0,         16,         16,        1,   152064, 0xff2f8204
+0,         17,         17,        1,   152064, 0xdc17c361
+0,         18,         18,        1,   152064, 0xd62b4994
+0,         19,         19,        1,   152064, 0x4abecb92
+0,         20,         20,        1,   152064, 0x4fd133cd
+0,         21,         21,        1,   152064, 0x7ddcd920
+0,         22,         22,        1,   152064, 0x56a51c8c
+0,         23,         23,        1,   152064, 0x8d40c7bd
+0,         24,         24,        1,   152064, 0xbacc8929
+0,         25,         25,        1,   152064, 0x742ccb13
+0,         26,         26,        1,   152064, 0xb44b6ab4
+0,         27,         27,        1,   152064, 0x56372838
+0,         28,         28,        1,   152064, 0xa69fa9d7
+0,         29,         29,        1,   152064, 0x45786818
+0,         30,         30,        1,   152064, 0x3367c650
+0,         31,         31,        1,   152064, 0xe58ea325
+0,         32,         32,        1,   152064, 0xd4d8e1d2
+0,         33,         33,        1,   152064, 0x4c67987b
+0,         34,         34,        1,   152064, 0xc9b253c9
+0,         35,         35,        1,   152064, 0x8cf58fff
+0,         36,         36,        1,   152064, 0xe61519af
+0,         37,         37,        1,   152064, 0x8f314fcb
+0,         38,         38,        1,   152064, 0xf307d0ed
+0,         39,         39,        1,   152064, 0x025f5670
+0,         40,         40,        1,   152064, 0x95eeef9b
+0,         41,         41,        1,   152064, 0x0f316da1
+0,         42,         42,        1,   152064, 0x668bfa05
+0,         43,         43,        1,   152064, 0xdb00700a
+0,         44,         44,        1,   152064, 0x7a64f502
+0,         45,         45,        1,   152064, 0xef3e681c
+0,         46,         46,        1,   152064, 0x6152e8d0
+0,         47,         47,        1,   152064, 0x958b5a48
+0,         48,         48,        1,   152064, 0x66f2d7e7
+0,         49,         49,        1,   152064, 0x47da49fd