diff --git a/Changelog b/Changelog index 1c3d91e5ce..a638c03250 100644 --- a/Changelog +++ b/Changelog @@ -12,6 +12,7 @@ version : - enable gdigrab device to grab a window using the hwnd=HANDLER syntax - IAMF raw demuxer and muxer - D3D12VA hardware accelerated H264, HEVC, VP9, AV1, MPEG-2 and VC1 decoding +- tiltandshift filter version 6.1: - libaribcaption decoder diff --git a/doc/filters.texi b/doc/filters.texi index 6d00ba2c3f..1d1634bf06 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -23186,6 +23186,69 @@ tile=3x2:nb_frames=5:padding=7:margin=2 @end example @end itemize +@section tiltandshift + +What happens when you invert time and space? + +Normally a video is composed of several frames that represent a different +instant of time and shows a scence that evolves in the space captured by the +frame. This filter is the antipode of that concept, taking inspiration by +tilt and shift photography. + +A filtered frame contains the whole timeline of events composing the sequence, +and this is obtained by placing a slice of pixels from each frame into a single +one. However, since there are no infinite-width frames, this is done up the +width of the input frame, and a video is recomposed by shifting away one +column for each subsequent frame. In order to map space to time, the filter +tilts each input frame as well, so that motion is preseved. This is accomplished +by progressively selecting a different column from each input frame. + +The end result is a sort of inverted parralax, so that far away objects move +much faster that the ones in the front. The ideal conditions for this video +effect are when there is either very little motion and the backgroud is static, +or when there is a lot of motion and a very wide depth of field (eg. wide +panorama, while moving on a train). + +The filter accepts the following parameters: + +@table @option + +@item tilt +Tilt video while shifting (default). When unset, video will be sliding a +static image, composed of the first column of each frame. + +@item start +What to do at the start of filtering (see below). + +@item end +What to do at the end of filtering (see below). + +@item hold +How many columns should pass through before start of filtering. + +@item pad +How many columns should be inserted before end of filtering. + +@end table + +Normally the filter shifts and tils from the very first frame, and stops when +the last one is received. However, before filtering starts, normal video may +be preseved, so that the effect is slowly shifted in its place. Similarly, +the last video frame may be reconstructed at the end. Alternatively it is +possible to just start and end with black. + +@table @samp +@item none +Filtering is starts immediately and ends when the last frame is received. + +@item frame +The first frames or the very last frame are kept intact during processing. + +@item black +Black is padded at the beginning or at the end of filtering. + +@end table + @section tinterlace Perform various types of temporal field interlacing. diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 63725f91b4..afc7bc1566 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -514,6 +514,7 @@ OBJS-$(CONFIG_THUMBNAIL_FILTER) += vf_thumbnail.o OBJS-$(CONFIG_THUMBNAIL_CUDA_FILTER) += vf_thumbnail_cuda.o vf_thumbnail_cuda.ptx.o \ cuda/load_helper.o OBJS-$(CONFIG_TILE_FILTER) += vf_tile.o +OBJS-$(CONFIG_TILTANDSHIFT_FILTER) += vf_tiltandshift.o OBJS-$(CONFIG_TINTERLACE_FILTER) += vf_tinterlace.o OBJS-$(CONFIG_TLUT2_FILTER) += vf_lut2.o framesync.o OBJS-$(CONFIG_TMEDIAN_FILTER) += vf_xmedian.o framesync.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index ed7c32be94..135794ba36 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -483,6 +483,7 @@ extern const AVFilter ff_vf_threshold; extern const AVFilter ff_vf_thumbnail; extern const AVFilter ff_vf_thumbnail_cuda; extern const AVFilter ff_vf_tile; +extern const AVFilter ff_vf_tiltandshift; extern const AVFilter ff_vf_tinterlace; extern const AVFilter ff_vf_tlut2; extern const AVFilter ff_vf_tmedian; diff --git a/libavfilter/version.h b/libavfilter/version.h index 7642b670d1..83931e11dd 100644 --- a/libavfilter/version.h +++ b/libavfilter/version.h @@ -32,7 +32,7 @@ #include "version_major.h" #define LIBAVFILTER_VERSION_MINOR 14 -#define LIBAVFILTER_VERSION_MICRO 100 +#define LIBAVFILTER_VERSION_MICRO 101 #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \ diff --git a/libavfilter/vf_tiltandshift.c b/libavfilter/vf_tiltandshift.c new file mode 100644 index 0000000000..0b1a03e29e --- /dev/null +++ b/libavfilter/vf_tiltandshift.c @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2014 Vittorio Giovara + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file vf_tiltandshift.c + * Simple time and space inverter. + */ + +#include + +#include "libavutil/common.h" +#include "libavutil/imgutils.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" +#include "libavutil/rational.h" + +#include "avfilter.h" +#include "formats.h" +#include "internal.h" +#include "video.h" + +enum PaddingOption { + TILT_NONE, + TILT_FRAME, + TILT_BLACK, + TILT_OPT_MAX, +}; + +typedef struct TiltandshiftContext { + const AVClass *class; + + /* set when all input frames have been processed and we have to + * empty buffers, pad and then return */ + int eof_recv; + + /* live or static sliding */ + int tilt; + + /* initial or final actions to perform (pad/hold a frame/black/nothing) */ + enum PaddingOption start; + enum PaddingOption end; + + /* columns to hold or pad at the beginning or at the end (respectively) */ + int hold; + int pad; + + /* buffers for black columns */ + uint8_t *black_buffers[4]; + int black_linesizes[4]; + + /* list containing all input frames */ + size_t input_size; + AVFrame *input; + AVFrame *prev; + + const AVPixFmtDescriptor *desc; +} TiltandshiftContext; + +static int list_add_frame(TiltandshiftContext *s, AVFrame *frame) +{ + if (s->input == NULL) { + s->input = frame; + } else { + AVFrame *head = s->input; + while (head->opaque) + head = head->opaque; + head->opaque = frame; + } + s->input_size++; + return 0; +} + +static void list_remove_head(TiltandshiftContext *s) +{ + AVFrame *head = s->input; + if (head) { + s->input = head->opaque; + av_frame_free(&head); + } + s->input_size--; +} + +static const enum AVPixelFormat pix_fmts[] = { + AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P, + AV_PIX_FMT_YUV410P, + AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P, + AV_PIX_FMT_YUVJ440P, + AV_PIX_FMT_NONE +}; + +static av_cold void uninit(AVFilterContext *ctx) +{ + TiltandshiftContext *s = ctx->priv; + while (s->input) + list_remove_head(s); + av_freep(&s->black_buffers); +} + +static int config_props(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + TiltandshiftContext *s = ctx->priv; + + outlink->w = ctx->inputs[0]->w; + outlink->h = ctx->inputs[0]->h; + outlink->format = ctx->inputs[0]->format; + + // when we have to pad black or a frame at the start, skip navigating + // the list and use either the frame or black for the requested value + if (s->start != TILT_NONE && !s->hold) + s->hold = outlink->w; + + // Init black buffers if we pad with black at the start or at the end. + // For the end, we always have to init on NONE and BLACK because we never + // know if there are going to be enough input frames to fill an output one. + if (s->start == TILT_BLACK || s->end != TILT_FRAME) { + int i, j, ret; + uint8_t black_data[] = { 0x10, 0x80, 0x80, 0x10 }; + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(outlink->format); + if (!desc) + return AVERROR_BUG; + + if (outlink->format == AV_PIX_FMT_YUVJ420P || + outlink->format == AV_PIX_FMT_YUVJ422P || + outlink->format == AV_PIX_FMT_YUVJ444P || + outlink->format == AV_PIX_FMT_YUVJ440P) + black_data[0] = black_data[3] = 0; + + ret = av_image_alloc(s->black_buffers, s->black_linesizes, 1, + outlink->h, outlink->format, 1); + if (ret < 0) + return ret; + + for (i = 0; i < FFMIN(desc->nb_components, 4); i++) + for (j = 0; j < (!i ? outlink->h + : -((-outlink->h) >> desc->log2_chroma_h)); j++) + memset(s->black_buffers[i] + j * s->black_linesizes[i], + black_data[i], 1); + + av_log(ctx, AV_LOG_VERBOSE, "Padding buffers initialized.\n"); + } + + s->desc = av_pix_fmt_desc_get(outlink->format); + if (!s->desc) + return AVERROR_BUG; + + return 0; +} + + +static void copy_column(AVFilterLink *outlink, + uint8_t *dst_data[4], int dst_linesizes[4], + const uint8_t *src_data[4], const int src_linesizes[4], + int ncol, int tilt) +{ + AVFilterContext *ctx = outlink->src; + TiltandshiftContext *s = ctx->priv; + uint8_t *dst[4]; + const uint8_t *src[4]; + + dst[0] = dst_data[0] + ncol; + dst[1] = dst_data[1] + (ncol >> s->desc->log2_chroma_h); + dst[2] = dst_data[2] + (ncol >> s->desc->log2_chroma_h); + + if (!tilt) + ncol = 0; + src[0] = src_data[0] + ncol; + src[1] = src_data[1] + (ncol >> s->desc->log2_chroma_h); + src[2] = src_data[2] + (ncol >> s->desc->log2_chroma_h); + + av_image_copy(dst, dst_linesizes, src, src_linesizes, outlink->format, 1, outlink->h); +} + +static int output_frame(AVFilterLink *outlink) +{ + TiltandshiftContext *s = outlink->src->priv; + AVFrame *head; + int ret; + + int ncol = 0; + AVFrame *dst = ff_get_video_buffer(outlink, outlink->w, outlink->h); + if (!dst) + return AVERROR(ENOMEM); + + // in case we have to do any initial black padding + if (s->start == TILT_BLACK) { + for ( ; ncol < s->hold; ncol++) + copy_column(outlink, dst->data, dst->linesize, + (const uint8_t **)s->black_buffers, s->black_linesizes, + ncol, 0); + } + + head = s->input; + // copy a column from each input frame + for ( ; ncol < s->input_size; ncol++) { + AVFrame *src = head; + + copy_column(outlink, dst->data, dst->linesize, + (const uint8_t **)src->data, src->linesize, + ncol, s->tilt); + + // keep track of the last known frame in case we need it below + s->prev = head; + // advance to the next frame unless we have to hold it + if (s->hold <= ncol) + head = head->opaque; + } + + // pad any remaining space with black or last frame + if (s->end == TILT_FRAME) { + for ( ; ncol < outlink->w; ncol++) + copy_column(outlink, dst->data, dst->linesize, + (const uint8_t **)s->prev->data, + s->prev->linesize, ncol, 1); + } else { // TILT_BLACK and TILT_NONE + for ( ; ncol < outlink->w; ncol++) + copy_column(outlink, dst->data, dst->linesize, + (const uint8_t **)s->black_buffers, s->black_linesizes, + ncol, 0); + } + + // set correct timestamps and props as long as there is proper input + ret = av_frame_copy_props(dst, s->input); + if (ret < 0) + return ret; + + // discard frame at the top of the list since it has been fully processed + list_remove_head(s); + // and it is safe to reduce the hold value (even if unused) + s->hold--; + + // output + return ff_filter_frame(outlink, dst); +} + +// This function just polls for new frames and queues them on a list +static int filter_frame(AVFilterLink *inlink, AVFrame *frame) +{ + AVFilterLink *outlink = inlink->dst->outputs[0]; + AVFilterContext *ctx = outlink->src; + TiltandshiftContext *s = inlink->dst->priv; + + int ret = list_add_frame(s, frame); + if (ret < 0) { + return ret; + } + + // load up enough frames to fill a frame and keep the queue filled on subsequent + // calls, until we receive EOF, and then we either pad or end + if (!s->eof_recv && s->input_size < outlink->w - s->pad) { + av_log(ctx, AV_LOG_DEBUG, "Not enough frames in the list (%zu/%d), waiting for more.\n", s->input_size, outlink->w - s->pad); + return 0; + } + + return output_frame(outlink); +} + +static int request_frame(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + TiltandshiftContext *s = ctx->priv; + int ret; + + // signal job finished when list is empty or when padding is either + // limited or disabled and eof was received + if ((s->input_size <= 0 || s->input_size == outlink->w - s->pad || s->end == TILT_NONE) && s->eof_recv) { + return AVERROR_EOF; + } + + ret = ff_request_frame(ctx->inputs[0]); + if (ret == AVERROR_EOF) { + s->eof_recv = 1; + } else if (ret < 0) { + return ret; + } + + if (s->eof_recv) { + while (s->input_size) { + av_log(ctx, AV_LOG_DEBUG, "Emptying buffers (%zu/%d).\n", s->input_size, outlink->w - s->pad); + ret = output_frame(outlink); + if (ret < 0) { + return ret; + } + } + } + + return 0; +} + +#define OFFSET(x) offsetof(TiltandshiftContext, x) +#define V AV_OPT_FLAG_VIDEO_PARAM +static const AVOption tiltandshift_options[] = { + { "tilt", "Tilt the video horizontally while shifting", OFFSET(tilt), AV_OPT_TYPE_INT, + { .i64 = 1 }, 0, 1, .flags = V, .unit = "tilt" }, + + { "start", "Action at the start of input", OFFSET(start), AV_OPT_TYPE_INT, + { .i64 = TILT_NONE }, 0, TILT_OPT_MAX, .flags = V, .unit = "start" }, + { "none", "Start immediately (default)", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_NONE }, INT_MIN, INT_MAX, .flags = V, .unit = "start" }, + { "frame", "Use the first frames", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_FRAME }, INT_MIN, INT_MAX, .flags = V, .unit = "start" }, + { "black", "Fill with black", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_BLACK }, INT_MIN, INT_MAX, .flags = V, .unit = "start" }, + + { "end", "Action at the end of input", OFFSET(end), AV_OPT_TYPE_INT, + { .i64 = TILT_NONE }, 0, TILT_OPT_MAX, .flags = V, .unit = "end" }, + { "none", "Do not pad at the end (default)", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_NONE }, INT_MIN, INT_MAX, .flags = V, .unit = "end" }, + { "frame", "Use the last frame", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_FRAME }, INT_MIN, INT_MAX, .flags = V, .unit = "end" }, + { "black", "Fill with black", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_BLACK }, INT_MIN, INT_MAX, .flags = V, .unit = "end" }, + + { "hold", "Number of columns to hold at the start of the video", OFFSET(hold), AV_OPT_TYPE_INT, + { .i64 = 0 }, 0, INT_MAX, .flags = V, .unit = "hold" }, + { "pad", "Number of columns to pad at the end of the video", OFFSET(pad), AV_OPT_TYPE_INT, + { .i64 = 0 }, 0, INT_MAX, .flags = V, .unit = "pad" }, + + { NULL }, +}; + +AVFILTER_DEFINE_CLASS(tiltandshift); + +static const AVFilterPad tiltandshift_inputs[] = { + { + .name = "in", + .type = AVMEDIA_TYPE_VIDEO, + .filter_frame = filter_frame, + }, +}; + +static const AVFilterPad tiltandshift_outputs[] = { + { + .name = "out", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = config_props, + .request_frame = request_frame, + }, +}; + +const AVFilter ff_vf_tiltandshift = { + .name = "tiltandshift", + .description = NULL_IF_CONFIG_SMALL("Generate a tilt-and-shift'd video."), + .priv_size = sizeof(TiltandshiftContext), + .priv_class = &tiltandshift_class, + .uninit = uninit, + FILTER_INPUTS(tiltandshift_inputs), + FILTER_OUTPUTS(tiltandshift_outputs), + FILTER_PIXFMTS_ARRAY(pix_fmts), +}; diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak index b57ef88c9b..a1bdcccfa0 100644 --- a/tests/fate/filter-video.mak +++ b/tests/fate/filter-video.mak @@ -544,6 +544,8 @@ fate-filter-pixelize-min: CMD = video_filter "pixelize=mode=min" FATE_FILTER_VSYNTH_VIDEO_FILTER-$(CONFIG_PIXELIZE_FILTER) += fate-filter-pixelize-max fate-filter-pixelize-max: CMD = video_filter "pixelize=mode=max" +FATE_FILTER_VSYNTH_VIDEO_FILTER-$(CONFIG_TILTANDSHIFT_FILTER) += fate-filter-tiltandshift +fate-filter-tiltandshift: CMD = framecrc -c:v pgmyuv -i $(SRC) -flags +bitexact -vf tiltandshift tests/pixfmts.mak: TAG = GEN tests/pixfmts.mak: ffmpeg$(PROGSSUF)$(EXESUF) | tests diff --git a/tests/ref/fate/filter-tiltandshift b/tests/ref/fate/filter-tiltandshift new file mode 100644 index 0000000000..bbb170e246 --- /dev/null +++ b/tests/ref/fate/filter-tiltandshift @@ -0,0 +1,55 @@ +#tb 0: 1/25 +#media_type 0: video +#codec_id 0: rawvideo +#dimensions 0: 352x288 +#sar 0: 0/1 +0, 0, 0, 1, 152064, 0x25250b4b +0, 1, 1, 1, 152064, 0x8d6ada7a +0, 2, 2, 1, 152064, 0xc033c3d7 +0, 3, 3, 1, 152064, 0x035b5d08 +0, 4, 4, 1, 152064, 0x5841e969 +0, 5, 5, 1, 152064, 0x76b809b2 +0, 6, 6, 1, 152064, 0x1cfcd9c3 +0, 7, 7, 1, 152064, 0x663a42ec +0, 8, 8, 1, 152064, 0x8230b191 +0, 9, 9, 1, 152064, 0xb9d627a3 +0, 10, 10, 1, 152064, 0xb3749370 +0, 11, 11, 1, 152064, 0x549f2eab +0, 12, 12, 1, 152064, 0x257d1476 +0, 13, 13, 1, 152064, 0xd4c76aea +0, 14, 14, 1, 152064, 0x315c9f3d +0, 15, 15, 1, 152064, 0x80dff6df +0, 16, 16, 1, 152064, 0xff2f8204 +0, 17, 17, 1, 152064, 0xdc17c361 +0, 18, 18, 1, 152064, 0xd62b4994 +0, 19, 19, 1, 152064, 0x4abecb92 +0, 20, 20, 1, 152064, 0x4fd133cd +0, 21, 21, 1, 152064, 0x7ddcd920 +0, 22, 22, 1, 152064, 0x56a51c8c +0, 23, 23, 1, 152064, 0x8d40c7bd +0, 24, 24, 1, 152064, 0xbacc8929 +0, 25, 25, 1, 152064, 0x742ccb13 +0, 26, 26, 1, 152064, 0xb44b6ab4 +0, 27, 27, 1, 152064, 0x56372838 +0, 28, 28, 1, 152064, 0xa69fa9d7 +0, 29, 29, 1, 152064, 0x45786818 +0, 30, 30, 1, 152064, 0x3367c650 +0, 31, 31, 1, 152064, 0xe58ea325 +0, 32, 32, 1, 152064, 0xd4d8e1d2 +0, 33, 33, 1, 152064, 0x4c67987b +0, 34, 34, 1, 152064, 0xc9b253c9 +0, 35, 35, 1, 152064, 0x8cf58fff +0, 36, 36, 1, 152064, 0xe61519af +0, 37, 37, 1, 152064, 0x8f314fcb +0, 38, 38, 1, 152064, 0xf307d0ed +0, 39, 39, 1, 152064, 0x025f5670 +0, 40, 40, 1, 152064, 0x95eeef9b +0, 41, 41, 1, 152064, 0x0f316da1 +0, 42, 42, 1, 152064, 0x668bfa05 +0, 43, 43, 1, 152064, 0xdb00700a +0, 44, 44, 1, 152064, 0x7a64f502 +0, 45, 45, 1, 152064, 0xef3e681c +0, 46, 46, 1, 152064, 0x6152e8d0 +0, 47, 47, 1, 152064, 0x958b5a48 +0, 48, 48, 1, 152064, 0x66f2d7e7 +0, 49, 49, 1, 152064, 0x47da49fd