avfilter: add colormap video filter

release/5.1
Paul B Mahol 3 years ago
parent b1b7249606
commit 43ea19fef4
  1. 1
      Changelog
  2. 33
      doc/filters.texi
  3. 1
      libavfilter/Makefile
  4. 1
      libavfilter/allfilters.c
  5. 2
      libavfilter/version.h
  6. 582
      libavfilter/vf_colormap.c

@ -11,6 +11,7 @@ version 5.1:
- avsynctest source filter
- feedback video filter
- pixelize video filter
- colormap video filter
version 5.0:

@ -8944,6 +8944,39 @@ colorlevels=romin=0.5:gomin=0.5:bomin=0.5
This filter supports the all above options as @ref{commands}.
@section colormap
Apply custom color maps to video stream.
This filter needs three input video streams.
First stream is video stream that is going to be filtered out.
Second and third video stream specify color patches for source
color to target color mapping.
The filter accepts the following options:
@table @option
@item patch_size
Set the source and target video stream patch size in pixels.
@item nb_patches
Set the max number of used patches from source and target video stream.
@item type
Set the adjustments used for target colors. Can be @code{relative} or @code{absolute}.
Defaults is @code{absolute}.
@item kernel
Set the kernel used to measure color differences between mapped colors.
The accepted values are:
@table @samp
@item euclidean
@item weuclidean
@end table
Default is @code{euclidean}.
@end table
@section colormatrix
Convert color matrix.

@ -218,6 +218,7 @@ OBJS-$(CONFIG_COLORKEY_OPENCL_FILTER) += vf_colorkey_opencl.o opencl.o \
opencl/colorkey.o
OBJS-$(CONFIG_COLORHOLD_FILTER) += vf_colorkey.o
OBJS-$(CONFIG_COLORLEVELS_FILTER) += vf_colorlevels.o
OBJS-$(CONFIG_COLORMAP_FILTER) += vf_colormap.o
OBJS-$(CONFIG_COLORMATRIX_FILTER) += vf_colormatrix.o
OBJS-$(CONFIG_COLORSPACE_FILTER) += vf_colorspace.o colorspace.o colorspacedsp.o
OBJS-$(CONFIG_COLORTEMPERATURE_FILTER) += vf_colortemperature.o

@ -204,6 +204,7 @@ extern const AVFilter ff_vf_colorkey;
extern const AVFilter ff_vf_colorkey_opencl;
extern const AVFilter ff_vf_colorhold;
extern const AVFilter ff_vf_colorlevels;
extern const AVFilter ff_vf_colormap;
extern const AVFilter ff_vf_colormatrix;
extern const AVFilter ff_vf_colorspace;
extern const AVFilter ff_vf_colortemperature;

@ -31,7 +31,7 @@
#include "version_major.h"
#define LIBAVFILTER_VERSION_MINOR 33
#define LIBAVFILTER_VERSION_MINOR 34
#define LIBAVFILTER_VERSION_MICRO 100

@ -0,0 +1,582 @@
/*
* Copyright (c) 2022 Paul B Mahol
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* Compute a look-up table from map of colors.
*/
#include "config_components.h"
#include "libavutil/attributes.h"
#include "libavutil/avstring.h"
#include "libavutil/common.h"
#include "libavutil/opt.h"
#include "libavutil/pixdesc.h"
#include "avfilter.h"
#include "formats.h"
#include "internal.h"
#include "framesync.h"
#include "video.h"
#define MAX_SIZE 64
enum KernelType {
EUCLIDEAN,
WEUCLIDEAN,
NB_KERNELS,
};
typedef struct ColorMapContext {
const AVClass *class;
int w, h;
int size;
int nb_maps;
int changed[2];
float source[MAX_SIZE][4];
float ttarget[MAX_SIZE][4];
float target[MAX_SIZE][4];
float icoeff[4][4];
float coeff[MAX_SIZE][4];
int target_type;
int kernel_type;
float (*kernel)(const float *x, const float *y);
FFFrameSync fs;
} ColorMapContext;
#define OFFSET(x) offsetof(ColorMapContext, x)
#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
static const AVOption colormap_options[] = {
{ "patch_size", "set patch size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "64x64"}, 0, 0, FLAGS },
{ "nb_patches", "set number of patches", OFFSET(size), AV_OPT_TYPE_INT, {.i64 = 8}, 1, MAX_SIZE, FLAGS },
{ "type", "set the target type used", OFFSET(target_type), AV_OPT_TYPE_INT, {.i64=1}, 0, 1, FLAGS, "type" },
{ "relative", "the target colors are relative", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 1, FLAGS, "type" },
{ "absolute", "the target colors are absolute", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 1, FLAGS, "type" },
{ "kernel", "set the kernel used for measuring color difference", OFFSET(kernel_type), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_KERNELS-1, FLAGS, "kernel" },
{ "euclidean", "square root of sum of squared differences", 0, AV_OPT_TYPE_CONST, {.i64=EUCLIDEAN}, 0, 0, FLAGS, "kernel" },
{ "weuclidean", "weighted square root of sum of squared differences",0, AV_OPT_TYPE_CONST, {.i64=WEUCLIDEAN}, 0, 0, FLAGS, "kernel" },
{ NULL }
};
static int gauss_make_triangular(double *A, int *p, int n)
{
p[n - 1] = n - 1;
for (int k = 0; k < n; k++) {
double t1;
int m = k;
for (int i = k + 1; i < n; i++)
if (fabs(A[k + n * i]) > fabs(A[k + n * m]))
m = i;
p[k] = m;
t1 = A[k + n * m];
A[k + n * m] = A[k + n * k];
A[k + n * k] = t1;
if (t1 != 0) {
for (int i = k + 1; i < n; i++)
A[k + n * i] /= -t1;
if (k != m)
for (int i = k + 1; i < n; i++) {
double t2 = A[i + n * m];
A[i + n * m] = A[i + n * k];
A[i + n * k] = t2;
}
for (int j = k + 1; j < n; j++)
for (int i = k + 1; i < n; i++)
A[i + n * j] += A[k + j * n] * A[i + k * n];
} else {
return 0;
}
}
return 1;
}
static void gauss_solve_triangular(const double *A, const int *p, double *b, int n)
{
for(int k = 0; k < n - 1; k++) {
int m = p[k];
double t = b[m];
b[m] = b[k];
b[k] = t;
for (int i = k + 1; i < n; i++)
b[i] += A[k + n * i] * t;
}
for(int k = n - 1; k > 0; k--) {
b[k] /= A[k + n * k];
double t = b[k];
for (int i = 0; i < k; i++)
b[i] -= A[k + n * i] * t;
}
b[0] /= A[0 + 0 * n];
}
static int gauss_solve(double *A, double *b, int n)
{
int *p = av_calloc(n, sizeof(*p));
if (!p)
return 1;
if (!gauss_make_triangular(A, p, n)) {
av_freep(&p);
return 1;
}
gauss_solve_triangular(A, p, b, n);
av_freep(&p);
return 0;
}
#define P2(x) ((x)*(x))
static float euclidean_kernel(const float *x, const float *y)
{
const float d2 = P2(x[0]-y[0]) +
P2(x[1]-y[1]) +
P2(x[2]-y[2]);
return sqrtf(d2);
}
static float weuclidean_kernel(const float *x, const float *y)
{
const float rm = (x[0] + y[0]) * 0.5f;
const float d2 = P2(x[0]-y[0]) * (2.f + rm) +
P2(x[1]-y[1]) * 4.f +
P2(x[2]-y[2]) * (3.f - rm);
return sqrtf(d2);
}
static void build_map(AVFilterContext *ctx)
{
ColorMapContext *s = ctx->priv;
for (int j = 0; j < s->nb_maps; j++) {
s->target[j][0] = s->target_type == 0 ? s->source[j][0] + s->ttarget[j][0] : s->ttarget[j][0];
s->target[j][1] = s->target_type == 0 ? s->source[j][1] + s->ttarget[j][1] : s->ttarget[j][1];
s->target[j][2] = s->target_type == 0 ? s->source[j][2] + s->ttarget[j][2] : s->ttarget[j][2];
}
for (int c = 0; c < 3; c++) {
for (int j = 0; j < s->nb_maps; j++)
s->coeff[j][c] = 0.f;
for (int j = 0; j < 4; j++) {
s->icoeff[j][c] = 0;
s->icoeff[j][c] = 0;
s->icoeff[j][c] = 0;
}
s->icoeff[c+1][c] = 1.f;
switch (s->nb_maps) {
case 1:
{
float div = fabsf(s->source[0][c]) < 1e-6f ? 1e-6f : s->source[0][c];
s->icoeff[c][1+c] = s->target[0][c] / div;
}
break;
case 2:
{
double A[2 * 2] = { 1, s->source[0][c],
1, s->source[1][c] };
double b[2] = { s->target[0][c], s->target[1][c] };
if (gauss_solve(A, b, 2))
continue;
s->icoeff[0 ][c] = b[0];
s->icoeff[1+c][c] = b[1];
}
break;
case 3:
{
const uint8_t idx[3][3] = {{ 0, 1, 2 },
{ 1, 0, 2 },
{ 2, 0, 1 }};
const uint8_t didx[3][4] = {{ 0, 1, 2, 2 },
{ 0, 2, 1, 2 },
{ 0, 2, 2, 1 }};
const int C0 = idx[c][0];
const int C1 = idx[c][1];
const int C2 = idx[c][2];
double A[3 * 3] = { 1, s->source[0][C0], s->source[0][C1] + s->source[0][C2],
1, s->source[1][C0], s->source[1][C1] + s->source[1][C2],
1, s->source[2][C0], s->source[2][C1] + s->source[2][C2] };
double b[3] = { s->target[0][c], s->target[1][c], s->target[2][c] };
if (gauss_solve(A, b, 3))
continue;
s->icoeff[0][c] = b[didx[c][0]];
s->icoeff[1][c] = b[didx[c][1]];
s->icoeff[2][c] = b[didx[c][2]];
s->icoeff[3][c] = b[didx[c][3]];
}
break;
case 4:
{
double A[4 * 4] = { 1, s->source[0][0], s->source[0][1], s->source[0][2],
1, s->source[1][0], s->source[1][1], s->source[1][2],
1, s->source[2][0], s->source[2][1], s->source[2][2],
1, s->source[3][0], s->source[3][1], s->source[3][2] };
double b[4] = { s->target[0][c], s->target[1][c], s->target[2][c], s->target[3][c] };
int pivot[4];
if (!gauss_make_triangular(A, pivot, 4))
continue;
gauss_solve_triangular(A, pivot, b, 4);
s->icoeff[0][c] = b[0];
s->icoeff[1][c] = b[1];
s->icoeff[2][c] = b[2];
s->icoeff[3][c] = b[3];
}
break;
default:
{
const int N = s->nb_maps;
const int N4 = N + 4;
double *A = av_calloc(sizeof(*A), N4 * N4);
double *b = av_calloc(sizeof(*b), N4);
int *pivot = NULL;
if (!A || !b)
goto error;
for (int j = 0; j < N; j++)
for (int i = j; i < N; i++)
A[j*N4+i] = A[i*N4+j] = s->kernel(s->source[i], s->source[j]);
for (int i = 0; i < N; i++)
A[i*N4+N+0] = A[(N+0)*N4+i] = 1;
for (int i = 0; i < N; i++)
A[i*N4+N+1] = A[(N+1)*N4+i] = s->source[i][0];
for (int i = 0; i < N; i++)
A[i*N4+N+2] = A[(N+2)*N4+i] = s->source[i][1];
for (int i = 0; i < N; i++)
A[i*N4+N+3] = A[(N+3)*N4+i] = s->source[i][2];
for (int j = N; j < N4; j++)
for (int i = N;i < N4; i++)
A[j * N4 + i] = 0.;
pivot = av_calloc(N4, sizeof(*pivot));
if (!pivot)
goto error;
if (gauss_make_triangular(A, pivot, N4)) {
for (int i = 0; i < N; i++)
b[i] = s->target[i][c];
for (int i = N; i < N + 4; i++)
b[i] = 0;
gauss_solve_triangular(A, pivot, b, N4);
for (int i = 0; i < N; i++)
s->coeff[i][c] = b[i];
for (int i = 0; i < 4; i++)
s->icoeff[i][c] = b[N + i];
}
error:
av_free(pivot);
av_free(b);
av_free(A);
}
}
}
}
typedef struct ThreadData {
AVFrame *in, *out;
} ThreadData;
static int colormap_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
{
ColorMapContext *s = ctx->priv;
ThreadData *td = arg;
AVFrame *in = td->in;
AVFrame *out = td->out;
const int maps = s->nb_maps;
const int width = out->width;
const int height = out->height;
const int slice_start = (height * jobnr) / nb_jobs;
const int slice_end = (height * (jobnr + 1)) / nb_jobs;
const int sr_linesize = in->linesize[2] / 4;
const int dr_linesize = out->linesize[2] / 4;
const int sg_linesize = in->linesize[0] / 4;
const int dg_linesize = out->linesize[0] / 4;
const int sb_linesize = in->linesize[1] / 4;
const int db_linesize = out->linesize[1] / 4;
const float *sr = (float *)in->data[2] + slice_start * sr_linesize;
const float *sg = (float *)in->data[0] + slice_start * sg_linesize;
const float *sb = (float *)in->data[1] + slice_start * sb_linesize;
float *r = (float *)out->data[2] + slice_start * dr_linesize;
float *g = (float *)out->data[0] + slice_start * dg_linesize;
float *b = (float *)out->data[1] + slice_start * db_linesize;
float (*kernel)(const float *x, const float *y) = s->kernel;
const float *icoeff[4] = { s->icoeff[0], s->icoeff[1], s->icoeff[2], s->icoeff[3] };
for (int y = slice_start; y < slice_end; y++) {
for (int x = 0; x < width; x++) {
const float input[3] = { sr[x], sg[x], sb[x] };
float srv, sgv, sbv;
float rv, gv, bv;
srv = sr[x];
sgv = sg[x];
sbv = sb[x];
rv = icoeff[0][0];
gv = icoeff[0][1];
bv = icoeff[0][2];
rv += icoeff[1][0] * srv + icoeff[2][0] * sgv + icoeff[3][0] * sbv;
gv += icoeff[1][1] * srv + icoeff[2][1] * sgv + icoeff[3][1] * sbv;
bv += icoeff[1][2] * srv + icoeff[2][2] * sgv + icoeff[3][2] * sbv;
for (int z = 0; z < maps && maps > 4; z++) {
const float *coeff = s->coeff[z];
const float cr = coeff[0];
const float cg = coeff[1];
const float cb = coeff[2];
const float f = kernel(input, s->source[z]);
rv += f * cr;
gv += f * cg;
bv += f * cb;
}
r[x] = rv;
g[x] = gv;
b[x] = bv;
}
sg += sg_linesize;
g += dg_linesize;
sb += sb_linesize;
b += db_linesize;
sr += sr_linesize;
r += dr_linesize;
}
return 0;
}
static int import_map(AVFilterLink *inlink, AVFrame *in)
{
AVFilterContext *ctx = inlink->dst;
ColorMapContext *s = ctx->priv;
const int is_target = FF_INLINK_IDX(inlink) > 1;
const int pw = s->w;
const int pw2 = s->w / 2;
const int ph = s->h;
const int ph2 = s->h / 2;
int changed = 0;
int idx;
for (int plane = 0; plane < 3; plane++) {
const int c = plane == 0 ? 1 : plane == 1 ? 2 : 0;
idx = 0;
for (int y = ph2; y < in->height && idx < MAX_SIZE; y += ph) {
const float *src = (const float *)(in->data[plane] + y * in->linesize[plane]);
for (int x = pw2; x < in->width && idx < MAX_SIZE; x += pw) {
float value = src[x];
if (is_target) {
if (s->ttarget[idx][c] != value)
changed = 1;
s->ttarget[idx][c] = value;
} else {
if (s->source[idx][c] != value)
changed = 1;
s->source[idx][c] = value;
}
idx++;
}
}
}
if (changed)
s->changed[is_target] = 1;
if (!is_target)
s->nb_maps = FFMIN(idx, s->size);
return 0;
}
static int process_frame(FFFrameSync *fs)
{
AVFilterContext *ctx = fs->parent;
ColorMapContext *s = fs->opaque;
AVFilterLink *outlink = ctx->outputs[0];
AVFrame *in, *out, *source, *target;
ThreadData td;
int ret;
switch (s->kernel_type) {
case EUCLIDEAN:
s->kernel = euclidean_kernel;
break;
case WEUCLIDEAN:
s->kernel = weuclidean_kernel;
break;
default:
return AVERROR_BUG;
}
if ((ret = ff_framesync_get_frame(&s->fs, 0, &in, 1)) < 0 ||
(ret = ff_framesync_get_frame(&s->fs, 1, &source, 0)) < 0 ||
(ret = ff_framesync_get_frame(&s->fs, 2, &target, 0)) < 0)
return ret;
import_map(ctx->inputs[1], source);
import_map(ctx->inputs[2], target);
if (s->changed[0] || s->changed[1]) {
build_map(ctx);
s->changed[0] = s->changed[1] = 0;
}
if (!ctx->is_disabled) {
if (av_frame_is_writable(in)) {
out = in;
} else {
out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
if (!out) {
av_frame_free(&in);
return AVERROR(ENOMEM);
}
av_frame_copy_props(out, in);
}
td.in = in;
td.out = out;
ff_filter_execute(ctx, colormap_slice, &td, NULL,
FFMIN(in->height, ff_filter_get_nb_threads(ctx)));
if (out != in)
av_frame_free(&in);
} else {
out = in;
}
out->pts = av_rescale_q(s->fs.pts, s->fs.time_base, outlink->time_base);
return ff_filter_frame(outlink, out);
}
static int config_output(AVFilterLink *outlink)
{
AVFilterContext *ctx = outlink->src;
ColorMapContext *s = ctx->priv;
AVFilterLink *inlink = ctx->inputs[0];
AVFilterLink *source = ctx->inputs[1];
AVFilterLink *target = ctx->inputs[2];
FFFrameSyncIn *in;
int ret;
outlink->time_base = inlink->time_base;
outlink->frame_rate = inlink->frame_rate;
outlink->sample_aspect_ratio = inlink->sample_aspect_ratio;
outlink->w = inlink->w;
outlink->h = inlink->h;
if ((ret = ff_framesync_init(&s->fs, ctx, 3)) < 0)
return ret;
in = s->fs.in;
in[0].time_base = inlink->time_base;
in[1].time_base = source->time_base;
in[2].time_base = target->time_base;
in[0].sync = 1;
in[0].before = EXT_STOP;
in[0].after = EXT_INFINITY;
in[1].sync = 1;
in[1].before = EXT_STOP;
in[1].after = EXT_INFINITY;
in[2].sync = 1;
in[2].before = EXT_STOP;
in[2].after = EXT_INFINITY;
s->fs.opaque = s;
s->fs.on_event = process_frame;
ret = ff_framesync_configure(&s->fs);
outlink->time_base = s->fs.time_base;
return ret;
}
static int activate(AVFilterContext *ctx)
{
ColorMapContext *s = ctx->priv;
return ff_framesync_activate(&s->fs);
}
static const AVFilterPad inputs[] = {
{
.name = "default",
.type = AVMEDIA_TYPE_VIDEO,
},
{
.name = "source",
.type = AVMEDIA_TYPE_VIDEO,
},
{
.name = "target",
.type = AVMEDIA_TYPE_VIDEO,
},
};
static const AVFilterPad outputs[] = {
{
.name = "default",
.type = AVMEDIA_TYPE_VIDEO,
.config_props = config_output,
},
};
AVFILTER_DEFINE_CLASS(colormap);
const AVFilter ff_vf_colormap = {
.name = "colormap",
.description = NULL_IF_CONFIG_SMALL("Apply custom Color Maps to video stream."),
.priv_class = &colormap_class,
.priv_size = sizeof(ColorMapContext),
.activate = activate,
FILTER_INPUTS(inputs),
FILTER_OUTPUTS(outputs),
FILTER_PIXFMTS(AV_PIX_FMT_GBRPF32, AV_PIX_FMT_GBRAPF32),
.flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL |
AVFILTER_FLAG_SLICE_THREADS,
.process_command = ff_filter_process_command,
};
Loading…
Cancel
Save