Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

lavfi/dnn: Remove DNN native backend

Browse Source 
According to discussion in
https://etherpad.mit.edu/p/FF_dev_meeting_20221202 and the proposal in
http://ffmpeg.org/pipermail/ffmpeg-devel/2022-December/304534.html,
the DNN native backend should be removed at first step.
All the DNN native backend related codes are deleted.

Signed-off-by: Ting Fu <ting.fu@intel.com>
pull/389/head
Ting Fu 2 years ago committed by Guo Yejun
parent a9fb141719
commit 78f95f1088
37 changed files with 4 additions and 4440 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      libavfilter/Makefile
  2. 10
      libavfilter/dnn/Makefile
  3. 561
      libavfilter/dnn/dnn_backend_native.c
  4. 149
      libavfilter/dnn/dnn_backend_native.h
  5. 147
      libavfilter/dnn/dnn_backend_native_layer_avgpool.c
  6. 69
      libavfilter/dnn/dnn_backend_native_layer_avgpool.h
  7. 265
      libavfilter/dnn/dnn_backend_native_layer_conv2d.c
  8. 68
      libavfilter/dnn/dnn_backend_native_layer_conv2d.h
  9. 151
      libavfilter/dnn/dnn_backend_native_layer_dense.c
  10. 65
      libavfilter/dnn/dnn_backend_native_layer_dense.h
  11. 102
      libavfilter/dnn/dnn_backend_native_layer_depth2space.c
  12. 72
      libavfilter/dnn/dnn_backend_native_layer_depth2space.h
  13. 193
      libavfilter/dnn/dnn_backend_native_layer_mathbinary.c
  14. 54
      libavfilter/dnn/dnn_backend_native_layer_mathbinary.h
  15. 156
      libavfilter/dnn/dnn_backend_native_layer_mathunary.c
  16. 92
      libavfilter/dnn/dnn_backend_native_layer_mathunary.h
  17. 83
      libavfilter/dnn/dnn_backend_native_layer_maximum.c
  18. 44
      libavfilter/dnn/dnn_backend_native_layer_maximum.h
  19. 268
      libavfilter/dnn/dnn_backend_native_layer_pad.c
  20. 43
      libavfilter/dnn/dnn_backend_native_layer_pad.h
  21. 42
      libavfilter/dnn/dnn_backend_native_layers.c
  22. 38
      libavfilter/dnn/dnn_backend_native_layers.h
  23. 368
      libavfilter/dnn/dnn_backend_tf.c
  24. 2
      libavfilter/dnn_interface.h
  25. 197
      libavfilter/tests/dnn-layer-avgpool.c
  26. 248
      libavfilter/tests/dnn-layer-conv2d.c
  27. 131
      libavfilter/tests/dnn-layer-dense.c
  28. 102
      libavfilter/tests/dnn-layer-depth2space.c
  29. 214
      libavfilter/tests/dnn-layer-mathbinary.c
  30. 148
      libavfilter/tests/dnn-layer-mathunary.c
  31. 71
      libavfilter/tests/dnn-layer-maximum.c
  32. 239
      libavfilter/tests/dnn-layer-pad.c
  33. 1
      libavfilter/vf_derain.c
  34. 1
      libavfilter/vf_dnn_processing.c
  35. 1
      libavfilter/vf_sr.c
  36. 1
      tests/Makefile
  37. 45
      tests/fate/dnn.mak

3
libavfilter/Makefile
Unescape View File

@ -635,9 +635,6 @@ SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_filter.h
TOOLS = graph2dot
TESTPROGS = drawutils filtfmts formats integral
TESTPROGS-$(CONFIG_DNN) += dnn-layer-avgpool dnn-layer-conv2d dnn-layer-dense \
dnn-layer-depth2space dnn-layer-mathbinary \
dnn-layer-mathunary dnn-layer-maximum dnn-layer-pad \
TOOLS-$(CONFIG_LIBZMQ) += zmqsend

10
libavfilter/dnn/Makefile
Unescape View File

@ -3,16 +3,6 @@ OBJS-$(CONFIG_DNN) += dnn/dnn_io_proc.o
OBJS-$(CONFIG_DNN) += dnn/queue.o
OBJS-$(CONFIG_DNN) += dnn/safe_queue.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_common.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layers.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_avgpool.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_dense.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_pad.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_conv2d.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_depth2space.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_maximum.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_mathbinary.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_mathunary.o
DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn/dnn_backend_tf.o
DNN-OBJS-$(CONFIG_LIBOPENVINO) += dnn/dnn_backend_openvino.o

561
libavfilter/dnn/dnn_backend_native.c
Unescape View File

@ -1,561 +0,0 @@
/*
* Copyright (c) 2018 Sergey Lavrushkin
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN native backend implementation.
*/
#include "dnn_backend_native.h"
#include "libavutil/avassert.h"
#include "dnn_backend_native_layer_conv2d.h"
#include "dnn_backend_native_layers.h"
#include "dnn_io_proc.h"
#include "dnn_backend_common.h"
#define OFFSET(x) offsetof(NativeContext, x)
#define FLAGS AV_OPT_FLAG_FILTERING_PARAM
static const AVOption dnn_native_options[] = {
{ "conv2d_threads", "threads num for conv2d layer", OFFSET(options.conv2d_threads), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS },
{ "async", "use DNN async inference", OFFSET(options.async), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
{ NULL },
};
static const AVClass dnn_native_class = {
.class_name = "dnn_native",
.item_name = av_default_item_name,
.option = dnn_native_options,
.version = LIBAVUTIL_VERSION_INT,
.category = AV_CLASS_CATEGORY_FILTER,
};
static int execute_model_native(Queue *lltask_queue);
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
{
NativeModel *native_model = task->model;
NativeContext *ctx = &native_model->ctx;
LastLevelTaskItem *lltask = av_malloc(sizeof(*lltask));
if (!lltask) {
av_log(ctx, AV_LOG_ERROR, "Unable to allocate space for LastLevelTaskItem\n");
return AVERROR(ENOMEM);
}
task->inference_todo = 1;
task->inference_done = 0;
lltask->task = task;
if (ff_queue_push_back(lltask_queue, lltask) < 0) {
av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
av_freep(&lltask);
return AVERROR(ENOMEM);
}
return 0;
}
static int get_input_native(void *model, DNNData *input, const char *input_name)
{
NativeModel *native_model = model;
NativeContext *ctx = &native_model->ctx;
for (int i = 0; i < native_model->operands_num; ++i) {
DnnOperand *oprd = &native_model->operands[i];
if (strcmp(oprd->name, input_name) == 0) {
if (oprd->type != DOT_INPUT) {
av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
return AVERROR(EINVAL);
}
input->dt = oprd->data_type;
av_assert0(oprd->dims[0] == 1);
input->height = oprd->dims[1];
input->width = oprd->dims[2];
input->channels = oprd->dims[3];
return 0;
}
}
// do not find the input operand
av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
return AVERROR(EINVAL);
}
static int get_output_native(void *model, const char *input_name, int input_width, int input_height,
const char *output_name, int *output_width, int *output_height)
{
int ret = 0;
NativeModel *native_model = model;
NativeContext *ctx = &native_model->ctx;
TaskItem task;
DNNExecBaseParams exec_params = {
.input_name = input_name,
.output_names = &output_name,
.nb_output = 1,
.in_frame = NULL,
.out_frame = NULL,
};
ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, native_model, input_height, input_width, ctx);
if (ret != 0) {
goto err;
}
ret = extract_lltask_from_task(&task, native_model->lltask_queue);
if (ret != 0) {
av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
goto err;
}
ret = execute_model_native(native_model->lltask_queue);
*output_width = task.out_frame->width;
*output_height = task.out_frame->height;
err:
av_frame_free(&task.out_frame);
av_frame_free(&task.in_frame);
return ret;
}
// Loads model and its parameters that are stored in a binary file with following structure:
// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
// For DEPTH_TO_SPACE layer: block_size
DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
{
#define DNN_NATIVE_MAGIC "FFMPEGDNNNATIVE"
DNNModel *model = NULL;
// sizeof - 1 to skip the terminating '\0' which is not written in the file
char buf[sizeof(DNN_NATIVE_MAGIC) - 1];
int version, header_size, major_version_expected = 1;
NativeModel *native_model = NULL;
AVIOContext *model_file_context;
int file_size, dnn_size, parsed_size;
int32_t layer;
DNNLayerType layer_type;
if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
return NULL;
}
file_size = avio_size(model_file_context);
model = av_mallocz(sizeof(DNNModel));
if (!model){
goto fail;
}
/**
* check file header with string and version
*/
if (avio_read(model_file_context, buf, sizeof(buf)) != sizeof(buf) ||
memcmp(buf, DNN_NATIVE_MAGIC, sizeof(buf)))
goto fail;
dnn_size = sizeof(buf);
version = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
if (version != major_version_expected) {
goto fail;
}
// currently no need to check minor version
version = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
header_size = dnn_size;
native_model = av_mallocz(sizeof(NativeModel));
if (!native_model){
goto fail;
}
model->model = native_model;
native_model->ctx.class = &dnn_native_class;
model->options = options;
if (av_opt_set_from_string(&native_model->ctx, model->options, NULL, "=", "&") < 0)
goto fail;
native_model->model = model;
if (native_model->ctx.options.async) {
av_log(&native_model->ctx, AV_LOG_WARNING, "Async not supported. Rolling back to sync\n");
native_model->ctx.options.async = 0;
}
#if !HAVE_PTHREAD_CANCEL
if (native_model->ctx.options.conv2d_threads > 1){
av_log(&native_model->ctx, AV_LOG_WARNING, "'conv2d_threads' option was set but it is not supported "
"on this build (pthread support is required)\n");
}
#endif
avio_seek(model_file_context, file_size - 8, SEEK_SET);
native_model->layers_num = (int32_t)avio_rl32(model_file_context);
native_model->operands_num = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
avio_seek(model_file_context, header_size, SEEK_SET);
native_model->layers = av_mallocz(native_model->layers_num * sizeof(Layer));
if (!native_model->layers){
goto fail;
}
native_model->operands = av_mallocz(native_model->operands_num * sizeof(DnnOperand));
if (!native_model->operands){
goto fail;
}
native_model->task_queue = ff_queue_create();
if (!native_model->task_queue) {
goto fail;
}
native_model->lltask_queue = ff_queue_create();
if (!native_model->lltask_queue) {
goto fail;
}
for (layer = 0; layer < native_model->layers_num; ++layer){
layer_type = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
if (layer_type >= DLT_COUNT) {
goto fail;
}
native_model->layers[layer].type = layer_type;
parsed_size = ff_layer_funcs[layer_type].pf_load(&native_model->layers[layer], model_file_context, file_size, native_model->operands_num);
if (!parsed_size) {
goto fail;
}
dnn_size += parsed_size;
}
for (int32_t i = 0; i < native_model->operands_num; ++i){
DnnOperand *oprd;
int32_t name_len;
int32_t operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
if (operand_index >= native_model->operands_num) {
goto fail;
}
oprd = &native_model->operands[operand_index];
name_len = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
dnn_size += name_len;
oprd->type = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
oprd->data_type = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
for (int32_t dim = 0; dim < 4; ++dim) {
oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
}
if (oprd->type == DOT_INPUT && oprd->dims[0] != 1)
goto fail;
oprd->isNHWC = 1;
}
avio_closep(&model_file_context);
if (dnn_size != file_size){
ff_dnn_free_model_native(&model);
return NULL;
}
model->get_input = &get_input_native;
model->get_output = &get_output_native;
model->filter_ctx = filter_ctx;
model->func_type = func_type;
return model;
fail:
ff_dnn_free_model_native(&model);
avio_closep(&model_file_context);
return NULL;
}
static int execute_model_native(Queue *lltask_queue)
{
NativeModel *native_model = NULL;
NativeContext *ctx = NULL;
int32_t layer;
DNNData input, output;
DnnOperand *oprd = NULL;
LastLevelTaskItem *lltask = NULL;
TaskItem *task = NULL;
int ret = 0;
lltask = ff_queue_pop_front(lltask_queue);
if (!lltask) {
av_log(NULL, AV_LOG_ERROR, "Failed to get LastLevelTaskItem\n");
ret = AVERROR(EINVAL);
goto err;
}
task = lltask->task;
native_model = task->model;
ctx = &native_model->ctx;
if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
ret = AVERROR(EINVAL);
goto err;
}
for (int i = 0; i < native_model->operands_num; ++i) {
oprd = &native_model->operands[i];
if (strcmp(oprd->name, task->input_name) == 0) {
if (oprd->type != DOT_INPUT) {
av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", task->input_name);
ret = AVERROR(EINVAL);
goto err;
}
break;
}
oprd = NULL;
}
if (!oprd) {
av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name);
ret = AVERROR(EINVAL);
goto err;
}
oprd->dims[1] = task->in_frame->height;
oprd->dims[2] = task->in_frame->width;
av_freep(&oprd->data);
oprd->length = ff_calculate_operand_data_length(oprd);
if (oprd->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n");
ret = AVERROR(EINVAL);
goto err;
}
oprd->data = av_malloc(oprd->length);
if (!oprd->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n");
ret = AVERROR(ENOMEM);
goto err;
}
input.height = oprd->dims[1];
input.width = oprd->dims[2];
input.channels = oprd->dims[3];
input.data = oprd->data;
input.dt = oprd->data_type;
if (task->do_ioproc) {
if (native_model->model->frame_pre_proc != NULL) {
native_model->model->frame_pre_proc(task->in_frame, &input, native_model->model->filter_ctx);
} else {
ff_proc_from_frame_to_dnn(task->in_frame, &input, ctx);
}
}
if (task->nb_output != 1) {
// currently, the filter does not need multiple outputs,
// so we just pending the support until we really need it.
avpriv_report_missing_feature(ctx, "multiple outputs");
ret = AVERROR(ENOSYS);
goto err;
}
for (layer = 0; layer < native_model->layers_num; ++layer){
DNNLayerType layer_type = native_model->layers[layer].type;
ret = ff_layer_funcs[layer_type].pf_exec(native_model->operands,
native_model->layers[layer].input_operand_indexes,
native_model->layers[layer].output_operand_index,
native_model->layers[layer].params,
&native_model->ctx);
if (ret != 0) {
av_log(ctx, AV_LOG_ERROR, "Failed to execute model\n");
goto err;
}
}
for (uint32_t i = 0; i < task->nb_output; ++i) {
DnnOperand *oprd = NULL;
const char *output_name = task->output_names[i];
for (int j = 0; j < native_model->operands_num; ++j) {
if (strcmp(native_model->operands[j].name, output_name) == 0) {
oprd = &native_model->operands[j];
break;
}
}
if (oprd == NULL) {
av_log(ctx, AV_LOG_ERROR, "Could not find output in model\n");
ret = AVERROR(EINVAL);
goto err;
}
output.data = oprd->data;
output.height = oprd->dims[1];
output.width = oprd->dims[2];
output.channels = oprd->dims[3];
output.dt = oprd->data_type;
if (task->do_ioproc) {
if (native_model->model->frame_post_proc != NULL) {
native_model->model->frame_post_proc(task->out_frame, &output, native_model->model->filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, &output, ctx);
}
} else {
task->out_frame->width = output.width;
task->out_frame->height = output.height;
}
}
task->inference_done++;
err:
av_freep(&lltask);
return ret;
}
int ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params)
{
NativeModel *native_model = model->model;
NativeContext *ctx = &native_model->ctx;
TaskItem *task;
int ret = 0;
ret = ff_check_exec_params(ctx, DNN_NATIVE, model->func_type, exec_params);
if (ret != 0) {
return ret;
}
task = av_malloc(sizeof(*task));
if (!task) {
av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
return AVERROR(ENOMEM);
}
ret = ff_dnn_fill_task(task, exec_params, native_model, ctx->options.async, 1);
if (ret != 0) {
av_freep(&task);
return ret;
}
if (ff_queue_push_back(native_model->task_queue, task) < 0) {
av_freep(&task);
av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
return AVERROR(ENOMEM);
}
ret = extract_lltask_from_task(task, native_model->lltask_queue);
if (ret != 0) {
av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
return ret;
}
return execute_model_native(native_model->lltask_queue);
}
int ff_dnn_flush_native(const DNNModel *model)
{
NativeModel *native_model = model->model;
if (ff_queue_size(native_model->lltask_queue) == 0) {
// no pending task need to flush
return 0;
}
// for now, use sync node with flush operation
// Switch to async when it is supported
return execute_model_native(native_model->lltask_queue);
}
DNNAsyncStatusType ff_dnn_get_result_native(const DNNModel *model, AVFrame **in, AVFrame **out)
{
NativeModel *native_model = model->model;
return ff_dnn_get_result_common(native_model->task_queue, in, out);
}
int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd)
{
int32_t result = 1;
for (int i = 0; i < 4; ++i)
result *= oprd->dims[i];
return result;
}
int32_t ff_calculate_operand_data_length(const DnnOperand* oprd)
{
// currently, we just support DNN_FLOAT
uint64_t len = sizeof(float);
for (int i = 0; i < 4; i++) {
len *= oprd->dims[i];
if (len > INT32_MAX)
return 0;
}
return len;
}
void ff_dnn_free_model_native(DNNModel **model)
{
NativeModel *native_model;
ConvolutionalParams *conv_params;
int32_t layer;
if (*model)
{
if ((*model)->model) {
native_model = (*model)->model;
if (native_model->layers) {
for (layer = 0; layer < native_model->layers_num; ++layer){
if (native_model->layers[layer].type == DLT_CONV2D){
conv_params = (ConvolutionalParams *)native_model->layers[layer].params;
av_freep(&conv_params->kernel);
av_freep(&conv_params->biases);
}
av_freep(&native_model->layers[layer].params);
}
av_freep(&native_model->layers);
}
if (native_model->operands) {
for (uint32_t operand = 0; operand < native_model->operands_num; ++operand)
av_freep(&native_model->operands[operand].data);
av_freep(&native_model->operands);
}
while (ff_queue_size(native_model->lltask_queue) != 0) {
LastLevelTaskItem *item = ff_queue_pop_front(native_model->lltask_queue);
av_freep(&item);
}
ff_queue_destroy(native_model->lltask_queue);
while (ff_queue_size(native_model->task_queue) != 0) {
TaskItem *item = ff_queue_pop_front(native_model->task_queue);
av_frame_free(&item->in_frame);
av_frame_free(&item->out_frame);
av_freep(&item);
}
ff_queue_destroy(native_model->task_queue);
av_freep(&native_model);
}
av_freep(model);
}
}

149
libavfilter/dnn/dnn_backend_native.h
Unescape View File

@ -1,149 +0,0 @@
/*
* Copyright (c) 2018 Sergey Lavrushkin
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN inference functions interface for native backend.
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_H
#include "../dnn_interface.h"
#include "libavformat/avio.h"
#include "libavutil/opt.h"
#include "queue.h"
/**
* the enum value of DNNLayerType should not be changed,
* the same values are used in convert_from_tensorflow.py
* and, it is used to index the layer execution/load function pointer.
*/
typedef enum {
DLT_INPUT = 0,
DLT_CONV2D = 1,
DLT_DEPTH_TO_SPACE = 2,
DLT_MIRROR_PAD = 3,
DLT_MAXIMUM = 4,
DLT_MATH_BINARY = 5,
DLT_MATH_UNARY = 6,
DLT_AVG_POOL = 7,
DLT_DENSE = 8,
DLT_COUNT
} DNNLayerType;
typedef enum {DOT_INPUT = 1, DOT_OUTPUT = 2, DOT_INTERMEDIATE = DOT_INPUT | DOT_OUTPUT} DNNOperandType;
typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNPaddingParam;
typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
typedef struct Layer{
DNNLayerType type;
/**
* a layer can have multiple inputs and one output.
* 4 is just a big enough number for input operands (increase it if necessary),
* do not use 'int32_t *input_operand_indexes', so we don't worry about mem leaks.
*/
int32_t input_operand_indexes[4];
int32_t output_operand_index;
void *params;
} Layer;
typedef struct DnnOperand{
/**
* there are two memory layouts, NHWC or NCHW, so we use dims,
* dims[0] is Number.
*/
int32_t dims[4];
/**
* input/output/intermediate operand of the network
*/
DNNOperandType type;
/**
* support different kinds of data type such as float, half float, int8 etc,
* first support float now.
*/
DNNDataType data_type;
/**
* NHWC if 1, otherwise NCHW.
* let's first support NHWC only, this flag is for extensive usage.
*/
int8_t isNHWC;
/**
* to avoid possible memory leak, do not use char *name
*/
char name[128];
/**
* data pointer with data length in bytes.
* usedNumbersLeft is only valid for intermediate operand,
* it means how many layers still depend on this operand,
* todo: the memory can be reused when usedNumbersLeft is zero.
*/
void *data;
int32_t length;
int32_t usedNumbersLeft;
}DnnOperand;
typedef struct InputParams{
int height, width, channels;
} InputParams;
typedef struct NativeOptions{
uint8_t async;
uint32_t conv2d_threads;
} NativeOptions;
typedef struct NativeContext {
const AVClass *class;
NativeOptions options;
} NativeContext;
// Represents simple feed-forward convolutional network.
typedef struct NativeModel{
NativeContext ctx;
DNNModel *model;
Layer *layers;
int32_t layers_num;
DnnOperand *operands;
int32_t operands_num;
Queue *task_queue;
Queue *lltask_queue;
} NativeModel;
DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
int ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params);
DNNAsyncStatusType ff_dnn_get_result_native(const DNNModel *model, AVFrame **in, AVFrame **out);
int ff_dnn_flush_native(const DNNModel *model);
void ff_dnn_free_model_native(DNNModel **model);
// NOTE: User must check for error (return value <= 0) to handle
// case like integer overflow.
int32_t ff_calculate_operand_data_length(const DnnOperand *oprd);
int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd);
#endif

147
libavfilter/dnn/dnn_backend_native_layer_avgpool.c
Unescape View File

@ -1,147 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN native backend implementation.
*/
#include "libavutil/avassert.h"
#include "dnn_backend_native_layer_avgpool.h"
int ff_dnn_load_layer_avg_pool(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
{
AvgPoolParams *avgpool_params;
int dnn_size = 0;
avgpool_params = av_malloc(sizeof(*avgpool_params));
if(!avgpool_params)
return 0;
avgpool_params->strides = (int32_t)avio_rl32(model_file_context);
avgpool_params->padding_method = (int32_t)avio_rl32(model_file_context);
avgpool_params->kernel_size = (int32_t)avio_rl32(model_file_context);
dnn_size += 12;
if (dnn_size > file_size || avgpool_params->kernel_size <= 0 || avgpool_params->strides <=0){
av_freep(&avgpool_params);
return 0;
}
layer->params = avgpool_params;
layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
return 0;
}
return dnn_size;
}
int ff_dnn_execute_layer_avg_pool(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx)
{
float *output;
int height_end, width_end, height_radius, width_radius, output_height, output_width, kernel_area;
int32_t input_operand_index = input_operand_indexes[0];
int number = operands[input_operand_index].dims[0];
int height = operands[input_operand_index].dims[1];
int width = operands[input_operand_index].dims[2];
int channel = operands[input_operand_index].dims[3];
const float *input = operands[input_operand_index].data;
const AvgPoolParams *avgpool_params = parameters;
int kernel_strides = avgpool_params->strides;
int src_linesize = width * channel;
DnnOperand *output_operand = &operands[output_operand_index];
/**
* When padding_method = SAME, the tensorflow will only padding the hald number of 0 pixels
* except the remainders.
* Eg: assuming the input height = 1080, the strides = 11, so the remainders = 1080 % 11 = 2
* and if ksize = 5: it will fill (5 - 2) >> 1 = 1 line before the first line of input image,
* and 5 - 2 - 1 = 2 lines after the last line of input image.
* and if ksize = 7: it will fill (7 - 2) >> 1 = 2 lines before the first line of input image,
* and 7 - 2 - 2 = 3 lines after the last line of input image.
*/
if (avgpool_params->padding_method == SAME) {
height_end = height;
width_end = width;
height_radius = avgpool_params->kernel_size - ((height - 1) % kernel_strides + 1);
width_radius = avgpool_params->kernel_size - ((width - 1) % kernel_strides + 1);
height_radius = height_radius < 0 ? 0 : height_radius >> 1;
width_radius = width_radius < 0 ? 0 : width_radius >> 1;
output_height = ceil(height / (kernel_strides * 1.0));
output_width = ceil(width / (kernel_strides * 1.0));
} else {
av_assert0(avgpool_params->padding_method == VALID);
height_end = height - avgpool_params->kernel_size + 1;
width_end = width - avgpool_params->kernel_size + 1;
height_radius = 0;
width_radius = 0;
output_height = ceil((height - avgpool_params->kernel_size + 1) / (kernel_strides * 1.0));
output_width = ceil((width - avgpool_params->kernel_size + 1) / (kernel_strides * 1.0));
}
output_operand->dims[0] = number;
output_operand->dims[1] = output_height;
output_operand->dims[2] = output_width;
// not support pooling in channel dimension now
output_operand->dims[3] = channel;
output_operand->data_type = operands[input_operand_index].data_type;
output_operand->length = ff_calculate_operand_data_length(output_operand);
if (output_operand->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
return AVERROR(EINVAL);
}
output_operand->data = av_realloc(output_operand->data, output_operand->length);
if (!output_operand->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
return AVERROR(ENOMEM);
}
output = output_operand->data;
for (int y = 0; y < height_end; y += kernel_strides) {
for (int x = 0; x < width_end; x += kernel_strides) {
for (int n_channel = 0; n_channel < channel; ++n_channel) {
output[n_channel] = 0.0;
kernel_area = 0;
for (int kernel_y = 0; kernel_y < avgpool_params->kernel_size; ++kernel_y) {
for (int kernel_x = 0; kernel_x < avgpool_params->kernel_size; ++kernel_x) {
float input_pel;
int y_pos = y + (kernel_y - height_radius);
int x_pos = x + (kernel_x - width_radius);
if (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) {
input_pel = 0.0;
} else {
kernel_area++;
input_pel = input[y_pos * src_linesize + x_pos * channel + n_channel];
}
output[n_channel] += input_pel;
}
}
output[n_channel] /= kernel_area;
}
output += channel;
}
}
return 0;
}

69
libavfilter/dnn/dnn_backend_native_layer_avgpool.h
Unescape View File

@ -1,69 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN inference functions interface for native backend.
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_AVGPOOL_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_AVGPOOL_H
#include "dnn_backend_native.h"
typedef struct AvgPoolParams{
int32_t strides, kernel_size;
DNNPaddingParam padding_method;
} AvgPoolParams;
/**
* @brief Load Average Pooling Layer.
*
* It assigns the Average Pooling layer with AvgPoolParams
* after parsing from the model file context.
*
* @param layer pointer to the DNN layer instance
* @param model_file_context pointer to model file context
* @param file_size model file size to check if data is read
* correctly from the model file
* @param operands_num operand count of the whole model to
* check if data is read correctly from the model file
* @return number of bytes read from the model file
* @retval 0 if out of memory or an error occurs
*/
int ff_dnn_load_layer_avg_pool(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num);
/**
* @brief Execute the Average Pooling Layer.
* Padding in channel dimensions is currently not supported.
*
* @param operands all operands for the model
* @param input_operand_indexes input operand indexes for this layer
* @param output_operand_index output operand index for this layer
* @param parameters average pooling parameters
* @param ctx pointer to Native model context for logging
* @retval 0 if the execution succeeds
* @retval AVERROR(ENOMEM) if memory allocation fails
* @retval AVERROR(EINVAL) for invalid arguments
*/
int ff_dnn_execute_layer_avg_pool(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx);
#endif

265
libavfilter/dnn/dnn_backend_native_layer_conv2d.c
Unescape View File

@ -1,265 +0,0 @@
/*
* Copyright (c) 2018 Sergey Lavrushkin
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/avassert.h"
#include "libavutil/thread.h"
#include "libavutil/cpu.h"
#include "dnn_backend_native_layer_conv2d.h"
#define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x)))
//struct to pass parameters
typedef struct ThreadCommonParam{
DnnOperand *operands;
const int32_t *input_operand_indexes;
int32_t output_operand_index;
const void *parameters;
NativeContext *ctx;
float *output_data;
} ThreadCommonParam;
typedef struct ThreadParam{
ThreadCommonParam *thread_common_param;
int thread_start, thread_end;
#if HAVE_PTHREAD_CANCEL
pthread_t thread;
#endif
} ThreadParam;
int ff_dnn_load_layer_conv2d(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
{
ConvolutionalParams *conv_params;
int kernel_size;
int dnn_size = 0;
conv_params = av_malloc(sizeof(*conv_params));
if (!conv_params)
return 0;
conv_params->dilation = (int32_t)avio_rl32(model_file_context);
conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
conv_params->activation = (int32_t)avio_rl32(model_file_context);
conv_params->input_num = (int32_t)avio_rl32(model_file_context);
conv_params->output_num = (int32_t)avio_rl32(model_file_context);
conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
conv_params->has_bias = (int32_t)avio_rl32(model_file_context);
dnn_size += 28;
kernel_size = conv_params->input_num * conv_params->output_num *
conv_params->kernel_size * conv_params->kernel_size;
dnn_size += kernel_size * 4;
if (conv_params->has_bias)
dnn_size += conv_params->output_num * 4;
if (dnn_size > file_size || conv_params->input_num <= 0 ||
conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
av_freep(&conv_params);
return 0;
}
conv_params->kernel = av_malloc_array(kernel_size, sizeof(*conv_params->kernel));
if (!conv_params->kernel) {
av_freep(&conv_params);
return 0;
}
for (int i = 0; i < kernel_size; ++i) {
conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context));
}
conv_params->biases = NULL;
if (conv_params->has_bias) {
conv_params->biases = av_malloc_array(conv_params->output_num, sizeof(*conv_params->biases));
if (!conv_params->biases){
av_freep(&conv_params->kernel);
av_freep(&conv_params);
return 0;
}
for (int i = 0; i < conv_params->output_num; ++i){
conv_params->biases[i] = av_int2float(avio_rl32(model_file_context));
}
}
layer->params = conv_params;
layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
return 0;
}
return dnn_size;
}
static void * dnn_execute_layer_conv2d_thread(void *threadarg)
{
//pass parameters
ThreadParam *thread_param = threadarg;
ThreadCommonParam *thread_common_param = thread_param->thread_common_param;
DnnOperand *operands = thread_common_param->operands;
int32_t input_operand_index = thread_common_param->input_operand_indexes[0];
int height = operands[input_operand_index].dims[1];
int width = operands[input_operand_index].dims[2];
int channel = operands[input_operand_index].dims[3];
const float *input = operands[input_operand_index].data;
const ConvolutionalParams *conv_params = thread_common_param->parameters;
int radius = conv_params->kernel_size >> 1;
int src_linesize = width * conv_params->input_num;
int filter_linesize = conv_params->kernel_size * conv_params->input_num;
int filter_size = conv_params->kernel_size * filter_linesize;
int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0;
float *output = thread_common_param->output_data;
output += (conv_params->output_num) * (width - 2 * pad_size) * (thread_param->thread_start - pad_size);
av_assert0(channel == conv_params->input_num);
for (int y = thread_param->thread_start; y < thread_param->thread_end; ++y) {
for (int x = pad_size; x < width - pad_size; ++x) {
for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) {
if (conv_params->has_bias)
output[n_filter] = conv_params->biases[n_filter];
else
output[n_filter] = 0.f;
for (int ch = 0; ch < conv_params->input_num; ++ch) {
for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) {
for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) {
float input_pel;
if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) {
int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height);
int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width);
input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
} else {
int y_pos = y + (kernel_y - radius) * conv_params->dilation;
int x_pos = x + (kernel_x - radius) * conv_params->dilation;
input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 :
input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
}
output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
kernel_x * conv_params->input_num + ch];
}
}
}
switch (conv_params->activation){
case RELU:
output[n_filter] = FFMAX(output[n_filter], 0.0);
break;
case TANH:
output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f;
break;
case SIGMOID:
output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter]));
break;
case NONE:
break;
case LEAKY_RELU:
output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0);
}
}
output += conv_params->output_num;
}
}
return NULL;
}
int ff_dnn_execute_layer_conv2d(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx)
{
#if HAVE_PTHREAD_CANCEL
int thread_num = (ctx->options.conv2d_threads <= 0 || ctx->options.conv2d_threads > av_cpu_count())
? (av_cpu_count() + 1) : (ctx->options.conv2d_threads);
int ret = 0, thread_stride;
ThreadParam *thread_param;
#else
ThreadParam thread_param = { 0 };
#endif
ThreadCommonParam thread_common_param;
const ConvolutionalParams *conv_params = parameters;
int height = operands[input_operand_indexes[0]].dims[1];
int width = operands[input_operand_indexes[0]].dims[2];
int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0;
DnnOperand *output_operand = &operands[output_operand_index];
void *tmp;
output_operand->dims[0] = operands[input_operand_indexes[0]].dims[0];
output_operand->dims[1] = height - pad_size * 2;
output_operand->dims[2] = width - pad_size * 2;
output_operand->dims[3] = conv_params->output_num;
output_operand->data_type = operands[input_operand_indexes[0]].data_type;
output_operand->length = ff_calculate_operand_data_length(output_operand);
if (output_operand->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
return AVERROR(EINVAL);
}
tmp = av_realloc(output_operand->data, output_operand->length);
if (!tmp) {
av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
return AVERROR(ENOMEM);
}
output_operand->data = tmp;
thread_common_param.output_data = output_operand->data;
thread_common_param.operands = operands;
thread_common_param.input_operand_indexes = input_operand_indexes;
thread_common_param.output_operand_index = output_operand_index;
thread_common_param.parameters = parameters;
thread_common_param.ctx = ctx;
#if HAVE_PTHREAD_CANCEL
thread_param = av_malloc_array(thread_num, sizeof(*thread_param));
if (!thread_param)
return AVERROR(ENOMEM);
thread_stride = (height - pad_size * 2) / thread_num;
//create threads
for (int i = 0; i < thread_num; i++){
int thread_ret = 0;
thread_param[i].thread_common_param = &thread_common_param;
thread_param[i].thread_start = thread_stride * i + pad_size;
thread_param[i].thread_end = (i == thread_num - 1) ? (height - pad_size) : (thread_param[i].thread_start + thread_stride);
thread_ret = pthread_create(&thread_param[i].thread, NULL,
dnn_execute_layer_conv2d_thread, &thread_param[i]);
if (thread_ret) {
thread_num = i;
ret = AVERROR(thread_ret);
break;
}
}
for (int i = 0; i < thread_num; i++){
pthread_join(thread_param[i].thread, NULL);
}
//release memory
av_freep(&thread_param);
return ret;
#else
thread_param.thread_common_param = &thread_common_param;
thread_param.thread_start = pad_size;
thread_param.thread_end = height - pad_size;
dnn_execute_layer_conv2d_thread(&thread_param);
return 0;
#endif
}

68
libavfilter/dnn/dnn_backend_native_layer_conv2d.h
Unescape View File

@ -1,68 +0,0 @@
/*
* Copyright (c) 2018 Sergey Lavrushkin
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_CONV2D_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_CONV2D_H
#include "dnn_backend_native.h"
typedef struct ConvolutionalParams{
int32_t input_num, output_num, kernel_size;
DNNActivationFunc activation;
DNNPaddingParam padding_method;
int32_t dilation;
int32_t has_bias;
float *kernel;
float *biases;
} ConvolutionalParams;
/**
* @brief Load the 2D Convolution Layer.
*
* It assigns the 2D convolution layer with ConvolutionalParams
* after parsing from the model file context.
*
* @param layer pointer to the DNN layer instance
* @param model_file_context pointer to model file context
* @param file_size model file size to check if data is read
* correctly from the model file
* @param operands_num operand count of the whole model to
* check if data is read correctly from the model file
* @return number of bytes read from the model file
* @retval 0 if out of memory or an error occurs
*/
int ff_dnn_load_layer_conv2d(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num);
/**
* @brief Execute the 2D Convolution Layer.
*
* @param operands all operands for the model
* @param input_operand_indexes input operand indexes for this layer
* @param output_operand_index output operand index for this layer
* @param parameters convolution parameters
* @param ctx pointer to Native model context for logging
* @retval 0 if the execution succeeds
* @retval AVERROR(ENOMEM) if memory allocation fails
* @retval AVERROR(EINVAL) for invalid arguments
*/
int ff_dnn_execute_layer_conv2d(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx);
#endif

151
libavfilter/dnn/dnn_backend_native_layer_dense.c
Unescape View File

@ -1,151 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/avassert.h"
#include "dnn_backend_native_layer_dense.h"
int ff_dnn_load_layer_dense(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
{
DenseParams *dense_params;
int kernel_size;
int dnn_size = 0;
dense_params = av_malloc(sizeof(*dense_params));
if (!dense_params)
return 0;
dense_params->activation = (int32_t)avio_rl32(model_file_context);
dense_params->input_num = (int32_t)avio_rl32(model_file_context);
dense_params->output_num = (int32_t)avio_rl32(model_file_context);
dense_params->has_bias = (int32_t)avio_rl32(model_file_context);
dnn_size += 16;
kernel_size = dense_params->input_num * dense_params->output_num;
dnn_size += kernel_size * 4;
if (dense_params->has_bias)
dnn_size += dense_params->output_num * 4;
if (dnn_size > file_size || dense_params->input_num <= 0 ||
dense_params->output_num <= 0){
av_freep(&dense_params);
return 0;
}
dense_params->kernel = av_malloc(kernel_size * sizeof(float));
if (!dense_params->kernel) {
av_freep(&dense_params);
return 0;
}
for (int i = 0; i < kernel_size; ++i) {
dense_params->kernel[i] = av_int2float(avio_rl32(model_file_context));
}
dense_params->biases = NULL;
if (dense_params->has_bias) {
dense_params->biases = av_malloc(dense_params->output_num * sizeof(float));
if (!dense_params->biases){
av_freep(&dense_params->kernel);
av_freep(&dense_params);
return 0;
}
for (int i = 0; i < dense_params->output_num; ++i){
dense_params->biases[i] = av_int2float(avio_rl32(model_file_context));
}
}
layer->params = dense_params;
layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
return 0;
}
return dnn_size;
}
int ff_dnn_execute_layer_dense(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx)
{
float *output;
int32_t input_operand_index = input_operand_indexes[0];
int number = operands[input_operand_index].dims[0];
int height = operands[input_operand_index].dims[1];
int width = operands[input_operand_index].dims[2];
int channel = operands[input_operand_index].dims[3];
const float *input = operands[input_operand_index].data;
const DenseParams *dense_params = parameters;
int src_linesize = width * channel;
DnnOperand *output_operand = &operands[output_operand_index];
output_operand->dims[0] = number;
output_operand->dims[1] = height;
output_operand->dims[2] = width;
output_operand->dims[3] = dense_params->output_num;
output_operand->data_type = operands[input_operand_index].data_type;
output_operand->length = ff_calculate_operand_data_length(output_operand);
if (output_operand->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
return AVERROR(EINVAL);
}
output_operand->data = av_realloc(output_operand->data, output_operand->length);
if (!output_operand->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
return AVERROR(ENOMEM);
}
output = output_operand->data;
av_assert0(channel == dense_params->input_num);
for (int y = 0; y < height; ++y) {
for (int x = 0; x < width; ++x) {
for (int n_filter = 0; n_filter < dense_params->output_num; ++n_filter) {
if (dense_params->has_bias)
output[n_filter] = dense_params->biases[n_filter];
else
output[n_filter] = 0.f;
for (int ch = 0; ch < dense_params->input_num; ++ch) {
float input_pel;
input_pel = input[y * src_linesize + x * dense_params->input_num + ch];
output[n_filter] += input_pel * dense_params->kernel[n_filter*dense_params->input_num + ch];
}
switch (dense_params->activation){
case RELU:
output[n_filter] = FFMAX(output[n_filter], 0.0);
break;
case TANH:
output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f;
break;
case SIGMOID:
output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter]));
break;
case NONE:
break;
case LEAKY_RELU:
output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0);
}
}
output += dense_params->output_num;
}
}
return 0;
}

65
libavfilter/dnn/dnn_backend_native_layer_dense.h
Unescape View File

@ -1,65 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_DENSE_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_DENSE_H
#include "dnn_backend_native.h"
typedef struct DenseParams{
int32_t input_num, output_num;
DNNActivationFunc activation;
int32_t has_bias;
float *kernel;
float *biases;
} DenseParams;
/**
* @brief Load the Densely-Connected Layer.
*
* It assigns the densely connected layer with DenseParams
* after parsing from the model file context.
*
* @param layer pointer to the DNN layer instance
* @param model_file_context pointer to model file context
* @param file_size model file size to check if data is read
* correctly from the model file
* @param operands_num operand count of the whole model to
* check if data is read correctly from the model file
* @return number of bytes read from the model file
* @retval 0 if out of memory or an error occurs
*/
int ff_dnn_load_layer_dense(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num);
/**
* @brief Execute the Densely-Connected Layer.
*
* @param operands all operands for the model
* @param input_operand_indexes input operand indexes for this layer
* @param output_operand_index output operand index for this layer
* @param parameters dense layer parameters
* @param ctx pointer to Native model context for logging
* @retval 0 if the execution succeeds
* @retval AVERROR(ENOMEM) if memory allocation fails
* @retval AVERROR(EINVAL) for invalid arguments
*/
int ff_dnn_execute_layer_dense(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx);
#endif

102
libavfilter/dnn/dnn_backend_native_layer_depth2space.c
Unescape View File

@ -1,102 +0,0 @@
/*
* Copyright (c) 2018 Sergey Lavrushkin
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN native backend implementation.
*/
#include "dnn_backend_native.h"
#include "dnn_backend_native_layer_depth2space.h"
int ff_dnn_load_layer_depth2space(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
{
DepthToSpaceParams *params;
int dnn_size = 0;
params = av_malloc(sizeof(*params));
if (!params)
return 0;
params->block_size = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
layer->params = params;
if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
return 0;
}
return dnn_size;
}
int ff_dnn_execute_layer_depth2space(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx)
{
float *output;
const DepthToSpaceParams *params = parameters;
int block_size = params->block_size;
int32_t input_operand_index = input_operand_indexes[0];
int number = operands[input_operand_index].dims[0];
int height = operands[input_operand_index].dims[1];
int width = operands[input_operand_index].dims[2];
int channels = operands[input_operand_index].dims[3];
const float *input = operands[input_operand_index].data;
int y, x, by, bx, ch;
int new_channels = channels / (block_size * block_size);
int output_linesize = width * channels;
int by_linesize = output_linesize / block_size;
int x_linesize = new_channels * block_size;
DnnOperand *output_operand = &operands[output_operand_index];
output_operand->dims[0] = number;
output_operand->dims[1] = height * block_size;
output_operand->dims[2] = width * block_size;
output_operand->dims[3] = new_channels;
output_operand->data_type = operands[input_operand_index].data_type;
output_operand->length = ff_calculate_operand_data_length(output_operand);
if (output_operand->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
return AVERROR(EINVAL);
}
output_operand->data = av_realloc(output_operand->data, output_operand->length);
if (!output_operand->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
return AVERROR(ENOMEM);
}
output = output_operand->data;
for (y = 0; y < height; ++y){
for (x = 0; x < width; ++x){
for (by = 0; by < block_size; ++by){
for (bx = 0; bx < block_size; ++bx){
for (ch = 0; ch < new_channels; ++ch){
output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch];
}
input += new_channels;
}
}
}
output += output_linesize;
}
return 0;
}

72
libavfilter/dnn/dnn_backend_native_layer_depth2space.h
Unescape View File

@ -1,72 +0,0 @@
/*
* Copyright (c) 2018 Sergey Lavrushkin
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN inference functions interface for native backend.
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_DEPTH2SPACE_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_DEPTH2SPACE_H
#include "../dnn_interface.h"
#include "libavformat/avio.h"
typedef struct DepthToSpaceParams{
int block_size;
} DepthToSpaceParams;
/**
* @brief Load the Depth to Space Layer.
*
* It assigns the depth to space layer with DepthToSpaceParams
* after parsing from the model file context.
*
* @param layer pointer to the DNN layer instance
* @param model_file_context pointer to model file context
* @param file_size model file size to check if data is read
* correctly from the model file
* @param operands_num operand count of the whole model to
* check if data is read correctly from the model file
* @return number of bytes read from the model file
* @retval 0 if an error occurs or out of memory
*/
int ff_dnn_load_layer_depth2space(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num);
/**
* @brief Execute the Depth to Space Layer.
*
* It rearranges the input data from depth into spatial
* form by applying Depth to Space transformation.
*
* @param operands all operands for the model
* @param input_operand_indexes input operand indexes for this layer
* @param output_operand_index output operand index for this layer
* @param parameters depth to space layer parameters
* @param ctx pointer to Native model context for logging
* @retval 0 if the execution succeeds
* @retval AVERROR(ENOMEM) if memory allocation fails
* @retval AVERROR(EINVAL) for invalid arguments
*/
int ff_dnn_execute_layer_depth2space(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx);
#endif

193
libavfilter/dnn/dnn_backend_native_layer_mathbinary.c
Unescape View File

@ -1,193 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN native backend implementation.
*/
#include "dnn_backend_native.h"
#include "dnn_backend_native_layer_mathbinary.h"
typedef float (*FunType)(float src0, float src1);
static float sub(float src0, float src1)
{
return src0 - src1;
}
static float add(float src0, float src1)
{
return src0 + src1;
}
static float mul(float src0, float src1)
{
return src0 * src1;
}
static float realdiv(float src0, float src1)
{
return src0 / src1;
}
static float minimum(float src0, float src1)
{
return FFMIN(src0, src1);
}
static float floormod(float src0, float src1)
{
return (float)((int)(src0) % (int)(src1));
}
static void math_binary_commutative(FunType pfun, const DnnLayerMathBinaryParams *params, const DnnOperand *input, DnnOperand *output, DnnOperand *operands, const int32_t *input_operand_indexes)
{
int dims_count;
const float *src;
float *dst;
dims_count = ff_calculate_operand_dims_count(output);
src = input->data;
dst = output->data;
if (params->input0_broadcast || params->input1_broadcast) {
for (int i = 0; i < dims_count; ++i) {
dst[i] = pfun(params->v, src[i]);
}
} else {
const DnnOperand *input1 = &operands[input_operand_indexes[1]];
const float *src1 = input1->data;
for (int i = 0; i < dims_count; ++i) {
dst[i] = pfun(src[i], src1[i]);
}
}
}
static void math_binary_not_commutative(FunType pfun, const DnnLayerMathBinaryParams *params, const DnnOperand *input, DnnOperand *output, DnnOperand *operands, const int32_t *input_operand_indexes)
{
int dims_count;
const float *src;
float *dst;
dims_count = ff_calculate_operand_dims_count(output);
src = input->data;
dst = output->data;
if (params->input0_broadcast) {
for (int i = 0; i < dims_count; ++i) {
dst[i] = pfun(params->v, src[i]);
}
} else if (params->input1_broadcast) {
for (int i = 0; i < dims_count; ++i) {
dst[i] = pfun(src[i], params->v);
}
} else {
const DnnOperand *input1 = &operands[input_operand_indexes[1]];
const float *src1 = input1->data;
for (int i = 0; i < dims_count; ++i) {
dst[i] = pfun(src[i], src1[i]);
}
}
}
int ff_dnn_load_layer_math_binary(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
{
DnnLayerMathBinaryParams params = { 0 };
int dnn_size = 0;
int input_index = 0;
params.bin_op = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
params.input0_broadcast = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
if (params.input0_broadcast) {
params.v = av_int2float(avio_rl32(model_file_context));
} else {
layer->input_operand_indexes[input_index] = (int32_t)avio_rl32(model_file_context);
if (layer->input_operand_indexes[input_index] >= operands_num) {
return 0;
}
input_index++;
}
dnn_size += 4;
params.input1_broadcast = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
if (params.input1_broadcast) {
params.v = av_int2float(avio_rl32(model_file_context));
} else {
layer->input_operand_indexes[input_index] = (int32_t)avio_rl32(model_file_context);
if (layer->input_operand_indexes[input_index] >= operands_num) {
return 0;
}
input_index++;
}
dnn_size += 4;
layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
if (layer->output_operand_index >= operands_num) {
return 0;
}
layer->params = av_memdup(&params, sizeof(params));
if (!layer->params)
return 0;
return dnn_size;
}
int ff_dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx)
{
const DnnOperand *input = &operands[input_operand_indexes[0]];
DnnOperand *output = &operands[output_operand_index];
const DnnLayerMathBinaryParams *params = parameters;
for (int i = 0; i < 4; ++i)
output->dims[i] = input->dims[i];
output->data_type = input->data_type;
output->length = ff_calculate_operand_data_length(output);
if (output->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
return AVERROR(EINVAL);
}
output->data = av_realloc(output->data, output->length);
if (!output->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
return AVERROR(ENOMEM);
}
switch (params->bin_op) {
case DMBO_SUB:
math_binary_not_commutative(sub, params, input, output, operands, input_operand_indexes);
return 0;
case DMBO_ADD:
math_binary_commutative(add, params, input, output, operands, input_operand_indexes);
return 0;
case DMBO_MUL:
math_binary_commutative(mul, params, input, output, operands, input_operand_indexes);
return 0;
case DMBO_REALDIV:
math_binary_not_commutative(realdiv, params, input, output, operands, input_operand_indexes);
return 0;
case DMBO_MINIMUM:
math_binary_commutative(minimum, params, input, output, operands, input_operand_indexes);
return 0;
case DMBO_FLOORMOD:
math_binary_not_commutative(floormod, params, input, output, operands, input_operand_indexes);
return 0;
default:
av_log(ctx, AV_LOG_ERROR, "Unmatch math binary operator\n");
return AVERROR(EINVAL);
}
}

54
libavfilter/dnn/dnn_backend_native_layer_mathbinary.h
Unescape View File

@ -1,54 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN inference functions interface for native backend.
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_MATHBINARY_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_MATHBINARY_H
#include "libavformat/avio.h"
#include "dnn_backend_native.h"
typedef enum {
DMBO_SUB = 0,
DMBO_ADD = 1,
DMBO_MUL = 2,
DMBO_REALDIV = 3,
DMBO_MINIMUM = 4,
DMBO_FLOORMOD = 5,
DMBO_COUNT
} DNNMathBinaryOperation;
typedef struct DnnLayerMathBinaryParams{
DNNMathBinaryOperation bin_op;
int input0_broadcast;
int input1_broadcast;
float v;
} DnnLayerMathBinaryParams;
int ff_dnn_load_layer_math_binary(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num);
int ff_dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx);
#endif

156
libavfilter/dnn/dnn_backend_native_layer_mathunary.c
Unescape View File

@ -1,156 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN native backend implementation.
*/
#include <math.h>
#include "dnn_backend_native.h"
#include "dnn_backend_native_layer_mathunary.h"
int ff_dnn_load_layer_math_unary(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
{
DnnLayerMathUnaryParams *params;
int dnn_size = 0;
params = av_malloc(sizeof(*params));
if(!params)
return 0;
params->un_op = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
layer->params = params;
layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
return 0;
}
return dnn_size;
}
int ff_dnn_execute_layer_math_unary(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx)
{
const DnnOperand *input = &operands[input_operand_indexes[0]];
DnnOperand *output = &operands[output_operand_index];
const DnnLayerMathUnaryParams *params = parameters;
int dims_count;
const float *src;
float *dst;
for (int i = 0; i < 4; ++i)
output->dims[i] = input->dims[i];
output->data_type = input->data_type;
output->length = ff_calculate_operand_data_length(output);
if (output->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
return AVERROR(EINVAL);
}
output->data = av_realloc(output->data, output->length);
if (!output->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
return AVERROR(ENOMEM);
}
dims_count = ff_calculate_operand_dims_count(output);
src = input->data;
dst = output->data;
switch (params->un_op) {
case DMUO_ABS:
for (int i = 0; i < dims_count; ++i)
dst[i] = FFABS(src[i]);
return 0;
case DMUO_SIN:
for (int i = 0; i < dims_count; ++i)
dst[i] = sin(src[i]);
return 0;
case DMUO_COS:
for (int i = 0; i < dims_count; ++i)
dst[i] = cos(src[i]);
return 0;
case DMUO_TAN:
for (int i = 0; i < dims_count; ++i)
dst[i] = tan(src[i]);
return 0;
case DMUO_ASIN:
for (int i = 0; i < dims_count; ++i)
dst[i] = asin(src[i]);
return 0;
case DMUO_ACOS:
for (int i = 0; i < dims_count; ++i)
dst[i] = acos(src[i]);
return 0;
case DMUO_ATAN:
for (int i = 0; i < dims_count; ++i)
dst[i] = atan(src[i]);
return 0;
case DMUO_SINH:
for (int i = 0; i < dims_count; ++i)
dst[i] = sinh(src[i]);
return 0;
case DMUO_COSH:
for (int i = 0; i < dims_count; ++i)
dst[i] = cosh(src[i]);
return 0;
case DMUO_TANH:
for (int i = 0; i < dims_count; ++i)
dst[i] = tanh(src[i]);
return 0;
case DMUO_ASINH:
for (int i = 0; i < dims_count; ++i)
dst[i] = asinh(src[i]);
return 0;
case DMUO_ACOSH:
for (int i = 0; i < dims_count; ++i)
dst[i] = acosh(src[i]);
return 0;
case DMUO_ATANH:
for (int i = 0; i < dims_count; ++i)
dst[i] = atanh(src[i]);
return 0;
case DMUO_CEIL:
for (int i = 0; i < dims_count; ++i)
dst[i] = ceil(src[i]);
return 0;
case DMUO_FLOOR:
for (int i = 0; i < dims_count; ++i)
dst[i] = floor(src[i]);
return 0;
case DMUO_ROUND:
for (int i = 0; i < dims_count; ++i)
dst[i] = round(src[i]);
return 0;
case DMUO_EXP:
for (int i = 0; i < dims_count; ++i)
dst[i] = exp(src[i]);
return 0;
default:
av_log(ctx, AV_LOG_ERROR, "Unmatch math unary operator\n");
return AVERROR(EINVAL);
}
}

92
libavfilter/dnn/dnn_backend_native_layer_mathunary.h
Unescape View File

@ -1,92 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN inference functions interface for native backend.
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_MATHUNARY_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_MATHUNARY_H
#include "libavformat/avio.h"
#include "dnn_backend_native.h"
typedef enum {
DMUO_ABS = 0,
DMUO_SIN = 1,
DMUO_COS = 2,
DMUO_TAN = 3,
DMUO_ASIN = 4,
DMUO_ACOS = 5,
DMUO_ATAN = 6,
DMUO_SINH = 7,
DMUO_COSH = 8,
DMUO_TANH = 9,
DMUO_ASINH = 10,
DMUO_ACOSH = 11,
DMUO_ATANH = 12,
DMUO_CEIL = 13,
DMUO_FLOOR = 14,
DMUO_ROUND = 15,
DMUO_EXP = 16,
DMUO_COUNT
} DNNMathUnaryOperation;
typedef struct DnnLayerMathUnaryParams{
DNNMathUnaryOperation un_op;
} DnnLayerMathUnaryParams;
/**
* @brief Load the Unary Math Layer.
*
* It assigns the unary math layer with DnnLayerMathUnaryParams
* after parsing from the model file context.
*
* @param layer pointer to the DNN layer instance
* @param model_file_context pointer to model file context
* @param file_size model file size to check if data is read
* correctly from the model file
* @param operands_num operand count of the whole model to
* check if data is read correctly from the model file
* @return number of bytes read from the model file
* @retval 0 if out of memory or an error occurs
*/
int ff_dnn_load_layer_math_unary(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num);
/**
* @brief Execute the Unary Math Layer.
*
* It applies the unary operator parsed while
* loading to the given input operands.
*
* @param operands all operands for the model
* @param input_operand_indexes input operand indexes for this layer
* @param output_operand_index output operand index for this layer
* @param parameters unary math layer parameters
* @param ctx pointer to Native model context for logging
* @retval 0 if the execution succeeds
* @retval AVERROR(ENOMEM) if memory allocation fails
* @retval AVERROR(EINVAL) for invalid arguments
*/
int ff_dnn_execute_layer_math_unary(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx);
#endif

83
libavfilter/dnn/dnn_backend_native_layer_maximum.c
Unescape View File

@ -1,83 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN native backend implementation.
*/
#include "dnn_backend_native.h"
#include "dnn_backend_native_layer_maximum.h"
int ff_dnn_load_layer_maximum(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
{
DnnLayerMaximumParams *params;
int dnn_size = 0;
params = av_malloc(sizeof(*params));
if (!params)
return 0;
params->val.u32 = avio_rl32(model_file_context);
dnn_size += 4;
layer->params = params;
layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
return 0;
}
return dnn_size;
}
int ff_dnn_execute_layer_maximum(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx)
{
const DnnOperand *input = &operands[input_operand_indexes[0]];
DnnOperand *output = &operands[output_operand_index];
const DnnLayerMaximumParams *params = parameters;
int dims_count;
const float *src;
float *dst;
for (int i = 0; i < 4; ++i)
output->dims[i] = input->dims[i];
output->data_type = input->data_type;
output->length = ff_calculate_operand_data_length(output);
if (output->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
return AVERROR(EINVAL);
}
output->data = av_realloc(output->data, output->length);
if (!output->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
return AVERROR(ENOMEM);
}
dims_count = ff_calculate_operand_dims_count(output);
src = input->data;
dst = output->data;
for (int i = 0; i < dims_count; ++i)
dst[i] = FFMAX(src[i], params->val.y);
return 0;
}

44
libavfilter/dnn/dnn_backend_native_layer_maximum.h
Unescape View File

@ -1,44 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN inference functions interface for native backend.
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_MAXIMUM_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_MAXIMUM_H
#include "libavformat/avio.h"
#include "dnn_backend_native.h"
typedef struct DnnLayerMaximumParams{
union {
uint32_t u32;
float y;
}val;
} DnnLayerMaximumParams;
int ff_dnn_load_layer_maximum(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num);
int ff_dnn_execute_layer_maximum(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx);
#endif

268
libavfilter/dnn/dnn_backend_native_layer_pad.c
Unescape View File

@ -1,268 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <string.h>
#include "libavutil/avassert.h"
#include "dnn_backend_native_layer_pad.h"
int ff_dnn_load_layer_pad(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
{
LayerPadParams *params;
int dnn_size = 0;
params = av_malloc(sizeof(*params));
if (!params)
return 0;
params->mode = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
for (int i = 0; i < 4; ++i) {
params->paddings[i][0] = avio_rl32(model_file_context);
params->paddings[i][1] = avio_rl32(model_file_context);
dnn_size += 8;
}
layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
layer->params = params;
if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
return 0;
}
return dnn_size;
}
static int before_get_buddy(int given, int paddings, LayerPadModeParam mode)
{
if (mode == LPMP_SYMMETRIC) {
return (2 * paddings - 1 - given);
} else if (mode == LPMP_REFLECT) {
return (2 * paddings - given);
} else {
av_assert0(!"should not reach here");
return 0;
}
}
static int after_get_buddy(int given, int border, LayerPadModeParam mode)
{
if (mode == LPMP_SYMMETRIC) {
int offset = given - border;
return (border - 1 - offset);
} else if (mode == LPMP_REFLECT) {
int offset = given - border;
return (border - 2 - offset);
} else {
av_assert0(!"should not reach here");
return 0;
}
}
int ff_dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx)
{
int32_t before_paddings;
int32_t after_paddings;
float* output;
const LayerPadParams *params = parameters;
// suppose format is <N, H, W, C>
int32_t input_operand_index = input_operand_indexes[0];
int number = operands[input_operand_index].dims[0];
int height = operands[input_operand_index].dims[1];
int width = operands[input_operand_index].dims[2];
int channel = operands[input_operand_index].dims[3];
const float *input = operands[input_operand_index].data;
int new_number = number + params->paddings[0][0] + params->paddings[0][1];
int new_height = height + params->paddings[1][0] + params->paddings[1][1];
int new_width = width + params->paddings[2][0] + params->paddings[2][1];
int new_channel = channel + params->paddings[3][0] + params->paddings[3][1];
int c_stride = channel;
int wc_stride = c_stride * width;
int hwc_stride = wc_stride * height;
int new_c_stride = new_channel;
int new_wc_stride = new_c_stride * new_width;
int new_hwc_stride = new_wc_stride * new_height;
DnnOperand *output_operand = &operands[output_operand_index];
output_operand->dims[0] = new_number;
output_operand->dims[1] = new_height;
output_operand->dims[2] = new_width;
output_operand->dims[3] = new_channel;
output_operand->data_type = operands[input_operand_index].data_type;
output_operand->length = ff_calculate_operand_data_length(output_operand);
if (output_operand->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
return AVERROR(EINVAL);
}
output_operand->data = av_realloc(output_operand->data, output_operand->length);
if (!output_operand->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
return AVERROR(ENOMEM);
}
output = output_operand->data;
// copy the original data
for (int n = 0; n < number; n++) {
for (int h = 0; h < height; h++) {
for (int w = 0; w < width; w++) {
const float *src = input + n * hwc_stride + h * wc_stride + w * c_stride;
float *dst = output + (n + params->paddings[0][0]) * new_hwc_stride
+ (h + params->paddings[1][0]) * new_wc_stride
+ (w + params->paddings[2][0]) * new_c_stride
+ params->paddings[3][0];
memcpy(dst, src, channel * sizeof(float));
}
}
}
// handle the first dimension
before_paddings = params->paddings[0][0];
after_paddings = params->paddings[0][1];
for (int n = 0; n < before_paddings; n++) {
float *dst = output + n * new_hwc_stride;
if (params->mode == LPMP_CONSTANT) {
for (int i = 0; i < new_hwc_stride; i++) {
dst[i] = params->constant_values;
}
}
else {
int buddy = before_get_buddy(n, before_paddings, params->mode);
float *src = output + buddy * new_hwc_stride;
memcpy(dst, src, new_hwc_stride * sizeof(float));
}
}
for (int n = 0; n < after_paddings; n++) {
int given = number + before_paddings + n;
float *dst = output + given * new_hwc_stride;
if (params->mode == LPMP_CONSTANT) {
for (int i = 0; i < new_hwc_stride; i++) {
dst[i] = params->constant_values;
}
} else {
int buddy = after_get_buddy(given, number + before_paddings, params->mode);
float *src = output + buddy * new_hwc_stride;
memcpy(dst, src, new_hwc_stride * sizeof(float));
}
}
// handle the second dimension
before_paddings = params->paddings[1][0];
after_paddings = params->paddings[1][1];
for (int n = 0; n < new_number; n++) {
float *start = output + n * new_hwc_stride;
for (int h = 0; h < before_paddings; h++) {
float *dst = start + h * new_wc_stride;
if (params->mode == LPMP_CONSTANT) {
for (int i = 0; i < new_wc_stride; i++) {
dst[i] = params->constant_values;
}
} else {
int buddy = before_get_buddy(h, before_paddings, params->mode);
float *src = start + buddy * new_wc_stride;
memcpy(dst, src, new_wc_stride * sizeof(float));
}
}
for (int h = 0; h < after_paddings; h++) {
int given = height + before_paddings + h;
float *dst = start + given * new_wc_stride;
if (params->mode == LPMP_CONSTANT) {
for (int i = 0; i < new_wc_stride; i++) {
dst[i] = params->constant_values;
}
} else {
int buddy = after_get_buddy(given, height + before_paddings, params->mode);
float *src = start + buddy * new_wc_stride;
memcpy(dst, src, new_wc_stride * sizeof(float));
}
}
}
// handle the third dimension
before_paddings = params->paddings[2][0];
after_paddings = params->paddings[2][1];
for (int n = 0; n < new_number; n++) {
for (int h = 0; h < new_height; h++) {
float *start = output + n * new_hwc_stride + h * new_wc_stride;
for (int w = 0; w < before_paddings; w++) {
float *dst = start + w * new_c_stride;
if (params->mode == LPMP_CONSTANT) {
for (int i = 0; i < new_c_stride; i++) {
dst[i] = params->constant_values;
}
} else {
int buddy = before_get_buddy(w, before_paddings, params->mode);
float *src = start + buddy * new_c_stride;
memcpy(dst, src, new_c_stride * sizeof(float));
}
}
for (int w = 0; w < after_paddings; w++) {
int given = width + before_paddings + w;
float *dst = start + given * new_c_stride;
if (params->mode == LPMP_CONSTANT) {
for (int i = 0; i < new_c_stride; i++) {
dst[i] = params->constant_values;
}
} else {
int buddy = after_get_buddy(given, width + before_paddings, params->mode);
float *src = start + buddy * new_c_stride;
memcpy(dst, src, new_c_stride * sizeof(float));
}
}
}
}
// handle the fourth dimension
before_paddings = params->paddings[3][0];
after_paddings = params->paddings[3][1];
for (int n = 0; n < new_number; n++) {
for (int h = 0; h < new_height; h++) {
for (int w = 0; w < new_width; w++) {
float *start = output + n * new_hwc_stride + h * new_wc_stride + w * new_c_stride;
for (int c = 0; c < before_paddings; c++) {
float *dst = start + c;
if (params->mode == LPMP_CONSTANT) {
*dst = params->constant_values;
} else {
int buddy = before_get_buddy(c, before_paddings, params->mode);
float *src = start + buddy;
*dst = *src;
}
}
for (int c = 0; c < after_paddings; c++) {
int given = channel + before_paddings + c;
float *dst = start + given;
if (params->mode == LPMP_CONSTANT) {
*dst = params->constant_values;
} else {
int buddy = after_get_buddy(given, channel + before_paddings, params->mode);
float *src = start + buddy;
*dst = *src;
}
}
}
}
}
return 0;
}

43
libavfilter/dnn/dnn_backend_native_layer_pad.h
Unescape View File

@ -1,43 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* layer pad (equivalent to tf.pad) for native backend.
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_PAD_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_PAD_H
#include <stdint.h>
#include "dnn_backend_native.h"
typedef enum {LPMP_CONSTANT, LPMP_REFLECT, LPMP_SYMMETRIC} LayerPadModeParam;
typedef struct LayerPadParams{
int32_t paddings[4][2];
LayerPadModeParam mode;
float constant_values;
} LayerPadParams;
int ff_dnn_load_layer_pad(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num);
int ff_dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx);
#endif

42
libavfilter/dnn/dnn_backend_native_layers.c
Unescape View File

@ -1,42 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <string.h>
#include "dnn_backend_native_layers.h"
#include "dnn_backend_native_layer_pad.h"
#include "dnn_backend_native_layer_conv2d.h"
#include "dnn_backend_native_layer_depth2space.h"
#include "dnn_backend_native_layer_maximum.h"
#include "dnn_backend_native_layer_mathbinary.h"
#include "dnn_backend_native_layer_mathunary.h"
#include "dnn_backend_native_layer_avgpool.h"
#include "dnn_backend_native_layer_dense.h"
const LayerFunc ff_layer_funcs[DLT_COUNT] = {
{NULL, NULL},
{ff_dnn_execute_layer_conv2d, ff_dnn_load_layer_conv2d},
{ff_dnn_execute_layer_depth2space, ff_dnn_load_layer_depth2space},
{ff_dnn_execute_layer_pad, ff_dnn_load_layer_pad},
{ff_dnn_execute_layer_maximum, ff_dnn_load_layer_maximum},
{ff_dnn_execute_layer_math_binary, ff_dnn_load_layer_math_binary},
{ff_dnn_execute_layer_math_unary, ff_dnn_load_layer_math_unary},
{ff_dnn_execute_layer_avg_pool, ff_dnn_load_layer_avg_pool},
{ff_dnn_execute_layer_dense, ff_dnn_load_layer_dense},
};

38
libavfilter/dnn/dnn_backend_native_layers.h
Unescape View File

@ -1,38 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYERS_H
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYERS_H
#include <stdint.h>
#include "dnn_backend_native.h"
typedef int (*LAYER_EXEC_FUNC)(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx);
typedef int (*LAYER_LOAD_FUNC)(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num);
typedef struct LayerFunc {
LAYER_EXEC_FUNC pf_exec;
LAYER_LOAD_FUNC pf_load;
}LayerFunc;
extern const LayerFunc ff_layer_funcs[DLT_COUNT];
#endif

368
libavfilter/dnn/dnn_backend_tf.c
Unescape View File

@ -24,17 +24,13 @@
*/
#include "dnn_backend_tf.h"
#include "dnn_backend_native.h"
#include "dnn_backend_native_layer_conv2d.h"
#include "dnn_backend_native_layer_depth2space.h"
#include "libavformat/avio.h"
#include "libavutil/avassert.h"
#include "libavutil/avstring.h"
#include "libavutil/cpu.h"
#include "libavutil/opt.h"
#include "libavcodec/defs.h"
#include "../internal.h"
#include "dnn_backend_native_layer_pad.h"
#include "dnn_backend_native_layer_maximum.h"
#include "dnn_io_proc.h"
#include "dnn_backend_common.h"
#include "safe_queue.h"
@ -481,363 +477,6 @@ static int load_tf_model(TFModel *tf_model, const char *model_filename)
#define NAME_BUFFER_SIZE 256
static int add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,
ConvolutionalParams* params, const int layer)
{
TFContext *ctx = &tf_model->ctx;
TF_Operation *op;
TF_OperationDescription *op_desc;
TF_Output input;
int64_t strides[] = {1, 1, 1, 1};
TF_Tensor *kernel_tensor = NULL, *biases_tensor = NULL;
int64_t dims[4];
int dims_len;
char name_buffer[NAME_BUFFER_SIZE];
int32_t size;
size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;
input.index = 0;
snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
dims[0] = params->output_num;
dims[1] = params->kernel_size;
dims[2] = params->kernel_size;
dims[3] = params->input_num;
dims_len = 4;
kernel_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
memcpy(TF_TensorData(kernel_tensor), params->kernel, size * sizeof(float));
TF_SetAttrTensor(op_desc, "value", kernel_tensor, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
goto err;
}
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);
input.oper = op;
TF_AddInput(op_desc, input);
input.oper = transpose_op;
TF_AddInput(op_desc, input);
TF_SetAttrType(op_desc, "T", TF_FLOAT);
TF_SetAttrType(op_desc, "Tperm", TF_INT32);
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);
input.oper = *cur_op;
TF_AddInput(op_desc, input);
input.oper = op;
TF_AddInput(op_desc, input);
TF_SetAttrType(op_desc, "T", TF_FLOAT);
TF_SetAttrIntList(op_desc, "strides", strides, 4);
TF_SetAttrString(op_desc, "padding", "VALID", 5);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
dims[0] = params->output_num;
dims_len = 1;
biases_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
memcpy(TF_TensorData(biases_tensor), params->biases, params->output_num * sizeof(float));
TF_SetAttrTensor(op_desc, "value", biases_tensor, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
goto err;
}
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);
input.oper = *cur_op;
TF_AddInput(op_desc, input);
input.oper = op;
TF_AddInput(op_desc, input);
TF_SetAttrType(op_desc, "T", TF_FLOAT);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
switch (params->activation){
case RELU:
op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);
break;
case TANH:
op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);
break;
case SIGMOID:
op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
break;
default:
avpriv_report_missing_feature(ctx, "convolutional activation function %d", params->activation);
return AVERROR(ENOSYS);
}
input.oper = *cur_op;
TF_AddInput(op_desc, input);
TF_SetAttrType(op_desc, "T", TF_FLOAT);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
goto err;
}
return 0;
err:
TF_DeleteTensor(kernel_tensor);
TF_DeleteTensor(biases_tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to add conv layer %d\n", layer);
return DNN_GENERIC_ERROR;
}
static int add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
DepthToSpaceParams *params, const int layer)
{
TFContext *ctx = &tf_model->ctx;
TF_OperationDescription *op_desc;
TF_Output input;
char name_buffer[NAME_BUFFER_SIZE];
snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);
input.oper = *cur_op;
input.index = 0;
TF_AddInput(op_desc, input);
TF_SetAttrType(op_desc, "T", TF_FLOAT);
TF_SetAttrInt(op_desc, "block_size", params->block_size);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
av_log(ctx, AV_LOG_ERROR, "Failed to add depth_to_space to layer %d\n", layer);
return DNN_GENERIC_ERROR;
}
return 0;
}
static int add_pad_layer(TFModel *tf_model, TF_Operation **cur_op,
LayerPadParams *params, const int layer)
{
TFContext *ctx = &tf_model->ctx;
TF_Operation *op;
TF_Tensor *tensor;
TF_OperationDescription *op_desc;
TF_Output input;
int32_t *pads;
int64_t pads_shape[] = {4, 2};
char name_buffer[NAME_BUFFER_SIZE];
snprintf(name_buffer, NAME_BUFFER_SIZE, "pad%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
TF_SetAttrType(op_desc, "dtype", TF_INT32);
tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));
pads = (int32_t *)TF_TensorData(tensor);
pads[0] = params->paddings[0][0];
pads[1] = params->paddings[0][1];
pads[2] = params->paddings[1][0];
pads[3] = params->paddings[1][1];
pads[4] = params->paddings[2][0];
pads[5] = params->paddings[2][1];
pads[6] = params->paddings[3][0];
pads[7] = params->paddings[3][1];
TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to set value for pad of layer %d\n", layer);
return DNN_GENERIC_ERROR;
}
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to add pad to layer %d\n", layer);
return DNN_GENERIC_ERROR;
}
op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");
input.oper = *cur_op;
input.index = 0;
TF_AddInput(op_desc, input);
input.oper = op;
TF_AddInput(op_desc, input);
TF_SetAttrType(op_desc, "T", TF_FLOAT);
TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);
TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to add mirror_pad to layer %d\n", layer);
return DNN_GENERIC_ERROR;
}
return 0;
}
static int add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op,
DnnLayerMaximumParams *params, const int layer)
{
TFContext *ctx = &tf_model->ctx;
TF_Operation *op;
TF_Tensor *tensor;
TF_OperationDescription *op_desc;
TF_Output input;
float *y;
char name_buffer[NAME_BUFFER_SIZE];
snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum/y%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
tensor = TF_AllocateTensor(TF_FLOAT, NULL, 0, TF_DataTypeSize(TF_FLOAT));
y = (float *)TF_TensorData(tensor);
*y = params->val.y;
TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to set value for maximum/y of layer %d", layer);
return DNN_GENERIC_ERROR;
}
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to add maximum/y to layer %d\n", layer);
return DNN_GENERIC_ERROR;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "Maximum", name_buffer);
input.oper = *cur_op;
input.index = 0;
TF_AddInput(op_desc, input);
input.oper = op;
TF_AddInput(op_desc, input);
TF_SetAttrType(op_desc, "T", TF_FLOAT);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to add maximum to layer %d\n", layer);
return DNN_GENERIC_ERROR;
}
return 0;
}
static int load_native_model(TFModel *tf_model, const char *model_filename)
{
TFContext *ctx = &tf_model->ctx;
int32_t layer;
TF_OperationDescription *op_desc;
TF_Operation *op;
TF_Operation *transpose_op;
TF_Tensor *tensor = NULL;
TF_Output input;
int32_t *transpose_perm;
int64_t transpose_perm_shape[] = {4};
int64_t input_shape[] = {1, -1, -1, -1};
int layer_add_res;
DNNModel *model = NULL;
NativeModel *native_model;
model = ff_dnn_load_model_native(model_filename, DFT_PROCESS_FRAME, NULL, NULL);
if (!model){
av_log(ctx, AV_LOG_ERROR, "Failed to load native model\n");
return AVERROR(EINVAL);
}
native_model = model->model;
tf_model->graph = TF_NewGraph();
tf_model->status = TF_NewStatus();
#define CLEANUP_ON_ERROR(tf_model) \
{ \
TF_DeleteTensor(tensor); \
TF_DeleteGraph(tf_model->graph); \
TF_DeleteStatus(tf_model->status); \
av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \
return DNN_GENERIC_ERROR; \
}
op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");
TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
TF_SetAttrShape(op_desc, "shape", input_shape, 4);
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
CLEANUP_ON_ERROR(tf_model);
}
op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");
TF_SetAttrType(op_desc, "dtype", TF_INT32);
tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));
transpose_perm = (int32_t *)TF_TensorData(tensor);
transpose_perm[0] = 1;
transpose_perm[1] = 2;
transpose_perm[2] = 3;
transpose_perm[3] = 0;
TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
CLEANUP_ON_ERROR(tf_model);
}
transpose_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
CLEANUP_ON_ERROR(tf_model);
}
for (layer = 0; layer < native_model->layers_num; ++layer){
switch (native_model->layers[layer].type){
case DLT_INPUT:
layer_add_res = 0;
break;
case DLT_CONV2D:
layer_add_res = add_conv_layer(tf_model, transpose_op, &op,
(ConvolutionalParams *)native_model->layers[layer].params, layer);
break;
case DLT_DEPTH_TO_SPACE:
layer_add_res = add_depth_to_space_layer(tf_model, &op,
(DepthToSpaceParams *)native_model->layers[layer].params, layer);
break;
case DLT_MIRROR_PAD:
layer_add_res = add_pad_layer(tf_model, &op,
(LayerPadParams *)native_model->layers[layer].params, layer);
break;
case DLT_MAXIMUM:
layer_add_res = add_maximum_layer(tf_model, &op,
(DnnLayerMaximumParams *)native_model->layers[layer].params, layer);
break;
default:
CLEANUP_ON_ERROR(tf_model);
}
if (layer_add_res != 0){
CLEANUP_ON_ERROR(tf_model);
}
}
op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");
input.oper = op;
input.index = 0;
TF_AddInput(op_desc, input);
TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
CLEANUP_ON_ERROR(tf_model);
}
ff_dnn_free_model_native(&model);
return 0;
}
DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
{
DNNModel *model = NULL;
@ -867,9 +506,8 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_
}
if (load_tf_model(tf_model, model_filename) != 0){
if (load_native_model(tf_model, model_filename) != 0){
goto err;
}
av_log(ctx, AV_LOG_ERROR, "Failed to load TensorFlow model: \"%s\"\n", model_filename);
goto err;
}
if (ctx->options.nireq <= 0) {

2
libavfilter/dnn_interface.h
Unescape View File

@ -32,7 +32,7 @@
#define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!')
typedef enum {DNN_NATIVE, DNN_TF, DNN_OV} DNNBackendType;
typedef enum {DNN_TF = 1, DNN_OV} DNNBackendType;
typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;

197
libavfilter/tests/dnn-layer-avgpool.c
Unescape View File

@ -1,197 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdio.h>
#include "libavfilter/dnn/dnn_backend_native_layer_avgpool.h"
#define EPSON 0.00001
static int test_with_same(void)
{
// the input data and expected data are generated with below python code.
/*
import tensorflow as tf
import numpy as np
x = tf.placeholder(tf.float32, shape=[1, None, None, 3])
y = tf.layers.average_pooling2d(x, pool_size=[2,2], strides=[1,1], padding='VALID')
data = np.random.rand(1, 5, 6, 3);
sess=tf.Session()
sess.run(tf.global_variables_initializer())
output = sess.run(y, feed_dict={x: data})
print("input:")
print(data.shape)
print(list(data.flatten()))
print("output:")
print(output.shape)
print(list(output.flatten()))
*/
AvgPoolParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*5*6*3] = {
0.7461309859908424, 0.7567538372797069, 0.07662743569678687, 0.8882112610336333, 0.9720443314026668, 0.3337200343220823, 0.4421032129780248,
0.14940809044964876, 0.6773177061961277, 0.9778844630669781, 0.6522650522626998, 0.0317651530878591, 0.31259897552911364, 0.6235936821891896,
0.40016094349542775, 0.4599222930032276, 0.7893807222960093, 0.8475986363538283, 0.5058802717647394, 0.7827005363222633, 0.3032188123727916,
0.8983728631302361, 0.20622408444965523, 0.22966072303869878, 0.09535751273161308, 0.8760709100995375, 0.9982324154558745, 0.7904595468621013,
0.13883671508879347, 0.9332751439533138, 0.0010861680752152214, 0.3607210449251048, 0.6600652759586171, 0.7629572058138805, 0.29441975810476106,
0.2683471432889405, 0.22574580829831536, 0.8893251976212904, 0.3907737043801005, 0.6421829842863968, 0.6670373870457297, 0.9383850793160277,
0.4120458907436003, 0.3589847212711481, 0.48047736550128983, 0.6428192648418949, 0.0313661686292348, 0.429357100401472, 0.5123413386514056,
0.8492446404097114, 0.9045286128486804, 0.8123708563814285, 0.3943245008451698, 0.9576713003177785, 0.5985610965938726, 0.9350833279543561,
0.8010079897491659, 0.45882114217642866, 0.35275037908941487, 0.4555844661432271, 0.12352455940255314, 0.37801756635035544, 0.2824056214573083,
0.6229462823245029, 0.7235305681391472, 0.5408259266122064, 0.12142224381781208, 0.34431198802873686, 0.7112823816321276, 0.6307144385115417,
0.8136734589018082, 0.842095618140585, 0.8602767724004784, 0.6649236853766185, 0.5184782829419623, 0.9119607270982825, 0.3084111974561645,
0.39460705638161364, 0.17710447526170836, 0.1715485945814199, 0.17277563576521882, 0.40188232428735704, 0.22847985411491878, 0.4135361701550696,
0.24621846601980057, 0.6576588108454774, 0.6063336087333997, 0.6452342242996931, 0.7071689702737508, 0.1973416063225648
};
float expected_output[] = {
0.75964886, 0.6794307, 0.23580676, 0.5810112, 0.5509369, 0.55973274, 0.5764512, 0.45414522, 0.6601476, 0.52050734, 0.44385415,
0.50631666, 0.38414115, 0.5170288, 0.544043, 0.61143976, 0.5419003, 0.5579729, 0.5680455, 0.6363218, 0.4655096, 0.51198983,
0.5270792, 0.66168886, 0.48517057, 0.3513146, 0.7103355, 0.48667657, 0.34504217, 0.7318065, 0.5221889, 0.4746775, 0.69765306,
0.78766406, 0.34437215, 0.6130092, 0.48132777, 0.7110491, 0.6464378, 0.40914366, 0.4391975, 0.5392131, 0.45033398, 0.37297475,
0.43326652, 0.4748823, 0.48711336, 0.64649844, 0.51921225, 0.60038865, 0.8538945, 0.7215426, 0.60399896, 0.89988345, 0.707405,
0.5652921, 0.54241943, 0.41785273, 0.30268195, 0.3263432, 0.3313644, 0.37539417, 0.35238582, 0.34811732, 0.48849532, 0.56799453,
0.41089734, 0.63070333, 0.5892633, 0.6379743, 0.7604212, 0.5197186, 0.88611877, 0.48666745, 0.45654267, 0.5445326, 0.2399799,
0.28369135, 0.28949338, 0.20001422, 0.2931559, 0.3240504, 0.44306934, 0.5099349, 0.44572634, 0.68241394, 0.40183762, 0.6452342,
0.707169, 0.1973416
};
float *output;
params.strides = 1;
params.kernel_size = 2;
params.padding_method = SAME;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 5;
operands[0].dims[2] = 6;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_avg_pool(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(expected_output) / sizeof(float); ++i) {
if (fabs(output[i] - expected_output[i]) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
static int test_with_valid(void)
{
// the input data and expected data are generated with below python code.
/*
import tensorflow as tf
import numpy as np
x = tf.placeholder(tf.float32, shape=[1, None, None, 3])
y = tf.layers.average_pooling2d(x, pool_size=[2,2], strides=[1,1], padding='VALID')
data = np.random.rand(1, 5, 6, 3);
sess=tf.Session()
sess.run(tf.global_variables_initializer())
output = sess.run(y, feed_dict={x: data})
print("input:")
print(data.shape)
print(list(data.flatten()))
print("output:")
print(output.shape)
print(list(output.flatten()))
*/
AvgPoolParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*5*6*3] = {
0.5046741692941682, 0.9273653202485155, 0.8193878359859937, 0.1904059431360905, 0.8664919633253656, 0.7484625128286059, 0.984534184632278,
0.31900804890072254, 0.3259426099940872, 0.05388974903570376, 0.7356610151331133, 0.46710858713311965, 0.718553768817036, 0.062478421853278676,
0.7813224786584609, 0.4826837517658389, 0.9748095400220147, 0.8078547703898341, 0.11976750668368585, 0.8713586777195065, 0.41447321551284355,
0.9818788239089807, 0.4335715767584073, 0.4059793452147419, 0.3677205907204525, 0.47919995923571, 0.8341395256258882, 0.7059726374074609,
0.5478504551919791, 0.8622900484790175, 0.8343709722511167, 0.05089827275068537, 0.6465283980840416, 0.544539116066677, 0.39812057257884337,
0.9578115576866337, 0.25012888117580145, 0.579333516024662, 0.5556732133051457, 0.6119862111181243, 0.0018736758772316398, 0.9795490254040474,
0.4488085008883018, 0.28947489777011737, 0.4834108668633247, 0.9280490084385024, 0.9895821458049648, 0.31777618554697606, 0.42679693258977847,
0.74447844466923, 0.9752225305081498, 0.17564130841849335, 0.22382692067314292, 0.009602884447469373, 0.5144884415025782, 0.031622570708844555,
0.8277532752502512, 0.4111593210409763, 0.5272084646575664, 0.28856508082905297, 0.11317726946036655, 0.7203328275540273, 0.8310055019972384,
0.8535951508685228, 0.40230347305233227, 0.2819703265132867, 0.6243143957791139, 0.7512463693822311, 0.7523056340495644, 0.8838077258040928,
0.5472240664033092, 0.2550538284454935, 0.5560317774456567, 0.8966847087518931, 0.6728358284165321, 0.30361297147530875, 0.464343925441822,
0.34507695659461224, 0.6333175615390685, 0.26661369038523497, 0.9926748632253231, 0.9994267301382666, 0.8684917986974414, 0.3598754806113009,
0.49550268625464666, 0.03652458679973214, 0.13469081713137177, 0.4579424049273835, 0.48641107969110353, 0.9670250266945365
};
float expected_output[1*4*5*3] = {
0.44918162, 0.7746969, 0.5970757, 0.63113487, 0.5245679, 0.578631, 0.52802926, 0.52042985, 0.6223702, 0.57819676, 0.34922206,
0.6893124, 0.64503694, 0.37157673, 0.7983793, 0.49094033, 0.47153437, 0.5889187, 0.6025985, 0.30103004, 0.6757697, 0.6126377,
0.5765268, 0.62440413, 0.7237974, 0.5832023, 0.7004543, 0.49533707, 0.35433105, 0.6472913, 0.44694072, 0.28500956, 0.6628852,
0.39628282, 0.38472247, 0.6456326, 0.58590746, 0.60042334, 0.47854072, 0.7081889, 0.7219026, 0.5818187, 0.5276401, 0.56669396,
0.49804622, 0.4463231, 0.4799649, 0.5335578, 0.36531678, 0.4946247, 0.6143306, 0.6498792, 0.5644355, 0.6163815, 0.7432098,
0.5146416, 0.38221055, 0.6153918, 0.45535153, 0.5272688
};
float *output;
params.strides = 1;
params.kernel_size = 2;
params.padding_method = VALID;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 5;
operands[0].dims[2] = 6;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_avg_pool(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(expected_output) / sizeof(float); ++i) {
if (fabs(output[i] - expected_output[i]) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
int main(int argc, char **argv)
{
if (test_with_same())
return 1;
if (test_with_valid())
return 1;
return 0;
}

248
libavfilter/tests/dnn-layer-conv2d.c
Unescape View File

@ -1,248 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "libavfilter/dnn/dnn_backend_native_layer_conv2d.h"
#define EPSON 0.00001
static int test_with_same_dilate(void)
{
// the input data and expected data are generated with below python code.
/*
x = tf.placeholder(tf.float32, shape=[1, None, None, 3])
y = tf.layers.conv2d(x, 2, 3, activation=tf.nn.tanh, padding='same', dilation_rate=(2, 2), bias_initializer=tf.keras.initializers.he_normal())
data = np.random.rand(1, 5, 6, 3);
sess=tf.Session()
sess.run(tf.global_variables_initializer())
weights = dict([(var.name, sess.run(var)) for var in tf.trainable_variables()])
kernel = weights['conv2d/kernel:0']
kernel = np.transpose(kernel, [3, 0, 1, 2])
print("kernel:")
print(kernel.shape)
print(list(kernel.flatten()))
bias = weights['conv2d/bias:0']
print("bias:")
print(bias.shape)
print(list(bias.flatten()))
output = sess.run(y, feed_dict={x: data})
print("input:")
print(data.shape)
print(list(data.flatten()))
print("output:")
print(output.shape)
print(list(output.flatten()))
*/
ConvolutionalParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*5*6*3] = {
0.7012556460308194, 0.4233847954643357, 0.19515900664313612, 0.16343083004926495, 0.5758261611052848, 0.9510767434014871, 0.11014085055947687,
0.906327053637727, 0.8136794715542507, 0.45371764543639526, 0.5768443343523952, 0.19543668786046986, 0.15648326047898609, 0.2099500241141279,
0.17658777090552413, 0.059335724777169196, 0.1729991838469117, 0.8150514704819208, 0.4435535466703049, 0.3752188477566878, 0.749936650421431,
0.6823494635284907, 0.10776389679424747, 0.34247481674596836, 0.5147867256244629, 0.9063709728129032, 0.12423605800856818, 0.6064872945412728,
0.5891681538551459, 0.9865836236466314, 0.9002163879294677, 0.003968273184274618, 0.8628374809643967, 0.1327176268279583, 0.8449799925703798,
0.1937671869354366, 0.41524410152707425, 0.02038786604756837, 0.49792466069597496, 0.8881874553848784, 0.9683921035597336, 0.4122972568010813,
0.843553550993252, 0.9588482762501964, 0.5190350762645546, 0.4283584264145317, 0.09781496073714646, 0.9501058833776156, 0.8665541760152776,
0.31669272550095806, 0.07133074675453632, 0.606438007334886, 0.7007157020538224, 0.4827996264130444, 0.5167615606392761, 0.6385043039312651,
0.23069664707810555, 0.058233497329354456, 0.06323892961591071, 0.24816458893245974, 0.8646369065257812, 0.24742185893094837, 0.09991225948167437,
0.625700606979606, 0.7678541502111257, 0.6215834594679912, 0.5623003956582483, 0.07389123942681242, 0.7659100715711249, 0.486061471642225,
0.9947455699829012, 0.9094911797643259, 0.7644355876253265, 0.05384315321492239, 0.13565394382783613, 0.9810628204953316, 0.007386389078887889,
0.226182754156241, 0.2609021390764772, 0.24182802076928933, 0.13264782451941648, 0.2035816485767682, 0.005504188177612557, 0.7014619934040155,
0.956215988391991, 0.5670398541013633, 0.9809764721750784, 0.6886338100487461, 0.5758152317218274, 0.7137823176776179
};
float expected_output[1*5*6*2] = {
-0.9480655, -0.7169147, -0.9404794, -0.5567385, -0.8991124, -0.8306558, -0.94487447, -0.8932543, -0.88238764, -0.7301602,
-0.8974813, -0.7026703, -0.8858988, -0.53203243, -0.92881465, -0.5648504, -0.8871471, -0.7000097, -0.91754407, -0.79684794,
-0.760465, -0.117928326, -0.88302773, -0.8975289, -0.70615053, 0.19231977, -0.8318776, -0.386184, -0.80698484, -0.8556624,
-0.7336671, -0.6168619, -0.7658234, -0.63449603, -0.73314047, -0.87502456, -0.58158904, -0.4184259, -0.52618927, -0.13613208,
-0.5093187, -0.21027721, -0.39455596, -0.44507834, -0.22269244, -0.73400885, -0.77655095, -0.74408925, -0.57313335, -0.15333457,
-0.74620694, -0.34858236, -0.42586932, -0.5240488, 0.1634339, -0.2447881, -0.57927346, -0.62732303, -0.82287043, -0.8474058
};
float *output;
float kernel[2*3*3*3] = {
0.26025516, 0.16536498, -0.24351254, 0.33892477, -0.34005195, 0.35202783, 0.34056443, 0.01422739, 0.13799345, 0.29489166,
0.2781723, 0.178585, 0.22122234, 0.044115514, 0.13134438, 0.31705368, 0.22527462, -0.021323413, 0.115134746, -0.18216397,
-0.21197563, -0.027848959, -0.01704529, -0.12401503, -0.23415318, -0.12661739, -0.35338148, 0.20049328, -0.076153606,
-0.23642601, -0.3125769, -0.025851756, -0.30006272, 0.050762743, 0.32003498, 0.3052225, -0.0017385483, 0.25337684, -0.25664508,
0.27846587, -0.3112659, 0.2066065, 0.31499845, 0.113178134, 0.09449363, -0.11828774, -0.12671001, -0.36259216, 0.2710235,
-0.19676702, 0.023612618, -0.2596915, -0.34949252, -0.108270735
};
float bias[2] = { -1.6574852, -0.72915393 };
NativeContext ctx;
ctx.class = NULL;
ctx.options.conv2d_threads = 1;
params.activation = TANH;
params.has_bias = 1;
params.biases = bias;
params.dilation = 2;
params.input_num = 3;
params.kernel = kernel;
params.kernel_size = 3;
params.output_num = 2;
params.padding_method = SAME;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 5;
operands[0].dims[2] = 6;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_conv2d(operands, input_indexes, 1, &params, &ctx);
output = operands[1].data;
for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
if (fabs(output[i] - expected_output[i]) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
static int test_with_valid(void)
{
// the input data and expected data are generated with below python code.
/*
x = tf.placeholder(tf.float32, shape=[1, None, None, 3])
y = tf.layers.conv2d(x, 2, 3, activation=tf.nn.tanh, padding='valid', bias_initializer=tf.keras.initializers.he_normal())
data = np.random.rand(1, 5, 6, 3);
sess=tf.Session()
sess.run(tf.global_variables_initializer())
weights = dict([(var.name, sess.run(var)) for var in tf.trainable_variables()])
kernel = weights['conv2d/kernel:0']
kernel = np.transpose(kernel, [3, 0, 1, 2])
print("kernel:")
print(kernel.shape)
print(list(kernel.flatten()))
bias = weights['conv2d/bias:0']
print("bias:")
print(bias.shape)
print(list(bias.flatten()))
output = sess.run(y, feed_dict={x: data})
print("input:")
print(data.shape)
print(list(data.flatten()))
print("output:")
print(output.shape)
print(list(output.flatten()))
*/
ConvolutionalParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*5*6*3] = {
0.26126657468269665, 0.42762216215337556, 0.7466274030131497, 0.802550266787863, 0.3709323443076644, 0.5919817068197668, 0.49274512279324967,
0.7170132295090351, 0.0911793215410649, 0.5134213878288361, 0.670132600785118, 0.49417034512633484, 0.03887389460089885, 0.436785102836845,
0.1490231658611978, 0.6413606121498127, 0.8595987991375995, 0.9132593077586231, 0.7075959004873255, 0.17754995944845464, 0.5212507214937141,
0.35379732738215475, 0.25205107358505296, 0.3928792840544273, 0.09485294189485782, 0.8685115437448666, 0.6489046799288605, 0.509253797582924,
0.8993255536791972, 0.18740056466602373, 0.34237617336313986, 0.3871438962989183, 0.1488532571774911, 0.5187002331293636, 0.8137098818752955,
0.521761863717401, 0.4622312310118274, 0.29038411334638825, 0.16194915718170566, 0.5175999923925211, 0.8852230040101133, 0.0218263385047206,
0.08482355352852367, 0.3463638568376264, 0.28627127120619733, 0.9553293378948409, 0.4803391055970835, 0.841635695030805, 0.3556828280031952,
0.06778527221541808, 0.28193560357091596, 0.8399957619031576, 0.03305536359456385, 0.6625039162109645, 0.9300552020023897, 0.8551529138204146,
0.6133216915522418, 0.222427800857393, 0.1315422686800336, 0.6189144989185527, 0.5346184916866876, 0.8348888624532548, 0.6544834567840291,
0.2844062293389934, 0.28780026600883324, 0.5372272015684924, 0.6250226011503823, 0.28119106062279453, 0.49655812908420094, 0.6451488959145951,
0.7362580606834843, 0.44815578616664087, 0.6454760235835586, 0.6794062414265861, 0.045378883014935756, 0.9008388543865096, 0.7949752851269782,
0.4179928876222264, 0.28733419007048644, 0.996902319501908, 0.5690851338677467, 0.9511814013279738, 0.025323788678181636, 0.5594359732604794,
0.1213732595086251, 0.7172624313368294, 0.6759328959074691, 0.07252138454885071, 0.17557735158403442, 0.5988895455048769
};
float expected_output[1*3*4*2] = {
-0.556947, -0.42143887, -0.092070885, 0.27404794, -0.41886684, 0.0862887, -0.25001016, -0.342721, 0.020730592, 0.04016919, -0.69839877,
-0.06136704, 0.14186388, -0.11655602, -0.23489095, -0.3845829, -0.19017771, 0.1595885, -0.18308741, -0.3071209, -0.5848686, -0.22509028,
-0.6023201, -0.14448485
};
float *output;
float kernel[2*3*3*3] = {
-0.25291282, 0.22402048, 0.028642118, -0.14615723, -0.27362752, -0.34801802, -0.2759148, 0.19594926, -0.25029412, 0.34606284, 0.10376671,
-0.1015394, 0.23616093, 0.2134214, 0.35285157, 0.05893758, 0.0024731457, -0.17143056, 0.35758412, 0.2186206, -0.28384736, -0.21206513,
-0.20871592, 0.27070445, 0.25878823, 0.11136332, -0.33737376, 0.08353335, -0.34290665, 0.041805506, -0.09738535, 0.3284936, -0.16838405,
-0.032494456, -0.29193437, 0.033259362, -0.09272635, -0.2802651, -0.28648436, 0.3542878, 0.2432127, -0.24551713, 0.27813476, 0.21024024,
-0.013690501, -0.1350077, -0.07826337, -0.34563828, 0.3220685, -0.07571727, 0.19420576, 0.20783454, 0.18738335, 0.16672492
};
float bias[2] = { -0.4773722, -0.19620377 };
NativeContext ctx;
ctx.class = NULL;
ctx.options.conv2d_threads = 1;
params.activation = TANH;
params.has_bias = 1;
params.biases = bias;
params.dilation = 1;
params.input_num = 3;
params.kernel = kernel;
params.kernel_size = 3;
params.output_num = 2;
params.padding_method = VALID;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 5;
operands[0].dims[2] = 6;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_conv2d(operands, input_indexes, 1, &params, &ctx);
output = operands[1].data;
for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
if (fabs(output[i] - expected_output[i]) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
int main(int argc, char **argv)
{
if (test_with_valid())
return 1;
if (test_with_same_dilate())
return 1;
return 0;
}

131
libavfilter/tests/dnn-layer-dense.c
Unescape View File

@ -1,131 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "libavfilter/dnn/dnn_backend_native_layer_dense.h"
#define EPSON 0.00001
static int test(void)
{
// the input data and expected data are generated with below python code.
/*
x = tf.placeholder(tf.float32, shape=[1, None, None, 3])
y = tf.layers.dense(input_x, 3, activation=tf.nn.sigmoid, bias_initializer=tf.keras.initializers.he_normal())
data = np.random.rand(1, 5, 6, 3);
sess=tf.Session()
sess.run(tf.global_variables_initializer())
weights = dict([(var.name, sess.run(var)) for var in tf.trainable_variables()])
kernel = weights['dense/kernel:0']
kernel = np.transpose(kernel, [1, 0])
print("kernel:")
print(kernel.shape)
print(list(kernel.flatten()))
bias = weights['dense/bias:0']
print("bias:")
print(bias.shape)
print(list(bias.flatten()))
output = sess.run(y, feed_dict={x: data})
print("input:")
print(data.shape)
print(list(data.flatten()))
print("output:")
print(output.shape)
print(list(output.flatten()))
*/
DenseParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*5*6*3] = {
0.5552418686576308, 0.20653189262022464, 0.31115120939398877, 0.5897014433221428, 0.37340078861060655, 0.6470921693941893, 0.8039950367872679, 0.8762700891949274,
0.6556655583829558, 0.5911096107039339, 0.18640250865290997, 0.2803248779238966, 0.31586613136402053, 0.9447300740056483, 0.9443980824873418, 0.8158851991115941,
0.5631010340387631, 0.9407402251929046, 0.6485434876551682, 0.5631376966470001, 0.17581924875609634, 0.7033802439103178, 0.04802402495561675, 0.9183681450194972,
0.46059317944364, 0.07964160481596883, 0.871787076270302, 0.973743142324361, 0.15923146943258415, 0.8212946080584571, 0.5415954459227064, 0.9552813822803975,
0.4908552668172057, 0.33723691635292274, 0.46588057864910026, 0.8994239961321776, 0.09845220457674186, 0.1713400292123486, 0.39570294912818826, 0.08018956486392803,
0.5290478278169032, 0.7141906125920976, 0.0320878067840098, 0.6412406575332606, 0.0075712007102423096, 0.7150828462386156, 0.1311989216968138, 0.4706847944253756,
0.5447610794883336, 0.3430923933318001, 0.536082357943209, 0.4371629342483694, 0.40227962985019927, 0.3553806249465469, 0.031806622424259245, 0.7053916426174,
0.3261570237309813, 0.419500213292063, 0.3155691223480851, 0.05664028113178088, 0.3636491555914486, 0.8502419746667123, 0.9836596530684955, 0.1628681802975801,
0.09410832912479894, 0.28407218939480294, 0.7983417928813697, 0.24132158596506748, 0.8154729498062224, 0.29173768373895637, 0.13407102008052096, 0.18705786678800385,
0.7167943621295573, 0.09222004247174376, 0.2319220738766018, 0.17708964382285064, 0.1391440370249517, 0.3254088083499256, 0.4013916894718289, 0.4819742663322323,
0.15080103744648077, 0.9302407847555013, 0.9397597961319524, 0.5719200825550793, 0.9538938024682824, 0.9583882089203861, 0.5168861091262276, 0.1926396841842669,
0.6781176744337578, 0.719366447288566
};
float expected_output[1*5*6*3] = {
-0.3921688, -0.9243112, -0.29659146, -0.64000785, -0.9466343, -0.62125254, -0.71759033, -0.9171336, -0.735589, -0.34365994,
-0.92100817, -0.23903961, -0.8962277, -0.9521279, -0.90962386, -0.7488303, -0.9563761, -0.7701762, -0.40800542, -0.87684774,
-0.3339763, -0.6354543, -0.97068924, -0.6246325, -0.6992075, -0.9706726, -0.6818918, -0.51864433, -0.9592881, -0.51187396,
-0.7423632, -0.89911884, -0.7457824, -0.82009757, -0.96402895, -0.8235518, -0.61980766, -0.94494647, -0.5410502, -0.8281218,
-0.95508635, -0.8201453, -0.5937325, -0.8679507, -0.500767, -0.39430764, -0.93967676, -0.32183182, -0.58913624, -0.939717,
-0.55179894, -0.55004454, -0.9214453, -0.4889004, -0.75294703, -0.9118363, -0.7200309, -0.3248641, -0.8878874, -0.18977344,
-0.8873837, -0.9571257, -0.90145934, -0.50521654, -0.93739635, -0.39051685, -0.61143184, -0.9591179, -0.605999, -0.40008977,
-0.92219675, -0.26732883, -0.19607787, -0.9172511, -0.07068595, -0.5409857, -0.9387041, -0.44181606, -0.4705004, -0.8899935,
-0.37997037, -0.66105115, -0.89754754, -0.68141997, -0.6324047, -0.886776, -0.65066385, -0.8334821, -0.94801456, -0.83297
};
float *output;
float kernel[3*3] = {
0.56611896, -0.5144603, -0.82600045, 0.19219112, 0.3835776, -0.7475352, 0.5209291, -0.6301091, -0.99442935};
float bias[3] = {-0.3654299, -1.5711838, -0.15546428};
params.activation = TANH;
params.has_bias = 1;
params.biases = bias;
params.input_num = 3;
params.kernel = kernel;
params.output_num = 3;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 5;
operands[0].dims[2] = 6;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_dense(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
if (fabs(output[i] - expected_output[i]) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
int main(int argc, char **argv)
{
if (test())
return 1;
return 0;
}

102
libavfilter/tests/dnn-layer-depth2space.c
Unescape View File

@ -1,102 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "libavfilter/dnn/dnn_backend_native.h"
#include "libavfilter/dnn/dnn_backend_native_layer_depth2space.h"
#define EPSON 0.00001
static int test(void)
{
// the input data and expected data are generated with below python code.
/*
x = tf.placeholder(tf.float32, shape=[1, None, None, 4])
y = tf.depth_to_space(x, 2)
data = np.random.rand(1, 5, 3, 4);
sess=tf.Session()
sess.run(tf.global_variables_initializer())
output = sess.run(y, feed_dict={x: data})
print("input:")
print(data.shape)
print(list(data.flatten()))
print("output:")
print(output.shape)
print(list(output.flatten()))
*/
DepthToSpaceParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*5*3*4] = {
0.09771065121566602, 0.6336807372403175, 0.5142416549709786, 0.8027206567330333, 0.2154276025069397, 0.12112878462616772, 0.913936596765778,
0.38881443647542646, 0.5850447615898835, 0.9311499327398275, 0.3613660929428246, 0.5420722002125493, 0.6002131190230359, 0.44800665702299525,
0.7271322557896777, 0.3869293511885826, 0.5144404769364138, 0.6910844856987723, 0.6142102742269762, 0.6249991371621018, 0.45663376215836626,
0.19523477129943423, 0.2483895888532045, 0.64326768256278, 0.5485877602998981, 0.45442067849873546, 0.529374943304256, 0.30439850391811885,
0.11961343361340993, 0.2909643484561082, 0.9810970344127848, 0.8886928489786549, 0.6112237084436409, 0.8852482695156674, 0.9110868043114374,
0.21242780027585217, 0.7101536973207572, 0.9709717457443375, 0.2702666770969332, 0.7718295953780221, 0.3957005164588574, 0.24383544252475453,
0.040143453532367035, 0.26358051835323115, 0.013130251443791319, 0.3016550481482074, 0.03582340459943956, 0.718025513612361, 0.09844204177633753,
0.04433767496953056, 0.6221895044119757, 0.6190414032940228, 0.8963550834625371, 0.5642449700064629, 0.2482982014723497, 0.17824909294583013,
0.024401882408643272, 0.21742800875253465, 0.6794724473181843, 0.4814830479242237
};
float expected_output[1*10*6*1] = {
0.097710654, 0.63368076, 0.2154276, 0.12112878, 0.58504474, 0.93114996, 0.51424164, 0.80272067, 0.9139366, 0.38881445,
0.3613661, 0.5420722, 0.6002131, 0.44800666, 0.5144405, 0.6910845, 0.45663378, 0.19523478, 0.72713226, 0.38692936,
0.61421025, 0.62499917, 0.24838959, 0.6432677, 0.54858774, 0.4544207, 0.11961343, 0.29096434, 0.6112237, 0.88524824,
0.52937496, 0.3043985, 0.98109704, 0.88869286, 0.9110868, 0.2124278, 0.7101537, 0.97097176, 0.3957005, 0.24383545,
0.013130251, 0.30165505, 0.27026668, 0.7718296, 0.040143453, 0.26358053, 0.035823405, 0.7180255, 0.09844204,
0.044337675, 0.8963551, 0.564245, 0.024401883, 0.21742801, 0.6221895, 0.6190414, 0.2482982, 0.17824909, 0.67947245, 0.48148304
};
float *output;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 5;
operands[0].dims[2] = 3;
operands[0].dims[3] = 4;
operands[1].data = NULL;
input_indexes[0] = 0;
params.block_size = 2;
ff_dnn_execute_layer_depth2space(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
if (fabs(output[i] - expected_output[i]) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
int main(int argc, char **argv)
{
return test();
}

214
libavfilter/tests/dnn-layer-mathbinary.c
Unescape View File

@ -1,214 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "libavfilter/dnn/dnn_backend_native_layer_mathbinary.h"
#include "libavutil/avassert.h"
#define EPSON 0.00005
static float get_expected(float f1, float f2, DNNMathBinaryOperation op)
{
switch (op)
{
case DMBO_SUB:
return f1 - f2;
case DMBO_ADD:
return f1 + f2;
case DMBO_MUL:
return f1 * f2;
case DMBO_REALDIV:
return f1 / f2;
case DMBO_MINIMUM:
return (f1 < f2) ? f1 : f2;
case DMBO_FLOORMOD:
return (float)((int)(f1) % (int)(f2));
default:
av_assert0(!"not supported yet");
return 0.f;
}
}
static int test_broadcast_input0(DNNMathBinaryOperation op)
{
DnnLayerMathBinaryParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*1*2*3] = {
-3, 2.5, 2, -2.1, 7.8, 100
};
float *output;
params.bin_op = op;
params.input0_broadcast = 1;
params.input1_broadcast = 0;
params.v = 7.28;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 1;
operands[0].dims[2] = 2;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_math_binary(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(input) / sizeof(float); i++) {
float expected_output = get_expected(params.v, input[i], op);
if (fabs(output[i] - expected_output) > EPSON) {
printf("op %d, at index %d, output: %f, expected_output: %f (%s:%d)\n",
op, i, output[i], expected_output, __FILE__, __LINE__);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
static int test_broadcast_input1(DNNMathBinaryOperation op)
{
DnnLayerMathBinaryParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*1*2*3] = {
-3, 2.5, 2, -2.1, 7.8, 100
};
float *output;
params.bin_op = op;
params.input0_broadcast = 0;
params.input1_broadcast = 1;
params.v = 7.28;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 1;
operands[0].dims[2] = 2;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_math_binary(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(input) / sizeof(float); i++) {
float expected_output = get_expected(input[i], params.v, op);
if (fabs(output[i] - expected_output) > EPSON) {
printf("op %d, at index %d, output: %f, expected_output: %f (%s:%d)\n",
op, i, output[i], expected_output, __FILE__, __LINE__);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
static int test_no_broadcast(DNNMathBinaryOperation op)
{
DnnLayerMathBinaryParams params;
DnnOperand operands[3];
int32_t input_indexes[2];
float input0[1*1*2*3] = {
-3, 2.5, 2, -2.1, 7.8, 100
};
float input1[1*1*2*3] = {
-1, 2, 3, -21, 8, 10.0
};
float *output;
params.bin_op = op;
params.input0_broadcast = 0;
params.input1_broadcast = 0;
operands[0].data = input0;
operands[0].dims[0] = 1;
operands[0].dims[1] = 1;
operands[0].dims[2] = 2;
operands[0].dims[3] = 3;
operands[1].data = input1;
operands[1].dims[0] = 1;
operands[1].dims[1] = 1;
operands[1].dims[2] = 2;
operands[1].dims[3] = 3;
operands[2].data = NULL;
input_indexes[0] = 0;
input_indexes[1] = 1;
ff_dnn_execute_layer_math_binary(operands, input_indexes, 2, &params, NULL);
output = operands[2].data;
for (int i = 0; i < sizeof(input0) / sizeof(float); i++) {
float expected_output = get_expected(input0[i], input1[i], op);
if (fabs(output[i] - expected_output) > EPSON) {
printf("op %d, at index %d, output: %f, expected_output: %f (%s:%d)\n",
op, i, output[i], expected_output, __FILE__, __LINE__);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
static int test(DNNMathBinaryOperation op)
{
if (test_broadcast_input0(op))
return 1;
if (test_broadcast_input1(op))
return 1;
if (test_no_broadcast(op))
return 1;
return 0;
}
int main(int argc, char **argv)
{
if (test(DMBO_SUB))
return 1;
if (test(DMBO_ADD))
return 1;
if (test(DMBO_MUL))
return 1;
if (test(DMBO_REALDIV))
return 1;
if (test(DMBO_MINIMUM))
return 1;
if (test(DMBO_FLOORMOD))
return 1;
return 0;
}

148
libavfilter/tests/dnn-layer-mathunary.c
Unescape View File

@ -1,148 +0,0 @@
/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "libavfilter/dnn/dnn_backend_native_layer_mathunary.h"
#include "libavutil/avassert.h"
#define EPS 0.00001
static float get_expected(float f, DNNMathUnaryOperation op)
{
switch (op)
{
case DMUO_ABS:
return (f >= 0) ? f : -f;
case DMUO_SIN:
return sin(f);
case DMUO_COS:
return cos(f);
case DMUO_TAN:
return tan(f);
case DMUO_ASIN:
return asin(f);
case DMUO_ACOS:
return acos(f);
case DMUO_ATAN:
return atan(f);
case DMUO_SINH:
return sinh(f);
case DMUO_COSH:
return cosh(f);
case DMUO_TANH:
return tanh(f);
case DMUO_ASINH:
return asinh(f);
case DMUO_ACOSH:
return acosh(f);
case DMUO_ATANH:
return atanh(f);
case DMUO_CEIL:
return ceil(f);
case DMUO_FLOOR:
return floor(f);
case DMUO_ROUND:
return round(f);
case DMUO_EXP:
return exp(f);
default:
av_assert0(!"not supported yet");
return 0.f;
}
}
static int test(DNNMathUnaryOperation op)
{
DnnLayerMathUnaryParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*1*3*3] = {
0.1, 0.5, 0.75, -3, 2.5, 2, -2.1, 7.8, 100};
float *output;
params.un_op = op;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 1;
operands[0].dims[2] = 3;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_math_unary(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(input) / sizeof(float); ++i) {
float expected_output = get_expected(input[i], op);
int output_nan = isnan(output[i]);
int expected_nan = isnan(expected_output);
if ((!output_nan && !expected_nan && fabs(output[i] - expected_output) > EPS) ||
(output_nan && !expected_nan) || (!output_nan && expected_nan)) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
int main(int agrc, char **argv)
{
if (test(DMUO_ABS))
return 1;
if (test(DMUO_SIN))
return 1;
if (test(DMUO_COS))
return 1;
if (test(DMUO_TAN))
return 1;
if (test(DMUO_ASIN))
return 1;
if (test(DMUO_ACOS))
return 1;
if (test(DMUO_ATAN))
return 1;
if (test(DMUO_SINH))
return 1;
if (test(DMUO_COSH))
return 1;
if (test(DMUO_TANH))
return 1;
if (test(DMUO_ASINH))
return 1;
if (test(DMUO_ACOSH))
return 1;
if (test(DMUO_ATANH))
return 1;
if (test(DMUO_CEIL))
return 1;
if (test(DMUO_FLOOR))
return 1;
if (test(DMUO_ROUND))
return 1;
if (test(DMUO_EXP))
return 1;
return 0;
}

71
libavfilter/tests/dnn-layer-maximum.c
Unescape View File

@ -1,71 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "libavfilter/dnn/dnn_backend_native_layer_maximum.h"
#define EPSON 0.00001
static int test(void)
{
DnnLayerMaximumParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*1*2*3] = {
-3, 2.5, 2, -2.1, 7.8, 100
};
float *output;
params.val.y = 2.3;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 1;
operands[0].dims[2] = 2;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_maximum(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(input) / sizeof(float); i++) {
float expected_output = input[i] > params.val.y ? input[i] : params.val.y;
if (fabs(output[i] - expected_output) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
int main(int argc, char **argv)
{
if (test())
return 1;
return 0;
}

239
libavfilter/tests/dnn-layer-pad.c
Unescape View File

@ -1,239 +0,0 @@
/*
* Copyright (c) 2019 Guo Yejun
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "libavfilter/dnn/dnn_backend_native_layer_pad.h"
#define EPSON 0.00001
static int test_with_mode_symmetric(void)
{
// the input data and expected data are generated with below python code.
/*
x = tf.placeholder(tf.float32, shape=[1, None, None, 3])
y = tf.pad(x, [[0, 0], [2, 3], [3, 2], [0, 0]], 'SYMMETRIC')
data = np.arange(48).reshape(1, 4, 4, 3);
sess=tf.Session()
sess.run(tf.global_variables_initializer())
output = sess.run(y, feed_dict={x: data})
print(list(data.flatten()))
print(list(output.flatten()))
print(data.shape)
print(output.shape)
*/
LayerPadParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*4*4*3] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47
};
float expected_output[1*9*9*3] = {
18.0, 19.0, 20.0, 15.0, 16.0, 17.0, 12.0, 13.0, 14.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 21.0, 22.0, 23.0, 18.0, 19.0, 20.0, 6.0, 7.0, 8.0, 3.0,
4.0, 5.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 9.0, 10.0, 11.0, 6.0, 7.0, 8.0, 6.0, 7.0, 8.0, 3.0, 4.0, 5.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 3.0,
4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 9.0, 10.0, 11.0, 6.0, 7.0, 8.0, 18.0, 19.0, 20.0, 15.0, 16.0, 17.0, 12.0, 13.0, 14.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0,
21.0, 22.0, 23.0, 21.0, 22.0, 23.0, 18.0, 19.0, 20.0, 30.0, 31.0, 32.0, 27.0, 28.0, 29.0, 24.0, 25.0, 26.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 33.0,
34.0, 35.0, 30.0, 31.0, 32.0, 42.0, 43.0, 44.0, 39.0, 40.0, 41.0, 36.0, 37.0, 38.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 45.0, 46.0, 47.0, 42.0, 43.0,
44.0, 42.0, 43.0, 44.0, 39.0, 40.0, 41.0, 36.0, 37.0, 38.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 45.0, 46.0, 47.0, 42.0, 43.0, 44.0, 30.0, 31.0, 32.0,
27.0, 28.0, 29.0, 24.0, 25.0, 26.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 33.0, 34.0, 35.0, 30.0, 31.0, 32.0, 18.0, 19.0, 20.0, 15.0, 16.0, 17.0, 12.0,
13.0, 14.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 21.0, 22.0, 23.0, 18.0, 19.0, 20.0
};
float *output;
params.mode = LPMP_SYMMETRIC;
params.paddings[0][0] = 0;
params.paddings[0][1] = 0;
params.paddings[1][0] = 2;
params.paddings[1][1] = 3;
params.paddings[2][0] = 3;
params.paddings[2][1] = 2;
params.paddings[3][0] = 0;
params.paddings[3][1] = 0;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 4;
operands[0].dims[2] = 4;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_pad(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
if (fabs(output[i] - expected_output[i]) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
static int test_with_mode_reflect(void)
{
// the input data and expected data are generated with below python code.
/*
x = tf.placeholder(tf.float32, shape=[3, None, None, 3])
y = tf.pad(x, [[1, 2], [0, 0], [0, 0], [0, 0]], 'REFLECT')
data = np.arange(36).reshape(3, 2, 2, 3);
sess=tf.Session()
sess.run(tf.global_variables_initializer())
output = sess.run(y, feed_dict={x: data})
print(list(data.flatten()))
print(list(output.flatten()))
print(data.shape)
print(output.shape)
*/
LayerPadParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[3*2*2*3] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35
};
float expected_output[6*2*2*3] = {
12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0,
35.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0
};
float *output;
params.mode = LPMP_REFLECT;
params.paddings[0][0] = 1;
params.paddings[0][1] = 2;
params.paddings[1][0] = 0;
params.paddings[1][1] = 0;
params.paddings[2][0] = 0;
params.paddings[2][1] = 0;
params.paddings[3][0] = 0;
params.paddings[3][1] = 0;
operands[0].data = input;
operands[0].dims[0] = 3;
operands[0].dims[1] = 2;
operands[0].dims[2] = 2;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_pad(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
if (fabs(output[i] - expected_output[i]) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
static int test_with_mode_constant(void)
{
// the input data and expected data are generated with below python code.
/*
x = tf.placeholder(tf.float32, shape=[1, None, None, 3])
y = tf.pad(x, [[0, 0], [1, 0], [0, 0], [1, 2]], 'CONSTANT', constant_values=728)
data = np.arange(12).reshape(1, 2, 2, 3);
sess=tf.Session()
sess.run(tf.global_variables_initializer())
output = sess.run(y, feed_dict={x: data})
print(list(data.flatten()))
print(list(output.flatten()))
print(data.shape)
print(output.shape)
*/
LayerPadParams params;
DnnOperand operands[2];
int32_t input_indexes[1];
float input[1*2*2*3] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
};
float expected_output[1*3*2*6] = {
728.0, 728.0, 728.0, 728.0, 728.0, 728.0, 728.0, 728.0, 728.0, 728.0, 728.0,
728.0, 728.0, 0.0, 1.0, 2.0, 728.0, 728.0, 728.0, 3.0, 4.0, 5.0, 728.0, 728.0,
728.0, 6.0, 7.0, 8.0, 728.0, 728.0, 728.0, 9.0, 10.0, 11.0, 728.0, 728.0
};
float *output;
params.mode = LPMP_CONSTANT;
params.constant_values = 728;
params.paddings[0][0] = 0;
params.paddings[0][1] = 0;
params.paddings[1][0] = 1;
params.paddings[1][1] = 0;
params.paddings[2][0] = 0;
params.paddings[2][1] = 0;
params.paddings[3][0] = 1;
params.paddings[3][1] = 2;
operands[0].data = input;
operands[0].dims[0] = 1;
operands[0].dims[1] = 2;
operands[0].dims[2] = 2;
operands[0].dims[3] = 3;
operands[1].data = NULL;
input_indexes[0] = 0;
ff_dnn_execute_layer_pad(operands, input_indexes, 1, &params, NULL);
output = operands[1].data;
for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
if (fabs(output[i] - expected_output[i]) > EPSON) {
printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
av_freep(&output);
return 1;
}
}
av_freep(&output);
return 0;
}
int main(int argc, char **argv)
{
if (test_with_mode_symmetric())
return 1;
if (test_with_mode_reflect())
return 1;
if (test_with_mode_constant())
return 1;
}

1
libavfilter/vf_derain.c
Unescape View File

@ -44,7 +44,6 @@ static const AVOption derain_options[] = {
{ "derain", "derain filter flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "type" },
{ "dehaze", "dehaze filter flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "type" },
{ "dnn_backend", "DNN backend", OFFSET(dnnctx.backend_type), AV_OPT_TYPE_INT, { .i64 = 1 }, 0, 1, FLAGS, "backend" },
{ "native", "native backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "backend" },
#if (CONFIG_LIBTENSORFLOW == 1)
{ "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
#endif

1
libavfilter/vf_dnn_processing.c
Unescape View File

@ -46,7 +46,6 @@ typedef struct DnnProcessingContext {
#define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
static const AVOption dnn_processing_options[] = {
{ "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 1 }, INT_MIN, INT_MAX, FLAGS, "backend" },
{ "native", "native backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "backend" },
#if (CONFIG_LIBTENSORFLOW == 1)
{ "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
#endif

1
libavfilter/vf_sr.c
Unescape View File

@ -47,7 +47,6 @@ typedef struct SRContext {
#define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
static const AVOption sr_options[] = {
{ "dnn_backend", "DNN backend used for model execution", OFFSET(dnnctx.backend_type), AV_OPT_TYPE_INT, { .i64 = 1 }, 0, 1, FLAGS, "backend" },
{ "native", "native backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "backend" },
#if (CONFIG_LIBTENSORFLOW == 1)
{ "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
#endif

1
tests/Makefile
Unescape View File

@ -172,7 +172,6 @@ include $(SRC_PATH)/tests/fate/cover-art.mak
include $(SRC_PATH)/tests/fate/dca.mak
include $(SRC_PATH)/tests/fate/demux.mak
include $(SRC_PATH)/tests/fate/dfa.mak
include $(SRC_PATH)/tests/fate/dnn.mak
include $(SRC_PATH)/tests/fate/dnxhd.mak
include $(SRC_PATH)/tests/fate/dpcm.mak
include $(SRC_PATH)/tests/fate/dvvideo.mak

45
tests/fate/dnn.mak
Unescape View File

@ -1,45 +0,0 @@
DNNTESTSDIR := libavfilter/tests
FATE_DNN += fate-dnn-layer-pad
fate-dnn-layer-pad: $(DNNTESTSDIR)/dnn-layer-pad$(EXESUF)
fate-dnn-layer-pad: CMD = run $(DNNTESTSDIR)/dnn-layer-pad$(EXESUF)
fate-dnn-layer-pad: CMP = null
FATE_DNN += fate-dnn-layer-conv2d
fate-dnn-layer-conv2d: $(DNNTESTSDIR)/dnn-layer-conv2d$(EXESUF)
fate-dnn-layer-conv2d: CMD = run $(DNNTESTSDIR)/dnn-layer-conv2d$(EXESUF)
fate-dnn-layer-conv2d: CMP = null
FATE_DNN += fate-dnn-layer-dense
fate-dnn-layer-dense: $(DNNTESTSDIR)/dnn-layer-dense$(EXESUF)
fate-dnn-layer-dense: CMD = run $(DNNTESTSDIR)/dnn-layer-dense$(EXESUF)
fate-dnn-layer-dense: CMP = null
FATE_DNN += fate-dnn-layer-depth2space
fate-dnn-layer-depth2space: $(DNNTESTSDIR)/dnn-layer-depth2space$(EXESUF)
fate-dnn-layer-depth2space: CMD = run $(DNNTESTSDIR)/dnn-layer-depth2space$(EXESUF)
fate-dnn-layer-depth2space: CMP = null
FATE_DNN += fate-dnn-layer-mathbinary
fate-dnn-layer-mathbinary: $(DNNTESTSDIR)/dnn-layer-mathbinary$(EXESUF)
fate-dnn-layer-mathbinary: CMD = run $(DNNTESTSDIR)/dnn-layer-mathbinary$(EXESUF)
fate-dnn-layer-mathbinary: CMP = null
FATE_DNN += fate-dnn-layer-maximum
fate-dnn-layer-maximum: $(DNNTESTSDIR)/dnn-layer-maximum$(EXESUF)
fate-dnn-layer-maximum: CMD = run $(DNNTESTSDIR)/dnn-layer-maximum$(EXESUF)
fate-dnn-layer-maximum: CMP = null
FATE_DNN += fate-dnn-layer-mathunary
fate-dnn-layer-mathunary: $(DNNTESTSDIR)/dnn-layer-mathunary$(EXESUF)
fate-dnn-layer-mathunary: CMD = run $(DNNTESTSDIR)/dnn-layer-mathunary$(EXESUF)
fate-dnn-layer-mathunary: CMP = null
FATE_DNN += fate-dnn-layer-avgpool
fate-dnn-layer-avgpool: $(DNNTESTSDIR)/dnn-layer-avgpool$(EXESUF)
fate-dnn-layer-avgpool: CMD = run $(DNNTESTSDIR)/dnn-layer-avgpool$(EXESUF)
fate-dnn-layer-avgpool: CMP = null
FATE-$(CONFIG_DNN) += $(FATE_DNN)
fate-dnn: $(FATE_DNN)
Write Preview
Loading…
Cancel
Save