Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

overlay_vulkan: port for the rewrite

Browse Source
pull/389/head
Lynne 2 years ago
parent 59f21c051d
commit 8e9ceb1efb
No known key found for this signature in database
GPG Key ID: A2FEA5F03F034464
1 changed files with 123 additions and 275 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 398
      libavfilter/vf_overlay_vulkan.c

398
libavfilter/vf_overlay_vulkan.c
Unescape View File

@ -1,4 +1,6 @@
/*
* Copyright (c) Lynne
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
@ -19,26 +21,26 @@
#include "libavutil/random_seed.h"
#include "libavutil/opt.h"
#include "vulkan_filter.h"
#include "vulkan_spirv.h"
#include "internal.h"
#include "framesync.h"
#define CGROUPS (int [3]){ 32, 32, 1 }
typedef struct OverlayVulkanContext {
FFVulkanContext vkctx;
FFFrameSync fs;
int initialized;
FFVulkanPipeline pl;
FFVkExecPool e;
FFVkQueueFamilyCtx qf;
FFVkExecContext *exec;
FFVulkanPipeline *pl;
FFFrameSync fs;
FFVkBuffer params_buf;
FFVkSPIRVShader shd;
VkSampler sampler;
/* Shader updators, must be in the main filter struct */
VkDescriptorImageInfo main_images[3];
VkDescriptorImageInfo overlay_images[3];
VkDescriptorImageInfo output_images[3];
VkDescriptorBufferInfo params_desc;
/* Push constants / options */
struct {
int32_t o_offset[2*3];
int32_t o_size[2*3];
} opts;
int overlay_x;
int overlay_y;
@ -80,279 +82,114 @@ static const char overlay_alpha[] = {
static av_cold int init_filter(AVFilterContext *ctx)
{
int err;
FFVkSampler *sampler;
uint8_t *spv_data;
size_t spv_len;
void *spv_opaque = NULL;
OverlayVulkanContext *s = ctx->priv;
FFVulkanContext *vkctx = &s->vkctx;
const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
ff_vk_qf_init(vkctx, &s->qf, VK_QUEUE_COMPUTE_BIT, 0);
sampler = ff_vk_init_sampler(vkctx, 1, VK_FILTER_NEAREST);
if (!sampler)
const int ialpha = av_pix_fmt_desc_get(s->vkctx.input_format)->flags & AV_PIX_FMT_FLAG_ALPHA;
const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(s->vkctx.output_format);
FFVkSPIRVShader *shd = &s->shd;
FFVkSPIRVCompiler *spv;
FFVulkanDescriptorSetBinding *desc;
spv = ff_vk_spirv_init();
if (!spv) {
av_log(ctx, AV_LOG_ERROR, "Unable to initialize SPIR-V compiler!\n");
return AVERROR_EXTERNAL;
s->pl = ff_vk_create_pipeline(vkctx, &s->qf);
if (!s->pl)
return AVERROR(ENOMEM);
{ /* Create the shader */
const int ialpha = av_pix_fmt_desc_get(s->vkctx.input_format)->flags & AV_PIX_FMT_FLAG_ALPHA;
FFVulkanDescriptorSetBinding desc_i[3] = {
{
.name = "main_img",
.type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.dimensions = 2,
.elems = planes,
.stages = VK_SHADER_STAGE_COMPUTE_BIT,
.updater = s->main_images,
.sampler = sampler,
},
{
.name = "overlay_img",
.type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.dimensions = 2,
.elems = planes,
.stages = VK_SHADER_STAGE_COMPUTE_BIT,
.updater = s->overlay_images,
.sampler = sampler,
},
{
.name = "output_img",
.type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
.mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format),
.mem_quali = "writeonly",
.dimensions = 2,
.elems = planes,
.stages = VK_SHADER_STAGE_COMPUTE_BIT,
.updater = s->output_images,
},
};
FFVulkanDescriptorSetBinding desc_b = {
.name = "params",
.type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.mem_quali = "readonly",
.mem_layout = "std430",
.stages = VK_SHADER_STAGE_COMPUTE_BIT,
.updater = &s->params_desc,
.buf_content = "ivec2 o_offset[3], o_size[3];",
};
FFVkSPIRVShader *shd = ff_vk_init_shader(s->pl, "overlay_compute",
VK_SHADER_STAGE_COMPUTE_BIT);
if (!shd)
return AVERROR(ENOMEM);
ff_vk_set_compute_shader_sizes(shd, CGROUPS);
RET(ff_vk_add_descriptor_set(vkctx, s->pl, shd, desc_i, FF_ARRAY_ELEMS(desc_i), 0)); /* set 0 */
RET(ff_vk_add_descriptor_set(vkctx, s->pl, shd, &desc_b, 1, 0)); /* set 1 */
GLSLD( overlay_noalpha );
GLSLD( overlay_alpha );
GLSLC(0, void main() );
GLSLC(0, { );
GLSLC(1, ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
GLSLF(1, int planes = %i; ,planes);
GLSLC(1, for (int i = 0; i < planes; i++) { );
if (ialpha)
GLSLC(2, overlay_alpha_opaque(i, pos); );
else
GLSLC(2, overlay_noalpha(i, pos); );
GLSLC(1, } );
GLSLC(0, } );
RET(ff_vk_compile_shader(vkctx, shd, "main"));
}
RET(ff_vk_init_pipeline_layout(vkctx, s->pl));
RET(ff_vk_init_compute_pipeline(vkctx, s->pl));
{ /* Create and update buffer */
const AVPixFmtDescriptor *desc;
/* NOTE: std430 requires the same identical struct layout, padding and
* alignment as C, so we're allowed to do this, as this will map
* exactly to what the shader recieves */
struct {
int32_t o_offset[2*3];
int32_t o_size[2*3];
} *par;
err = ff_vk_create_buf(vkctx, &s->params_buf,
sizeof(*par), NULL,
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
if (err)
return err;
err = ff_vk_map_buffers(vkctx, &s->params_buf, (uint8_t **)&par, 1, 0);
if (err)
return err;
desc = av_pix_fmt_desc_get(s->vkctx.output_format);
par->o_offset[0] = s->overlay_x;
par->o_offset[1] = s->overlay_y;
par->o_offset[2] = par->o_offset[0] >> desc->log2_chroma_w;
par->o_offset[3] = par->o_offset[1] >> desc->log2_chroma_h;
par->o_offset[4] = par->o_offset[0] >> desc->log2_chroma_w;
par->o_offset[5] = par->o_offset[1] >> desc->log2_chroma_h;
par->o_size[0] = s->overlay_w;
par->o_size[1] = s->overlay_h;
par->o_size[2] = par->o_size[0] >> desc->log2_chroma_w;
par->o_size[3] = par->o_size[1] >> desc->log2_chroma_h;
par->o_size[4] = par->o_size[0] >> desc->log2_chroma_w;
par->o_size[5] = par->o_size[1] >> desc->log2_chroma_h;
err = ff_vk_unmap_buffers(vkctx, &s->params_buf, 1, 1);
if (err)
return err;
s->params_desc.buffer = s->params_buf.buf;
s->params_desc.range = VK_WHOLE_SIZE;
ff_vk_update_descriptor_set(vkctx, s->pl, 1);
}
/* Execution context */
RET(ff_vk_create_exec_ctx(vkctx, &s->exec, &s->qf));
ff_vk_qf_init(vkctx, &s->qf, VK_QUEUE_COMPUTE_BIT);
RET(ff_vk_exec_pool_init(vkctx, &s->qf, &s->e, s->qf.nb_queues*4, 0, 0, 0, NULL));
RET(ff_vk_init_sampler(vkctx, &s->sampler, 1, VK_FILTER_NEAREST));
RET(ff_vk_shader_init(&s->pl, &s->shd, "overlay_compute",
VK_SHADER_STAGE_COMPUTE_BIT, 0));
ff_vk_shader_set_compute_sizes(&s->shd, 32, 32, 1);
GLSLC(0, layout(push_constant, std430) uniform pushConstants { );
GLSLC(1, ivec2 o_offset[3]; );
GLSLC(1, ivec2 o_size[3]; );
GLSLC(0, }; );
GLSLC(0, );
ff_vk_add_push_constant(&s->pl, 0, sizeof(s->opts),
VK_SHADER_STAGE_COMPUTE_BIT);
desc = (FFVulkanDescriptorSetBinding []) {
{
.name = "main_img",
.type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.dimensions = 2,
.elems = planes,
.stages = VK_SHADER_STAGE_COMPUTE_BIT,
.samplers = DUP_SAMPLER(s->sampler),
},
{
.name = "overlay_img",
.type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.dimensions = 2,
.elems = planes,
.stages = VK_SHADER_STAGE_COMPUTE_BIT,
.samplers = DUP_SAMPLER(s->sampler),
},
{
.name = "output_img",
.type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
.mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format),
.mem_quali = "writeonly",
.dimensions = 2,
.elems = planes,
.stages = VK_SHADER_STAGE_COMPUTE_BIT,
},
};
RET(ff_vk_pipeline_descriptor_set_add(vkctx, &s->pl, shd, desc, 3, 0, 0));
GLSLD( overlay_noalpha );
GLSLD( overlay_alpha );
GLSLC(0, void main() );
GLSLC(0, { );
GLSLC(1, ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
GLSLF(1, int planes = %i; ,planes);
GLSLC(1, for (int i = 0; i < planes; i++) { );
if (ialpha)
GLSLC(2, overlay_alpha_opaque(i, pos); );
else
GLSLC(2, overlay_noalpha(i, pos); );
GLSLC(1, } );
GLSLC(0, } );
RET(spv->compile_shader(spv, ctx, shd, &spv_data, &spv_len, "main",
&spv_opaque));
RET(ff_vk_shader_create(vkctx, shd, spv_data, spv_len, "main"));
RET(ff_vk_init_compute_pipeline(vkctx, &s->pl, shd));
RET(ff_vk_exec_pipeline_register(vkctx, &s->e, &s->pl));
s->opts.o_offset[0] = s->overlay_x;
s->opts.o_offset[1] = s->overlay_y;
s->opts.o_offset[2] = s->opts.o_offset[0] >> pix_desc->log2_chroma_w;
s->opts.o_offset[3] = s->opts.o_offset[1] >> pix_desc->log2_chroma_h;
s->opts.o_offset[4] = s->opts.o_offset[0] >> pix_desc->log2_chroma_w;
s->opts.o_offset[5] = s->opts.o_offset[1] >> pix_desc->log2_chroma_h;
s->opts.o_size[0] = s->overlay_w;
s->opts.o_size[1] = s->overlay_h;
s->opts.o_size[2] = s->opts.o_size[0] >> pix_desc->log2_chroma_w;
s->opts.o_size[3] = s->opts.o_size[1] >> pix_desc->log2_chroma_h;
s->opts.o_size[4] = s->opts.o_size[0] >> pix_desc->log2_chroma_w;
s->opts.o_size[5] = s->opts.o_size[1] >> pix_desc->log2_chroma_h;
s->initialized = 1;
return 0;
fail:
return err;
}
static int process_frames(AVFilterContext *avctx, AVFrame *out_f,
AVFrame *main_f, AVFrame *overlay_f)
{
int err;
VkCommandBuffer cmd_buf;
OverlayVulkanContext *s = avctx->priv;
FFVulkanContext *vkctx = &s->vkctx;
FFVulkanFunctions *vk = &vkctx->vkfn;
int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
AVVkFrame *out = (AVVkFrame *)out_f->data[0];
AVVkFrame *main = (AVVkFrame *)main_f->data[0];
AVVkFrame *overlay = (AVVkFrame *)overlay_f->data[0];
AVHWFramesContext *main_fc = (AVHWFramesContext*)main_f->hw_frames_ctx->data;
AVHWFramesContext *overlay_fc = (AVHWFramesContext*)overlay_f->hw_frames_ctx->data;
const VkFormat *output_formats = av_vkfmt_from_pixfmt(s->vkctx.output_format);
const VkFormat *main_sw_formats = av_vkfmt_from_pixfmt(main_fc->sw_format);
const VkFormat *overlay_sw_formats = av_vkfmt_from_pixfmt(overlay_fc->sw_format);
/* Update descriptors and init the exec context */
ff_vk_start_exec_recording(vkctx, s->exec);
cmd_buf = ff_vk_get_exec_buf(s->exec);
for (int i = 0; i < planes; i++) {
RET(ff_vk_create_imageview(vkctx, s->exec,
&s->main_images[i].imageView, main->img[i],
main_sw_formats[i],
ff_comp_identity_map));
RET(ff_vk_create_imageview(vkctx, s->exec,
&s->overlay_images[i].imageView, overlay->img[i],
overlay_sw_formats[i],
ff_comp_identity_map));
RET(ff_vk_create_imageview(vkctx, s->exec,
&s->output_images[i].imageView, out->img[i],
output_formats[i],
ff_comp_identity_map));
s->main_images[i].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
s->overlay_images[i].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
s->output_images[i].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
}
ff_vk_update_descriptor_set(vkctx, s->pl, 0);
for (int i = 0; i < planes; i++) {
VkImageMemoryBarrier bar[3] = {
{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.srcAccessMask = 0,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
.oldLayout = main->layout[i],
.newLayout = s->main_images[i].imageLayout,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = main->img[i],
.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.subresourceRange.levelCount = 1,
.subresourceRange.layerCount = 1,
},
{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.srcAccessMask = 0,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
.oldLayout = overlay->layout[i],
.newLayout = s->overlay_images[i].imageLayout,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = overlay->img[i],
.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.subresourceRange.levelCount = 1,
.subresourceRange.layerCount = 1,
},
{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.srcAccessMask = 0,
.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
.oldLayout = out->layout[i],
.newLayout = s->output_images[i].imageLayout,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = out->img[i],
.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.subresourceRange.levelCount = 1,
.subresourceRange.layerCount = 1,
},
};
vk->CmdPipelineBarrier(cmd_buf, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0,
0, NULL, 0, NULL, FF_ARRAY_ELEMS(bar), bar);
main->layout[i] = bar[0].newLayout;
main->access[i] = bar[0].dstAccessMask;
overlay->layout[i] = bar[1].newLayout;
overlay->access[i] = bar[1].dstAccessMask;
out->layout[i] = bar[2].newLayout;
out->access[i] = bar[2].dstAccessMask;
}
ff_vk_bind_pipeline_exec(vkctx, s->exec, s->pl);
vk->CmdDispatch(cmd_buf,
FFALIGN(s->vkctx.output_width, CGROUPS[0])/CGROUPS[0],
FFALIGN(s->vkctx.output_height, CGROUPS[1])/CGROUPS[1], 1);
ff_vk_add_exec_dep(vkctx, s->exec, main_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
ff_vk_add_exec_dep(vkctx, s->exec, overlay_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
ff_vk_add_exec_dep(vkctx, s->exec, out_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
err = ff_vk_submit_exec_queue(vkctx, s->exec);
if (err)
return err;
if (spv_opaque)
spv->free_shader(spv, &spv_opaque);
if (spv)
spv->uninit(&spv);
ff_vk_qf_rotate(&s->qf);
return err;
fail:
ff_vk_discard_exec_deps(s->exec);
return err;
}
@ -394,7 +231,9 @@ static int overlay_vulkan_blend(FFFrameSync *fs)
goto fail;
}
RET(process_frames(ctx, out, input_main, input_overlay));
RET(ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->pl,
out, (AVFrame *[]){ input_main, input_overlay }, 2,
s->sampler, &s->opts, sizeof(s->opts)));
err = av_frame_copy_props(out, input_main);
if (err < 0)
@ -443,8 +282,17 @@ static av_cold int overlay_vulkan_init(AVFilterContext *avctx)
static void overlay_vulkan_uninit(AVFilterContext *avctx)
{
OverlayVulkanContext *s = avctx->priv;
FFVulkanContext *vkctx = &s->vkctx;
FFVulkanFunctions *vk = &vkctx->vkfn;
ff_vk_exec_pool_free(vkctx, &s->e);
ff_vk_pipeline_free(vkctx, &s->pl);
ff_vk_shader_free(vkctx, &s->shd);
if (s->sampler)
vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler,
vkctx->hwctx->alloc);
ff_vk_free_buf(&s->vkctx, &s->params_buf);
ff_vk_uninit(&s->vkctx);
ff_framesync_uninit(&s->fs);

Write Preview
Loading…
Cancel
Save