hwcontext_vulkan: rewrite to support multiplane surfaces

This commit adds proper handling of multiplane images throughout
all of the hwcontext code. To avoid breakage of individual
components, the change is performed as a single commit.

libavutil/hwcontext_vulkan.h

@@ -164,6 +164,10 @@ typedef enum AVVkFrameFlags {
     /* DEPRECATED: does nothing. Replaced by multiplane images. */
     AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY = (1ULL << 1),
 #endif
+
+    /* Disables multiplane images.
+     * This is required to export/import images from CUDA. */
+    AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE = (1ULL << 2),
 } AVVkFrameFlags;
 
 /**
@@ -171,26 +175,32 @@ typedef enum AVVkFrameFlags {
  */
 typedef struct AVVulkanFramesContext {
     /**
-     * Controls the tiling of allocated frames. If left as optimal tiling,
+     * Controls the tiling of allocated frames.
-     * then during av_hwframe_ctx_init() will decide based on whether the device
+     * If left as VK_IMAGE_TILING_OPTIMAL (0), will use optimal tiling.
-     * supports DRM modifiers, or if the linear_images flag is set, otherwise
+     * Can be set to VK_IMAGE_TILING_LINEAR to force linear images,
-     * will allocate optimally-tiled images.
+     * or VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT to force DMABUF-backed
+     * images.
+     * @note Imported frames from other APIs ignore this.
      */
     VkImageTiling tiling;
 
     /**
-     * Defines extra usage of output frames. If left as 0, the following bits
+     * Defines extra usage of output frames. If non-zero, all flags MUST be
-     * are set: TRANSFER_SRC, TRANSFER_DST. SAMPLED and STORAGE.
+     * supported by the VkFormat. Otherwise, will use supported flags amongst:
+     * - VK_IMAGE_USAGE_SAMPLED_BIT
+     * - VK_IMAGE_USAGE_STORAGE_BIT
+     * - VK_IMAGE_USAGE_TRANSFER_SRC_BIT
+     * - VK_IMAGE_USAGE_TRANSFER_DST_BIT
      */
     VkImageUsageFlagBits usage;
 
     /**
      * Extension data for image creation.
-     * If VkImageDrmFormatModifierListCreateInfoEXT is present in the chain,
+     * If DRM tiling is used, a VkImageDrmFormatModifierListCreateInfoEXT structure
-     * and the device supports DRM modifiers, then images will be allocated
+     * can be added to specify the exact modifier to use.
-     * with the specific requested DRM modifiers.
+     *
      * Additional structures may be added at av_hwframe_ctx_init() time,
-     * which will be freed automatically on uninit(), so users need only free
+     * which will be freed automatically on uninit(), so users must only free
      * any structures they've allocated themselves.
      */
     void *create_pnext;
@@ -211,6 +221,25 @@ typedef struct AVVulkanFramesContext {
      */
     AVVkFrameFlags flags;
 
+    /**
+     * Flags to set during image creation. If unset, defaults to
+     * VK_IMAGE_CREATE_ALIAS_BIT.
+     */
+    VkImageCreateFlags img_flags;
+
+    /**
+     * Vulkan format for each image. MUST be compatible with the pixel format.
+     * If unset, will be automatically set.
+     * There are at most two compatible formats for a frame - a multiplane
+     * format, and a single-plane multi-image format.
+     */
+    VkFormat format[AV_NUM_DATA_POINTERS];
+
+    /**
+     * Number of layers each image will have.
+     */
+    int nb_layers;
+
     /**
      * Locks a frame, preventing other threads from changing frame properties.
      * Users SHOULD only ever lock just before command submission in order
@@ -228,14 +257,7 @@ typedef struct AVVulkanFramesContext {
 } AVVulkanFramesContext;
 
 /*
- * Frame structure, the VkFormat of the image will always match
+ * Frame structure.
- * the pool's sw_format.
- * All frames, imported or allocated, will be created with the
- * VK_IMAGE_CREATE_ALIAS_BIT flag set, so the memory may be aliased if needed.
- *
- * If all queue family indices in the device context are the same,
- * images will be created with the EXCLUSIVE sharing mode. Otherwise, all images
- * will be created using the CONCURRENT sharing mode.
  *
  * @note the size of this structure is not part of the ABI, to allocate
  * you must use @av_vk_frame_alloc().
@@ -243,18 +265,20 @@ typedef struct AVVulkanFramesContext {
 struct AVVkFrame {
     /**
      * Vulkan images to which the memory is bound to.
+     * May be one for multiplane formats, or multiple.
      */
     VkImage img[AV_NUM_DATA_POINTERS];
 
     /**
-     * The same tiling must be used for all images in the frame.
+     * Tiling for the frame.
      */
     VkImageTiling tiling;
 
     /**
-     * Memory backing the images. Could be less than the amount of planes,
+     * Memory backing the images. Either one, or as many as there are planes
-     * in which case the offset value will indicate the binding offset of
+     * in the sw_format.
-     * each plane in the memory.
+     * In case of having multiple VkImages, but one memory, the offset field
+     * will indicate the bound offset for each image.
      */
     VkDeviceMemory mem[AV_NUM_DATA_POINTERS];
     size_t size[AV_NUM_DATA_POINTERS];
@@ -265,13 +289,13 @@ struct AVVkFrame {
     VkMemoryPropertyFlagBits flags;
 
     /**
-     * Updated after every barrier
+     * Updated after every barrier. One per VkImage.
      */
     VkAccessFlagBits access[AV_NUM_DATA_POINTERS];
     VkImageLayout layout[AV_NUM_DATA_POINTERS];
 
     /**
-     * Synchronization timeline semaphores, one for each sw_format plane.
+     * Synchronization timeline semaphores, one for each VkImage.
      * Must not be freed manually. Must be waited on at every submission using
      * the value in sem_value, and must be signalled at every submission,
      * using an incremented value.
@@ -280,6 +304,7 @@ struct AVVkFrame {
 
     /**
      * Up to date semaphore value at which each image becomes accessible.
+     * One per VkImage.
      * Clients must wait on this value when submitting a command queue,
      * and increment it when signalling.
      */
@@ -291,16 +316,18 @@ struct AVVkFrame {
     struct AVVkFrameInternal *internal;
 
     /**
-     * Describes the binding offset of each plane to the VkDeviceMemory.
+     * Describes the binding offset of each image to the VkDeviceMemory.
+     * One per VkImage.
      */
     ptrdiff_t offset[AV_NUM_DATA_POINTERS];
 
     /**
      * Queue family of the images. Must be VK_QUEUE_FAMILY_IGNORED if
      * the image was allocated with the CONCURRENT concurrency option.
+     * One per VkImage.
      */
     uint32_t queue_family[AV_NUM_DATA_POINTERS];
-} AVVkFrame;
+};
 
 /**
  * Allocates a single AVVkFrame and initializes everything as 0.
@@ -309,7 +336,8 @@ struct AVVkFrame {
 AVVkFrame *av_vk_frame_alloc(void);
 
 /**
- * Returns the format of each image up to the number of planes for a given sw_format.
+ * Returns the optimal per-plane Vulkan format for a given sw_format,
+ * one for each plane.
  * Returns NULL on unsupported formats.
  */
 const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p);