hwcontext_d3d11va: implement av_hwdevice_get_hwframe_constraints()

D3D11 has rather fine grained per format capabilities for different uses
that can be queried at runtime. Since we don't know what the user wants
to do with the formats when av_hwdevice_get_hwframe_constraints() is
called, we simply return all formats that have the most basic support.

libavutil/hwcontext_d3d11va.c

@@ -120,6 +120,38 @@ static void d3d11va_frames_uninit(AVHWFramesContext *ctx)
     s->staging_texture = NULL;
 }
 
+static int d3d11va_frames_get_constraints(AVHWDeviceContext *ctx,
+                                          const void *hwconfig,
+                                          AVHWFramesConstraints *constraints)
+{
+    AVD3D11VADeviceContext *device_hwctx = ctx->hwctx;
+    int nb_sw_formats = 0;
+    HRESULT hr;
+    int i;
+
+    constraints->valid_sw_formats = av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1,
+                                                    sizeof(*constraints->valid_sw_formats));
+    if (!constraints->valid_sw_formats)
+        return AVERROR(ENOMEM);
+
+    for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
+        UINT format_support = 0;
+        hr = ID3D11Device_CheckFormatSupport(device_hwctx->device, supported_formats[i].d3d_format, &format_support);
+        if (SUCCEEDED(hr) && (format_support & D3D11_FORMAT_SUPPORT_TEXTURE2D))
+            constraints->valid_sw_formats[nb_sw_formats++] = supported_formats[i].pix_fmt;
+    }
+    constraints->valid_sw_formats[nb_sw_formats] = AV_PIX_FMT_NONE;
+
+    constraints->valid_hw_formats = av_malloc_array(2, sizeof(*constraints->valid_hw_formats));
+    if (!constraints->valid_hw_formats)
+        return AVERROR(ENOMEM);
+
+    constraints->valid_hw_formats[0] = AV_PIX_FMT_D3D11;
+    constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
+
+    return 0;
+}
+
 static void free_texture(void *opaque, uint8_t *data)
 {
     ID3D11Texture2D_Release((ID3D11Texture2D *)opaque);
@@ -576,6 +608,7 @@ const HWContextType ff_hwcontext_type_d3d11va = {
     .device_create        = d3d11va_device_create,
     .device_init          = d3d11va_device_init,
     .device_uninit        = d3d11va_device_uninit,
+    .frames_get_constraints = d3d11va_frames_get_constraints,
     .frames_init          = d3d11va_frames_init,
     .frames_uninit        = d3d11va_frames_uninit,
     .frames_get_buffer    = d3d11va_get_buffer,