hwcontext_vaapi: Try to support the VDPAU wrapper

The driver is somewhat bitrotten (not updated for years) but is still
usable for decoding with this change.  To support it, this adds a new
driver quirk to indicate no support at all for surface attributes.

Based on a patch by wm4 <nfxjfg@googlemail.com>.

(cherry picked from commit e791b915c7)

libavutil/hwcontext_vaapi.c

@@ -155,7 +155,8 @@ static int vaapi_frames_get_constraints(AVHWDeviceContext *hwdev,
     unsigned int fourcc;
     int err, i, j, attr_count, pix_fmt_count;
 
-    if (config) {
+    if (config &&
+        !(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES)) {
         attr_count = 0;
         vas = vaQuerySurfaceAttributes(hwctx->display, config->config_id,
                                        0, &attr_count);
@@ -273,6 +274,11 @@ static const struct {
         "ubit",
         AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE,
     },
+    {
+        "VDPAU wrapper",
+        "Splitted-Desktop Systems VDPAU backend for VA-API",
+        AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES,
+    },
 };
 
 static int vaapi_device_init(AVHWDeviceContext *hwdev)
@@ -451,6 +457,7 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc)
     }
 
     if (!hwfc->pool) {
+        if (!(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES)) {
             int need_memory_type = !(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE);
             int need_pixel_format = 1;
             for (i = 0; i < avfc->nb_attributes; i++) {
@@ -488,6 +495,10 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc)
                 };
             }
             av_assert0(i == ctx->nb_attributes);
+        } else {
+            ctx->attributes = NULL;
+            ctx->nb_attributes = 0;
+        }
 
         ctx->rt_format = rt_format;
 

libavutil/hwcontext_vaapi.h

@@ -51,6 +51,13 @@ enum {
      * so the surface allocation code will not try to use it.
      */
     AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE = (1 << 2),
+
+    /**
+     * The driver does not support surface attributes at all.
+     * The surface allocation code will never pass them to surface allocation,
+     * and the results of the vaQuerySurfaceAttributes() call will be faked.
+     */
+    AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES = (1 << 3),
 };
 
 /**