vdpau: add support for 4:2:2 and 4:4:4 chroma sampling

Signed-off-by: Rémi Denis-Courmont <remi@remlab.net>
Signed-off-by: Anton Khirnov <anton@khirnov.net>

libavcodec/h264_slice.c

@@ -970,6 +970,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h)
             *fmt++ = AV_PIX_FMT_YUV420P10;
         break;
     case 8:
+#if CONFIG_H264_VDPAU_HWACCEL
+        *fmt++ = AV_PIX_FMT_VDPAU;
+#endif
         if (CHROMA444(h)) {
             if (h->avctx->colorspace == AVCOL_SPC_RGB)
                 *fmt++ = AV_PIX_FMT_GBRP;
@@ -992,9 +995,6 @@ static enum AVPixelFormat get_pixel_format(H264Context *h)
 #if CONFIG_H264_VDA_HWACCEL
             *fmt++ = AV_PIX_FMT_VDA_VLD;
             *fmt++ = AV_PIX_FMT_VDA;
-#endif
-#if CONFIG_H264_VDPAU_HWACCEL
-            *fmt++ = AV_PIX_FMT_VDPAU;
 #endif
             if (h->avctx->codec->pix_fmts)
                 choices = h->avctx->codec->pix_fmts;

libavcodec/vdpau.c

@@ -142,6 +142,10 @@ int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
     if (av_vdpau_get_surface_parameters(avctx, &type, &width, &height))
         return AVERROR(ENOSYS);
 
+    if (!(hwctx->flags & AV_HWACCEL_FLAG_ALLOW_HIGH_DEPTH) &&
+        type != VDP_CHROMA_TYPE_420)
+        return AVERROR(ENOSYS);
+
     status = vdctx->get_proc_address(vdctx->device,
                                      VDP_FUNC_ID_VIDEO_SURFACE_QUERY_CAPABILITIES,
                                      &func);
@@ -371,7 +375,7 @@ int av_vdpau_bind_context(AVCodecContext *avctx, VdpDevice device,
 {
     VDPAUHWContext *hwctx;
 
-    if (flags & ~AV_HWACCEL_FLAG_IGNORE_LEVEL)
+    if (flags & ~(AV_HWACCEL_FLAG_IGNORE_LEVEL|AV_HWACCEL_FLAG_ALLOW_HIGH_DEPTH))
         return AVERROR(EINVAL);
 
     if (av_reallocp(&avctx->hwaccel_context, sizeof(*hwctx)))