lavc/libvpx: Fix support for RGB colorspace.

Reported and tested by Nicolas George.
Fixes ticket #5249.

libavcodec/libvpx.c

@@ -38,6 +38,9 @@ static const enum AVPixelFormat vp9_pix_fmts_highcol[] = {
     AV_PIX_FMT_YUV422P,
     AV_PIX_FMT_YUV440P,
     AV_PIX_FMT_YUV444P,
+#if VPX_IMAGE_ABI_VERSION >= 3
+    AV_PIX_FMT_GBRP,
+#endif
     AV_PIX_FMT_NONE
 };
 
@@ -54,6 +57,11 @@ static const enum AVPixelFormat vp9_pix_fmts_highbd[] = {
     AV_PIX_FMT_YUV422P12,
     AV_PIX_FMT_YUV440P12,
     AV_PIX_FMT_YUV444P12,
+#if VPX_IMAGE_ABI_VERSION >= 3
+    AV_PIX_FMT_GBRP,
+    AV_PIX_FMT_GBRP10,
+    AV_PIX_FMT_GBRP12,
+#endif
     AV_PIX_FMT_NONE
 };
 #endif

libavcodec/libvpxenc.c

@@ -106,6 +106,7 @@ typedef struct VP8EncoderContext {
     int aq_mode;
     int drop_threshold;
     int noise_sensitivity;
+    int vpx_cs;
 } VP8Context;
 
 /** String mappings for enum vp8e_enc_control_id */
@@ -277,6 +278,7 @@ static int set_pix_fmt(AVCodecContext *avctx, vpx_codec_caps_t codec_caps,
                        struct vpx_codec_enc_cfg *enccfg, vpx_codec_flags_t *flags,
                        vpx_img_fmt_t *img_fmt)
 {
+    VP8Context *ctx = avctx->priv_data;
 #ifdef VPX_IMG_FMT_HIGHBITDEPTH
     enccfg->g_bit_depth = enccfg->g_input_bit_depth = 8;
 #endif
@@ -294,6 +296,8 @@ static int set_pix_fmt(AVCodecContext *avctx, vpx_codec_caps_t codec_caps,
         enccfg->g_profile = 1;
         *img_fmt = VPX_IMG_FMT_I440;
         return 0;
+    case AV_PIX_FMT_GBRP:
+        ctx->vpx_cs = VPX_CS_SRGB;
 #endif
     case AV_PIX_FMT_YUV444P:
         enccfg->g_profile = 1;
@@ -334,12 +338,16 @@ static int set_pix_fmt(AVCodecContext *avctx, vpx_codec_caps_t codec_caps,
             return 0;
         }
         break;
+    case AV_PIX_FMT_GBRP10:
+    case AV_PIX_FMT_GBRP12:
+        ctx->vpx_cs = VPX_CS_SRGB;
 #endif
     case AV_PIX_FMT_YUV444P10:
     case AV_PIX_FMT_YUV444P12:
         if (codec_caps & VPX_CODEC_CAP_HIGHBITDEPTH) {
             enccfg->g_bit_depth = enccfg->g_input_bit_depth =
-                avctx->pix_fmt == AV_PIX_FMT_YUV444P10 ? 10 : 12;
+                avctx->pix_fmt == AV_PIX_FMT_YUV444P10 ||
+                avctx->pix_fmt == AV_PIX_FMT_GBRP10 ? 10 : 12;
             enccfg->g_profile = 3;
             *img_fmt = VPX_IMG_FMT_I44416;
             *flags |= VPX_CODEC_USE_HIGHBITDEPTH;
@@ -358,7 +366,11 @@ static int set_pix_fmt(AVCodecContext *avctx, vpx_codec_caps_t codec_caps,
 static void set_colorspace(AVCodecContext *avctx)
 {
     enum vpx_color_space vpx_cs;
+    VP8Context *ctx = avctx->priv_data;
 
+    if (ctx->vpx_cs) {
+        vpx_cs = ctx->vpx_cs;
+    } else {
     switch (avctx->colorspace) {
     case AVCOL_SPC_RGB:         vpx_cs = VPX_CS_SRGB;      break;
     case AVCOL_SPC_BT709:       vpx_cs = VPX_CS_BT_709;    break;
@@ -373,6 +385,7 @@ static void set_colorspace(AVCodecContext *avctx)
                avctx->colorspace);
         return;
     }
+    }
     codecctl_int(avctx, VP9E_SET_COLOR_SPACE, vpx_cs);
 }
 #endif

libavcodec/version.h

@@ -30,7 +30,7 @@
 
 #define LIBAVCODEC_VERSION_MAJOR  57
 #define LIBAVCODEC_VERSION_MINOR  24
-#define LIBAVCODEC_VERSION_MICRO 104
+#define LIBAVCODEC_VERSION_MICRO 105
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \