avcodec/libaom: Support monochrome encoding with libaom >= 2.0.1

Monochrome encoding with libaom was buggy for a long time, but this was
finally sorted out in libaom 2.0.1 (2.0.0 is almost there but was still
buggy in realtime mode).

We'll keep support for libaom 1.x around until the LTS distros that
include it are EOL (which is still a long time from now).

Fixes: https://trac.ffmpeg.org/ticket/7599

Changelog

@@ -51,6 +51,7 @@ version <next>:
 - asubcut filter
 - Microsoft Paint (MSP) version 2 decoder
 - Microsoft Paint (MSP) demuxer
+- AV1 monochrome encoding support via libaom >= 2.0.1
 
 
 version 4.3:

libavcodec/libaomenc.c

@@ -338,6 +338,9 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->pix_fmt);
     enccfg->g_bit_depth = enccfg->g_input_bit_depth = desc->comp[0].depth;
     switch (avctx->pix_fmt) {
+    case AV_PIX_FMT_GRAY8:
+        enccfg->monochrome = 1;
+        /* Fall-through */
     case AV_PIX_FMT_YUV420P:
         enccfg->g_profile = FF_PROFILE_AV1_MAIN;
         *img_fmt = AOM_IMG_FMT_I420;
@@ -351,6 +354,10 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
         enccfg->g_profile = FF_PROFILE_AV1_HIGH;
         *img_fmt = AOM_IMG_FMT_I444;
         return 0;
+    case AV_PIX_FMT_GRAY10:
+    case AV_PIX_FMT_GRAY12:
+        enccfg->monochrome = 1;
+        /* Fall-through */
     case AV_PIX_FMT_YUV420P10:
     case AV_PIX_FMT_YUV420P12:
         if (codec_caps & AOM_CODEC_CAP_HIGHBITDEPTH) {
@@ -1158,6 +1165,15 @@ static const enum AVPixelFormat av1_pix_fmts[] = {
     AV_PIX_FMT_NONE
 };
 
+static const enum AVPixelFormat av1_pix_fmts_with_gray[] = {
+    AV_PIX_FMT_YUV420P,
+    AV_PIX_FMT_YUV422P,
+    AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_GBRP,
+    AV_PIX_FMT_GRAY8,
+    AV_PIX_FMT_NONE
+};
+
 static const enum AVPixelFormat av1_pix_fmts_highbd[] = {
     AV_PIX_FMT_YUV420P,
     AV_PIX_FMT_YUV422P,
@@ -1174,13 +1190,35 @@ static const enum AVPixelFormat av1_pix_fmts_highbd[] = {
     AV_PIX_FMT_NONE
 };
 
+static const enum AVPixelFormat av1_pix_fmts_highbd_with_gray[] = {
+    AV_PIX_FMT_YUV420P,
+    AV_PIX_FMT_YUV422P,
+    AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_GBRP,
+    AV_PIX_FMT_YUV420P10,
+    AV_PIX_FMT_YUV422P10,
+    AV_PIX_FMT_YUV444P10,
+    AV_PIX_FMT_YUV420P12,
+    AV_PIX_FMT_YUV422P12,
+    AV_PIX_FMT_YUV444P12,
+    AV_PIX_FMT_GBRP10,
+    AV_PIX_FMT_GBRP12,
+    AV_PIX_FMT_GRAY8,
+    AV_PIX_FMT_GRAY10,
+    AV_PIX_FMT_GRAY12,
+    AV_PIX_FMT_NONE
+};
+
 static av_cold void av1_init_static(AVCodec *codec)
 {
+    int supports_monochrome = aom_codec_version() >= 20001;
     aom_codec_caps_t codec_caps = aom_codec_get_caps(aom_codec_av1_cx());
     if (codec_caps & AOM_CODEC_CAP_HIGHBITDEPTH)
-        codec->pix_fmts = av1_pix_fmts_highbd;
+        codec->pix_fmts = supports_monochrome ? av1_pix_fmts_highbd_with_gray :
+                                                av1_pix_fmts_highbd;
     else
-        codec->pix_fmts = av1_pix_fmts;
+        codec->pix_fmts = supports_monochrome ? av1_pix_fmts_with_gray :
+                                                av1_pix_fmts;
 
     if (aom_codec_version_major() < 2)
         codec->capabilities |= AV_CODEC_CAP_EXPERIMENTAL;

libavcodec/version.h

@@ -29,7 +29,7 @@
 
 #define LIBAVCODEC_VERSION_MAJOR  58
 #define LIBAVCODEC_VERSION_MINOR 115
-#define LIBAVCODEC_VERSION_MICRO 101
+#define LIBAVCODEC_VERSION_MICRO 102
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \