avcodec/libx265: ignore user set alpha x265-param

It makes no difference when the input has an alpha plane, and may end up in
crashes or undefined behavior if it doesn't.

Signed-off-by: James Almer <jamrial@gmail.com>

libavcodec/cbs_h2645.c

@@ -499,9 +499,9 @@ static int cbs_h2645_fragment_add_nals(CodedBitstreamContext *ctx,
         size_t size = nal->size;
         enum AVCodecID codec_id = ctx->codec->codec_id;
 
-        if (codec_id == AV_CODEC_ID_HEVC && nal->nuh_layer_id > 0 &&
+//        if (codec_id == AV_CODEC_ID_HEVC && nal->nuh_layer_id > 0 &&
-            (nal->type < HEVC_NAL_VPS || nal->type > HEVC_NAL_PPS))
+  //          (nal->type < HEVC_NAL_VPS || nal->type > HEVC_NAL_PPS))
-            continue;
+    //        continue;
 
         // Remove trailing zeroes.
         while (size > 0 && nal->data[size - 1] == 0)

libavcodec/libx265.c

@@ -505,8 +505,22 @@ FF_ENABLE_DEPRECATION_WARNINGS
     {
         const AVDictionaryEntry *en = NULL;
         while ((en = av_dict_iterate(ctx->x265_opts, en))) {
-            int parse_ret = ctx->api->param_parse(ctx->params, en->key, en->value);
+            int parse_ret;
 
+            // ignore forced alpha option. The pixel format is all we need.
+            if (!strncmp(en->key, "alpha", 5)) {
+                if (desc->nb_components == 4) {
+                    av_log(avctx, AV_LOG_WARNING,
+                           "Ignoring redundant \"alpha\" option.\n");
+                    continue;
+                }
+                av_log(avctx, AV_LOG_ERROR,
+                       "Alpha encoding was requested through an unsupported "
+                       "option when no alpha plane is present\n");
+                return AVERROR(EINVAL);
+            }
+
+            parse_ret = ctx->api->param_parse(ctx->params, en->key, en->value);
             switch (parse_ret) {
             case X265_PARAM_BAD_NAME:
                 av_log(avctx, AV_LOG_WARNING,