fftools/ffmpeg_opt: try to propagate the requested output channel layout

Don't silently replace it with the default layout for the amount of channels
from the requested layout.

Should fix ticket #9869

Signed-off-by: James Almer <jamrial@gmail.com>

fftools/ffmpeg_opt.c

@@ -2867,6 +2867,42 @@ static void of_add_metadata(AVFormatContext *oc, const OptionsContext *o)
         }
     }
 }
+static void set_channel_layout(OutputFilter *f, OutputStream *ost)
+{
+    int i, err;
+
+    if (ost->enc_ctx->ch_layout.order != AV_CHANNEL_ORDER_UNSPEC) {
+        /* Pass the layout through for all orders but UNSPEC */
+        err = av_channel_layout_copy(&f->ch_layout, &ost->enc_ctx->ch_layout);
+        if (err < 0)
+            exit_program(1);
+        return;
+    }
+
+    /* Requested layout is of order UNSPEC */
+    if (!ost->enc->ch_layouts) {
+        /* Use the default native layout for the requested amount of channels when the
+           encoder doesn't have a list of supported layouts */
+        av_channel_layout_default(&f->ch_layout, ost->enc_ctx->ch_layout.nb_channels);
+        return;
+    }
+    /* Encoder has a list of supported layouts. Pick the first layout in it with the
+       same amount of channels as the requested layout */
+    for (i = 0; ost->enc->ch_layouts[i].nb_channels; i++) {
+        if (ost->enc->ch_layouts[i].nb_channels == ost->enc_ctx->ch_layout.nb_channels)
+            break;
+    }
+    if (ost->enc->ch_layouts[i].nb_channels) {
+        /* Use it if one is found */
+        err = av_channel_layout_copy(&f->ch_layout, &ost->enc->ch_layouts[i]);
+        if (err < 0)
+            exit_program(1);
+        return;
+    }
+    /* If no layout for the amount of channels requested was found, use the default
+       native layout for it. */
+    av_channel_layout_default(&f->ch_layout, ost->enc_ctx->ch_layout.nb_channels);
+}
 
 static int open_output_file(OptionsContext *o, const char *filename)
 {
@@ -3057,7 +3093,7 @@ static int open_output_file(OptionsContext *o, const char *filename)
                     f->sample_rates = ost->enc->supported_samplerates;
                 }
                 if (ost->enc_ctx->ch_layout.nb_channels) {
-                    av_channel_layout_default(&f->ch_layout, ost->enc_ctx->ch_layout.nb_channels);
+                    set_channel_layout(f, ost);
                 } else if (ost->enc->ch_layouts) {
                     f->ch_layouts = ost->enc->ch_layouts;
                 }