fftools/ffmpeg_mux_init: move check for mixing simple/complex filters

Do it in ost_get_filters() together with other similar checks. Will be
useful in following commits.

Also, improve the log message.

fftools/ffmpeg_mux_init.c

@@ -438,6 +438,19 @@ static int ost_get_filters(const OptionsContext *o, AVFormatContext *oc,
         return 0;
     }
 
+    if (!ost->ist) {
+        if (ost->filters_script || ost->filters) {
+            av_log(ost, AV_LOG_ERROR,
+                   "%s '%s' was specified for a stream fed from a complex "
+                   "filtergraph. Simple and complex filtering cannot be used "
+                   "together for the same stream.\n",
+                   ost->filters ? "Filtergraph" : "Filtergraph script",
+                   ost->filters ? ost->filters : ost->filters_script);
+            return AVERROR(EINVAL);
+        }
+        return 0;
+    }
+
     if (ost->filters_script && ost->filters) {
         av_log(ost, AV_LOG_ERROR, "Both -filter and -filter_script set\n");
         exit_program(1);
@@ -1263,18 +1276,6 @@ static void init_output_filter(OutputFilter *ofilter, const OptionsContext *o,
         exit_program(1);
     }
 
-    if (ost->avfilter && (ost->filters || ost->filters_script)) {
-        const char *opt = ost->filters ? "-vf/-af/-filter" : "-filter_script";
-        av_log(ost, AV_LOG_ERROR,
-               "%s '%s' was specified through the %s option "
-               "for output stream %d:%d, which is fed from a complex filtergraph.\n"
-               "%s and -filter_complex cannot be used together for the same stream.\n",
-               ost->filters ? "Filtergraph" : "Filtergraph script",
-               ost->filters ? ost->filters : ost->filters_script,
-               opt, ost->file_index, ost->index, opt);
-        exit_program(1);
-    }
-
     avfilter_inout_free(&ofilter->out_tmp);
 }