libavfilter: vf_scale: Properly take in->color_range into account

While swscale can be reconfigured with sws_setColorspaceDetails,
the in/out ranges also need to be set before calling
sws_init_context, otherwise the initialization might choose
fastpaths that don't take the ranges into account.

Therefore, look at in->color_range too, when deciding on whether
the scaler needs to be reconfigured.

Add a new member variable for keeping track of this, for being
able to differentiate between whether the scale filter parameter
"in_range" has been set (which should override whatever the input
frame has set) or whether it has been configured based on the
latest frame (which should trigger reconfiguring the scaler if
the input frame ranges change).

Fixes: Ticket #9576

Signed-off-by: Martin Storsjö <martin@martin.st>

libavfilter/vf_scale.c

@@ -138,6 +138,7 @@ typedef struct ScaleContext {
     char *out_color_matrix;
 
     int in_range;
+    int in_frame_range;
     int out_range;
 
     int out_h_chr_pos;
@@ -322,6 +323,8 @@ static av_cold int init_dict(AVFilterContext *ctx, AVDictionary **opts)
     scale->opts = *opts;
     *opts = NULL;
 
+    scale->in_frame_range = AVCOL_RANGE_UNSPECIFIED;
+
     return 0;
 }
 
@@ -544,6 +547,9 @@ static int config_props(AVFilterLink *outlink)
             if (scale->in_range != AVCOL_RANGE_UNSPECIFIED)
                 av_opt_set_int(s, "src_range",
                                scale->in_range == AVCOL_RANGE_JPEG, 0);
+            else if (scale->in_frame_range != AVCOL_RANGE_UNSPECIFIED)
+                av_opt_set_int(s, "src_range",
+                               scale->in_frame_range == AVCOL_RANGE_JPEG, 0);
             if (scale->out_range != AVCOL_RANGE_UNSPECIFIED)
                 av_opt_set_int(s, "dst_range",
                                scale->out_range == AVCOL_RANGE_JPEG, 0);
@@ -690,6 +696,13 @@ static int scale_frame(AVFilterLink *link, AVFrame *in, AVFrame **frame_out)
                     in->sample_aspect_ratio.den != link->sample_aspect_ratio.den ||
                     in->sample_aspect_ratio.num != link->sample_aspect_ratio.num;
 
+    if (in->color_range != AVCOL_RANGE_UNSPECIFIED &&
+        scale->in_range == AVCOL_RANGE_UNSPECIFIED &&
+        in->color_range != scale->in_frame_range) {
+        scale->in_frame_range = in->color_range;
+        frame_changed = 1;
+    }
+
     if (scale->eval_mode == EVAL_MODE_FRAME || frame_changed) {
         unsigned vars_w[VARS_NB] = { 0 }, vars_h[VARS_NB] = { 0 };