diff --git a/libavfilter/vf_scale.c b/libavfilter/vf_scale.c
index 01f0020f05..89a80ee2aa 100644
--- a/libavfilter/vf_scale.c
+++ b/libavfilter/vf_scale.c
@@ -535,6 +535,8 @@ static int filter_frame(AVFilterLink *link, AVFrame *in)
             inv_table = parse_yuv_type(scale->in_color_matrix, av_frame_get_colorspace(in));
         if (scale->out_color_matrix)
             table     = parse_yuv_type(scale->out_color_matrix, AVCOL_SPC_UNSPECIFIED);
+        else if (scale->in_color_matrix)
+            table = inv_table;
 
         if (scale-> in_range != AVCOL_RANGE_UNSPECIFIED)
             in_full  = (scale-> in_range == AVCOL_RANGE_JPEG);