avfilter/vf_lut: add planar RGB support

libavfilter/vf_lut.c

@@ -66,6 +66,7 @@ typedef struct LutContext {
     int hsub, vsub;
     double var_values[VAR_VARS_NB];
     int is_rgb, is_yuv;
+    int is_planar;
     int is_16bit;
     int step;
     int negate_alpha; /* only used by negate */
@@ -126,7 +127,12 @@ static av_cold void uninit(AVFilterContext *ctx)
     AV_PIX_FMT_ARGB,         AV_PIX_FMT_RGBA,         \
     AV_PIX_FMT_ABGR,         AV_PIX_FMT_BGRA,         \
     AV_PIX_FMT_RGB24,        AV_PIX_FMT_BGR24,        \
-    AV_PIX_FMT_RGB48LE,      AV_PIX_FMT_RGBA64LE
+    AV_PIX_FMT_RGB48LE,      AV_PIX_FMT_RGBA64LE,     \
+    AV_PIX_FMT_GBRP,         AV_PIX_FMT_GBRAP,        \
+    AV_PIX_FMT_GBRP9,        AV_PIX_FMT_GBRP10,       \
+    AV_PIX_FMT_GBRP12,       AV_PIX_FMT_GBRP14,       \
+    AV_PIX_FMT_GBRP16,       AV_PIX_FMT_GBRAP12,      \
+    AV_PIX_FMT_GBRAP16
 
 static const enum AVPixelFormat yuv_pix_fmts[] = { YUV_FORMATS, AV_PIX_FMT_NONE };
 static const enum AVPixelFormat rgb_pix_fmts[] = { RGB_FORMATS, AV_PIX_FMT_NONE };
@@ -268,10 +274,11 @@ static int config_props(AVFilterLink *inlink)
         break;
     default:
         min[0] = min[1] = min[2] = min[3] = 0;
-        max[0] = max[1] = max[2] = max[3] = 255;
+        max[0] = max[1] = max[2] = max[3] = 255 * (1 << (desc->comp[0].depth - 8));
     }
 
     s->is_yuv = s->is_rgb = 0;
+    s->is_planar = desc->flags & AV_PIX_FMT_FLAG_PLANAR;
     if      (ff_fmt_is_in(inlink->format, yuv_pix_fmts)) s->is_yuv = 1;
     else if (ff_fmt_is_in(inlink->format, rgb_pix_fmts)) s->is_rgb = 1;
 
@@ -345,7 +352,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
         av_frame_copy_props(out, in);
     }
 
-    if (s->is_rgb && s->is_16bit) {
+    if (s->is_rgb && s->is_16bit && !s->is_planar) {
         /* packed, 16-bit */
         uint16_t *inrow, *outrow, *inrow0, *outrow0;
         const int w = inlink->w;
@@ -382,7 +389,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
             inrow0  += in_linesize;
             outrow0 += out_linesize;
         }
-    } else if (s->is_rgb) {
+    } else if (s->is_rgb && !s->is_planar) {
         /* packed */
         uint8_t *inrow, *outrow, *inrow0, *outrow0;
         const int w = inlink->w;
@@ -412,7 +419,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
             outrow0 += out_linesize;
         }
     } else if (s->is_16bit) {
-        // planar yuv >8 bit depth
+        // planar >8 bit depth
         uint16_t *inrow, *outrow;
 
         for (plane = 0; plane < 4 && in->data[plane] && in->linesize[plane]; plane++) {

tests/ref/fate/filter-pixfmts-lut

@@ -2,6 +2,12 @@ abgr                0a932e831efd4ec22f68b25278bac402
 argb                4f575be3cd02799389f581df99c4de38
 bgr24               fa43e3b2abfde8d9e60e157a9acc553d
 bgra                4e2e689897ee7a8e42b16234597bab35
+gbrap               0d1eb2c39e291c53c57302cdc653c2fc
+gbrp                e572d53183f3f2ed3951aa9940d440a1
+gbrp10le            a8fd1ebbc36a477e2b134241fed91687
+gbrp12le            c5a4b89571f7095eb737ad9fd6b1ee08
+gbrp14le            bdfdfd6f36c60497d1cdae791f3cc117
+gbrp9le             a8c4e29f4cb627db81ba053e0853e702
 rgb24               a356171207723a580e7d277078072005
 rgb48le             5c7dd8575836d18c91e09f1915cf9aa9
 rgba                7bc854c2698b78af3e9159a19c2d9d21