avfilter/vidstab: add option for file format specification

The vidstab library added support in Nov 2020 for writing/reading
the transforms data in binary in addition to ASCII. The library default
was changed to binary format but no changes were made to the AVfilters
resulting in data file for writing or reading being always opened as text.
This effectively broke the filters.

Option added to vidstabdetect to specify file format and open files in
both filters with the correct attributes.

doc/filters.texi

@@ -24618,6 +24618,19 @@ If set to 0, it is disabled. The frames are counted starting from 1.
 Show fields and transforms in the resulting frames. It accepts an
 integer in the range 0-2. Default value is 0, which disables any
 visualization.
+
+@item fileformat
+Format for the transforms data file to be written.
+Acceptable values are
+
+@table @samp
+@item ascii
+Human-readable plain text
+
+@item binary
+Binary format, roughly 40% smaller than @code{ascii}. (@emph{default})
+@end table
+
 @end table
 
 @subsection Examples

libavfilter/vf_vidstabdetect.c

@@ -40,6 +40,7 @@ typedef struct StabData {
     VSMotionDetectConfig conf;
 
     char *result;
+    int fileformat;
     FILE *f;
 } StabData;
 
@@ -58,6 +59,11 @@ static const AVOption vidstabdetect_options[] = {
     {"show",        "0: draw nothing; 1,2: show fields and transforms",              OFFSETC(show),              AV_OPT_TYPE_INT,    {.i64 = 0},      0,   2, FLAGS},
     {"tripod",      "virtual tripod mode (if >0): motion is compared to a reference"
                     " reference frame (frame # is the value)",                       OFFSETC(virtualTripod),     AV_OPT_TYPE_INT,    {.i64 = 0}, 0, INT_MAX, FLAGS},
+#ifdef LIBVIDSTAB_FILE_FORMAT_VERSION
+    { "fileformat",   "transforms data file format",  OFFSET(fileformat),  AV_OPT_TYPE_INT,  {.i64 = BINARY_SERIALIZATION_MODE},  ASCII_SERIALIZATION_MODE,  BINARY_SERIALIZATION_MODE,  FLAGS,  "file_format"},
+    { "ascii",        "ASCII text",  0,  AV_OPT_TYPE_CONST,  {.i64 = ASCII_SERIALIZATION_MODE },  0,  0,  FLAGS,  "file_format"},
+    { "binary",       "binary",      0,  AV_OPT_TYPE_CONST,  {.i64 = BINARY_SERIALIZATION_MODE},  0,  0,  FLAGS,  "file_format"},
+#endif
     {NULL}
 };
 
@@ -94,6 +100,13 @@ static int config_input(AVFilterLink *inlink)
     VSFrameInfo fi;
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
     int is_planar = desc->flags & AV_PIX_FMT_FLAG_PLANAR;
+    const char *file_mode = "w";
+
+#ifdef LIBVIDSTAB_FILE_FORMAT_VERSION
+    md->serializationMode = s->fileformat;
+    if (s->fileformat == BINARY_SERIALIZATION_MODE)
+        file_mode = "wb";
+#endif
 
     vsFrameInfoInit(&fi, inlink->w, inlink->h,
                     ff_av2vs_pixfmt(ctx, inlink->format));
@@ -129,7 +142,7 @@ static int config_input(AVFilterLink *inlink)
     av_log(ctx, AV_LOG_INFO, "          show = %d\n", s->conf.show);
     av_log(ctx, AV_LOG_INFO, "        result = %s\n", s->result);
 
-    s->f = avpriv_fopen_utf8(s->result, "w");
+    s->f = avpriv_fopen_utf8(s->result, file_mode);
     if (s->f == NULL) {
         av_log(ctx, AV_LOG_ERROR, "cannot open transform file %s\n", s->result);
         return AVERROR(EINVAL);

libavfilter/vf_vidstabtransform.c

@@ -193,7 +193,7 @@ static int config_input(AVFilterLink *inlink)
         av_log(ctx, AV_LOG_INFO, "    zoomspeed = %g\n", tc->conf.zoomSpeed);
     av_log(ctx, AV_LOG_INFO, "    interpol  = %s\n", getInterpolationTypeName(tc->conf.interpolType));
 
-    f = avpriv_fopen_utf8(tc->input, "r");
+    f = avpriv_fopen_utf8(tc->input, "rb");
     if (!f) {
         int ret = AVERROR(errno);
         av_log(ctx, AV_LOG_ERROR, "cannot open input file %s\n", tc->input);