avformat/hlsenc: size and duration in segment filenames

1st:
This patch makes it possible to put actual segment file size (measured
in bytes) and/or duration (calculated in microseconds) into segment
filenames. This feature is useful when post-processing live streaming
access log files. New behaviour works only when -use_localtime option
is set and second_level_segment_size or/and
second_level_segment_duration new hls_flags are specified. %%s is the
placeholder for size and %%t for duration in hls_segment_filename
option. Fix sized trailing zeropadding also works eg. %%09s or %%023t.

A command to test new features:
./ffmpeg -loglevel info -y -f lavfi -i color=c=red:size=640x480:r=25 -f
lavfi -i sine=f=440:b=4:r=44100 -c:v mpeg2video -g 25 -acodec aac
-cutoff 20000 -ac 2 -ar 44100 -ab 192k -f hls -hls_time 3 -hls_list_size
5 -hls_flags
second_level_segment_index+second_level_segment_size+second_level_segment_duration
-use_localtime 1 -use_localtime_mkdir 1 -hls_segment_filename
"segment_%Y%m%d%H%M%S_%%04d_%%08s_%%013t.ts" stream.m3u8

2nd:
doc/muxers: beside second_level_segment_duration and second_level_segment_size,
added some more details and example to hls_segment_filename,
use_localtime, use_localtime_mkdir, hls_flags. hls_flags option list
reformatted to table

Signed-off-by: Bela Bodecs <bodecsb@vivanet.hu>
Signed-off-by: Steven Liu <lq@chinaffmpeg.org>

doc/muxers.texi

@@ -441,8 +441,15 @@ ffmpeg -i in.nut -hls_segment_filename 'file%03d.ts' out.m3u8
 This example will produce the playlist, @file{out.m3u8}, and segment files:
 @file{file000.ts}, @file{file001.ts}, @file{file002.ts}, etc.
 
+@var{filename} may contain full path or relative path specification,
+but only the file name part without any path info will be contained in the m3u8 segment list.
+Should a relative path be specified, the path of the created segment
+files will be relative to the current working directory.
+When use_localtime_mkdir is set, the whole expanded value of @var{filename} will be written into the m3u8 segment list.
+
+
 @item use_localtime
-Use strftime on @var{filename} to expand the segment filename with localtime.
+Use strftime() on @var{filename} to expand the segment filename with localtime.
 The segment number is also available in this mode, but to use it, you need to specify second_level_segment_index
 hls_flag and %%d will be the specifier.
 @example
@@ -450,6 +457,8 @@ ffmpeg -i in.nut -use_localtime 1 -hls_segment_filename 'file-%Y%m%d-%s.ts' out.
 @end example
 This example will produce the playlist, @file{out.m3u8}, and segment files:
 @file{file-20160215-1455569023.ts}, @file{file-20160215-1455569024.ts}, etc.
+Note: On some systems/environments, the @code{%s} specifier is not available. See
+  @code{strftime()} documentation.
 @example
 ffmpeg -i in.nut -use_localtime 1 -hls_flags second_level_segment_index -hls_segment_filename 'file-%Y%m%d-%%04d.ts' out.m3u8
 @end example
@@ -457,14 +466,21 @@ This example will produce the playlist, @file{out.m3u8}, and segment files:
 @file{file-20160215-0001.ts}, @file{file-20160215-0002.ts}, etc.
 
 @item use_localtime_mkdir
-Used together with -use_localtime, it will create up to one subdirectory which
+Used together with -use_localtime, it will create all subdirectories which
 is expanded in @var{filename}.
 @example
 ffmpeg -i in.nut -use_localtime 1 -use_localtime_mkdir 1 -hls_segment_filename '%Y%m%d/file-%Y%m%d-%s.ts' out.m3u8
 @end example
 This example will create a directory 201560215 (if it does not exist), and then
 produce the playlist, @file{out.m3u8}, and segment files:
-@file{201560215/file-20160215-1455569023.ts}, @file{201560215/file-20160215-1455569024.ts}, etc.
+@file{20160215/file-20160215-1455569023.ts}, @file{20160215/file-20160215-1455569024.ts}, etc.
+
+@example
+ffmpeg -i in.nut -use_localtime 1 -use_localtime_mkdir 1 -hls_segment_filename '%Y/%m/%d/file-%Y%m%d-%s.ts' out.m3u8
+@end example
+This example will create a directory hierarchy 2016/02/15 (if any of them do not exist), and then
+produce the playlist, @file{out.m3u8}, and segment files:
+@file{2016/02/15/file-20160215-1455569023.ts}, @file{2016/02/15/file-20160215-1455569024.ts}, etc.
 
 
 @item hls_key_info_file @var{key_info_file}
@@ -523,7 +539,12 @@ ffmpeg -f lavfi -re -i testsrc -c:v h264 -hls_flags delete_segments \
   -hls_key_info_file file.keyinfo out.m3u8
 @end example
 
-@item hls_flags single_file
+
+@item hls_flags @var{flags}
+Possible values:
+
+@table @samp
+@item single_file
 If this flag is set, the muxer will store all segments in a single MPEG-TS
 file, and will use byte ranges in the playlist. HLS playlists generated with
 this way will have the version number 4.
@@ -534,36 +555,60 @@ ffmpeg -i in.nut -hls_flags single_file out.m3u8
 Will produce the playlist, @file{out.m3u8}, and a single segment file,
 @file{out.ts}.
 
-@item hls_flags delete_segments
+@item delete_segments
 Segment files removed from the playlist are deleted after a period of time
 equal to the duration of the segment plus the duration of the playlist.
 
-@item hls_flags append_list
+@item append_list
 Append new segments into the end of old segment list,
 and remove the @code{#EXT-X-ENDLIST} from the old segment list.
 
-@item hls_flags round_durations
+@item round_durations
 Round the duration info in the playlist file segment info to integer
 values, instead of using floating point.
 
-@item hls_flags discont_starts
+@item discont_starts
 Add the @code{#EXT-X-DISCONTINUITY} tag to the playlist, before the
 first segment's information.
 
-@item hls_flags omit_endlist
+@item omit_endlist
 Do not append the @code{EXT-X-ENDLIST} tag at the end of the playlist.
 
-@item hls_flags split_by_time
+@item split_by_time
 Allow segments to start on frames other than keyframes. This improves
 behavior on some players when the time between keyframes is inconsistent,
 but may make things worse on others, and can cause some oddities during
 seeking. This flag should be used with the @code{hls_time} option.
 
-@item hls_flags program_date_time
+@item program_date_time
 Generate @code{EXT-X-PROGRAM-DATE-TIME} tags.
 
-@item hls_flags second_level_segment_index
+@item second_level_segment_index
-Makes it possible to use segment indexes as %%d besides date/time values when use_localtime is on.
+Makes it possible to use segment indexes as %%d in hls_segment_filename expression
+besides date/time values when use_localtime is on.
+To get fixed width numbers with trailing zeroes, %%0xd format is available where x is the required width.
+
+@item second_level_segment_size
+Makes it possible to use segment sizes (counted in bytes) as %%s in hls_segment_filename
+expression besides date/time values when use_localtime is on.
+To get fixed width numbers with trailing zeroes, %%0xs format is available where x is the required width.
+
+@item second_level_segment_duration
+Makes it possible to use segment duration (calculated  in microseconds) as %%t in hls_segment_filename
+expression besides date/time values when use_localtime is on.
+To get fixed width numbers with trailing zeroes, %%0xt format is available where x is the required width.
+
+@example
+ffmpeg -i sample.mpeg \
+   -f hls -hls_time 3 -hls_list_size 5 \
+   -hls_flags second_level_segment_index+second_level_segment_size+second_level_segment_duration \
+   -use_localtime 1 -use_localtime_mkdir 1 -hls_segment_filename "segment_%Y%m%d%H%M%S_%%04d_%%08s_%%013t.ts" stream.m3u8
+@end example
+This will produce segments like this:
+@file{segment_20170102194334_0003_00122200_0000003000000.ts}, @file{segment_20170102194334_0004_00120072_0000003000000.ts} etc.
+
+
+@end table
 
 @item hls_playlist_type event
 Emit @code{#EXT-X-PLAYLIST-TYPE:EVENT} in the m3u8 header. Forces

libavformat/hlsenc.c

@@ -67,6 +67,8 @@ typedef enum HLSFlags {
     HLS_APPEND_LIST = (1 << 6),
     HLS_PROGRAM_DATE_TIME = (1 << 7),
     HLS_SECOND_LEVEL_SEGMENT_INDEX = (1 << 8), // include segment index in segment filenames when use_localtime  e.g.: %%03d
+    HLS_SECOND_LEVEL_SEGMENT_DURATION = (1 << 9), // include segment duration (microsec) in segment filenames when use_localtime  e.g.: %%09t
+    HLS_SECOND_LEVEL_SEGMENT_SIZE = (1 << 10), // include segment size (bytes) in segment filenames when use_localtime  e.g.: %%014s
 } HLSFlags;
 
 typedef enum {
@@ -134,6 +136,7 @@ typedef struct HLSContext {
     char *method;
 
     double initial_prog_date_time;
+    char current_segment_final_filename_fmt[1024]; // when renaming segments
 } HLSContext;
 
 static int mkdir_p(const char *path) {
@@ -169,6 +172,58 @@ static int mkdir_p(const char *path) {
     return ret;
 }
 
+static int replace_int_data_in_filename(char *buf, int buf_size, const char *filename, char placeholder, int64_t number)
+{
+    const char *p;
+    char *q, buf1[20], c;
+    int nd, len, addchar_count;
+    int found_count = 0;
+
+    q = buf;
+    p = filename;
+    for (;;) {
+        c = *p;
+        if (c == '\0')
+            break;
+        if (c == '%' && *(p+1) == '%')  // %%
+            addchar_count = 2;
+        else if (c == '%' && (av_isdigit(*(p+1)) || *(p+1) == placeholder)) {
+            nd = 0;
+            addchar_count = 1;
+            while (av_isdigit(*(p + addchar_count))) {
+                nd = nd * 10 + *(p + addchar_count) - '0';
+                addchar_count++;
+            }
+
+            if (*(p + addchar_count) == placeholder) {
+                len = snprintf(buf1, sizeof(buf1), "%0*"PRId64, (number < 0) ? nd : nd++, number);
+                if (len < 1)  // returned error or empty buf1
+                    goto fail;
+                if ((q - buf + len) > buf_size - 1)
+                    goto fail;
+                memcpy(q, buf1, len);
+                q += len;
+                p += (addchar_count + 1);
+                addchar_count = 0;
+                found_count++;
+            }
+
+        } else
+            addchar_count = 1;
+
+        while (addchar_count--)
+            if ((q - buf) < buf_size - 1)
+                *q++ = *p++;
+            else
+                goto fail;
+    }
+    *q = '\0';
+    return found_count;
+fail:
+    *q = '\0';
+    return -1;
+}
+
 static int hls_delete_old_segments(HLSContext *hls) {
 
     HLSSegment *segment, *previous_segment = NULL;
@@ -388,6 +443,47 @@ static int hls_append_segment(struct AVFormatContext *s, HLSContext *hls, double
     if (!en)
         return AVERROR(ENOMEM);
 
+    if ((hls->flags & (HLS_SECOND_LEVEL_SEGMENT_SIZE | HLS_SECOND_LEVEL_SEGMENT_DURATION)) &&
+        strlen(hls->current_segment_final_filename_fmt)) {
+        char * old_filename = av_strdup(hls->avf->filename);  // %%s will be %s after strftime
+        av_strlcpy(hls->avf->filename, hls->current_segment_final_filename_fmt, sizeof(hls->avf->filename));
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_SIZE) {
+            char * filename = av_strdup(hls->avf->filename);  // %%s will be %s after strftime
+            if (!filename)
+                return AVERROR(ENOMEM);
+            if (replace_int_data_in_filename(hls->avf->filename, sizeof(hls->avf->filename),
+                filename, 's', pos + size) < 1) {
+                av_log(hls, AV_LOG_ERROR,
+                       "Invalid second level segment filename template '%s', "
+                        "you can try to remove second_level_segment_size flag\n",
+                       filename);
+                av_free(filename);
+                av_free(old_filename);
+                return AVERROR(EINVAL);
+            }
+            av_free(filename);
+        }
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_DURATION) {
+            char * filename = av_strdup(hls->avf->filename);  // %%t will be %t after strftime
+            if (!filename)
+                return AVERROR(ENOMEM);
+            if (replace_int_data_in_filename(hls->avf->filename, sizeof(hls->avf->filename),
+                filename, 't',  (int64_t)round(1000000 * duration)) < 1) {
+                av_log(hls, AV_LOG_ERROR,
+                       "Invalid second level segment filename template '%s', "
+                        "you can try to remove second_level_segment_time flag\n",
+                       filename);
+                av_free(filename);
+                av_free(old_filename);
+                return AVERROR(EINVAL);
+            }
+            av_free(filename);
+        }
+        ff_rename(old_filename, hls->avf->filename, hls);
+        av_free(old_filename);
+    }
+
+
     filename = av_basename(hls->avf->filename);
 
     if (hls->use_localtime_mkdir) {
@@ -709,15 +805,49 @@ static int hls_start(AVFormatContext *s)
                 char * filename = av_strdup(oc->filename);  // %%d will be %d after strftime
                 if (!filename)
                     return AVERROR(ENOMEM);
-                if (av_get_frame_filename2(oc->filename, sizeof(oc->filename),
+                if (replace_int_data_in_filename(oc->filename, sizeof(oc->filename),
-                    filename, c->wrap ? c->sequence % c->wrap : c->sequence,
+                    filename, 'd', c->wrap ? c->sequence % c->wrap : c->sequence) < 1) {
-                    AV_FRAME_FILENAME_FLAGS_MULTIPLE) < 0) {
+                    av_log(c, AV_LOG_ERROR,
-                    av_log(c, AV_LOG_ERROR, "Invalid second level segment filename template '%s', you can try to remove second_level_segment_index flag\n", filename);
+                           "Invalid second level segment filename template '%s', "
+                            "you can try to remove second_level_segment_index flag\n",
+                           filename);
                     av_free(filename);
                     return AVERROR(EINVAL);
                 }
                 av_free(filename);
             }
+            if (c->flags & (HLS_SECOND_LEVEL_SEGMENT_SIZE | HLS_SECOND_LEVEL_SEGMENT_DURATION)) {
+                av_strlcpy(c->current_segment_final_filename_fmt, oc->filename,
+                           sizeof(c->current_segment_final_filename_fmt));
+                if (c->flags & HLS_SECOND_LEVEL_SEGMENT_SIZE) {
+                    char * filename = av_strdup(oc->filename);  // %%s will be %s after strftime
+                    if (!filename)
+                        return AVERROR(ENOMEM);
+                    if (replace_int_data_in_filename(oc->filename, sizeof(oc->filename), filename, 's', 0) < 1) {
+                        av_log(c, AV_LOG_ERROR,
+                               "Invalid second level segment filename template '%s', "
+                                "you can try to remove second_level_segment_size flag\n",
+                               filename);
+                        av_free(filename);
+                        return AVERROR(EINVAL);
+                    }
+                    av_free(filename);
+                }
+                if (c->flags & HLS_SECOND_LEVEL_SEGMENT_DURATION) {
+                    char * filename = av_strdup(oc->filename);  // %%t will be %t after strftime
+                    if (!filename)
+                        return AVERROR(ENOMEM);
+                    if (replace_int_data_in_filename(oc->filename, sizeof(oc->filename), filename, 't', 0) < 1) {
+                        av_log(c, AV_LOG_ERROR,
+                               "Invalid second level segment filename template '%s', "
+                                "you can try to remove second_level_segment_time flag\n",
+                               filename);
+                        av_free(filename);
+                        return AVERROR(EINVAL);
+                    }
+                    av_free(filename);
+                }
+            }
             if (c->use_localtime_mkdir) {
                 const char *dir;
                 char *fn_copy = av_strdup(oc->filename);
@@ -832,6 +962,7 @@ static int hls_write_header(AVFormatContext *s)
     hls->sequence       = hls->start_sequence;
     hls->recording_time = (hls->init_time ? hls->init_time : hls->time) * AV_TIME_BASE;
     hls->start_pts      = AV_NOPTS_VALUE;
+    hls->current_segment_final_filename_fmt[0] = '\0';
 
     if (hls->flags & HLS_PROGRAM_DATE_TIME) {
         time_t now0;
@@ -906,10 +1037,41 @@ static int hls_write_header(AVFormatContext *s)
             av_strlcat(hls->basename, pattern, basename_size);
         }
     }
-    if (!hls->use_localtime && (hls->flags & HLS_SECOND_LEVEL_SEGMENT_INDEX)) {
+    if (!hls->use_localtime) {
-        av_log(hls, AV_LOG_ERROR, "second_level_segment_index hls_flag requires use_localtime to be true\n");
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_DURATION) {
-        ret = AVERROR(EINVAL);
+             av_log(hls, AV_LOG_ERROR,
-        goto fail;
+                    "second_level_segment_duration hls_flag requires use_localtime to be true\n");
+             ret = AVERROR(EINVAL);
+             goto fail;
+        }
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_SIZE) {
+             av_log(hls, AV_LOG_ERROR,
+                    "second_level_segment_size hls_flag requires use_localtime to be true\n");
+             ret = AVERROR(EINVAL);
+             goto fail;
+        }
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_INDEX) {
+            av_log(hls, AV_LOG_ERROR,
+                   "second_level_segment_index hls_flag requires use_localtime to be true\n");
+            ret = AVERROR(EINVAL);
+            goto fail;
+        }
+    } else {
+        const char *proto = avio_find_protocol_name(hls->basename);
+        int segment_renaming_ok = proto && !strcmp(proto, "file");
+
+        if ((hls->flags & HLS_SECOND_LEVEL_SEGMENT_DURATION) && !segment_renaming_ok) {
+             av_log(hls, AV_LOG_ERROR,
+                    "second_level_segment_duration hls_flag works only with file protocol segment names\n");
+             ret = AVERROR(EINVAL);
+             goto fail;
+        }
+        if ((hls->flags & HLS_SECOND_LEVEL_SEGMENT_SIZE) && !segment_renaming_ok) {
+             av_log(hls, AV_LOG_ERROR,
+                    "second_level_segment_size hls_flag works only with file protocol segment names\n");
+             ret = AVERROR(EINVAL);
+             goto fail;
+        }
     }
     if(hls->has_subtitle) {
 
@@ -1167,6 +1329,8 @@ static const AVOption options[] = {
     {"append_list", "append the new segments into old hls segment list", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_APPEND_LIST }, 0, UINT_MAX,   E, "flags"},
     {"program_date_time", "add EXT-X-PROGRAM-DATE-TIME", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_PROGRAM_DATE_TIME }, 0, UINT_MAX,   E, "flags"},
     {"second_level_segment_index", "include segment index in segment filenames when use_localtime", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SECOND_LEVEL_SEGMENT_INDEX }, 0, UINT_MAX,   E, "flags"},
+    {"second_level_segment_duration", "include segment duration in segment filenames when use_localtime", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SECOND_LEVEL_SEGMENT_DURATION }, 0, UINT_MAX,   E, "flags"},
+    {"second_level_segment_size", "include segment size in segment filenames when use_localtime", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SECOND_LEVEL_SEGMENT_SIZE }, 0, UINT_MAX,   E, "flags"},
     {"use_localtime", "set filename expansion with strftime at segment creation", OFFSET(use_localtime), AV_OPT_TYPE_BOOL, {.i64 = 0 }, 0, 1, E },
     {"use_localtime_mkdir", "create last directory component in strftime-generated filename", OFFSET(use_localtime_mkdir), AV_OPT_TYPE_BOOL, {.i64 = 0 }, 0, 1, E },
     {"hls_playlist_type", "set the HLS playlist type", OFFSET(pl_type), AV_OPT_TYPE_INT, {.i64 = PLAYLIST_TYPE_NONE }, 0, PLAYLIST_TYPE_NB-1, E, "pl_type" },