avformat/hlsenc: Add WebVtt support in hls

Signed-off-by: Anshul Maheshwari <er.anshul.maheshwari@gmail.com> Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
10 years ago · 379ddc6af4
parent ec7f04c13a
commit 379ddc6af4
1 changed files with 192 additions and 10 deletions
--- a/libavformat/hlsenc.c
+++ b/libavformat/hlsenc.c
@ -42,6 +42,7 @@

 typedef struct HLSSegment {
    char filename[1024];
+    char sub_filename[1024];
    double duration; /* in seconds */
    int64_t pos;
    int64_t size;
@ -67,8 +68,10 @@ typedef struct HLSContext {
    int64_t sequence;
    int64_t start_sequence;
    AVOutputFormat *oformat;
+    AVOutputFormat *vtt_oformat;

    AVFormatContext *avf;
+    AVFormatContext *vtt_avf;

    float time;            // Set by a private option.
    int max_nb_segments;   // Set by a private option.
@ -79,6 +82,7 @@ typedef struct HLSContext {
    int allowcache;
    int64_t recording_time;
    int has_video;
+    int has_subtitle;
    int64_t start_pts;
    int64_t end_pts;
    double duration;      // last segment duration computed so far, in seconds
@ -92,8 +96,12 @@ typedef struct HLSContext {
    HLSSegment *old_segments;

    char *basename;
+    char *vtt_basename;
+    char *vtt_m3u8_name;
    char *baseurl;
    char *format_options_str;
+    char *vtt_format_options_str;
+    char *subtitle_filename;
    AVDictionary *format_options;

    char *key_info_file;
@ -101,14 +109,16 @@ typedef struct HLSContext {
    char key_uri[LINE_BUFFER_SIZE + 1];
    char key_string[KEYSIZE*2 + 1];
    char iv_string[KEYSIZE*2 + 1];
+    AVDictionary *vtt_format_options;
+
 } HLSContext;

 static int hls_delete_old_segments(HLSContext *hls) {

    HLSSegment *segment, *previous_segment = NULL;
    float playlist_duration = 0.0f;
-    int ret = 0, path_size;
-    char *dirname = NULL, *p, *path;
+    int ret = 0, path_size, sub_path_size;
+    char *dirname = NULL, *p, *path, *sub_path;

    segment = hls->segments;
    while (segment) {
@ -150,13 +160,28 @@ static int hls_delete_old_segments(HLSContext *hls) {
            ret = AVERROR(ENOMEM);
            goto fail;
        }
+        sub_path_size = strlen(dirname) + strlen(segment->sub_filename) + 1;
+        sub_path = av_malloc(sub_path_size);
+        if (!sub_path) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
+
        av_strlcpy(path, dirname, path_size);
        av_strlcat(path, segment->filename, path_size);
        if (unlink(path) < 0) {
            av_log(hls, AV_LOG_ERROR, "failed to delete old segment %s: %s\n",
                                     path, strerror(errno));
        }
+
+        av_strlcpy(sub_path, dirname, sub_path_size);
+        av_strlcat(sub_path, segment->sub_filename, sub_path_size);
+        if (unlink(sub_path) < 0) {
+            av_log(hls, AV_LOG_ERROR, "failed to delete old segment %s: %s\n",
+                                     sub_path, strerror(errno));
+        }
        av_free(path);
+        av_free(sub_path);
        previous_segment = segment;
        segment = previous_segment->next;
        av_free(previous_segment);
@ -226,6 +251,7 @@ static int hls_mux_init(AVFormatContext *s)
 {
    HLSContext *hls = s->priv_data;
    AVFormatContext *oc;
+    AVFormatContext *vtt_oc;
    int i, ret;

    ret = avformat_alloc_output_context2(&hls->avf, hls->oformat, NULL, NULL);
@ -238,9 +264,24 @@ static int hls_mux_init(AVFormatContext *s)
    oc->max_delay          = s->max_delay;
    av_dict_copy(&oc->metadata, s->metadata, 0);

+    if(hls->vtt_oformat) {
+        ret = avformat_alloc_output_context2(&hls->vtt_avf, hls->vtt_oformat, NULL, NULL);
+        if (ret < 0)
+            return ret;
+        vtt_oc          = hls->vtt_avf;
+        vtt_oc->oformat = hls->vtt_oformat;
+        av_dict_copy(&vtt_oc->metadata, s->metadata, 0);
+    }
+
    for (i = 0; i < s->nb_streams; i++) {
        AVStream *st;
-        if (!(st = avformat_new_stream(oc, NULL)))
+        AVFormatContext *loc;
+        if (s->streams[i]->codec->codec_type == AVMEDIA_TYPE_SUBTITLE)
+            loc = vtt_oc;
+        else
+            loc = oc;
+
+        if (!(st = avformat_new_stream(loc, NULL)))
            return AVERROR(ENOMEM);
        avcodec_copy_context(st->codec, s->streams[i]->codec);
        st->sample_aspect_ratio = s->streams[i]->sample_aspect_ratio;
@ -263,6 +304,9 @@ static int hls_append_segment(HLSContext *hls, double duration, int64_t pos,

    av_strlcpy(en->filename, av_basename(hls->avf->filename), sizeof(en->filename));

+    if(hls->has_subtitle)
+        av_strlcpy(en->sub_filename, av_basename(hls->vtt_avf->filename), sizeof(en->sub_filename));
+
    en->duration = duration;
    en->pos      = pos;
    en->size     = size;
@ -317,6 +361,7 @@ static int hls_window(AVFormatContext *s, int last)
    int target_duration = 0;
    int ret = 0;
    AVIOContext *out = NULL;
+    AVIOContext *sub_out = NULL;
    char temp_filename[1024];
    int64_t sequence = FFMAX(hls->start_sequence, hls->sequence - hls->nb_entries);
    int version = hls->flags & HLS_SINGLE_FILE ? 4 : 3;
@ -380,8 +425,39 @@ static int hls_window(AVFormatContext *s, int last)
    if (last && (hls->flags & HLS_OMIT_ENDLIST)==0)
        avio_printf(out, "#EXT-X-ENDLIST\n");

+    if( hls->vtt_m3u8_name ) {
+        if ((ret = avio_open2(&sub_out, hls->vtt_m3u8_name, AVIO_FLAG_WRITE,
+                          &s->interrupt_callback, NULL)) < 0)
+            goto fail;
+        avio_printf(sub_out, "#EXTM3U\n");
+        avio_printf(sub_out, "#EXT-X-VERSION:%d\n", version);
+        if (hls->allowcache == 0 || hls->allowcache == 1) {
+            avio_printf(sub_out, "#EXT-X-ALLOW-CACHE:%s\n", hls->allowcache == 0 ? "NO" : "YES");
+        }
+        avio_printf(sub_out, "#EXT-X-TARGETDURATION:%d\n", target_duration);
+        avio_printf(sub_out, "#EXT-X-MEDIA-SEQUENCE:%"PRId64"\n", sequence);
+
+        av_log(s, AV_LOG_VERBOSE, "EXT-X-MEDIA-SEQUENCE:%"PRId64"\n",
+               sequence);
+
+        for (en = hls->segments; en; en = en->next) {
+            avio_printf(sub_out, "#EXTINF:%f,\n", en->duration);
+            if (hls->flags & HLS_SINGLE_FILE)
+                 avio_printf(sub_out, "#EXT-X-BYTERANGE:%"PRIi64"@%"PRIi64"\n",
+                         en->size, en->pos);
+            if (hls->baseurl)
+                avio_printf(sub_out, "%s", hls->baseurl);
+            avio_printf(sub_out, "%s\n", en->sub_filename);
+        }
+
+        if (last)
+            avio_printf(sub_out, "#EXT-X-ENDLIST\n");
+
+    }
+
 fail:
    avio_closep(&out);
+    avio_closep(&sub_out);
    if (ret >= 0 && use_rename)
        ff_rename(temp_filename, s->filename, s);
    return ret;
@ -391,19 +467,31 @@ static int hls_start(AVFormatContext *s)
 {
    HLSContext *c = s->priv_data;
    AVFormatContext *oc = c->avf;
+    AVFormatContext *vtt_oc = c->vtt_avf;
    AVDictionary *options = NULL;
    char *filename, iv_string[KEYSIZE*2 + 1];
    int err = 0;

-    if (c->flags & HLS_SINGLE_FILE)
+    if (c->flags & HLS_SINGLE_FILE) {
        av_strlcpy(oc->filename, c->basename,
                   sizeof(oc->filename));
-    else
+        if (c->vtt_basename)
+            av_strlcpy(vtt_oc->filename, c->vtt_basename,
+                  sizeof(vtt_oc->filename));
+    } else {
        if (av_get_frame_filename(oc->filename, sizeof(oc->filename),
                                  c->basename, c->wrap ? c->sequence % c->wrap : c->sequence) < 0) {
            av_log(oc, AV_LOG_ERROR, "Invalid segment filename template '%s'\n", c->basename);
            return AVERROR(EINVAL);
        }
+        if( c->vtt_basename) {
+            if (av_get_frame_filename(vtt_oc->filename, sizeof(vtt_oc->filename),
+                              c->vtt_basename, c->wrap ? c->sequence % c->wrap : c->sequence) < 0) {
+                av_log(vtt_oc, AV_LOG_ERROR, "Invalid segment filename template '%s'\n", c->vtt_basename);
+                return AVERROR(EINVAL);
+            }
+       }
+    }
    c->number++;

    if (c->key_info_file) {
@ -433,10 +521,18 @@ static int hls_start(AVFormatContext *s)
        if ((err = avio_open2(&oc->pb, oc->filename, AVIO_FLAG_WRITE,
                          &s->interrupt_callback, NULL)) < 0)
            return err;
+    if (c->vtt_basename) {
+        if ((err = avio_open2(&vtt_oc->pb, vtt_oc->filename, AVIO_FLAG_WRITE,
+                         &s->interrupt_callback, NULL)) < 0)
+            return err;
+    }

    if (oc->oformat->priv_class && oc->priv_data)
        av_opt_set(oc->priv_data, "mpegts_flags", "resend_headers", 0);

+    if (c->vtt_basename)
+        avformat_write_header(vtt_oc,NULL);
+
    return 0;
 }

@ -446,8 +542,10 @@ static int hls_write_header(AVFormatContext *s)
    int ret, i;
    char *p;
    const char *pattern = "%d.ts";
+    const char *vtt_pattern = "%d.vtt";
    AVDictionary *options = NULL;
    int basename_size;
+    int vtt_basename_size;

    hls->sequence       = hls->start_sequence;
    hls->recording_time = hls->time * AV_TIME_BASE;
@ -461,9 +559,12 @@ static int hls_write_header(AVFormatContext *s)
        }
    }

-    for (i = 0; i < s->nb_streams; i++)
+    for (i = 0; i < s->nb_streams; i++) {
        hls->has_video +=
            s->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO;
+        hls->has_subtitle +=
+            s->streams[i]->codec->codec_type == AVMEDIA_TYPE_SUBTITLE;
+    }

    if (hls->has_video > 1)
        av_log(s, AV_LOG_WARNING,
@ -477,6 +578,14 @@ static int hls_write_header(AVFormatContext *s)
        goto fail;
    }

+    if(hls->has_subtitle) {
+        hls->vtt_oformat = av_guess_format("webvtt", NULL, NULL);
+        if (!hls->oformat) {
+            ret = AVERROR_MUXER_NOT_FOUND;
+            goto fail;
+        }
+    }
+
    if (hls->segment_filename) {
        hls->basename = av_strdup(hls->segment_filename);
        if (!hls->basename) {
@ -502,6 +611,35 @@ static int hls_write_header(AVFormatContext *s)
        av_strlcat(hls->basename, pattern, basename_size);
    }

+    if(hls->has_subtitle) {
+
+        if (hls->flags & HLS_SINGLE_FILE)
+            vtt_pattern = ".vtt";
+        vtt_basename_size = strlen(s->filename) + strlen(vtt_pattern) + 1;
+        hls->vtt_basename = av_malloc(vtt_basename_size);
+        if (!hls->vtt_basename) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
+        hls->vtt_m3u8_name = av_malloc(vtt_basename_size);
+        if (!hls->vtt_m3u8_name ) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
+        av_strlcpy(hls->vtt_basename, s->filename, vtt_basename_size);
+        p = strrchr(hls->vtt_basename, '.');
+        if (p)
+            *p = '\0';
+
+        if( hls->subtitle_filename ) {
+            strcpy(hls->vtt_m3u8_name, hls->subtitle_filename);
+        } else {
+            strcpy(hls->vtt_m3u8_name, hls->vtt_basename);
+            av_strlcat(hls->vtt_m3u8_name, "_vtt.m3u8", vtt_basename_size);
+        }
+        av_strlcat(hls->vtt_basename, vtt_pattern, vtt_basename_size);
+    }
+
    if ((ret = hls_mux_init(s)) < 0)
        goto fail;

@ -515,10 +653,19 @@ static int hls_write_header(AVFormatContext *s)
        ret = AVERROR(EINVAL);
        goto fail;
    }
-    av_assert0(s->nb_streams == hls->avf->nb_streams);
+    //av_assert0(s->nb_streams == hls->avf->nb_streams);
    for (i = 0; i < s->nb_streams; i++) {
-        AVStream *inner_st  = hls->avf->streams[i];
+        AVStream *inner_st;
        AVStream *outer_st = s->streams[i];
+        if (outer_st->codec->codec_type != AVMEDIA_TYPE_SUBTITLE)
+            inner_st = hls->avf->streams[i];
+        else if (hls->vtt_avf)
+            inner_st = hls->vtt_avf->streams[0];
+        else {
+            /* We have subtitle stream, when user dont want */
+            inner_st = NULL;
+            continue;
+        }
        avpriv_set_pts_info(outer_st, inner_st->pts_wrap_bits, inner_st->time_base.num, inner_st->time_base.den);
    }
 fail:
@ -526,8 +673,12 @@ fail:
    av_dict_free(&options);
    if (ret < 0) {
        av_freep(&hls->basename);
+        av_freep(&hls->vtt_basename);
        if (hls->avf)
            avformat_free_context(hls->avf);
+        if (hls->vtt_avf)
+            avformat_free_context(hls->vtt_avf);
+
    }
    return ret;
 }
@ -535,12 +686,20 @@ fail:
 static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
 {
    HLSContext *hls = s->priv_data;
-    AVFormatContext *oc = hls->avf;
+    AVFormatContext *oc = NULL;
    AVStream *st = s->streams[pkt->stream_index];
    int64_t end_pts = hls->recording_time * hls->number;
    int is_ref_pkt = 1;
    int ret, can_split = 1;
+    int stream_index = 0;

+    if( st->codec->codec_type == AVMEDIA_TYPE_SUBTITLE ) {
+        oc = hls->vtt_avf;
+        stream_index = 0;
+    } else {
+        oc = hls->avf;
+        stream_index = pkt->stream_index;
+    }
    if (hls->start_pts == AV_NOPTS_VALUE) {
        hls->start_pts = pkt->pts;
        hls->end_pts   = pkt->pts;
@ -579,6 +738,8 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
            hls->number++;
        } else {
            avio_closep(&oc->pb);
+            if (hls->vtt_avf)
+                avio_close(hls->vtt_avf->pb);

            ret = hls_start(s);
        }
@ -586,13 +747,16 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
        if (ret < 0)
            return ret;

+        if( st->codec->codec_type == AVMEDIA_TYPE_SUBTITLE )
+            oc = hls->vtt_avf;
+        else
        oc = hls->avf;

        if ((ret = hls_window(s, 0)) < 0)
            return ret;
    }

-    ret = ff_write_chained(oc, pkt->stream_index, pkt, s, 0);
+    ret = ff_write_chained(oc, stream_index, pkt, s, 0);

    return ret;
 }
@ -601,6 +765,7 @@ static int hls_write_trailer(struct AVFormatContext *s)
 {
    HLSContext *hls = s->priv_data;
    AVFormatContext *oc = hls->avf;
+    AVFormatContext *vtt_oc = hls->vtt_avf;

    av_write_trailer(oc);
    if (oc->pb) {
@ -608,8 +773,22 @@ static int hls_write_trailer(struct AVFormatContext *s)
        avio_closep(&oc->pb);
        hls_append_segment(hls, hls->duration, hls->start_pos, hls->size);
    }
+
+    if (vtt_oc) {
+        if (vtt_oc->pb)
+            av_write_trailer(vtt_oc);
+        hls->size = avio_tell(hls->vtt_avf->pb) - hls->start_pos;
+        avio_closep(&vtt_oc->pb);
+    }
    av_freep(&hls->basename);
    avformat_free_context(oc);
+
+    if (vtt_oc) {
+        av_freep(&hls->vtt_basename);
+        av_freep(&hls->vtt_m3u8_name);
+        avformat_free_context(vtt_oc);
+    }
+
    hls->avf = NULL;
    hls_window(s, 1);

@ -625,11 +804,13 @@ static const AVOption options[] = {
    {"hls_time",      "set segment length in seconds",           OFFSET(time),    AV_OPT_TYPE_FLOAT,  {.dbl = 2},     0, FLT_MAX, E},
    {"hls_list_size", "set maximum number of playlist entries",  OFFSET(max_nb_segments),    AV_OPT_TYPE_INT,    {.i64 = 5},     0, INT_MAX, E},
    {"hls_ts_options","set hls mpegts list of options for the container format used for hls", OFFSET(format_options_str), AV_OPT_TYPE_STRING, {.str = NULL},  0, 0,    E},
+    {"hls_vtt_options","set hls vtt list of options for the container format used for hls", OFFSET(vtt_format_options_str), AV_OPT_TYPE_STRING, {.str = NULL},  0, 0,    E},
    {"hls_wrap",      "set number after which the index wraps",  OFFSET(wrap),    AV_OPT_TYPE_INT,    {.i64 = 0},     0, INT_MAX, E},
    {"hls_allow_cache", "explicitly set whether the client MAY (1) or MUST NOT (0) cache media segments", OFFSET(allowcache), AV_OPT_TYPE_INT, {.i64 = -1}, INT_MIN, INT_MAX, E},
    {"hls_base_url",  "url to prepend to each playlist entry",   OFFSET(baseurl), AV_OPT_TYPE_STRING, {.str = NULL},  0, 0,       E},
    {"hls_segment_filename", "filename template for segment files", OFFSET(segment_filename),   AV_OPT_TYPE_STRING, {.str = NULL},            0,       0,         E},
    {"hls_key_info_file",    "file with key URI and key file path", OFFSET(key_info_file),      AV_OPT_TYPE_STRING, {.str = NULL},            0,       0,         E},
+    {"hls_subtitle_path",     "set path of hls subtitles", OFFSET(subtitle_filename), AV_OPT_TYPE_STRING, {.str = NULL},  0, 0,    E},
    {"hls_flags",     "set flags affecting HLS playlist and media file generation", OFFSET(flags), AV_OPT_TYPE_FLAGS, {.i64 = 0 }, 0, UINT_MAX, E, "flags"},
    {"single_file",   "generate a single media file indexed with byte ranges", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SINGLE_FILE }, 0, UINT_MAX,   E, "flags"},
    {"delete_segments", "delete segment files that are no longer part of the playlist", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_DELETE_SEGMENTS }, 0, UINT_MAX,   E, "flags"},
@ -655,6 +836,7 @@ AVOutputFormat ff_hls_muxer = {
    .priv_data_size = sizeof(HLSContext),
    .audio_codec    = AV_CODEC_ID_AAC,
    .video_codec    = AV_CODEC_ID_H264,
+    .subtitle_codec = AV_CODEC_ID_WEBVTT,
    .flags          = AVFMT_NOFILE | AVFMT_ALLOW_FLUSH,
    .write_header   = hls_write_header,
    .write_packet   = hls_write_packet,