diff --git a/libavformat/flvenc.c b/libavformat/flvenc.c
index 7873a0080f..721f062811 100644
--- a/libavformat/flvenc.c
+++ b/libavformat/flvenc.c
@@ -117,12 +117,9 @@ typedef struct FLVContext {
     AVCodecParameters *data_par;
 
     int flags;
+    int64_t last_ts[FLV_STREAM_TYPE_NB];
 } FLVContext;
 
-typedef struct FLVStreamContext {
-    int64_t last_ts;    ///< last timestamp for each stream
-} FLVStreamContext;
-
 static int get_audio_flags(AVFormatContext *s, AVCodecParameters *par)
 {
     int flags = (par->bits_per_coded_sample == 16) ? FLV_SAMPLESSIZE_16BIT
@@ -609,9 +606,15 @@ static int flv_init(struct AVFormatContext *s)
     int i;
     FLVContext *flv = s->priv_data;
 
+    if (s->nb_streams > FLV_STREAM_TYPE_NB) {
+        av_log(s, AV_LOG_ERROR, "invalid number of streams %d\n",
+                s->nb_streams);
+        return AVERROR(EINVAL);
+    }
+
     for (i = 0; i < s->nb_streams; i++) {
         AVCodecParameters *par = s->streams[i]->codecpar;
-        FLVStreamContext *sc;
+
         switch (par->codec_type) {
         case AVMEDIA_TYPE_VIDEO:
             if (s->streams[i]->avg_frame_rate.den &&
@@ -675,12 +678,7 @@ static int flv_init(struct AVFormatContext *s)
             return AVERROR(EINVAL);
         }
         avpriv_set_pts_info(s->streams[i], 32, 1, 1000); /* 32 bit pts in ms */
-
-        sc = av_mallocz(sizeof(FLVStreamContext));
-        if (!sc)
-            return AVERROR(ENOMEM);
-        s->streams[i]->priv_data = sc;
-        sc->last_ts = -1;
+        flv->last_ts[i] = -1;
     }
 
     flv->delay = AV_NOPTS_VALUE;
@@ -783,10 +781,9 @@ end:
         /* Add EOS tag */
         for (i = 0; i < s->nb_streams; i++) {
             AVCodecParameters *par = s->streams[i]->codecpar;
-            FLVStreamContext *sc = s->streams[i]->priv_data;
             if (par->codec_type == AVMEDIA_TYPE_VIDEO &&
                     (par->codec_id == AV_CODEC_ID_H264 || par->codec_id == AV_CODEC_ID_MPEG4))
-                put_eos_tag(pb, sc->last_ts, par->codec_id);
+                put_eos_tag(pb, flv->last_ts[i], par->codec_id);
         }
     }
 
@@ -821,7 +818,6 @@ static int flv_write_packet(AVFormatContext *s, AVPacket *pkt)
     AVIOContext *pb      = s->pb;
     AVCodecParameters *par = s->streams[pkt->stream_index]->codecpar;
     FLVContext *flv      = s->priv_data;
-    FLVStreamContext *sc = s->streams[pkt->stream_index]->priv_data;
     unsigned ts;
     int size = pkt->size;
     uint8_t *data = NULL;
@@ -919,13 +915,13 @@ static int flv_write_packet(AVFormatContext *s, AVPacket *pkt)
     }
 
     /* check Speex packet duration */
-    if (par->codec_id == AV_CODEC_ID_SPEEX && ts - sc->last_ts > 160)
+    if (par->codec_id == AV_CODEC_ID_SPEEX && ts - flv->last_ts[pkt->stream_index] > 160)
         av_log(s, AV_LOG_WARNING, "Warning: Speex stream has more than "
                                   "8 frames per packet. Adobe Flash "
                                   "Player cannot handle this!\n");
 
-    if (sc->last_ts < ts)
-        sc->last_ts = ts;
+    if (flv->last_ts[pkt->stream_index] < ts)
+        flv->last_ts[pkt->stream_index] = ts;
 
     if (size + flags_size >= 1<<24) {
         av_log(s, AV_LOG_ERROR, "Too large packet with size %u >= %u\n",