avformat/avidec: Prevent entity expansion attacks

Fixes: Timeout
Fixes no testcase, this is the same idea as similar attacks against XML parsers

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/avidec.c

@@ -82,6 +82,8 @@ typedef struct AVIContext {
     int stream_index;
     DVDemuxContext *dv_demux;
     int odml_depth;
+    int64_t odml_read;
+    int64_t odml_max_pos;
     int use_odml;
 #define MAX_ODML_DEPTH 1000
     int64_t dts_max;
@@ -200,7 +202,7 @@ static int read_odml_index(AVFormatContext *s, int64_t frame_num)
     st  = s->streams[stream_id];
     ast = st->priv_data;
 
-    if (index_sub_type)
+    if (index_sub_type || entries_in_use < 0)
         return AVERROR_INVALIDDATA;
 
     avio_rl32(pb);
@@ -221,11 +223,18 @@ static int read_odml_index(AVFormatContext *s, int64_t frame_num)
     }
 
     for (i = 0; i < entries_in_use; i++) {
+        avi->odml_max_pos = FFMAX(avi->odml_max_pos, avio_tell(pb));
+
+        // If we read more than there are bytes then we must have been reading something twice
+        if (avi->odml_read > avi->odml_max_pos)
+            return AVERROR_INVALIDDATA;
+
         if (index_type) {
             int64_t pos = avio_rl32(pb) + base - 8;
             int len     = avio_rl32(pb);
             int key     = len >= 0;
             len &= 0x7FFFFFFF;
+            avi->odml_read += 8;
 
             av_log(s, AV_LOG_TRACE, "pos:%"PRId64", len:%X\n", pos, len);
 
@@ -244,6 +253,7 @@ static int read_odml_index(AVFormatContext *s, int64_t frame_num)
             int64_t offset, pos;
             int duration;
             int ret;
+            avi->odml_read += 16;
 
             offset = avio_rl64(pb);
             avio_rl32(pb);       /* size */