avformat/rpl: Use 64bit for total_audio_size and check it

Fixes: 62276/clusterfuzz-testcase-minimized-ffmpeg_dem_RPL_fuzzer-4677434693517312
Fixes: signed integer overflow: 5555555555555555556 * 8 cannot be represented in type 'long long'

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 878625812f)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/rpl.c

@@ -117,7 +117,7 @@ static int rpl_read_header(AVFormatContext *s)
     AVIOContext *pb = s->pb;
     RPLContext *rpl = s->priv_data;
     AVStream *vst = NULL, *ast = NULL;
-    int total_audio_size;
+    int64_t total_audio_size;
     int error = 0;
     const char *endptr;
     char audio_type[RPL_LINE_LENGTH];
@@ -302,6 +302,8 @@ static int rpl_read_header(AVFormatContext *s)
         if (ast)
             av_add_index_entry(ast, offset + video_size, total_audio_size,
                                audio_size, audio_size * 8, 0);
+        if (total_audio_size/8 + (uint64_t)audio_size >= INT64_MAX/8)
+            return AVERROR_INVALIDDATA;
         total_audio_size += audio_size * 8;
     }