avformat/bintext: Check avio_size() return

Fixes: CID1604503 Overflowed constant
Fixes: CID1604566 Overflowed constant

Sponsored-by: Sovereign Tech Fund
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit bf61f811e7)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/bintext.c

@@ -93,9 +93,12 @@ static int next_tag_read(AVFormatContext *avctx, uint64_t *fsize)
     AVIOContext *pb = avctx->pb;
     char buf[36];
     int len;
-    uint64_t start_pos = avio_size(pb) - 256;
+    int64_t start_pos = avio_size(pb);
 
-    avio_seek(pb, start_pos, SEEK_SET);
+    if (start_pos < 256)
+        return AVERROR_INVALIDDATA;
+
+    avio_seek(pb, start_pos - 256, SEEK_SET);
     if (avio_read(pb, buf, sizeof(next_magic)) != sizeof(next_magic))
         return -1;
     if (memcmp(buf, next_magic, sizeof(next_magic)))
@@ -245,7 +248,10 @@ static int xbin_read_header(AVFormatContext *s)
         return AVERROR(EIO);
 
     if (pb->seekable & AVIO_SEEKABLE_NORMAL) {
-        bin->fsize = avio_size(pb) - 9 - st->codecpar->extradata_size;
+        int64_t fsize =  avio_size(pb);
+        if (fsize < 9 + st->codecpar->extradata_size)
+            return 0;
+        bin->fsize = fsize - 9 - st->codecpar->extradata_size;
         ff_sauce_read(s, &bin->fsize, NULL, 0);
         avio_seek(pb, 9 + st->codecpar->extradata_size, SEEK_SET);
     }
@@ -285,7 +291,10 @@ static int adf_read_header(AVFormatContext *s)
 
     if (pb->seekable & AVIO_SEEKABLE_NORMAL) {
         int got_width = 0;
-        bin->fsize = avio_size(pb) - 1 - 192 - 4096;
+        int64_t fsize =  avio_size(pb);
+        if (fsize < 1 + 192 + 4096)
+            return 0;
+        bin->fsize = fsize - 1 - 192 - 4096;
         st->codecpar->width = 80<<3;
         ff_sauce_read(s, &bin->fsize, &got_width, 0);
         if (st->codecpar->width < 8)
@@ -318,6 +327,7 @@ static int idf_read_header(AVFormatContext *s)
     AVIOContext *pb = s->pb;
     AVStream *st;
     int got_width = 0, ret;
+    int64_t fsize;
 
     if (!(pb->seekable & AVIO_SEEKABLE_NORMAL))
         return AVERROR(EIO);
@@ -332,14 +342,18 @@ static int idf_read_header(AVFormatContext *s)
     st->codecpar->extradata[0] = 16;
     st->codecpar->extradata[1] = BINTEXT_PALETTE|BINTEXT_FONT;
 
-    avio_seek(pb, avio_size(pb) - 4096 - 48, SEEK_SET);
+    fsize = avio_size(pb);
+    if (fsize < 12 + 4096 + 48)
+        return AVERROR_INVALIDDATA;
+    bin->fsize = fsize - 12 - 4096 - 48;
+
+    avio_seek(pb, bin->fsize + 12, SEEK_SET);
 
     if (avio_read(pb, st->codecpar->extradata + 2 + 48, 4096) < 0)
         return AVERROR(EIO);
     if (avio_read(pb, st->codecpar->extradata + 2, 48) < 0)
         return AVERROR(EIO);
 
-    bin->fsize = avio_size(pb) - 12 - 4096 - 48;
     ff_sauce_read(s, &bin->fsize, &got_width, 0);
     if (st->codecpar->width < 8)
         return AVERROR_INVALIDDATA;