avcodec/dvdsubdec: Check for fully transparent rectangles earlier

Testcase with large transparent rectangles changes from 67 sec to 3 sec decode time

Fixes: Timeout
Fixes: 8728/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_DVDSUB_fuzzer-5190088756559872

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/dvdsubdec.c

@@ -42,6 +42,7 @@ typedef struct DVDSubContext
   uint8_t  buf[0x10000];
   int      buf_size;
   int      forced_subs_only;
+  uint8_t  used_color[256];
 #ifdef DEBUG
   int sub_id;
 #endif
@@ -100,7 +101,7 @@ static int decode_run_8bit(GetBitContext *gb, int *color)
     return len;
 }
 
-static int decode_rle(uint8_t *bitmap, int linesize, int w, int h,
+static int decode_rle(uint8_t *bitmap, int linesize, int w, int h, uint8_t used_color[256],
                       const uint8_t *buf, int start, int buf_size, int is_8bit)
 {
     GetBitContext gb;
@@ -129,6 +130,7 @@ static int decode_rle(uint8_t *bitmap, int linesize, int w, int h,
             len = decode_run_2bit(&gb, &color);
         len = FFMIN(len, w - x);
         memset(d + x, color, len);
+        used_color[color] = 1;
         x += len;
         if (x >= w) {
             y++;
@@ -370,7 +372,7 @@ static int decode_dvd_subtitles(DVDSubContext *ctx, AVSubtitle *sub_header,
                 h = 0;
             if (w > 0 && h > 1) {
                 reset_rects(sub_header);
-
+                memset(ctx->used_color, 0, sizeof(ctx->used_color));
                 sub_header->rects = av_mallocz(sizeof(*sub_header->rects));
                 if (!sub_header->rects)
                     goto fail;
@@ -381,10 +383,10 @@ static int decode_dvd_subtitles(DVDSubContext *ctx, AVSubtitle *sub_header,
                 bitmap = sub_header->rects[0]->data[0] = av_malloc(w * h);
                 if (!bitmap)
                     goto fail;
-                if (decode_rle(bitmap, w * 2, w, (h + 1) / 2,
+                if (decode_rle(bitmap, w * 2, w, (h + 1) / 2, ctx->used_color,
                                buf, offset1, buf_size, is_8bit) < 0)
                     goto fail;
-                if (decode_rle(bitmap + w, w * 2, w, h / 2,
+                if (decode_rle(bitmap + w, w * 2, w, h / 2, ctx->used_color,
                                buf, offset2, buf_size, is_8bit) < 0)
                     goto fail;
                 sub_header->rects[0]->data[1] = av_mallocz(AVPALETTE_SIZE);
@@ -448,19 +450,24 @@ static int is_transp(const uint8_t *buf, int pitch, int n,
 }
 
 /* return 0 if empty rectangle, 1 if non empty */
-static int find_smallest_bounding_rectangle(AVSubtitle *s)
+static int find_smallest_bounding_rectangle(DVDSubContext *ctx, AVSubtitle *s)
 {
     uint8_t transp_color[256] = { 0 };
     int y1, y2, x1, x2, y, w, h, i;
     uint8_t *bitmap;
+    int transparent = 1;
 
     if (s->num_rects == 0 || !s->rects || s->rects[0]->w <= 0 || s->rects[0]->h <= 0)
         return 0;
 
     for(i = 0; i < s->rects[0]->nb_colors; i++) {
-        if ((((uint32_t *)s->rects[0]->data[1])[i] >> 24) == 0)
+        if ((((uint32_t *)s->rects[0]->data[1])[i] >> 24) == 0) {
             transp_color[i] = 1;
+        } else if (ctx->used_color[i])
+            transparent = 0;
     }
+    if (transparent)
+        return 0;
     y1 = 0;
     while (y1 < s->rects[0]->h && is_transp(s->rects[0]->data[0] + y1 * s->rects[0]->linesize[0],
                                   1, s->rects[0]->w, transp_color))
@@ -595,7 +602,7 @@ static int dvdsub_decode(AVCodecContext *avctx,
 
         return buf_size;
     }
-    if (!is_menu && find_smallest_bounding_rectangle(sub) == 0)
+    if (!is_menu && find_smallest_bounding_rectangle(ctx, sub) == 0)
         goto no_subtitle;
 
     if (ctx->forced_subs_only && !(sub->rects[0]->flags & AV_SUBTITLE_FLAG_FORCED))