tools/target_dec_fuzzer: Limit number off all pixels decoded

This should reduces the number of uninteresting timeouts encountered

A single threshold for all codecs did not work

Fixes: 13979/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_QTRLE_fuzzer-5629872380051456 (14sec -> 4sec)
Fixes: 14709/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_GDV_fuzzer-5704215281795072 (179sec -> 7sec)
Fixes: 16296/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_HNM4_VIDEO_fuzzer-5756304521428992 (108sec -> 9sec)
Fixes: 15620/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_GIF_fuzzer-5657214435459072 (26sec -> 26ms)

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

tools/target_dec_fuzzer.c

@@ -128,6 +128,8 @@ static void FDBPrepare(FuzzDataBuffer *FDB, AVPacket *dst, const uint8_t *data,
 
 // Ensure we don't loop forever
 const uint32_t maxiteration = 8096;
+const uint64_t maxpixels_per_frame = 4096 * 4096;
+uint64_t maxpixels;
 
 static const uint64_t FUZZ_TAG = 0x4741542D5A5A5546ULL;
 
@@ -165,13 +167,24 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     case AVMEDIA_TYPE_VIDEO   : decode_handler = avcodec_decode_video2; break;
     case AVMEDIA_TYPE_SUBTITLE: decode_handler = subtitle_handler     ; break;
     }
+    maxpixels = maxpixels_per_frame * maxiteration;
+    switch (c->id) {
+        // Allows a small input to generate gigantic output
+    case AV_CODEC_ID_QTRLE:     maxpixels /= 16;
+    case AV_CODEC_ID_GIF:       maxpixels /= 16;
+        // Performs slow frame rescaling in C
+    case AV_CODEC_ID_GDV:       maxpixels /= 256;
+        // Postprocessing in C
+    case AV_CODEC_ID_HNM4_VIDEO:maxpixels /= 128;
+    }
+
 
     AVCodecContext* ctx = avcodec_alloc_context3(NULL);
     AVCodecContext* parser_avctx = avcodec_alloc_context3(NULL);
     if (!ctx || !parser_avctx)
         error("Failed memory allocation");
 
-    ctx->max_pixels = 4096 * 4096; //To reduce false positive OOM and hangs
+    ctx->max_pixels = maxpixels_per_frame; //To reduce false positive OOM and hangs
 
     if (size > 1024) {
         GetByteContext gbc;
@@ -260,6 +273,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
             ec_pixels += ctx->width * ctx->height;
             if (it > 20 || ec_pixels > 4 * ctx->max_pixels)
                 ctx->error_concealment = 0;
+            if (ec_pixels > maxpixels)
+                goto maximums_reached;
 
             if (ret <= 0 || ret > avpkt.size)
                break;
@@ -270,6 +285,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
           }
         }
     }
+maximums_reached:
 
     av_init_packet(&avpkt);
     avpkt.data = NULL;