exr: rle decompression

Signed-off-by: Paul B Mahol <onemda@gmail.com>

libavcodec/exr.c

@@ -34,6 +34,7 @@
 
 #include "avcodec.h"
 #include "bytestream.h"
+#include "mathops.h"
 #include "libavutil/imgutils.h"
 
 enum ExrCompr {
@@ -183,6 +184,42 @@ static void reorder_pixels(uint8_t *src, uint8_t *dst, int size)
     }
 }
 
+static int rle_uncompress(const uint8_t *src, int ssize, uint8_t *dst, int dsize)
+{
+    int8_t *d = (int8_t *)dst;
+    int8_t *s = (int8_t *)src;
+    int8_t *dend = d + dsize;
+    int count;
+
+    while (ssize > 0) {
+        count = *s++;
+
+        if (count < 0) {
+            count = -count;
+
+            if ((dsize -= count    ) < 0 ||
+                (ssize -= count + 1) < 0)
+                return -1;
+
+            while (count--)
+                *d++ = *s++;
+        } else {
+            count++;
+
+            if ((dsize -= count) < 0 ||
+                (ssize -= 2    ) < 0)
+                return -1;
+
+            while (count--)
+                *d++ = *s;
+
+            s++;
+        }
+    }
+
+    return dend != d;
+}
+
 static int decode_frame(AVCodecContext *avctx,
                         void *data,
                         int *data_size,
@@ -366,10 +403,10 @@ static int decode_frame(AVCodecContext *avctx,
             s->compr = *buf;
             switch (s->compr) {
             case EXR_RAW:
+            case EXR_RLE:
             case EXR_ZIP1:
             case EXR_ZIP16:
                 break;
-            case EXR_RLE:
             case EXR_PIZ:
             case EXR_B44:
             default:
@@ -431,6 +468,7 @@ static int decode_frame(AVCodecContext *avctx,
 
     switch (s->compr) {
     case EXR_RAW:
+    case EXR_RLE:
     case EXR_ZIP1:
         scan_lines_per_block = 1;
         break;
@@ -508,7 +546,14 @@ static int decode_frame(AVCodecContext *avctx,
                         av_log(avctx, AV_LOG_ERROR, "error during zlib decompression\n");
                         return AVERROR(EINVAL);
                     }
+                } else if (s->compr == EXR_RLE && data_size < uncompressed_size) {
+                    if (rle_uncompress(avpkt->data + line_offset, data_size, s->tmp, uncompressed_size)) {
+                        av_log(avctx, AV_LOG_ERROR, "error during rle decompression\n");
+                        return AVERROR(EINVAL);
+                    }
+                }
 
+                if (s->compr != EXR_RAW && data_size < uncompressed_size) {
                     predictor(s->tmp, uncompressed_size);
                     reorder_pixels(s->tmp, s->uncompressed_data, uncompressed_size);