j2k/jpeg2000: check cblk size

This is based on j2k FCD which is ambigous thus the limit
might be tighter.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/j2kdec.c

@@ -233,8 +233,15 @@ static int get_cox(Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c)
     if (bytestream2_get_bytes_left(&s->g) < 5)
         return AVERROR(EINVAL);
           c->nreslevels = bytestream2_get_byteu(&s->g) + 1; // num of resolution levels - 1
-     c->log2_cblk_width = bytestream2_get_byteu(&s->g) + 2; // cblk width
+
-    c->log2_cblk_height = bytestream2_get_byteu(&s->g) + 2; // cblk height
+    c->log2_cblk_width  = (bytestream2_get_byteu(&s->g) & 15) + 2; // cblk width
+    c->log2_cblk_height = (bytestream2_get_byteu(&s->g) & 15) + 2; // cblk height
+
+    if (c->log2_cblk_width > 10 || c->log2_cblk_height > 10 ||
+        c->log2_cblk_width + c->log2_cblk_height > 14) {
+        av_log(s->avctx, AV_LOG_ERROR, "cblk size invalid\n");
+        return AVERROR_INVALIDDATA;
+    }
 
     c->cblk_style = bytestream2_get_byteu(&s->g);
     if (c->cblk_style != 0) { // cblk style

libavcodec/jpeg2000dec.c

@@ -263,8 +263,14 @@ static int get_cox(Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c)
     else
         c->nreslevels2decode = c->nreslevels - s->reduction_factor;
 
-    c->log2_cblk_width  = bytestream2_get_byteu(&s->g) + 2; // cblk width
+    c->log2_cblk_width  = (bytestream2_get_byteu(&s->g) & 15) + 2; // cblk width
-    c->log2_cblk_height = bytestream2_get_byteu(&s->g) + 2; // cblk height
+    c->log2_cblk_height = (bytestream2_get_byteu(&s->g) & 15) + 2; // cblk height
+
+    if (c->log2_cblk_width > 10 || c->log2_cblk_height > 10 ||
+        c->log2_cblk_width + c->log2_cblk_height > 14) {
+        av_log(s->avctx, AV_LOG_ERROR, "cblk size invalid\n");
+        return AVERROR_INVALIDDATA;
+    }
 
     c->cblk_style = bytestream2_get_byteu(&s->g);
     if (c->cblk_style != 0) { // cblk style