avcodec/speedhq: fix decoding non 444 subsampling when width is not multiple of 16

libavcodec/speedhq.c

@@ -276,6 +276,78 @@ static inline int decode_dct_block(const SHQContext *s, GetBitContext *gb, int l
     return 0;
 }
 
+static int decode_speedhq_border(const SHQContext *s, GetBitContext *gb, AVFrame *frame, int field_number, int line_stride)
+{
+    int linesize_y  = frame->linesize[0] * line_stride;
+    int linesize_cb = frame->linesize[1] * line_stride;
+    int linesize_cr = frame->linesize[2] * line_stride;
+    int linesize_a;
+    int ret;
+
+    if (s->alpha_type != SHQ_NO_ALPHA)
+        linesize_a = frame->linesize[3] * line_stride;
+
+    for (int y = 0; y < frame->height; y += 16 * line_stride) {
+        int last_dc[4] = { 1024, 1024, 1024, 1024 };
+        uint8_t *dest_y, *dest_cb, *dest_cr, *dest_a;
+        uint8_t last_alpha[16];
+        int x = frame->width - 8;
+
+        dest_y = frame->data[0] + frame->linesize[0] * (y + field_number) + x;
+        if (s->subsampling == SHQ_SUBSAMPLING_420) {
+            dest_cb = frame->data[1] + frame->linesize[1] * (y/2 + field_number) + x / 2;
+            dest_cr = frame->data[2] + frame->linesize[2] * (y/2 + field_number) + x / 2;
+        } else if (s->subsampling == SHQ_SUBSAMPLING_422) {
+            dest_cb = frame->data[1] + frame->linesize[1] * (y + field_number) + x / 2;
+            dest_cr = frame->data[2] + frame->linesize[2] * (y + field_number) + x / 2;
+        }
+        if (s->alpha_type != SHQ_NO_ALPHA) {
+            memset(last_alpha, 255, sizeof(last_alpha));
+            dest_a = frame->data[3] + frame->linesize[3] * (y + field_number) + x;
+        }
+
+        if ((ret = decode_dct_block(s, gb, last_dc, 0, dest_y, linesize_y)) < 0)
+            return ret;
+        if ((ret = decode_dct_block(s, gb, last_dc, 0, dest_y + 8, linesize_y)) < 0)
+            return ret;
+        if ((ret = decode_dct_block(s, gb, last_dc, 0, dest_y + 8 * linesize_y, linesize_y)) < 0)
+            return ret;
+        if ((ret = decode_dct_block(s, gb, last_dc, 0, dest_y + 8 * linesize_y + 8, linesize_y)) < 0)
+            return ret;
+        if ((ret = decode_dct_block(s, gb, last_dc, 1, dest_cb, linesize_cb)) < 0)
+            return ret;
+        if ((ret = decode_dct_block(s, gb, last_dc, 2, dest_cr, linesize_cr)) < 0)
+            return ret;
+
+        if (s->subsampling != SHQ_SUBSAMPLING_420) {
+            if ((ret = decode_dct_block(s, gb, last_dc, 1, dest_cb + 8 * linesize_cb, linesize_cb)) < 0)
+                return ret;
+            if ((ret = decode_dct_block(s, gb, last_dc, 2, dest_cr + 8 * linesize_cr, linesize_cr)) < 0)
+                return ret;
+        }
+
+        if (s->alpha_type == SHQ_RLE_ALPHA) {
+            /* Alpha coded using 16x8 RLE blocks. */
+            if ((ret = decode_alpha_block(s, gb, last_alpha, dest_a, linesize_a)) < 0)
+                return ret;
+            if ((ret = decode_alpha_block(s, gb, last_alpha, dest_a + 8 * linesize_a, linesize_a)) < 0)
+                return ret;
+        } else if (s->alpha_type == SHQ_DCT_ALPHA) {
+            /* Alpha encoded exactly like luma. */
+            if ((ret = decode_dct_block(s, gb, last_dc, 3, dest_a, linesize_a)) < 0)
+                return ret;
+            if ((ret = decode_dct_block(s, gb, last_dc, 3, dest_a + 8, linesize_a)) < 0)
+                return ret;
+            if ((ret = decode_dct_block(s, gb, last_dc, 3, dest_a + 8 * linesize_a, linesize_a)) < 0)
+                return ret;
+            if ((ret = decode_dct_block(s, gb, last_dc, 3, dest_a + 8 * linesize_a + 8, linesize_a)) < 0)
+                return ret;
+        }
+    }
+
+    return 0;
+}
+
 static int decode_speedhq_field(const SHQContext *s, const uint8_t *buf, int buf_size, AVFrame *frame, int field_number, int start, int end, int line_stride)
 {
     int ret, slice_number, slice_offsets[5];
@@ -283,6 +355,7 @@ static int decode_speedhq_field(const SHQContext *s, const uint8_t *buf, int buf
     int linesize_cb = frame->linesize[1] * line_stride;
     int linesize_cr = frame->linesize[2] * line_stride;
     int linesize_a;
+    GetBitContext gb;
 
     if (s->alpha_type != SHQ_NO_ALPHA)
         linesize_a = frame->linesize[3] * line_stride;
@@ -304,7 +377,6 @@ static int decode_speedhq_field(const SHQContext *s, const uint8_t *buf, int buf
     }
 
     for (slice_number = 0; slice_number < 4; slice_number++) {
-        GetBitContext gb;
         uint32_t slice_begin, slice_end;
         int x, y;
 
@@ -333,7 +405,7 @@ static int decode_speedhq_field(const SHQContext *s, const uint8_t *buf, int buf
                 dest_a = frame->data[3] + frame->linesize[3] * (y + field_number);
             }
 
-            for (x = 0; x < frame->width; x += 16) {
+            for (x = 0; x < frame->width - 8 * (s->subsampling != SHQ_SUBSAMPLING_444); x += 16) {
                 /* Decode the four luma blocks. */
                 if ((ret = decode_dct_block(s, &gb, last_dc, 0, dest_y, linesize_y)) < 0)
                     return ret;
@@ -402,6 +474,9 @@ static int decode_speedhq_field(const SHQContext *s, const uint8_t *buf, int buf
         }
     }
 
+    if (s->subsampling != SHQ_SUBSAMPLING_444 && (frame->width & 15))
+        return decode_speedhq_border(s, &gb, frame, field_number, line_stride);
+
     return 0;
 }