Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

qtrle: Use bytestream2 functions to prevent buffer overreads.

Browse Source 
Signed-off-by: Ronald S. Bultje <rsbultje@gmail.com>
pull/3/merge
Aneesh Dogra 13 years ago committed by Ronald S. Bultje
parent 341404f753
commit de64d8cf17
1 changed files with 74 additions and 113 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 187
      libavcodec/qtrle.c

187
libavcodec/qtrle.c
Unescape View File

@ -35,27 +35,17 @@
#include <stdlib.h>
#include <string.h>
#include "libavutil/intreadwrite.h"
#include "avcodec.h"
#include "bytestream.h"
typedef struct QtrleContext {
AVCodecContext *avctx;
AVFrame frame;
const unsigned char *buf;
int size;
GetByteContext g;
uint32_t pal[256];
} QtrleContext;
#define CHECK_STREAM_PTR(n) \
if ((stream_ptr + n) > s->size) { \
av_log (s->avctx, AV_LOG_INFO, "Problem: stream_ptr out of bounds (%d >= %d)\n", \
stream_ptr + n, s->size); \
return; \
}
#define CHECK_PIXEL_PTR(n) \
if ((pixel_ptr + n > pixel_limit) || (pixel_ptr + n < 0)) { \
av_log (s->avctx, AV_LOG_INFO, "Problem: pixel_ptr = %d, pixel_limit = %d\n", \
@ -63,7 +53,7 @@ typedef struct QtrleContext {
return; \
} \
static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_1bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr = 0;
@ -74,9 +64,8 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int skip;
while (lines_to_change) {
CHECK_STREAM_PTR(2);
skip = s->buf[stream_ptr++];
rle_code = (signed char)s->buf[stream_ptr++];
skip = bytestream2_get_byte(&s->g);
rle_code = (signed char)bytestream2_get_byte(&s->g);
if (rle_code == 0)
break;
if(skip & 0x80) {
@ -92,9 +81,9 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
rle_code = -rle_code;
/* get the next 2 bytes from the stream, treat them as groups
* of 8 pixels, and output them rle_code times */
CHECK_STREAM_PTR(2);
pi0 = s->buf[stream_ptr++];
pi1 = s->buf[stream_ptr++];
pi0 = bytestream2_get_byte(&s->g);
pi1 = bytestream2_get_byte(&s->g);
CHECK_PIXEL_PTR(rle_code * 2);
while (rle_code--) {
@ -104,17 +93,16 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
} else {
/* copy the same pixel directly to output 2 times */
rle_code *= 2;
CHECK_STREAM_PTR(rle_code);
CHECK_PIXEL_PTR(rle_code);
while (rle_code--)
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
}
}
}
static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
int row_ptr, int lines_to_change, int bpp)
static inline void qtrle_decode_2n4bpp(QtrleContext *s, int row_ptr,
int lines_to_change, int bpp)
{
int rle_code, i;
int pixel_ptr;
@ -125,24 +113,21 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
int num_pixels = (bpp == 4) ? 8 : 16;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (num_pixels * (s->buf[stream_ptr++] - 1));
pixel_ptr = row_ptr + (num_pixels * (bytestream2_get_byte(&s->g) - 1));
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (num_pixels * (s->buf[stream_ptr++] - 1));
pixel_ptr += (num_pixels * (bytestream2_get_byte(&s->g) - 1));
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
/* get the next 4 bytes from the stream, treat them as palette
* indexes, and output them rle_code times */
CHECK_STREAM_PTR(4);
for (i = num_pixels-1; i >= 0; i--) {
pi[num_pixels-1-i] = (s->buf[stream_ptr] >> ((i*bpp) & 0x07)) & ((1<<bpp)-1);
stream_ptr+= ((i & ((num_pixels>>2)-1)) == 0);
pi[num_pixels-1-i] = (bytestream2_peek_byte(&s->g) >> ((i*bpp) & 0x07)) & ((1<<bpp)-1);
bytestream2_skip(&s->g, ((i & ((num_pixels>>2)-1)) == 0));
}
CHECK_PIXEL_PTR(rle_code * num_pixels);
while (rle_code--) {
@ -152,17 +137,18 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
} else {
/* copy the same pixel directly to output 4 times */
rle_code *= 4;
CHECK_STREAM_PTR(rle_code);
CHECK_PIXEL_PTR(rle_code*(num_pixels>>2));
while (rle_code--) {
if(bpp == 4) {
rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 4) & 0x0f;
rgb[pixel_ptr++] = (s->buf[stream_ptr++]) & 0x0f;
int x = bytestream2_get_byte(&s->g);
rgb[pixel_ptr++] = (x >> 4) & 0x0f;
rgb[pixel_ptr++] = x & 0x0f;
} else {
rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 6) & 0x03;
rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 4) & 0x03;
rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 2) & 0x03;
rgb[pixel_ptr++] = (s->buf[stream_ptr++]) & 0x03;
int x = bytestream2_get_byte(&s->g);
rgb[pixel_ptr++] = (x >> 6) & 0x03;
rgb[pixel_ptr++] = (x >> 4) & 0x03;
rgb[pixel_ptr++] = (x >> 2) & 0x03;
rgb[pixel_ptr++] = x & 0x03;
}
}
}
@ -171,7 +157,7 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
}
}
static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_8bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr;
@ -181,25 +167,22 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int pixel_limit = s->frame.linesize[0] * s->avctx->height;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (4 * (s->buf[stream_ptr++] - 1));
pixel_ptr = row_ptr + (4 * (bytestream2_get_byte(&s->g) - 1));
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (4 * (s->buf[stream_ptr++] - 1));
pixel_ptr += (4 * (bytestream2_get_byte(&s->g) - 1));
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
/* get the next 4 bytes from the stream, treat them as palette
* indexes, and output them rle_code times */
CHECK_STREAM_PTR(4);
pi1 = s->buf[stream_ptr++];
pi2 = s->buf[stream_ptr++];
pi3 = s->buf[stream_ptr++];
pi4 = s->buf[stream_ptr++];
pi1 = bytestream2_get_byte(&s->g);
pi2 = bytestream2_get_byte(&s->g);
pi3 = bytestream2_get_byte(&s->g);
pi4 = bytestream2_get_byte(&s->g);
CHECK_PIXEL_PTR(rle_code * 4);
@ -212,11 +195,10 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
} else {
/* copy the same pixel directly to output 4 times */
rle_code *= 4;
CHECK_STREAM_PTR(rle_code);
CHECK_PIXEL_PTR(rle_code);
while (rle_code--) {
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
}
}
}
@ -224,7 +206,7 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
}
}
static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_16bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr;
@ -234,21 +216,17 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int pixel_limit = s->frame.linesize[0] * s->avctx->height;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 2;
pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 2;
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (s->buf[stream_ptr++] - 1) * 2;
pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 2;
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
CHECK_STREAM_PTR(2);
rgb16 = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 2;
rgb16 = bytestream2_get_be16(&s->g);
CHECK_PIXEL_PTR(rle_code * 2);
@ -257,13 +235,11 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
pixel_ptr += 2;
}
} else {
CHECK_STREAM_PTR(rle_code * 2);
CHECK_PIXEL_PTR(rle_code * 2);
/* copy pixels directly to output */
while (rle_code--) {
rgb16 = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 2;
rgb16 = bytestream2_get_be16(&s->g);
*(unsigned short *)(&rgb[pixel_ptr]) = rgb16;
pixel_ptr += 2;
}
@ -273,7 +249,7 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
}
}
static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_24bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr;
@ -283,22 +259,19 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int pixel_limit = s->frame.linesize[0] * s->avctx->height;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 3;
pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 3;
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (s->buf[stream_ptr++] - 1) * 3;
pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 3;
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
CHECK_STREAM_PTR(3);
r = s->buf[stream_ptr++];
g = s->buf[stream_ptr++];
b = s->buf[stream_ptr++];
r = bytestream2_get_byte(&s->g);
g = bytestream2_get_byte(&s->g);
b = bytestream2_get_byte(&s->g);
CHECK_PIXEL_PTR(rle_code * 3);
@ -308,14 +281,13 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
rgb[pixel_ptr++] = b;
}
} else {
CHECK_STREAM_PTR(rle_code * 3);
CHECK_PIXEL_PTR(rle_code * 3);
/* copy pixels directly to output */
while (rle_code--) {
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = s->buf[stream_ptr++];
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
}
}
}
@ -323,7 +295,7 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
}
}
static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
static void qtrle_decode_32bpp(QtrleContext *s, int row_ptr, int lines_to_change)
{
int rle_code;
int pixel_ptr;
@ -333,21 +305,17 @@ static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
int pixel_limit = s->frame.linesize[0] * s->avctx->height;
while (lines_to_change--) {
CHECK_STREAM_PTR(2);
pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 4;
pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 4;
while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
if (rle_code == 0) {
/* there's another skip code in the stream */
CHECK_STREAM_PTR(1);
pixel_ptr += (s->buf[stream_ptr++] - 1) * 4;
pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 4;
CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
} else if (rle_code < 0) {
/* decode the run length code */
rle_code = -rle_code;
CHECK_STREAM_PTR(4);
argb = AV_RB32(s->buf + stream_ptr);
stream_ptr += 4;
argb = bytestream2_get_be32(&s->g);
CHECK_PIXEL_PTR(rle_code * 4);
@ -356,14 +324,12 @@ static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
pixel_ptr += 4;
}
} else {
CHECK_STREAM_PTR(rle_code * 4);
CHECK_PIXEL_PTR(rle_code * 4);
/* copy pixels directly to output */
while (rle_code--) {
argb = AV_RB32(s->buf + stream_ptr);
argb = bytestream2_get_be32(&s->g);
AV_WN32A(rgb + pixel_ptr, argb);
stream_ptr += 4;
pixel_ptr += 4;
}
}
@ -419,16 +385,12 @@ static int qtrle_decode_frame(AVCodecContext *avctx,
void *data, int *data_size,
AVPacket *avpkt)
{
const uint8_t *buf = avpkt->data;
int buf_size = avpkt->size;
QtrleContext *s = avctx->priv_data;
int header, start_line;
int stream_ptr, height, row_ptr;
int height, row_ptr;
int has_palette = 0;
s->buf = buf;
s->size = buf_size;
bytestream2_init(&s->g, avpkt->data, avpkt->size);
s->frame.reference = 1;
s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE |
FF_BUFFER_HINTS_REUSABLE | FF_BUFFER_HINTS_READABLE;
@ -438,64 +400,63 @@ static int qtrle_decode_frame(AVCodecContext *avctx,
}
/* check if this frame is even supposed to change */
if (s->size < 8)
if (avpkt->size < 8)
goto done;
/* start after the chunk size */
stream_ptr = 4;
bytestream2_seek(&s->g, 4, SEEK_SET);
/* fetch the header */
header = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 2;
header = bytestream2_get_be16(&s->g);
/* if a header is present, fetch additional decoding parameters */
if (header & 0x0008) {
if(s->size < 14)
if (avpkt->size < 14)
goto done;
start_line = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 4;
height = AV_RB16(&s->buf[stream_ptr]);
stream_ptr += 4;
start_line = bytestream2_get_be16(&s->g);
bytestream2_skip(&s->g, 2);
height = bytestream2_get_be16(&s->g);
bytestream2_skip(&s->g, 2);
} else {
start_line = 0;
height = s->avctx->height;
height = s->avctx->height;
}
row_ptr = s->frame.linesize[0] * start_line;
switch (avctx->bits_per_coded_sample) {
case 1:
case 33:
qtrle_decode_1bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_1bpp(s, row_ptr, height);
break;
case 2:
case 34:
qtrle_decode_2n4bpp(s, stream_ptr, row_ptr, height, 2);
qtrle_decode_2n4bpp(s, row_ptr, height, 2);
has_palette = 1;
break;
case 4:
case 36:
qtrle_decode_2n4bpp(s, stream_ptr, row_ptr, height, 4);
qtrle_decode_2n4bpp(s, row_ptr, height, 4);
has_palette = 1;
break;
case 8:
case 40:
qtrle_decode_8bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_8bpp(s, row_ptr, height);
has_palette = 1;
break;
case 16:
qtrle_decode_16bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_16bpp(s, row_ptr, height);
break;
case 24:
qtrle_decode_24bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_24bpp(s, row_ptr, height);
break;
case 32:
qtrle_decode_32bpp(s, stream_ptr, row_ptr, height);
qtrle_decode_32bpp(s, row_ptr, height);
break;
default:
@ -521,7 +482,7 @@ done:
*(AVFrame*)data = s->frame;
/* always report that the buffer was completely consumed */
return buf_size;
return avpkt->size;
}
static av_cold int qtrle_decode_end(AVCodecContext *avctx)

Write Preview
Loading…
Cancel
Save