hqdn3d: check memory allocations and propagate errors

pull/109/head
Vittorio Giovara 10 years ago
parent 70d246d5cc
commit 22b985d59c
  1. 42
      libavfilter/vf_hqdn3d.c

@ -121,11 +121,11 @@ static void denoise_spatial(HQDN3DContext *s,
} }
av_always_inline av_always_inline
static void denoise_depth(HQDN3DContext *s, static int denoise_depth(HQDN3DContext *s,
uint8_t *src, uint8_t *dst, uint8_t *src, uint8_t *dst,
uint16_t *line_ant, uint16_t **frame_ant_ptr, uint16_t *line_ant, uint16_t **frame_ant_ptr,
int w, int h, int sstride, int dstride, int w, int h, int sstride, int dstride,
int16_t *spatial, int16_t *temporal, int depth) int16_t *spatial, int16_t *temporal, int depth)
{ {
// FIXME: For 16bit depth, frame_ant could be a pointer to the previous // FIXME: For 16bit depth, frame_ant could be a pointer to the previous
// filtered frame rather than a separate buffer. // filtered frame rather than a separate buffer.
@ -134,6 +134,8 @@ static void denoise_depth(HQDN3DContext *s,
if (!frame_ant) { if (!frame_ant) {
uint8_t *frame_src = src; uint8_t *frame_src = src;
*frame_ant_ptr = frame_ant = av_malloc(w*h*sizeof(uint16_t)); *frame_ant_ptr = frame_ant = av_malloc(w*h*sizeof(uint16_t));
if (!frame_ant)
return AVERROR(ENOMEM);
for (y = 0; y < h; y++, src += sstride, frame_ant += w) for (y = 0; y < h; y++, src += sstride, frame_ant += w)
for (x = 0; x < w; x++) for (x = 0; x < w; x++)
frame_ant[x] = LOAD(x); frame_ant[x] = LOAD(x);
@ -148,15 +150,25 @@ static void denoise_depth(HQDN3DContext *s,
denoise_temporal(src, dst, frame_ant, denoise_temporal(src, dst, frame_ant,
w, h, sstride, dstride, temporal, depth); w, h, sstride, dstride, temporal, depth);
emms_c(); emms_c();
return 0;
} }
#define denoise(...) \ #define denoise(...) \
switch (s->depth) {\ do { \
case 8: denoise_depth(__VA_ARGS__, 8); break;\ int ret = AVERROR_INVALIDDATA; \
case 9: denoise_depth(__VA_ARGS__, 9); break;\ switch (s->depth) { \
case 10: denoise_depth(__VA_ARGS__, 10); break;\ case 8: ret = denoise_depth(__VA_ARGS__, 8); break; \
case 16: denoise_depth(__VA_ARGS__, 16); break;\ case 9: ret = denoise_depth(__VA_ARGS__, 9); break; \
} case 10: ret = denoise_depth(__VA_ARGS__, 10); break; \
case 16: ret = denoise_depth(__VA_ARGS__, 16); break; \
} \
if (ret < 0) { \
av_frame_free(&out); \
if (!direct) \
av_frame_free(&in); \
return ret; \
} \
} while (0)
static int16_t *precalc_coefs(double dist25, int depth) static int16_t *precalc_coefs(double dist25, int depth)
{ {
@ -280,13 +292,11 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
HQDN3DContext *s = inlink->dst->priv; HQDN3DContext *s = inlink->dst->priv;
AVFilterLink *outlink = inlink->dst->outputs[0]; AVFilterLink *outlink = inlink->dst->outputs[0];
AVFrame *out; AVFrame *out;
int direct, c; int c, direct = av_frame_is_writable(in);
if (av_frame_is_writable(in)) { if (direct) {
direct = 1;
out = in; out = in;
} else { } else {
direct = 0;
out = ff_get_video_buffer(outlink, outlink->w, outlink->h); out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
if (!out) { if (!out) {
av_frame_free(&in); av_frame_free(&in);

Loading…
Cancel
Save