hqdn3d: check memory allocations and propagate errors

pull/109/head
Vittorio Giovara 10 years ago
parent 70d246d5cc
commit 22b985d59c
  1. 42
      libavfilter/vf_hqdn3d.c

@ -121,11 +121,11 @@ static void denoise_spatial(HQDN3DContext *s,
}
av_always_inline
static void denoise_depth(HQDN3DContext *s,
uint8_t *src, uint8_t *dst,
uint16_t *line_ant, uint16_t **frame_ant_ptr,
int w, int h, int sstride, int dstride,
int16_t *spatial, int16_t *temporal, int depth)
static int denoise_depth(HQDN3DContext *s,
uint8_t *src, uint8_t *dst,
uint16_t *line_ant, uint16_t **frame_ant_ptr,
int w, int h, int sstride, int dstride,
int16_t *spatial, int16_t *temporal, int depth)
{
// FIXME: For 16bit depth, frame_ant could be a pointer to the previous
// filtered frame rather than a separate buffer.
@ -134,6 +134,8 @@ static void denoise_depth(HQDN3DContext *s,
if (!frame_ant) {
uint8_t *frame_src = src;
*frame_ant_ptr = frame_ant = av_malloc(w*h*sizeof(uint16_t));
if (!frame_ant)
return AVERROR(ENOMEM);
for (y = 0; y < h; y++, src += sstride, frame_ant += w)
for (x = 0; x < w; x++)
frame_ant[x] = LOAD(x);
@ -148,15 +150,25 @@ static void denoise_depth(HQDN3DContext *s,
denoise_temporal(src, dst, frame_ant,
w, h, sstride, dstride, temporal, depth);
emms_c();
return 0;
}
#define denoise(...) \
switch (s->depth) {\
case 8: denoise_depth(__VA_ARGS__, 8); break;\
case 9: denoise_depth(__VA_ARGS__, 9); break;\
case 10: denoise_depth(__VA_ARGS__, 10); break;\
case 16: denoise_depth(__VA_ARGS__, 16); break;\
}
#define denoise(...) \
do { \
int ret = AVERROR_INVALIDDATA; \
switch (s->depth) { \
case 8: ret = denoise_depth(__VA_ARGS__, 8); break; \
case 9: ret = denoise_depth(__VA_ARGS__, 9); break; \
case 10: ret = denoise_depth(__VA_ARGS__, 10); break; \
case 16: ret = denoise_depth(__VA_ARGS__, 16); break; \
} \
if (ret < 0) { \
av_frame_free(&out); \
if (!direct) \
av_frame_free(&in); \
return ret; \
} \
} while (0)
static int16_t *precalc_coefs(double dist25, int depth)
{
@ -280,13 +292,11 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
HQDN3DContext *s = inlink->dst->priv;
AVFilterLink *outlink = inlink->dst->outputs[0];
AVFrame *out;
int direct, c;
int c, direct = av_frame_is_writable(in);
if (av_frame_is_writable(in)) {
direct = 1;
if (direct) {
out = in;
} else {
direct = 0;
out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
if (!out) {
av_frame_free(&in);

Loading…
Cancel
Save