diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c
index 97f78e4629..6e958e7141 100644
--- a/libavfilter/avfilter.c
+++ b/libavfilter/avfilter.c
@@ -81,12 +81,41 @@ AVFilterBufferRef *avfilter_ref_buffer(AVFilterBufferRef *ref, int pmask)
     return ret;
 }
 
+static void free_pool(AVFilterPool *pool)
+{
+    int i;
+
+    av_assert0(pool->refcount > 0);
+
+    for (i = 0; i < POOL_SIZE; i++) {
+        if (pool->pic[i]) {
+            AVFilterBufferRef *picref = pool->pic[i];
+            /* free buffer: picrefs stored in the pool are not
+             * supposed to contain a free callback */
+            av_freep(&picref->buf->data[0]);
+            av_freep(&picref->buf);
+
+            av_freep(&picref->audio);
+            av_freep(&picref->video);
+            av_freep(&pool->pic[i]);
+            pool->count--;
+        }
+    }
+    pool->draining = 1;
+
+    if (!--pool->refcount) {
+        av_assert0(!pool->count);
+        av_free(pool);
+    }
+}
+
 static void store_in_pool(AVFilterBufferRef *ref)
 {
     int i;
     AVFilterPool *pool= ref->buf->priv;
 
     av_assert0(ref->buf->data[0]);
+    av_assert0(pool->refcount>0);
 
     if (pool->count == POOL_SIZE) {
         AVFilterBufferRef *ref1 = pool->pic[0];
@@ -107,6 +136,10 @@ static void store_in_pool(AVFilterBufferRef *ref)
             break;
         }
     }
+    if (pool->draining) {
+        free_pool(pool);
+    } else
+        --pool->refcount;
 }
 
 void avfilter_unref_buffer(AVFilterBufferRef *ref)
@@ -181,24 +214,9 @@ void avfilter_link_free(AVFilterLink **link)
     if (!*link)
         return;
 
-    if ((*link)->pool) {
-        int i;
-        for (i = 0; i < POOL_SIZE; i++) {
-            if ((*link)->pool->pic[i]) {
-                AVFilterBufferRef *picref = (*link)->pool->pic[i];
-                /* free buffer: picrefs stored in the pool are not
-                 * supposed to contain a free callback */
-                av_freep(&picref->buf->data[0]);
-                av_freep(&picref->buf);
-
-                av_freep(&picref->audio);
-                av_freep(&picref->video);
-                av_freep(&(*link)->pool->pic[i]);
-            }
-        }
-        (*link)->pool->count = 0;
-//        av_freep(&(*link)->pool);
-    }
+    if ((*link)->pool)
+        free_pool((*link)->pool);
+
     av_freep(link);
 }
 
diff --git a/libavfilter/defaults.c b/libavfilter/defaults.c
index 5f18168374..c083195b4f 100644
--- a/libavfilter/defaults.c
+++ b/libavfilter/defaults.c
@@ -57,11 +57,14 @@ AVFilterBufferRef *avfilter_default_get_video_buffer(AVFilterLink *link, int per
                 pic->refcount = 1;
                 memcpy(picref->data,     pic->data,     sizeof(picref->data));
                 memcpy(picref->linesize, pic->linesize, sizeof(picref->linesize));
+                pool->refcount++;
                 return picref;
             }
         }
-    } else
+    } else {
         pool = link->pool = av_mallocz(sizeof(AVFilterPool));
+        pool->refcount = 1;
+    }
 
     // align: +2 is needed for swscaler, +16 to be SIMD-friendly
     if ((i = av_image_alloc(data, linesize, w, h, link->format, 32)) < 0)
@@ -77,6 +80,7 @@ AVFilterBufferRef *avfilter_default_get_video_buffer(AVFilterLink *link, int per
 
     picref->buf->priv = pool;
     picref->buf->free = NULL;
+    pool->refcount++;
 
     return picref;
 }
diff --git a/libavfilter/internal.h b/libavfilter/internal.h
index 227fe6243d..575bcff364 100644
--- a/libavfilter/internal.h
+++ b/libavfilter/internal.h
@@ -31,6 +31,8 @@
 typedef struct AVFilterPool {
     AVFilterBufferRef *pic[POOL_SIZE];
     int count;
+    int refcount;
+    int draining;
 } AVFilterPool;
 
 typedef struct AVFilterCommand {