diff --git a/libavfilter/vf_hwmap.c b/libavfilter/vf_hwmap.c
index b28cb21456..d5ad2768f2 100644
--- a/libavfilter/vf_hwmap.c
+++ b/libavfilter/vf_hwmap.c
@@ -138,7 +138,9 @@ static int hwmap_config_output(AVFilterLink *outlink)
             frames->sw_format = hwfc->sw_format;
             frames->width     = hwfc->width;
             frames->height    = hwfc->height;
-            frames->initial_pool_size = 64;
+
+            if (avctx->extra_hw_frames >= 0)
+                frames->initial_pool_size = 2 + avctx->extra_hw_frames;
 
             err = av_hwframe_ctx_init(ctx->hwframes_ref);
             if (err < 0) {
@@ -218,6 +220,9 @@ static int hwmap_config_output(AVFilterLink *outlink)
         hwfc->width     = inlink->w;
         hwfc->height    = inlink->h;
 
+        if (avctx->extra_hw_frames >= 0)
+            hwfc->initial_pool_size = 2 + avctx->extra_hw_frames;
+
         err = av_hwframe_ctx_init(ctx->hwframes_ref);
         if (err < 0) {
             av_log(avctx, AV_LOG_ERROR, "Failed to create frame "
diff --git a/libavfilter/vf_hwupload.c b/libavfilter/vf_hwupload.c
index 8cca9f42e2..af4ff9b81f 100644
--- a/libavfilter/vf_hwupload.c
+++ b/libavfilter/vf_hwupload.c
@@ -133,6 +133,9 @@ static int hwupload_config_output(AVFilterLink *outlink)
     ctx->hwframes->width     = inlink->w;
     ctx->hwframes->height    = inlink->h;
 
+    if (avctx->extra_hw_frames >= 0)
+        ctx->hwframes->initial_pool_size = 2 + avctx->extra_hw_frames;
+
     err = av_hwframe_ctx_init(ctx->hwframes_ref);
     if (err < 0)
         goto fail;