diff --git a/libavfilter/audio.c b/libavfilter/audio.c
index 35270c14d2..652b7ff3b8 100644
--- a/libavfilter/audio.c
+++ b/libavfilter/audio.c
@@ -26,6 +26,7 @@
 
 #include "audio.h"
 #include "avfilter.h"
+#include "avfilter_internal.h"
 #include "framepool.h"
 #include "internal.h"
 
@@ -44,6 +45,7 @@ AVFrame *ff_null_get_audio_buffer(AVFilterLink *link, int nb_samples)
 AVFrame *ff_default_get_audio_buffer(AVFilterLink *link, int nb_samples)
 {
     AVFrame *frame = NULL;
+    FilterLinkInternal *const li = ff_link_internal(link);
     int channels = link->ch_layout.nb_channels;
     int align = av_cpu_max_align();
 #if FF_API_OLD_CHANNEL_LAYOUT
@@ -54,10 +56,10 @@ FF_DISABLE_DEPRECATION_WARNINGS
 FF_ENABLE_DEPRECATION_WARNINGS
 #endif
 
-    if (!link->frame_pool) {
-        link->frame_pool = ff_frame_pool_audio_init(av_buffer_allocz, channels,
-                                                    nb_samples, link->format, align);
-        if (!link->frame_pool)
+    if (!li->frame_pool) {
+        li->frame_pool = ff_frame_pool_audio_init(av_buffer_allocz, channels,
+                                                  nb_samples, link->format, align);
+        if (!li->frame_pool)
             return NULL;
     } else {
         int pool_channels = 0;
@@ -65,7 +67,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
         int pool_align = 0;
         enum AVSampleFormat pool_format = AV_SAMPLE_FMT_NONE;
 
-        if (ff_frame_pool_get_audio_config(link->frame_pool,
+        if (ff_frame_pool_get_audio_config(li->frame_pool,
                                            &pool_channels, &pool_nb_samples,
                                            &pool_format, &pool_align) < 0) {
             return NULL;
@@ -74,15 +76,15 @@ FF_ENABLE_DEPRECATION_WARNINGS
         if (pool_channels != channels || pool_nb_samples < nb_samples ||
             pool_format != link->format || pool_align != align) {
 
-            ff_frame_pool_uninit((FFFramePool **)&link->frame_pool);
-            link->frame_pool = ff_frame_pool_audio_init(av_buffer_allocz, channels,
-                                                        nb_samples, link->format, align);
-            if (!link->frame_pool)
+            ff_frame_pool_uninit(&li->frame_pool);
+            li->frame_pool = ff_frame_pool_audio_init(av_buffer_allocz, channels,
+                                                      nb_samples, link->format, align);
+            if (!li->frame_pool)
                 return NULL;
         }
     }
 
-    frame = ff_frame_pool_get(link->frame_pool);
+    frame = ff_frame_pool_get(li->frame_pool);
     if (!frame)
         return NULL;
 
diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c
index 41dc774ace..3aa20085ed 100644
--- a/libavfilter/avfilter.c
+++ b/libavfilter/avfilter.c
@@ -201,7 +201,7 @@ void avfilter_link_free(AVFilterLink **link)
     li = ff_link_internal(*link);
 
     ff_framequeue_free(&li->fifo);
-    ff_frame_pool_uninit((FFFramePool**)&(*link)->frame_pool);
+    ff_frame_pool_uninit(&li->frame_pool);
     av_channel_layout_uninit(&(*link)->ch_layout);
 
     av_freep(link);
diff --git a/libavfilter/avfilter.h b/libavfilter/avfilter.h
index 9252713ae2..5eff35b836 100644
--- a/libavfilter/avfilter.h
+++ b/libavfilter/avfilter.h
@@ -666,11 +666,6 @@ struct AVFilterLink {
      */
     int64_t sample_count_in, sample_count_out;
 
-    /**
-     * A pointer to a FFFramePool struct.
-     */
-    void *frame_pool;
-
     /**
      * True if a frame is currently wanted on the output of this filter.
      * Set when ff_request_frame() is called by the output,
diff --git a/libavfilter/avfilter_internal.h b/libavfilter/avfilter_internal.h
index 0101512843..2c31c3e7de 100644
--- a/libavfilter/avfilter_internal.h
+++ b/libavfilter/avfilter_internal.h
@@ -33,6 +33,8 @@
 typedef struct FilterLinkInternal {
     AVFilterLink l;
 
+    struct FFFramePool *frame_pool;
+
     /**
      * Queue of frames waiting to be filtered.
      */
diff --git a/libavfilter/video.c b/libavfilter/video.c
index 243762c8fd..bbd1193835 100644
--- a/libavfilter/video.c
+++ b/libavfilter/video.c
@@ -29,6 +29,7 @@
 #include "libavutil/imgutils.h"
 
 #include "avfilter.h"
+#include "avfilter_internal.h"
 #include "framepool.h"
 #include "internal.h"
 #include "video.h"
@@ -47,6 +48,7 @@ AVFrame *ff_null_get_video_buffer(AVFilterLink *link, int w, int h)
 
 AVFrame *ff_default_get_video_buffer2(AVFilterLink *link, int w, int h, int align)
 {
+    FilterLinkInternal *const li = ff_link_internal(link);
     AVFrame *frame = NULL;
     int pool_width = 0;
     int pool_height = 0;
@@ -68,13 +70,13 @@ AVFrame *ff_default_get_video_buffer2(AVFilterLink *link, int w, int h, int alig
         return frame;
     }
 
-    if (!link->frame_pool) {
-        link->frame_pool = ff_frame_pool_video_init(av_buffer_allocz, w, h,
-                                                    link->format, align);
-        if (!link->frame_pool)
+    if (!li->frame_pool) {
+        li->frame_pool = ff_frame_pool_video_init(av_buffer_allocz, w, h,
+                                                  link->format, align);
+        if (!li->frame_pool)
             return NULL;
     } else {
-        if (ff_frame_pool_get_video_config(link->frame_pool,
+        if (ff_frame_pool_get_video_config(li->frame_pool,
                                            &pool_width, &pool_height,
                                            &pool_format, &pool_align) < 0) {
             return NULL;
@@ -83,15 +85,15 @@ AVFrame *ff_default_get_video_buffer2(AVFilterLink *link, int w, int h, int alig
         if (pool_width != w || pool_height != h ||
             pool_format != link->format || pool_align != align) {
 
-            ff_frame_pool_uninit((FFFramePool **)&link->frame_pool);
-            link->frame_pool = ff_frame_pool_video_init(av_buffer_allocz, w, h,
-                                                        link->format, align);
-            if (!link->frame_pool)
+            ff_frame_pool_uninit(&li->frame_pool);
+            li->frame_pool = ff_frame_pool_video_init(av_buffer_allocz, w, h,
+                                                      link->format, align);
+            if (!li->frame_pool)
                 return NULL;
         }
     }
 
-    frame = ff_frame_pool_get(link->frame_pool);
+    frame = ff_frame_pool_get(li->frame_pool);
     if (!frame)
         return NULL;