diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 133f4b780c..50fd45a741 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -715,7 +715,6 @@ int ff_h264_get_slice_type(const H264SliceContext *sl);
 int ff_h264_alloc_tables(H264Context *h);
 
 int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl);
-void ff_h264_fill_mbaff_ref_list(H264SliceContext *sl);
 void ff_h264_remove_all_refs(H264Context *h);
 
 /**
diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index b6bb8f2a87..15957a37e0 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -270,6 +270,31 @@ static int pic_num_extract(const H264Context *h, int pic_num, int *structure)
     return pic_num;
 }
 
+static void h264_fill_mbaff_ref_list(H264SliceContext *sl)
+{
+    int list, i, j;
+    for (list = 0; list < sl->list_count; list++) {
+        for (i = 0; i < sl->ref_count[list]; i++) {
+            H264Ref *frame = &sl->ref_list[list][i];
+            H264Ref *field = &sl->ref_list[list][16 + 2 * i];
+
+            field[0] = *frame;
+
+            for (j = 0; j < 3; j++)
+                field[0].linesize[j] <<= 1;
+            field[0].reference = PICT_TOP_FIELD;
+            field[0].poc       = field[0].parent->field_poc[0];
+
+            field[1] = field[0];
+
+            for (j = 0; j < 3; j++)
+                field[1].data[j] += frame->parent->f->linesize[j];
+            field[1].reference = PICT_BOTTOM_FIELD;
+            field[1].poc       = field[1].parent->field_poc[1];
+        }
+    }
+}
+
 int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl)
 {
     int list, index, pic_structure;
@@ -398,32 +423,10 @@ int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl)
         }
     }
 
-    return 0;
-}
-
-void ff_h264_fill_mbaff_ref_list(H264SliceContext *sl)
-{
-    int list, i, j;
-    for (list = 0; list < sl->list_count; list++) {
-        for (i = 0; i < sl->ref_count[list]; i++) {
-            H264Ref *frame = &sl->ref_list[list][i];
-            H264Ref *field = &sl->ref_list[list][16 + 2 * i];
-
-            field[0] = *frame;
-
-            for (j = 0; j < 3; j++)
-                field[0].linesize[j] <<= 1;
-            field[0].reference = PICT_TOP_FIELD;
-            field[0].poc       = field[0].parent->field_poc[0];
+    if (FRAME_MBAFF(h))
+        h264_fill_mbaff_ref_list(sl);
 
-            field[1] = field[0];
-
-            for (j = 0; j < 3; j++)
-                field[1].data[j] += frame->parent->f->linesize[j];
-            field[1].reference = PICT_BOTTOM_FIELD;
-            field[1].poc       = field[1].parent->field_poc[1];
-        }
-    }
+    return 0;
 }
 
 /**
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 497a99cdca..666d99f837 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -1629,8 +1629,6 @@ static int h264_slice_header_parse(H264Context *h, H264SliceContext *sl)
     }
 
     if (FRAME_MBAFF(h)) {
-        ff_h264_fill_mbaff_ref_list(sl);
-
         if (pps->weighted_bipred_idc == 2 && sl->slice_type_nos == AV_PICTURE_TYPE_B) {
             implicit_weight_table(h, sl, 0);
             implicit_weight_table(h, sl, 1);