diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 2e9587997d..0c38121827 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -1347,9 +1347,6 @@ static int h264_slice_header_parse(H264Context *h, H264SliceContext *sl)
            sl->ref_count[1] = sl->ref_count[0] = 0;
            return ret;
        }
-       ret = ff_h264_build_ref_list(h, sl);
-       if (ret < 0)
-           return ret;
     }
 
     sl->pwt.use_weight = 0;
@@ -1449,6 +1446,10 @@ int ff_h264_decode_slice_header(H264Context *h, H264SliceContext *sl)
     if (ret < 0)
         return ret;
 
+    ret = ff_h264_build_ref_list(h, sl);
+    if (ret < 0)
+        return ret;
+
     if (h->ps.pps->weighted_bipred_idc == 2 &&
         sl->slice_type_nos == AV_PICTURE_TYPE_B) {
         implicit_weight_table(h, sl, -1);