diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 124215c279..4f38379021 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1726,6 +1726,8 @@ int ff_h264_frame_start(H264Context *h)
     h->cur_pic.f.extended_data = h->cur_pic.f.data;
 
     ff_er_frame_start(&h->er);
+    h->er.last_pic =
+    h->er.next_pic = NULL;
 
     assert(h->linesize && h->uvlinesize);
 
@@ -2737,8 +2739,6 @@ static int field_end(H264Context *h, int in_setup)
      */
     if (!FIELD_PICTURE && h->current_slice && !h->sps.new) {
         h->er.cur_pic  = h->cur_pic_ptr;
-        h->er.last_pic = h->ref_count[0] ? &h->ref_list[0][0] : NULL;
-        h->er.next_pic = h->ref_count[1] ? &h->ref_list[1][0] : NULL;
         ff_er_frame_end(&h->er);
     }
     emms_c();
@@ -3720,6 +3720,9 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
                              (h->ref_list[j][i].f.reference & 3);
     }
 
+    if (h->ref_count[0]) h->er.last_pic = &h->ref_list[0][0];
+    if (h->ref_count[1]) h->er.next_pic = &h->ref_list[1][0];
+
     if (h->avctx->debug & FF_DEBUG_PICT_INFO) {
         av_log(h->avctx, AV_LOG_DEBUG,
                "slice:%d %s mb:%d %c%s%s pps:%u frame:%d poc:%d/%d ref:%d/%d qp:%d loop:%d:%d:%d weight:%d%s %s\n",