diff --git a/libavcodec/vorbisdec.c b/libavcodec/vorbisdec.c
index dd856a6dfe..e9dad4ef4f 100644
--- a/libavcodec/vorbisdec.c
+++ b/libavcodec/vorbisdec.c
@@ -38,6 +38,7 @@
 #include "codec_internal.h"
 #include "decode.h"
 #include "get_bits.h"
+#include "internal.h"
 #include "vorbis.h"
 #include "vorbisdsp.h"
 #include "vorbis_data.h"
@@ -134,7 +135,6 @@ typedef struct vorbis_context_s {
     av_tx_fn      mdct_fn[2];
 
     uint8_t       first_frame;
-    int64_t       initial_pts;
     uint32_t      version;
     uint8_t       audio_channels;
     uint32_t      audio_samplerate;
@@ -1839,13 +1839,7 @@ static int vorbis_decode_frame(AVCodecContext *avctx, AVFrame *frame,
 
     if (!vc->first_frame) {
         vc->first_frame = 1;
-        vc->initial_pts = frame->pts;
-    }
-
-    if (frame->pts == vc->initial_pts) {
-        *got_frame_ptr = 0;
-        av_frame_unref(frame);
-        return buf_size;
+        avctx->internal->skip_samples = len;
     }
 
     ff_dlog(NULL, "parsed %d bytes %d bits, returned %d samples (*ch*bits) \n",
@@ -1877,6 +1871,7 @@ static av_cold void vorbis_decode_flush(AVCodecContext *avctx)
                              sizeof(*vc->saved));
     }
     vc->previous_window = -1;
+    vc->first_frame = 0;
 }
 
 const FFCodec ff_vorbis_decoder = {