diff --git a/libavformat/rtpdec_xiph.c b/libavformat/rtpdec_xiph.c
index 4c9cad2877..bcf583fd0a 100644
--- a/libavformat/rtpdec_xiph.c
+++ b/libavformat/rtpdec_xiph.c
@@ -172,6 +172,11 @@ static int xiph_handle_packet(AVFormatContext * ctx,
             av_log(ctx, AV_LOG_ERROR, "RTP timestamps don't match!\n");
             return AVERROR_INVALIDDATA;
         }
+        if (!data->fragment) {
+            av_log(ctx, AV_LOG_WARNING,
+                   "Received packet without a start fragment; dropping.\n");
+            return AVERROR(EAGAIN);
+        }
 
         // copy data to fragment buffer
         put_buffer(data->fragment, buf, pkt_len);