diff --git a/libavformat/mov.c b/libavformat/mov.c
index 67b3e11eb9..8a91239243 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -3332,6 +3332,8 @@ static void mov_estimate_video_delay(MOVContext *c, AVStream* st) {
                 if (pts_buf[j] < pts_buf[r]) {
                     FFSWAP(int64_t, pts_buf[j], pts_buf[r]);
                     ++num_swaps;
+                } else {
+                    break;
                 }
                 j = r;
             }