diff --git a/libavformat/matroskaenc.c b/libavformat/matroskaenc.c
index 2cd5933da8..5e0346d341 100644
--- a/libavformat/matroskaenc.c
+++ b/libavformat/matroskaenc.c
@@ -2332,6 +2332,7 @@ static int mkv_write_packet(AVFormatContext *s, AVPacket *pkt)
     if (ret < 0)
         return ret;
 
+    if (mkv->cluster_pos != -1) {
     if (mkv->tracks[pkt->stream_index].write_dts)
         cluster_time = pkt->dts - mkv->cluster_pts;
     else
@@ -2359,9 +2360,10 @@ static int mkv_write_packet(AVFormatContext *s, AVPacket *pkt)
         start_new_cluster = 0;
     }
 
-    if (mkv->cluster_pos != -1 && start_new_cluster) {
+    if (start_new_cluster) {
         mkv_end_cluster(s);
     }
+    }
 
     if (!mkv->cluster_pos)
         avio_write_marker(s->pb,