diff --git a/libavformat/mux.c b/libavformat/mux.c
index 3fb4a012cf..a09e33cfdd 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -557,13 +557,13 @@ static int write_packet(AVFormatContext *s, AVPacket *pkt)
         AVStream *st = s->streams[pkt->stream_index];
         int64_t offset = st->mux_ts_offset;
 
-        if (!s->offset && pkt->dts != AV_NOPTS_VALUE &&
+        if (s->offset == AV_NOPTS_VALUE && pkt->dts != AV_NOPTS_VALUE &&
             (pkt->dts < 0 || s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_MAKE_ZERO)) {
             s->offset = -pkt->dts;
             s->offset_timebase = st->time_base;
         }
 
-        if (s->offset && !offset) {
+        if (s->offset != AV_NOPTS_VALUE && !offset) {
             offset = st->mux_ts_offset =
                 av_rescale_q_rnd(s->offset,
                                  s->offset_timebase,
diff --git a/libavformat/options.c b/libavformat/options.c
index e0d6df6fbd..504404312c 100644
--- a/libavformat/options.c
+++ b/libavformat/options.c
@@ -110,6 +110,7 @@ AVFormatContext *avformat_alloc_context(void)
     ic = av_malloc(sizeof(AVFormatContext));
     if (!ic) return ic;
     avformat_get_context_defaults(ic);
+    ic->offset = AV_NOPTS_VALUE;
 
     ic->internal = av_mallocz(sizeof(*ic->internal));
     if (!ic->internal) {
diff --git a/tests/ref/lavf/ismv b/tests/ref/lavf/ismv
index ab2975f0bf..f29b5ff3ef 100644
--- a/tests/ref/lavf/ismv
+++ b/tests/ref/lavf/ismv
@@ -1,9 +1,9 @@
-b1106bfcdbd1bb12be9161320351dd00 *./tests/data/lavf/lavf.ismv
+a9ccbb4cd1436d222ef4425567b4e03d *./tests/data/lavf/lavf.ismv
 312542 ./tests/data/lavf/lavf.ismv
-./tests/data/lavf/lavf.ismv CRC=0xb7d29d29
-c8bea6c84b6274f14042381045b0f532 *./tests/data/lavf/lavf.ismv
+./tests/data/lavf/lavf.ismv CRC=0x9d9a638a
+440d85f9fd5b9f63c2676638782b5c15 *./tests/data/lavf/lavf.ismv
 321448 ./tests/data/lavf/lavf.ismv
-./tests/data/lavf/lavf.ismv CRC=0x33834815
-b1106bfcdbd1bb12be9161320351dd00 *./tests/data/lavf/lavf.ismv
+./tests/data/lavf/lavf.ismv CRC=0xe8130120
+a9ccbb4cd1436d222ef4425567b4e03d *./tests/data/lavf/lavf.ismv
 312542 ./tests/data/lavf/lavf.ismv
-./tests/data/lavf/lavf.ismv CRC=0xb7d29d29
+./tests/data/lavf/lavf.ismv CRC=0x9d9a638a