diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index 74aba28a9b..f65ff879c7 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -948,9 +948,6 @@ static int check_output_constraints(InputStream *ist, OutputStream *ost)
 {
     OutputFile *of = output_files[ost->file_index];
 
-    if (ost->ist != ist)
-        return 0;
-
     if (ost->finished & MUXER_FINISHED)
         return 0;
 
@@ -1468,7 +1465,8 @@ static int process_subtitle(InputStream *ist, AVSubtitle *subtitle, int *got_out
     if (!subtitle->num_rects)
         goto out;
 
-    for (OutputStream *ost = ost_iter(NULL); ost; ost = ost_iter(ost)) {
+    for (int oidx = 0; oidx < ist->nb_outputs; oidx++) {
+        OutputStream *ost = ist->outputs[oidx];
         if (!check_output_constraints(ist, ost) || !ost->enc_ctx
             || ost->enc_ctx->codec_type != AVMEDIA_TYPE_SUBTITLE)
             continue;
@@ -1830,7 +1828,8 @@ static int process_input_packet(InputStream *ist, const AVPacket *pkt, int no_eo
     } else if (!ist->decoding_needed)
         eof_reached = 1;
 
-    for (OutputStream *ost = ost_iter(NULL); ost; ost = ost_iter(ost)) {
+    for (int oidx = 0; oidx < ist->nb_outputs; oidx++) {
+        OutputStream *ost = ist->outputs[oidx];
         if (!check_output_constraints(ist, ost) || ost->enc_ctx ||
             (!pkt && no_eof))
             continue;
@@ -2577,13 +2576,11 @@ static int process_input(int file_index)
             }
 
             /* mark all outputs that don't go through lavfi as finished */
-            for (OutputStream *ost = ost_iter(NULL); ost; ost = ost_iter(ost)) {
-                if (ost->ist == ist &&
-                    (!ost->enc_ctx || ost->enc_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE)) {
-                    OutputFile *of = output_files[ost->file_index];
-                    close_output_stream(ost);
-                    of_output_packet(of, ost->pkt, ost, 1);
-                }
+            for (int oidx = 0; oidx < ist->nb_outputs; oidx++) {
+                OutputStream *ost = ist->outputs[oidx];
+                OutputFile    *of = output_files[ost->file_index];
+                close_output_stream(ost);
+                of_output_packet(of, ost->pkt, ost, 1);
             }
         }
 
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 80a1192ed0..aef2bbc0df 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -416,6 +416,14 @@ typedef struct InputStream {
     InputFilter **filters;
     int        nb_filters;
 
+    /*
+     * Output targets that do not go through lavfi, i.e. subtitles or
+     * streamcopy. Those two cases are distinguished by the OutputStream
+     * having an encoder or not.
+     */
+    struct OutputStream **outputs;
+    int                nb_outputs;
+
     int reinit_filters;
 
     /* hwaccel options */
@@ -867,6 +875,8 @@ void ifile_close(InputFile **f);
  */
 int ifile_get_packet(InputFile *f, AVPacket **pkt);
 
+void ist_output_add(InputStream *ist, OutputStream *ost);
+
 /* iterate over all input streams in all input files;
  * pass NULL to start iteration */
 InputStream *ist_iter(InputStream *prev);
diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c
index 93eed28809..f2da0826ad 100644
--- a/fftools/ffmpeg_demux.c
+++ b/fftools/ffmpeg_demux.c
@@ -528,6 +528,7 @@ static void ist_free(InputStream **pist)
     avsubtitle_free(&ist->prev_sub.subtitle);
     av_frame_free(&ist->sub2video.frame);
     av_freep(&ist->filters);
+    av_freep(&ist->outputs);
     av_freep(&ist->hwaccel_device);
     av_freep(&ist->dts_buffer);
 
@@ -559,6 +560,12 @@ void ifile_close(InputFile **pf)
     av_freep(pf);
 }
 
+void ist_output_add(InputStream *ist, OutputStream *ost)
+{
+    GROW_ARRAY(ist->outputs, ist->nb_outputs);
+    ist->outputs[ist->nb_outputs - 1] = ost;
+}
+
 static const AVCodec *choose_decoder(const OptionsContext *o, AVFormatContext *s, AVStream *st,
                                      enum HWAccelID hwaccel_id, enum AVHWDeviceType hwaccel_device_type)
 
diff --git a/fftools/ffmpeg_mux_init.c b/fftools/ffmpeg_mux_init.c
index 4b6c704046..62e5643a04 100644
--- a/fftools/ffmpeg_mux_init.c
+++ b/fftools/ffmpeg_mux_init.c
@@ -647,6 +647,9 @@ static OutputStream *new_output_stream(Muxer *mux, const OptionsContext *o,
     if (ost->ist) {
         ost->ist->discard = 0;
         ost->ist->st->discard = ost->ist->user_set_discard;
+
+        if (!(ost->enc && (type == AVMEDIA_TYPE_VIDEO || type == AVMEDIA_TYPE_AUDIO)))
+            ist_output_add(ost->ist, ost);
     }
     ost->last_mux_dts = AV_NOPTS_VALUE;