diff --git a/libavfilter/af_silenceremove.c b/libavfilter/af_silenceremove.c
index 166961442e..6f3250bbb1 100644
--- a/libavfilter/af_silenceremove.c
+++ b/libavfilter/af_silenceremove.c
@@ -792,17 +792,20 @@ silence_copy:
                 } else if (!threshold) {
                     for (j = 0; j < outlink->channels; j++) {
                         s->update(s, in, j, nb_samples_read);
-                        if (s->stop_silence) {
+                        if (s->stop_silence)
                             s->copy(s, s->stop_silence_hold, in, j, s->stop_silence_offset, nb_samples_read);
-                            s->stop_silence_end = FFMIN(s->stop_silence_end + 1, s->stop_silence);
-                            if (s->stop_silence_offset >= s->stop_silence) {
-                                s->stop_silence_offset = 0;
-                            }
-                        }
 
                         s->copy(s, s->stop_holdoff, in, j, s->stop_holdoff_end, nb_samples_read);
                     }
 
+                    if (s->stop_silence) {
+                        s->stop_silence_offset++;
+                        s->stop_silence_end = FFMIN(s->stop_silence_end + 1, s->stop_silence);
+                        if (s->stop_silence_offset >= s->stop_silence) {
+                            s->stop_silence_offset = 0;
+                        }
+                    }
+
                     s->window_offset++;
                     if (s->window_offset >= s->window_duration)
                         s->window_offset = 0;