diff --git a/libavcodec/mlpdec.c b/libavcodec/mlpdec.c
index e74d146af0..596e0a2c79 100644
--- a/libavcodec/mlpdec.c
+++ b/libavcodec/mlpdec.c
@@ -446,7 +446,10 @@ static int read_filter_params(MLPDecodeContext *m, GetBitContext *gbp,
     // Filter is 0 for FIR, 1 for IIR.
     assert(filter < 2);
 
-    m->filter_changed[channel][filter]++;
+    if (m->filter_changed[channel][filter]++ > 1) {
+        av_log(m->avctx, AV_LOG_ERROR, "Filters may change only once per access unit.\n");
+        return -1;
+    }
 
     order = get_bits(gbp, 4);
     if (order > max_order) {
@@ -508,8 +511,12 @@ static int read_matrix_params(MLPDecodeContext *m, SubStream *s, GetBitContext *
 {
     unsigned int mat, ch;
 
+    if (m->matrix_changed++ > 1) {
+        av_log(m->avctx, AV_LOG_ERROR, "Matrices may change only once per access unit.\n");
+        return -1;
+    }
+
     s->num_primitive_matrices = get_bits(gbp, 4);
-    m->matrix_changed++;
 
     for (mat = 0; mat < s->num_primitive_matrices; mat++) {
         int frac_bits, max_chan;
@@ -1013,8 +1020,6 @@ static int read_access_unit(AVCodecContext *avctx, void* data, int *data_size,
 
         s->blockpos = 0;
         do {
-            unsigned int ch;
-
             if (get_bits1(&gb)) {
                 if (get_bits1(&gb)) {
                     /* A restart header should be present. */
@@ -1031,17 +1036,6 @@ static int read_access_unit(AVCodecContext *avctx, void* data, int *data_size,
                     goto next_substr;
             }
 
-            if (m->matrix_changed > 1) {
-                av_log(m->avctx, AV_LOG_ERROR, "Matrices may change only once per access unit.\n");
-                goto next_substr;
-            }
-            for (ch = 0; ch < s->max_channel; ch++)
-                if (m->filter_changed[ch][FIR] > 1 ||
-                    m->filter_changed[ch][IIR] > 1) {
-                    av_log(m->avctx, AV_LOG_ERROR, "Filters may change only once per access unit.\n");
-                    goto next_substr;
-                }
-
             if (!s->restart_seen) {
                 goto next_substr;
             }