diff --git a/libavcodec/aacdec_template.c b/libavcodec/aacdec_template.c
index 10fba3d3b2..119976aa19 100644
--- a/libavcodec/aacdec_template.c
+++ b/libavcodec/aacdec_template.c
@@ -3418,9 +3418,6 @@ static int aac_decode_frame_int(AVCodecContext *avctx, AVFrame *frame,
         ac->oc[1].status = OC_LOCKED;
     }
 
-    if (multiplier)
-        avctx->internal->skip_samples_multiplier = 2;
-
     if (!ac->frame->data[0] && samples) {
         av_log(avctx, AV_LOG_ERROR, "no frame data found\n");
         err = AVERROR_INVALIDDATA;
diff --git a/libavcodec/avcodec.c b/libavcodec/avcodec.c
index fb11440e5d..4bc18183a9 100644
--- a/libavcodec/avcodec.c
+++ b/libavcodec/avcodec.c
@@ -161,8 +161,6 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, const AVCodec *code
         goto free_and_end;
     }
 
-    avci->skip_samples_multiplier = 1;
-
     if (codec2->priv_data_size > 0) {
         if (!avctx->priv_data) {
             avctx->priv_data = av_mallocz(codec2->priv_data_size);
diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index 14ce4c2e2f..0613681f89 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -359,7 +359,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
 
         side= av_packet_get_side_data(avci->last_pkt_props, AV_PKT_DATA_SKIP_SAMPLES, &side_size);
         if(side && side_size>=10) {
-            avci->skip_samples = AV_RL32(side) * avci->skip_samples_multiplier;
+            avci->skip_samples = AV_RL32(side);
             avci->skip_samples = FFMAX(0, avci->skip_samples);
             discard_padding = AV_RL32(side + 4);
             av_log(avctx, AV_LOG_DEBUG, "skip %d / discard %d samples due to side data\n",
diff --git a/libavcodec/internal.h b/libavcodec/internal.h
index 17e1de8127..6fb4e1b9af 100644
--- a/libavcodec/internal.h
+++ b/libavcodec/internal.h
@@ -133,8 +133,6 @@ typedef struct AVCodecInternal {
 
     int showed_multi_packet_warning;
 
-    int skip_samples_multiplier;
-
     /* to prevent infinite loop on errors when draining */
     int nb_draining_errors;
 
diff --git a/tests/ref/fate/audiomatch-afconvert-16000-mono-he-m4a b/tests/ref/fate/audiomatch-afconvert-16000-mono-he-m4a
index f6f8df5e00..b8fd03942c 100644
--- a/tests/ref/fate/audiomatch-afconvert-16000-mono-he-m4a
+++ b/tests/ref/fate/audiomatch-afconvert-16000-mono-he-m4a
@@ -1 +1 @@
-presig: 481 postsig:223 c: 0.9835 lenerr:704
+presig: 1537 postsig:223 c: 0.9835 lenerr:1760
diff --git a/tests/ref/fate/audiomatch-afconvert-16000-stereo-he-m4a b/tests/ref/fate/audiomatch-afconvert-16000-stereo-he-m4a
index 9ba0ef2338..b0381bfa0f 100644
--- a/tests/ref/fate/audiomatch-afconvert-16000-stereo-he-m4a
+++ b/tests/ref/fate/audiomatch-afconvert-16000-stereo-he-m4a
@@ -1 +1 @@
-presig: 962 postsig:446 c: 0.9895 lenerr:1408
+presig: 3074 postsig:446 c: 0.9895 lenerr:3520
diff --git a/tests/ref/fate/audiomatch-afconvert-16000-stereo-he2-m4a b/tests/ref/fate/audiomatch-afconvert-16000-stereo-he2-m4a
index d4fc21bda5..0b16a523f7 100644
--- a/tests/ref/fate/audiomatch-afconvert-16000-stereo-he2-m4a
+++ b/tests/ref/fate/audiomatch-afconvert-16000-stereo-he2-m4a
@@ -1 +1 @@
-presig: 962 postsig:446 c: 0.9839 lenerr:1408
+presig: 3074 postsig:446 c: 0.9839 lenerr:3520
diff --git a/tests/ref/fate/audiomatch-afconvert-44100-mono-he-m4a b/tests/ref/fate/audiomatch-afconvert-44100-mono-he-m4a
index 0ad4911ff8..34921ed8b5 100644
--- a/tests/ref/fate/audiomatch-afconvert-44100-mono-he-m4a
+++ b/tests/ref/fate/audiomatch-afconvert-44100-mono-he-m4a
@@ -1 +1 @@
-presig: 962 postsig:822 c: 0.9911 lenerr:1784
+presig: 3074 postsig:822 c: 0.9911 lenerr:3896
diff --git a/tests/ref/fate/audiomatch-afconvert-44100-stereo-he-m4a b/tests/ref/fate/audiomatch-afconvert-44100-stereo-he-m4a
index ff5540b572..3bab086939 100644
--- a/tests/ref/fate/audiomatch-afconvert-44100-stereo-he-m4a
+++ b/tests/ref/fate/audiomatch-afconvert-44100-stereo-he-m4a
@@ -1 +1 @@
-presig: 1924 postsig:1644 c: 0.9890 lenerr:3568
+presig: 6148 postsig:1644 c: 0.9890 lenerr:7792
diff --git a/tests/ref/fate/audiomatch-afconvert-44100-stereo-he2-m4a b/tests/ref/fate/audiomatch-afconvert-44100-stereo-he2-m4a
index f8214cfbdc..27341f54e2 100644
--- a/tests/ref/fate/audiomatch-afconvert-44100-stereo-he2-m4a
+++ b/tests/ref/fate/audiomatch-afconvert-44100-stereo-he2-m4a
@@ -1 +1 @@
-presig: 1924 postsig:1644 c: 0.9909 lenerr:3568
+presig: 6148 postsig:1644 c: 0.9909 lenerr:7792
diff --git a/tests/ref/fate/audiomatch-nero-16000-mono-he-m4a b/tests/ref/fate/audiomatch-nero-16000-mono-he-m4a
index 4ee764e967..fb62ae0c36 100644
--- a/tests/ref/fate/audiomatch-nero-16000-mono-he-m4a
+++ b/tests/ref/fate/audiomatch-nero-16000-mono-he-m4a
@@ -1 +1 @@
-presig: -4 postsig:196 c: 0.9736 lenerr:192
+presig: 2332 postsig:196 c: 0.9736 lenerr:2528
diff --git a/tests/ref/fate/audiomatch-nero-16000-stereo-he-m4a b/tests/ref/fate/audiomatch-nero-16000-stereo-he-m4a
index cdc31c12ef..4a30beca9d 100644
--- a/tests/ref/fate/audiomatch-nero-16000-stereo-he-m4a
+++ b/tests/ref/fate/audiomatch-nero-16000-stereo-he-m4a
@@ -1 +1 @@
-presig: -8 postsig:392 c: 0.9777 lenerr:384
+presig: 4664 postsig:392 c: 0.9777 lenerr:5056
diff --git a/tests/ref/fate/audiomatch-nero-16000-stereo-he2-m4a b/tests/ref/fate/audiomatch-nero-16000-stereo-he2-m4a
index 74d549780d..22aed2aa03 100644
--- a/tests/ref/fate/audiomatch-nero-16000-stereo-he2-m4a
+++ b/tests/ref/fate/audiomatch-nero-16000-stereo-he2-m4a
@@ -1 +1 @@
-presig: 2 postsig:2590 c: 0.9934 lenerr:2592
+presig: 5618 postsig:2590 c: 0.9934 lenerr:8208
diff --git a/tests/ref/fate/audiomatch-nero-44100-mono-he-m4a b/tests/ref/fate/audiomatch-nero-44100-mono-he-m4a
index b5dcd9daeb..66ebd92730 100644
--- a/tests/ref/fate/audiomatch-nero-44100-mono-he-m4a
+++ b/tests/ref/fate/audiomatch-nero-44100-mono-he-m4a
@@ -1 +1 @@
-presig: 0 postsig:1336 c: 0.9973 lenerr:1336
+presig: 2336 postsig:1336 c: 0.9973 lenerr:3672
diff --git a/tests/ref/fate/audiomatch-nero-44100-stereo-he-m4a b/tests/ref/fate/audiomatch-nero-44100-stereo-he-m4a
index 42a87c08e3..a7d1d9f249 100644
--- a/tests/ref/fate/audiomatch-nero-44100-stereo-he-m4a
+++ b/tests/ref/fate/audiomatch-nero-44100-stereo-he-m4a
@@ -1 +1 @@
-presig: -2 postsig:2674 c: 0.9986 lenerr:2672
+presig: 4670 postsig:2674 c: 0.9986 lenerr:7344
diff --git a/tests/ref/fate/audiomatch-nero-44100-stereo-he2-m4a b/tests/ref/fate/audiomatch-nero-44100-stereo-he2-m4a
index 3fa2cdba2b..4da8519ee7 100644
--- a/tests/ref/fate/audiomatch-nero-44100-stereo-he2-m4a
+++ b/tests/ref/fate/audiomatch-nero-44100-stereo-he2-m4a
@@ -1 +1 @@
-presig: 2 postsig:782 c: 0.9980 lenerr:784
+presig: 5618 postsig:782 c: 0.9980 lenerr:6400