diff --git a/libavcodec/aac/aacdec_usac.c b/libavcodec/aac/aacdec_usac.c
index aa89b83182..100203ff71 100644
--- a/libavcodec/aac/aacdec_usac.c
+++ b/libavcodec/aac/aacdec_usac.c
@@ -971,7 +971,7 @@ static void apply_noise_fill(AACDecContext *ac, SingleChannelElement *sce,
             }
 
             if (band_quantized_to_zero)
-                sce->sf[g*ics->max_sfb + sfb] += noise_offset;
+                sce->sfo[g*ics->max_sfb + sfb] += noise_offset;
         }
         coef += g_len << 7;
     }
@@ -987,6 +987,9 @@ static void spectrum_scale(AACDecContext *ac, SingleChannelElement *sce,
     if (ue->noise.level)
         apply_noise_fill(ac, sce, ue);
 
+    /* Noise filling may apply an offset to the scalefactor offset */
+    ac->dsp.dequant_scalefactors(sce);
+
     /* Apply scalefactors */
     coef = sce->coeffs;
     for (int g = 0; g < ics->num_window_groups; g++) {
@@ -1371,8 +1374,6 @@ static int decode_usac_core_coder(AACDecContext *ac, AACUSACConfig *usac,
         if (ret < 0)
             return ret;
 
-        ac->dsp.dequant_scalefactors(sce);
-
         if (ue->tns_data_present) {
             sce->tns.present = 1;
             ret = ff_aac_decode_tns(ac, &sce->tns, gb, ics);