diff --git a/libavcodec/hevc/filter.c b/libavcodec/hevc/filter.c
index 18c73dc2fa..78f408f170 100644
--- a/libavcodec/hevc/filter.c
+++ b/libavcodec/hevc/filter.c
@@ -264,7 +264,8 @@ static void restore_tqb_pixels(const HEVCContext *s,
 
 #define CTB(tab, x, y) ((tab)[(y) * sps->ctb_width + (x)])
 
-static void sao_filter_CTB(HEVCLocalContext *lc, const HEVCContext *s,
+static void sao_filter_CTB(HEVCLocalContext *lc, const HEVCLayerContext *l,
+                           const HEVCContext *s,
                            const HEVCPPS *pps, const HEVCSPS *sps,
                            int x, int y)
 {
@@ -275,7 +276,7 @@ static void sao_filter_CTB(HEVCLocalContext *lc, const HEVCContext *s,
     int y_ctb                = y >> sps->log2_ctb_size;
     int ctb_addr_rs          = y_ctb * sps->ctb_width + x_ctb;
     int ctb_addr_ts          = pps->ctb_addr_rs_to_ts[ctb_addr_rs];
-    SAOParams *sao           = &CTB(s->sao, x_ctb, y_ctb);
+    SAOParams *sao           = &CTB(l->sao, x_ctb, y_ctb);
     // flags indicating unfilterable edges
     uint8_t vert_edge[]      = { 0, 0 };
     uint8_t horiz_edge[]     = { 0, 0 };
@@ -385,17 +386,17 @@ static void sao_filter_CTB(HEVCLocalContext *lc, const HEVCContext *s,
                 src1[1] = s->sao_pixel_buffer_h[c_idx] + (((2 * y_ctb - 1) * w + x0 - left) << sh);
                 pos = 0;
                 if (left) {
-                    src_idx = (CTB(s->sao, x_ctb-1, y_ctb-1).type_idx[c_idx] ==
+                    src_idx = (CTB(l->sao, x_ctb-1, y_ctb-1).type_idx[c_idx] ==
                                SAO_APPLIED);
                     copy_pixel(dst1, src1[src_idx], sh);
                     pos += (1 << sh);
                 }
-                src_idx = (CTB(s->sao, x_ctb, y_ctb-1).type_idx[c_idx] ==
+                src_idx = (CTB(l->sao, x_ctb, y_ctb-1).type_idx[c_idx] ==
                            SAO_APPLIED);
                 memcpy(dst1 + pos, src1[src_idx] + pos, width << sh);
                 if (right) {
                     pos += width << sh;
-                    src_idx = (CTB(s->sao, x_ctb+1, y_ctb-1).type_idx[c_idx] ==
+                    src_idx = (CTB(l->sao, x_ctb+1, y_ctb-1).type_idx[c_idx] ==
                                SAO_APPLIED);
                     copy_pixel(dst1 + pos, src1[src_idx] + pos, sh);
                 }
@@ -412,24 +413,24 @@ static void sao_filter_CTB(HEVCLocalContext *lc, const HEVCContext *s,
                 src1[1] = s->sao_pixel_buffer_h[c_idx] + (((2 * y_ctb + 2) * w + x0 - left) << sh);
                 pos = 0;
                 if (left) {
-                    src_idx = (CTB(s->sao, x_ctb-1, y_ctb+1).type_idx[c_idx] ==
+                    src_idx = (CTB(l->sao, x_ctb-1, y_ctb+1).type_idx[c_idx] ==
                                SAO_APPLIED);
                     copy_pixel(dst1, src1[src_idx], sh);
                     pos += (1 << sh);
                 }
-                src_idx = (CTB(s->sao, x_ctb, y_ctb+1).type_idx[c_idx] ==
+                src_idx = (CTB(l->sao, x_ctb, y_ctb+1).type_idx[c_idx] ==
                            SAO_APPLIED);
                 memcpy(dst1 + pos, src1[src_idx] + pos, width << sh);
                 if (right) {
                     pos += width << sh;
-                    src_idx = (CTB(s->sao, x_ctb+1, y_ctb+1).type_idx[c_idx] ==
+                    src_idx = (CTB(l->sao, x_ctb+1, y_ctb+1).type_idx[c_idx] ==
                                SAO_APPLIED);
                     copy_pixel(dst1 + pos, src1[src_idx] + pos, sh);
                 }
             }
             left_pixels = 0;
             if (!left_edge) {
-                if (CTB(s->sao, x_ctb-1, y_ctb).type_idx[c_idx] == SAO_APPLIED) {
+                if (CTB(l->sao, x_ctb-1, y_ctb).type_idx[c_idx] == SAO_APPLIED) {
                     copy_vert(dst - (1 << sh),
                               s->sao_pixel_buffer_v[c_idx] + (((2 * x_ctb - 1) * h + y0) << sh),
                               sh, height, stride_dst, 1 << sh);
@@ -439,7 +440,7 @@ static void sao_filter_CTB(HEVCLocalContext *lc, const HEVCContext *s,
             }
             right_pixels = 0;
             if (!right_edge) {
-                if (CTB(s->sao, x_ctb+1, y_ctb).type_idx[c_idx] == SAO_APPLIED) {
+                if (CTB(l->sao, x_ctb+1, y_ctb).type_idx[c_idx] == SAO_APPLIED) {
                     copy_vert(dst + (width << sh),
                               s->sao_pixel_buffer_v[c_idx] + (((2 * x_ctb + 2) * h + y0) << sh),
                               sh, height, stride_dst, 1 << sh);
@@ -889,16 +890,16 @@ void ff_hevc_hls_filter(HEVCLocalContext *lc, const HEVCLayerContext *l,
     if (sps->sao_enabled && !skip) {
         int y_end = y >= sps->height - ctb_size;
         if (y && x)
-            sao_filter_CTB(lc, s, pps, sps, x - ctb_size, y - ctb_size);
+            sao_filter_CTB(lc, l, s, pps, sps, x - ctb_size, y - ctb_size);
         if (x && y_end)
-            sao_filter_CTB(lc, s, pps, sps, x - ctb_size, y);
+            sao_filter_CTB(lc, l, s, pps, sps, x - ctb_size, y);
         if (y && x_end) {
-            sao_filter_CTB(lc, s, pps, sps, x, y - ctb_size);
+            sao_filter_CTB(lc, l, s, pps, sps, x, y - ctb_size);
             if (s->avctx->active_thread_type & FF_THREAD_FRAME )
                 ff_progress_frame_report(&s->cur_frame->tf, y);
         }
         if (x_end && y_end) {
-            sao_filter_CTB(lc, s, pps, sps, x , y);
+            sao_filter_CTB(lc, l, s, pps, sps, x , y);
             if (s->avctx->active_thread_type & FF_THREAD_FRAME )
                 ff_progress_frame_report(&s->cur_frame->tf, y + ctb_size);
         }
diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c
index 4978aa745c..0bb4d43ed4 100644
--- a/libavcodec/hevc/hevcdec.c
+++ b/libavcodec/hevc/hevcdec.c
@@ -66,9 +66,9 @@ static const uint8_t hevc_pel_weight[65] = { [2] = 0, [4] = 1, [6] = 2, [8] = 3,
  */
 
 /* free everything allocated  by pic_arrays_init() */
-static void pic_arrays_free(HEVCContext *s)
+static void pic_arrays_free(HEVCContext *s, HEVCLayerContext *l)
 {
-    av_freep(&s->sao);
+    av_freep(&l->sao);
     av_freep(&s->deblock);
 
     av_freep(&s->skip_flag);
@@ -103,9 +103,9 @@ static int pic_arrays_init(HEVCContext *s, HEVCLayerContext *l, const HEVCSPS *s
     l->bs_width  = (width  >> 2) + 1;
     l->bs_height = (height >> 2) + 1;
 
-    s->sao           = av_calloc(ctb_count, sizeof(*s->sao));
+    l->sao           = av_calloc(ctb_count, sizeof(*l->sao));
     s->deblock       = av_calloc(ctb_count, sizeof(*s->deblock));
-    if (!s->sao || !s->deblock)
+    if (!l->sao || !s->deblock)
         goto fail;
 
     s->skip_flag    = av_malloc_array(sps->min_cb_height, sps->min_cb_width);
@@ -140,7 +140,7 @@ static int pic_arrays_init(HEVCContext *s, HEVCLayerContext *l, const HEVCSPS *s
     return 0;
 
 fail:
-    pic_arrays_free(s);
+    pic_arrays_free(s, l);
     return AVERROR(ENOMEM);
 }
 
@@ -531,7 +531,7 @@ static int set_sps(HEVCContext *s, HEVCLayerContext *l, const HEVCSPS *sps)
 {
     int ret, i;
 
-    pic_arrays_free(s);
+    pic_arrays_free(s, l);
     s->ps.sps = NULL;
     ff_refstruct_unref(&s->vps);
 
@@ -576,7 +576,7 @@ static int set_sps(HEVCContext *s, HEVCLayerContext *l, const HEVCSPS *sps)
     return 0;
 
 fail:
-    pic_arrays_free(s);
+    pic_arrays_free(s, l);
     for (i = 0; i < 3; i++) {
         av_freep(&s->sao_pixel_buffer_h[i]);
         av_freep(&s->sao_pixel_buffer_v[i]);
@@ -990,21 +990,21 @@ do {                                                    \
     if (!sao_merge_up_flag && !sao_merge_left_flag)     \
         sao->elem = value;                              \
     else if (sao_merge_left_flag)                       \
-        sao->elem = CTB(s->sao, rx-1, ry).elem;         \
+        sao->elem = CTB(l->sao, rx-1, ry).elem;         \
     else if (sao_merge_up_flag)                         \
-        sao->elem = CTB(s->sao, rx, ry-1).elem;         \
+        sao->elem = CTB(l->sao, rx, ry-1).elem;         \
     else                                                \
         sao->elem = 0;                                  \
 } while (0)
 
-static void hls_sao_param(HEVCLocalContext *lc,
+static void hls_sao_param(HEVCLocalContext *lc, const HEVCLayerContext *l,
                           const HEVCPPS *pps, const HEVCSPS *sps,
                           int rx, int ry)
 {
     const HEVCContext *const s = lc->parent;
     int sao_merge_left_flag = 0;
     int sao_merge_up_flag   = 0;
-    SAOParams *sao          = &CTB(s->sao, rx, ry);
+    SAOParams *sao          = &CTB(l->sao, rx, ry);
     int c_idx, i;
 
     if (s->sh.slice_sample_adaptive_offset_flag[0] ||
@@ -2556,7 +2556,7 @@ static int hls_decode_entry(HEVCContext *s, GetBitContext *gb)
             return ret;
         }
 
-        hls_sao_param(lc, pps, sps,
+        hls_sao_param(lc, l, pps, sps,
                       x_ctb >> sps->log2_ctb_size, y_ctb >> sps->log2_ctb_size);
 
         s->deblock[ctb_addr_rs].beta_offset = s->sh.beta_offset;
@@ -2624,7 +2624,7 @@ static int hls_decode_entry_wpp(AVCodecContext *avctx, void *hevc_lclist,
         ret = ff_hevc_cabac_init(lc, pps, ctb_addr_ts, data, data_size, 1);
         if (ret < 0)
             goto error;
-        hls_sao_param(lc, pps, sps,
+        hls_sao_param(lc, l, pps, sps,
                       x_ctb >> sps->log2_ctb_size, y_ctb >> sps->log2_ctb_size);
         more_data = hls_coding_quadtree(lc, l, pps, sps, x_ctb, y_ctb, sps->log2_ctb_size, 0);
 
@@ -3518,7 +3518,8 @@ static av_cold int hevc_decode_free(AVCodecContext *avctx)
     HEVCContext       *s = avctx->priv_data;
     int i;
 
-    pic_arrays_free(s);
+    for (int i = 0; i < FF_ARRAY_ELEMS(s->layers); i++)
+        pic_arrays_free(s, &s->layers[i]);
 
     ff_refstruct_unref(&s->vps);
     ff_refstruct_unref(&s->pps);
diff --git a/libavcodec/hevc/hevcdec.h b/libavcodec/hevc/hevcdec.h
index 18b15e7dc1..e0b026eef6 100644
--- a/libavcodec/hevc/hevcdec.h
+++ b/libavcodec/hevc/hevcdec.h
@@ -443,6 +443,8 @@ typedef struct HEVCLayerContext {
 
     int                     bs_width;
     int                     bs_height;
+
+    SAOParams              *sao;
 } HEVCLayerContext;
 
 typedef struct HEVCContext {
@@ -476,7 +478,6 @@ typedef struct HEVCContext {
     const HEVCVPS *vps; ///< RefStruct reference
     const HEVCPPS *pps; ///< RefStruct reference
     SliceHeader sh;
-    SAOParams *sao;
     DBParams *deblock;
     enum HEVCNALUnitType nal_unit_type;
     int temporal_id;  ///< temporal_id_plus1 - 1