diff --git a/libavcodec/hevc_ps.c b/libavcodec/hevc_ps.c
index d90f172c46..a6b0021bc3 100644
--- a/libavcodec/hevc_ps.c
+++ b/libavcodec/hevc_ps.c
@@ -213,7 +213,7 @@ int ff_hevc_decode_short_term_rps(GetBitContext *gb, AVCodecContext *avctx,
             int prev = 0;
 
             for (i = 0; i < rps->num_negative_pics; i++) {
-                delta_poc = rps->delta_poc_s0[i] = get_ue_golomb_long(gb) + 1;
+                delta_poc = get_ue_golomb_long(gb) + 1;
                 if (delta_poc < 1 || delta_poc > 32768) {
                     av_log(avctx, AV_LOG_ERROR,
                         "Invalid value of delta_poc: %d\n",
@@ -226,7 +226,7 @@ int ff_hevc_decode_short_term_rps(GetBitContext *gb, AVCodecContext *avctx,
             }
             prev = 0;
             for (i = 0; i < nb_positive_pics; i++) {
-                delta_poc = rps->delta_poc_s1[i] = get_ue_golomb_long(gb) + 1;
+                delta_poc = get_ue_golomb_long(gb) + 1;
                 if (delta_poc < 1 || delta_poc > 32768) {
                     av_log(avctx, AV_LOG_ERROR,
                         "Invalid value of delta_poc: %d\n",
diff --git a/libavcodec/hevc_ps.h b/libavcodec/hevc_ps.h
index 3cd2eac923..1d3bdca4c6 100644
--- a/libavcodec/hevc_ps.h
+++ b/libavcodec/hevc_ps.h
@@ -78,8 +78,6 @@ typedef struct ShortTermRPS {
     unsigned int num_negative_pics;
     int num_delta_pocs;
     int rps_idx_num_delta_pocs;
-    int32_t delta_poc_s0[32];
-    int32_t delta_poc_s1[32];
     int32_t delta_poc[32];
     uint8_t used[32];
 } ShortTermRPS;
diff --git a/libavcodec/vulkan_hevc.c b/libavcodec/vulkan_hevc.c
index df86049d22..21cf49c0ec 100644
--- a/libavcodec/vulkan_hevc.c
+++ b/libavcodec/vulkan_hevc.c
@@ -351,6 +351,8 @@ static void set_sps(const HEVCSPS *sps, int sps_idx,
             pal->PredictorPaletteEntries[i][j] = sps->sps_palette_predictor_initializer[i][j];
 
     for (int i = 0; i < sps->nb_st_rps; i++) {
+        const ShortTermRPS *st_rps = &sps->st_rps[i];
+
         str[i] = (StdVideoH265ShortTermRefPicSet) {
             .flags = (StdVideoH265ShortTermRefPicSetFlags) {
                 .inter_ref_pic_set_prediction_flag = sps->st_rps[i].rps_predict,
@@ -375,13 +377,14 @@ static void set_sps(const HEVCSPS *sps, int sps_idx,
             str[i].used_by_curr_pic_flag |= sps->st_rps[i].used[j] << j;
 
         for (int j = 0; j < str[i].num_negative_pics; j++) {
-            str[i].delta_poc_s0_minus1[j] = sps->st_rps[i].delta_poc_s0[j] - 1;
+            str[i].delta_poc_s0_minus1[j] = st_rps->delta_poc[j] - (j ? st_rps->delta_poc[j - 1] : 0) - 1;
             str[i].used_by_curr_pic_s0_flag |= sps->st_rps[i].used[j] << j;
         }
 
         for (int j = 0; j < str[i].num_positive_pics; j++) {
-            str[i].delta_poc_s1_minus1[j] = sps->st_rps[i].delta_poc_s1[j] - 1;
-            str[i].used_by_curr_pic_s0_flag |= sps->st_rps[i].used[str[i].num_negative_pics + j] << j;
+            str[i].delta_poc_s1_minus1[j] = st_rps->delta_poc[st_rps->num_negative_pics + j] -
+                                            (j ? st_rps->delta_poc[st_rps->num_negative_pics + j - 1] : 0) - 1;
+            str[i].used_by_curr_pic_s1_flag |= sps->st_rps[i].used[str[i].num_negative_pics + j] << j;
         }
     }