From a93801b6266c2170842275271c64a40d015cd03b Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@haasn.dev>
Date: Tue, 16 Jul 2024 15:11:27 +0200
Subject: [PATCH] avcodec/dovi_rpudec: implement validation for compression

Add some error checking. I've limited it to AV_EF_CAREFUL and
AV_EF_COMPLIANT for now, because we can technically decode such RPUs
just fine.
---
 libavcodec/dovi_rpudec.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/libavcodec/dovi_rpudec.c b/libavcodec/dovi_rpudec.c
index 375e6e560b..bf6e5075d1 100644
--- a/libavcodec/dovi_rpudec.c
+++ b/libavcodec/dovi_rpudec.c
@@ -314,6 +314,7 @@ int ff_dovi_rpu_parse(DOVIContext *s, const uint8_t *rpu, size_t rpu_size,
     uint8_t use_prev_vdr_rpu;
     uint8_t use_nlq;
     uint8_t profile;
+    uint8_t compression = s->cfg.dv_profile ? s->cfg.dv_md_compression : 0;
 
     if (rpu_size < 5)
         return AVERROR_INVALIDDATA;
@@ -459,6 +460,20 @@ int ff_dovi_rpu_parse(DOVIContext *s, const uint8_t *rpu, size_t rpu_size,
         return AVERROR_INVALIDDATA;
     }
 
+    if (err_recognition & (AV_EF_COMPLIANT | AV_EF_CAREFUL)) {
+        if (profile < 8 && compression) {
+            av_log(s->logctx, AV_LOG_ERROR, "Profile %d RPUs should not use "
+                   "metadata compression.", profile);
+            return AVERROR_INVALIDDATA;
+        }
+
+        if (use_prev_vdr_rpu && !compression) {
+            av_log(s->logctx, AV_LOG_ERROR, "Uncompressed RPUs should not have "
+                   "use_prev_vdr_rpu=1\n");
+            return AVERROR_INVALIDDATA;
+        }
+    }
+
     if (use_prev_vdr_rpu) {
         int prev_vdr_rpu_id = get_ue_golomb_31(gb);
         VALIDATE(prev_vdr_rpu_id, 0, DOVI_MAX_DM_ID);