libavformat/vvc: Make probe more conservative

Reduce false positives for VVC files by adding additional checks in
`vvc_probe`. Specifically, `nuh_temporal_id_plus1` is tested for valid
values in extra cases depending on the NAL unit type, as per ITU-T H.266
section 7.4.2.2.

Resolves trac #10703.

Signed-off-by: Frank Plowman <post@frankplowman.com>
Signed-off-by: Zhao Zhili <zhilizhao@tencent.com>
(cherry picked from commit 7d7ba2175c9e139b594b3cdf686c29bc116154a8)

libavformat/vvcdec.c

@@ -24,6 +24,22 @@
 #include "avformat.h"
 #include "rawdec.h"
 
+static int check_temporal_id(uint8_t nuh_temporal_id_plus1, int type)
+{
+    if (nuh_temporal_id_plus1 == 0)
+        return 0;
+
+    if (nuh_temporal_id_plus1 != 1) {
+        if (type >= VVC_IDR_W_RADL && type <= VVC_RSV_IRAP_11
+                || type == VVC_DCI_NUT || type == VVC_OPI_NUT
+                || type == VVC_VPS_NUT || type == VVC_SPS_NUT
+                || type == VVC_EOS_NUT || type == VVC_EOB_NUT)
+            return 0;
+    }
+
+    return 1;
+}
+
 static int vvc_probe(const AVProbeData *p)
 {
     uint32_t code = -1;
@@ -39,7 +55,7 @@ static int vvc_probe(const AVProbeData *p)
             if (code & 0x80) // forbidden_zero_bit
                 return 0;
 
-            if ((nal2 & 0x7) == 0) // nuh_temporal_id_plus1
+            if (!check_temporal_id(nal2 & 0x7, type))
                 return 0;
 
             switch (type) {