avcodec: h264: Extract decoder methods

Extract two methods from decode_registered_user_data in order to improve
code readability. Also make the constant holding the allocation size a
64-bit unsigned integer so that the size comparison against INT_MAX makes
sense.

Bug-Id: CID1312090

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

libavcodec/h264_sei.c

@@ -106,11 +106,78 @@ static int decode_picture_timing(H264Context *h)
     return 0;
 }
 
+static int decode_registered_user_data_afd(H264Context *h, int size)
+{
+    int flag;
+
+    if (size-- < 1)
+        return AVERROR_INVALIDDATA;
+    skip_bits(&h->gb, 1);               // 0
+    flag = get_bits(&h->gb, 1);         // active_format_flag
+    skip_bits(&h->gb, 6);               // reserved
+
+    if (flag) {
+        if (size-- < 1)
+            return AVERROR_INVALIDDATA;
+        skip_bits(&h->gb, 4);           // reserved
+        h->active_format_description   = get_bits(&h->gb, 4);
+        h->sei_reguserdata_afd_present = 1;
+    }
+
+    return 0;
+}
+
+static int decode_registered_user_data_closed_caption(H264Context *h, int size)
+{
+    int flag;
+    int user_data_type_code;
+    int cc_count;
+
+    if (size < 3)
+        return AVERROR(EINVAL);
+
+    user_data_type_code = get_bits(&h->gb, 8);
+    if (user_data_type_code == 0x3) {
+        skip_bits(&h->gb, 1);           // reserved
+
+        flag = get_bits(&h->gb, 1);     // process_cc_data_flag
+        if (flag) {
+            skip_bits(&h->gb, 1);       // zero bit
+            cc_count = get_bits(&h->gb, 5);
+            skip_bits(&h->gb, 8);       // reserved
+            size -= 2;
+
+            if (cc_count && size >= cc_count * 3) {
+                const uint64_t new_size = (h->a53_caption_size + cc_count
+                                           * UINT64_C(3));
+                int i, ret;
+
+                if (new_size > INT_MAX)
+                    return AVERROR(EINVAL);
+
+                /* Allow merging of the cc data from two fields. */
+                ret = av_reallocp(&h->a53_caption, new_size);
+                if (ret < 0)
+                    return ret;
+
+                for (i = 0; i < cc_count; i++) {
+                    h->a53_caption[h->a53_caption_size++] = get_bits(&h->gb, 8);
+                    h->a53_caption[h->a53_caption_size++] = get_bits(&h->gb, 8);
+                    h->a53_caption[h->a53_caption_size++] = get_bits(&h->gb, 8);
+                }
+
+                skip_bits(&h->gb, 8);   // marker_bits
+            }
+        }
+    }
+
+    return 0;
+}
+
 static int decode_registered_user_data(H264Context *h, int size)
 {
     uint32_t country_code;
     uint32_t user_identifier;
-    int flag, user_data_type_code, cc_count;
 
     if (size < 7)
         return AVERROR_INVALIDDATA;
@@ -129,60 +196,9 @@ static int decode_registered_user_data(H264Context *h, int size)
 
     switch (user_identifier) {
         case MKBETAG('D', 'T', 'G', '1'):       // afd_data
-            if (size-- < 1)
-                return AVERROR_INVALIDDATA;
-            skip_bits(&h->gb, 1);               // 0
-            flag = get_bits(&h->gb, 1);         // active_format_flag
-            skip_bits(&h->gb, 6);               // reserved
-
-            if (flag) {
-                if (size-- < 1)
-                    return AVERROR_INVALIDDATA;
-                skip_bits(&h->gb, 4);           // reserved
-                h->active_format_description   = get_bits(&h->gb, 4);
-                h->sei_reguserdata_afd_present = 1;
-            }
-            break;
+            return decode_registered_user_data_afd(h, size);
         case MKBETAG('G', 'A', '9', '4'):       // closed captions
-            if (size < 3)
-                return AVERROR(EINVAL);
-
-            user_data_type_code = get_bits(&h->gb, 8);
-            if (user_data_type_code == 0x3) {
-                skip_bits(&h->gb, 1);           // reserved
-
-                flag = get_bits(&h->gb, 1);     // process_cc_data_flag
-                if (flag) {
-                    skip_bits(&h->gb, 1);       // zero bit
-                    cc_count = get_bits(&h->gb, 5);
-                    skip_bits(&h->gb, 8);       // reserved
-                    size -= 2;
-
-                    if (cc_count && size >= cc_count * 3) {
-                        int i, ret;
-                        int new_size = (int64_t) h->a53_caption_size +
-                                       (int64_t) cc_count * 3;
-
-                        if (new_size > INT_MAX)
-                            return AVERROR(EINVAL);
-
-                        /* Allow merging of the cc data from two fields. */
-                        ret = av_reallocp(&h->a53_caption,
-                                          h->a53_caption_size + cc_count * 3);
-                        if (ret < 0)
-                            return ret;
-
-                        for (i = 0; i < cc_count; i++) {
-                            h->a53_caption[h->a53_caption_size++] = get_bits(&h->gb, 8);
-                            h->a53_caption[h->a53_caption_size++] = get_bits(&h->gb, 8);
-                            h->a53_caption[h->a53_caption_size++] = get_bits(&h->gb, 8);
-                        }
-
-                        skip_bits(&h->gb, 8);   // marker_bits
-                    }
-                }
-            }
-            break;
+            return decode_registered_user_data_closed_caption(h, size);
         default:
             skip_bits(&h->gb, size * 8);
             break;