lavc/videotoolbox: fix avcC extradata creation

Fixes properly ticket #5638.

Since 772ad7142d PPS and SPS contain the
nal type byte.

This reverts commit b8d754c5d0.
This reverts commit 8e47a99f57.

libavcodec/videotoolbox.c

@@ -84,7 +84,7 @@ CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx)
     H264Context *h     = avctx->priv_data;
     CFDataRef data = NULL;
     uint8_t *p;
-    int vt_extradata_size = 6 + 3 + h->ps.sps->data_size + 4 + h->ps.pps->data_size;
+    int vt_extradata_size = 6 + 2 + h->ps.sps->data_size + 3 + h->ps.pps->data_size;
     uint8_t *vt_extradata = av_malloc(vt_extradata_size);
     if (!vt_extradata)
         return NULL;
@@ -92,21 +92,19 @@ CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx)
     p = vt_extradata;
 
     AV_W8(p + 0, 1); /* version */
-    AV_W8(p + 1, h->ps.sps->data[0]); /* profile */
-    AV_W8(p + 2, h->ps.sps->data[1]); /* profile compat */
-    AV_W8(p + 3, h->ps.sps->data[2]); /* level */
+    AV_W8(p + 1, h->ps.sps->data[1]); /* profile */
+    AV_W8(p + 2, h->ps.sps->data[2]); /* profile compat */
+    AV_W8(p + 3, h->ps.sps->data[3]); /* level */
     AV_W8(p + 4, 0xff); /* 6 bits reserved (111111) + 2 bits nal size length - 3 (11) */
     AV_W8(p + 5, 0xe1); /* 3 bits reserved (111) + 5 bits number of sps (00001) */
-    AV_WB16(p + 6, h->ps.sps->data_size + 1);
-    AV_W8(p + 8, NAL_SPS | (3 << 5)); // NAL unit header
-    memcpy(p + 9, h->ps.sps->data, h->ps.sps->data_size);
-    p += 9 + h->ps.sps->data_size;
+    AV_WB16(p + 6, h->ps.sps->data_size);
+    memcpy(p + 8, h->ps.sps->data, h->ps.sps->data_size);
+    p += 8 + h->ps.sps->data_size;
     AV_W8(p + 0, 1); /* number of pps */
-    AV_WB16(p + 1, h->ps.pps->data_size + 1);
-    AV_W8(p + 3, NAL_PPS | (3 << 5)); // NAL unit header
-    memcpy(p + 4, h->ps.pps->data, h->ps.pps->data_size);
+    AV_WB16(p + 1, h->ps.pps->data_size);
+    memcpy(p + 3, h->ps.pps->data, h->ps.pps->data_size);
 
-    p += 4 + h->ps.pps->data_size;
+    p += 3 + h->ps.pps->data_size;
     av_assert0(p - vt_extradata == vt_extradata_size);
 
     data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
@@ -487,58 +485,23 @@ static CFDictionaryRef videotoolbox_buffer_attributes_create(int width,
     return buffer_attributes;
 }
 
-static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(AVCodecContext *avctx,
-                                                                   CMVideoCodecType codec_type,
+static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type,
                                                                    CFDictionaryRef decoder_spec,
                                                                    int width,
                                                                    int height)
 {
-    CMFormatDescriptionRef cm_fmt_desc = NULL;
-    int status;
-
-#if TARGET_OS_IPHONE || defined(__MAC_10_9)
-    H264Context *h = codec_type == kCMVideoCodecType_H264 ? avctx->priv_data : NULL;
-
-    if (h && h->ps.sps->data_size && h->ps.pps->data_size) {
-        int ps_count = 2;
-        const uint8_t **ps_data = av_malloc(sizeof(uint8_t*) * ps_count);
-        size_t *ps_sizes = av_malloc(sizeof(size_t)  * ps_count);
-
-        ps_data[0]  = h->ps.sps->data;
-        ps_sizes[0] = h->ps.sps->data_size;
-
-        ps_data[1]  = h->ps.pps->data;
-        ps_sizes[1] = h->ps.pps->data_size;
+    CMFormatDescriptionRef cm_fmt_desc;
+    OSStatus status;
 
-        status = CMVideoFormatDescriptionCreateFromH264ParameterSets(NULL,
-                                                                     ps_count,
-                                                                     ps_data,
-                                                                     ps_sizes,
-                                                                     4,
-                                                                     &cm_fmt_desc);
-        av_freep(&ps_sizes);
-        av_freep(&ps_data);
+    status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
+                                            codec_type,
+                                            width,
+                                            height,
+                                            decoder_spec, // Dictionary of extension
+                                            &cm_fmt_desc);
 
-        if (status) {
-            av_log(avctx, AV_LOG_ERROR, "Error creating H.264 format description: %d\n", status);
-            return NULL;
-        }
-    } else {
-#endif
-        status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
-                                                codec_type,
-                                                width,
-                                                height,
-                                                decoder_spec, // Dictionary of extension
-                                                &cm_fmt_desc);
-
-        if (status) {
-            av_log(avctx, AV_LOG_ERROR, "Error creating format description: %d\n", status);
-            return NULL;
-        }
-#if TARGET_OS_IPHONE || defined(__MAC_10_9)
-    }
-#endif
+    if (status)
+        return NULL;
 
     return cm_fmt_desc;
 }
@@ -578,8 +541,7 @@ static int videotoolbox_default_init(AVCodecContext *avctx)
 
     decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx);
 
-    videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(avctx,
-                                                                videotoolbox->cm_codec_type,
+    videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type,
                                                                 decoder_spec,
                                                                 avctx->width,
                                                                 avctx->height);