rtpenc: Add support for mp4 format h264

Signed-off-by: Martin Storsjö <martin@martin.st>

libavformat/rtpenc_h264.c

@@ -29,6 +29,21 @@
 #include "avc.h"
 #include "rtpenc.h"
 
+static const uint8_t *avc_mp4_find_startcode(const uint8_t *start, const uint8_t *end, int nal_length_size)
+{
+    int res = 0;
+
+    if (end - start < nal_length_size)
+        return NULL;
+    while (nal_length_size--)
+        res = (res << 8) | *start++;
+
+    if (start + res > end || res < 0 || start + res < start)
+        return NULL;
+
+    return start + res;
+}
+
 static void nal_send(AVFormatContext *s1, const uint8_t *buf, int size, int last)
 {
     RTPMuxContext *s = s1->priv_data;
@@ -66,12 +81,22 @@ void ff_rtp_send_h264(AVFormatContext *s1, const uint8_t *buf1, int size)
     RTPMuxContext *s = s1->priv_data;
 
     s->timestamp = s->cur_timestamp;
-    r = ff_avc_find_startcode(buf1, end);
+    if (s->nal_length_size)
+        r = avc_mp4_find_startcode(buf1, end, s->nal_length_size) ? buf1 : end;
+    else
+        r = ff_avc_find_startcode(buf1, end);
     while (r < end) {
         const uint8_t *r1;
 
-        while(!*(r++));
-        r1 = ff_avc_find_startcode(r, end);
+        if (s->nal_length_size) {
+            r1 = avc_mp4_find_startcode(r, end, s->nal_length_size);
+            if (!r1)
+                r1 = end;
+            r += s->nal_length_size;
+        } else {
+            while (!*(r++));
+            r1 = ff_avc_find_startcode(r, end);
+        }
         nal_send(s1, r, r1 - r, r1 == end);
         r = r1;
     }