ADTS Muxer: Refuse to write illegal ADTS files by checking validity of header

members and erroring out if invalid

Patch by Alex Converse ( alex converse gmail com )

Originally committed as revision 17193 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavformat/adtsenc.c

@@ -32,7 +32,7 @@ typedef struct {
     int channel_conf;
 } ADTSContext;
 
-static int decode_extradata(ADTSContext *adts, uint8_t *buf, int size)
+static int decode_extradata(AVFormatContext *s, ADTSContext *adts, uint8_t *buf, int size)
 {
     GetBitContext gb;
 
@@ -41,6 +41,19 @@ static int decode_extradata(ADTSContext *adts, uint8_t *buf, int size)
     adts->sample_rate_index = get_bits(&gb, 4);
     adts->channel_conf = get_bits(&gb, 4);
 
+    if (adts->objecttype > 3) {
+        av_log(s, AV_LOG_ERROR, "MPEG-4 AOT %d is not allowed in ADTS\n", adts->objecttype);
+        return -1;
+    }
+    if (adts->sample_rate_index == 15) {
+        av_log(s, AV_LOG_ERROR, "Escape sample rate index illegal in ADTS\n");
+        return -1;
+    }
+    if (adts->channel_conf == 0) {
+        ff_log_missing_feature(s, "PCE based channel configuration", 0);
+        return -1;
+    }
+
     adts->write_adts = 1;
 
     return 0;
@@ -51,8 +64,9 @@ static int adts_write_header(AVFormatContext *s)
     ADTSContext *adts = s->priv_data;
     AVCodecContext *avc = s->streams[0]->codec;
 
-    if(avc->extradata_size > 0)
+    if(avc->extradata_size > 0 &&
-        decode_extradata(adts, avc->extradata, avc->extradata_size);
+            decode_extradata(s, adts, avc->extradata, avc->extradata_size) < 0)
+        return -1;
 
     return 0;
 }