avcodec/rangecoder: Add and test ff_rac_check_termination()

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/rangecoder.c

@@ -121,3 +121,22 @@ int ff_rac_terminate(RangeCoder *c, int version)
 
     return c->bytestream - c->bytestream_start;
 }
+
+int ff_rac_check_termination(RangeCoder *c, int version)
+{
+    if (version == 1) {
+        RangeCoder tmp = *c;
+        get_rac(c, (uint8_t[]) { 129 });
+
+        if (c->bytestream == tmp.bytestream && c->bytestream > c->bytestream_start)
+            tmp.low -= *--tmp.bytestream;
+        tmp.bytestream_end = tmp.bytestream;
+
+        if (get_rac(&tmp, (uint8_t[]) { 129 }))
+            return AVERROR_INVALIDDATA;
+    } else {
+        if (c->bytestream_end != c->bytestream)
+            return AVERROR_INVALIDDATA;
+    }
+    return 0;
+}

libavcodec/rangecoder.h

@@ -57,6 +57,15 @@ void ff_init_range_decoder(RangeCoder *c, const uint8_t *buf, int buf_size);
  */
 int ff_rac_terminate(RangeCoder *c, int version);
 
+/**
+ * Check if at the current position there is a valid looking termination
+ * @param version version 0 requires the decoder to know the data size in bytes
+ *                version 1 needs about 1 bit more space but does not need to
+ *                          carry the size from encoder to decoder
+ * @returns negative AVERROR code on error or non negative.
+ */
+int ff_rac_check_termination(RangeCoder *c, int version);
+
 void ff_build_rac_states(RangeCoder *c, int factor, int max_p);
 
 static inline void renorm_encoder(RangeCoder *c)

libavcodec/tests/rangecoder.c

@@ -60,8 +60,11 @@ int main(void)
                     av_log(NULL, AV_LOG_ERROR, "rac failure at %d pass %d version %d\n", i, p, version);
                     return 1;
                 }
-            if(version)
+
-                get_rac(&c, (uint8_t[]) { 129 });
+            if (ff_rac_check_termination(&c, version) < 0) {
+                av_log(NULL, AV_LOG_ERROR, "rac failure at termination pass %d version %d\n", p, version);
+                return 1;
+            }
             if (c.bytestream - c.bytestream_start - actual_length != version) {
                 av_log(NULL, AV_LOG_ERROR, "rac failure at pass %d version %d\n", p, version);
                 return 1;