mathematics: remove asserts from av_rescale_rnd()

It is a public function, it must not assert on its parameters.

libavutil/mathematics.c

@@ -23,7 +23,6 @@
  * miscellaneous math routines and tables
  */
 
-#include <assert.h>
 #include <stdint.h>
 #include <limits.h>
 
@@ -58,9 +57,9 @@ int64_t av_gcd(int64_t a, int64_t b){
 
 int64_t av_rescale_rnd(int64_t a, int64_t b, int64_t c, enum AVRounding rnd){
     int64_t r=0;
-    assert(c > 0);
-    assert(b >=0);
-    assert((unsigned)rnd<=5 && rnd!=4);
+
+    if (c <= 0 || b < 0 || rnd == 4 || rnd > 5)
+        return INT64_MIN;
 
     if(a<0 && a != INT64_MIN) return -av_rescale_rnd(-a, b, c, rnd ^ ((rnd>>1)&1));