avutil/softfloat: Fix overflows in shifts in av_cmp_sf() and av_gt_sf()

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
9 years ago · cee3c9d29a
parent df2a2117d2
commit cee3c9d29a
1 changed files with 8 additions and 4 deletions
--- a/libavutil/softfloat.h
+++ b/libavutil/softfloat.h
@ -119,15 +119,19 @@ static inline av_const SoftFloat av_div_sf(SoftFloat a, SoftFloat b){

 static inline av_const int av_cmp_sf(SoftFloat a, SoftFloat b){
    int t= a.exp - b.exp;
-    if(t<0) return (a.mant >> (-t)) -  b.mant      ;
-    else    return  a.mant          - (b.mant >> t);
+    if      (t <-31) return                  -  b.mant      ;
+    else if (t <  0) return (a.mant >> (-t)) -  b.mant      ;
+    else if (t < 32) return  a.mant          - (b.mant >> t);
+    else             return  a.mant                         ;
 }

 static inline av_const int av_gt_sf(SoftFloat a, SoftFloat b)
 {
    int t= a.exp - b.exp;
-    if(t<0) return (a.mant >> (-t)) >  b.mant      ;
-    else    return  a.mant          > (b.mant >> t);
+    if      (t <-31) return 0;
+    else if (t <  0) return (a.mant >> (-t)) >  b.mant      ;
+    else if (t < 32) return  a.mant          > (b.mant >> t);
+    else             return 1;
 }

 static inline av_const SoftFloat av_add_sf(SoftFloat a, SoftFloat b){