avfilter/scale_eval: Use 64bit, check values in ff_scale_adjust_dimensions()

Found by reviewing CID1513722 Operands don't affect result

Sponsored-by: Sovereign Tech Fund
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavfilter/scale_eval.c

@@ -114,7 +114,7 @@ int ff_scale_adjust_dimensions(AVFilterLink *inlink,
     int *ret_w, int *ret_h,
     int force_original_aspect_ratio, int force_divisible_by)
 {
-    int w, h;
+    int64_t w, h;
     int factor_w, factor_h;
 
     w = *ret_w;
@@ -149,9 +149,9 @@ int ff_scale_adjust_dimensions(AVFilterLink *inlink,
      * unless force_divisible_by is defined as well */
     if (force_original_aspect_ratio) {
         // Including force_divisible_by here rounds to the nearest multiple of it.
-        int tmp_w = av_rescale(h, inlink->w, inlink->h * (int64_t)force_divisible_by)
+        int64_t tmp_w = av_rescale(h, inlink->w, inlink->h * (int64_t)force_divisible_by)
                     * force_divisible_by;
-        int tmp_h = av_rescale(w, inlink->h, inlink->w * (int64_t)force_divisible_by)
+        int64_t tmp_h = av_rescale(w, inlink->h, inlink->w * (int64_t)force_divisible_by)
                     * force_divisible_by;
 
         if (force_original_aspect_ratio == 1) {
@@ -173,6 +173,9 @@ int ff_scale_adjust_dimensions(AVFilterLink *inlink,
         }
     }
 
+    if ((int32_t)w != w || (int32_t)h != h)
+        return AVERROR(EINVAL);
+
     *ret_w = w;
     *ret_h = h;
 

libavfilter/scale_eval.h

@@ -41,7 +41,7 @@ int ff_scale_eval_dimensions(void *ctx,
  * force_original_aspect_ratio is set. force_divisible_by is used only when
  * force_original_aspect_ratio is set and must be at least 1.
  *
- * Returns 0.
+ * Returns negative error code on error or non negative on success
  */
 int ff_scale_adjust_dimensions(AVFilterLink *inlink,
     int *ret_w, int *ret_h,