avformat/rpl: Fix check for negative values

Fixes: signed integer overflow: 10 * -1923267925333400000 cannot be represented in type 'int64_t' (aka 'long')
Fixes: 378891963/clusterfuzz-testcase-minimized-fuzzer_loadfile_direct-5714338935013376
Found-by: ossfuzz
Reported-by: Kacper Michajlow <kasper93@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/rpl.c

@@ -102,7 +102,7 @@ static AVRational read_fps(const char* line, int* error)
         line++;
     for (; *line>='0' && *line<='9'; line++) {
         // Truncate any numerator too large to fit into an int64_t
-        if (num > (INT64_MAX - 9) / 10 || den > INT64_MAX / 10)
+        if (num > (INT64_MAX - 9) / 10ULL || den > INT64_MAX / 10ULL)
             break;
         num  = 10 * num + (*line - '0');
         den *= 10;