avcodec/mpegaudiodec: Eliminate many undefined operations

Fixes: 625/clusterfuzz-testcase-4574924406521856
Fixes: 626/clusterfuzz-testcase-4738718621499392

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/dct32_template.c

@@ -21,17 +21,28 @@
 
 #include "dct32.h"
 #include "mathops.h"
+#include "libavutil/internal.h"
+
+#ifdef CHECKED
+#define SUINT   int
+#define SUINT32 int32_t
+#else
+#define SUINT   unsigned
+#define SUINT32 uint32_t
+#endif
 
 #if DCT32_FLOAT
 #   define dct32 ff_dct32_float
 #   define FIXHR(x)       ((float)(x))
 #   define MULH3(x, y, s) ((s)*(y)*(x))
 #   define INTFLOAT float
+#   define SUINTFLOAT float
 #else
 #   define dct32 ff_dct32_fixed
 #   define FIXHR(a)       ((int)((a) * (1LL<<32) + 0.5))
 #   define MULH3(x, y, s) MULH((s)*(x), y)
 #   define INTFLOAT int
+#   define SUINTFLOAT SUINT
 #endif
 
 
@@ -114,9 +125,9 @@
 /* DCT32 without 1/sqrt(2) coef zero scaling. */
 void dct32(INTFLOAT *out, const INTFLOAT *tab)
 {
-    INTFLOAT tmp0, tmp1;
+    SUINTFLOAT tmp0, tmp1;
 
-    INTFLOAT val0 , val1 , val2 , val3 , val4 , val5 , val6 , val7 ,
+    SUINTFLOAT val0 , val1 , val2 , val3 , val4 , val5 , val6 , val7 ,
              val8 , val9 , val10, val11, val12, val13, val14, val15,
              val16, val17, val18, val19, val20, val21, val22, val23,
              val24, val25, val26, val27, val28, val29, val30, val31;

libavcodec/mpegaudio.h

@@ -31,6 +31,7 @@
 #endif
 
 #include <stdint.h>
+#include "libavutil/internal.h"
 
 /* max frame size, in samples */
 #define MPA_FRAME_SIZE 1152
@@ -58,16 +59,27 @@
 
 #define FIX(a)   ((int)((a) * FRAC_ONE))
 
+#ifdef CHECKED
+#define SUINT   int
+#define SUINT32 int32_t
+#else
+#define SUINT   unsigned
+#define SUINT32 uint32_t
+#endif
+
 #if USE_FLOATS
 #   define INTFLOAT float
+#   define SUINTFLOAT float
 typedef float MPA_INT;
 typedef float OUT_INT;
 #elif FRAC_BITS <= 15
 #   define INTFLOAT int
+#   define SUINTFLOAT SUINT
 typedef int16_t MPA_INT;
 typedef int16_t OUT_INT;
 #else
 #   define INTFLOAT int
+#   define SUINTFLOAT SUINT
 typedef int32_t MPA_INT;
 typedef int16_t OUT_INT;
 #endif

libavcodec/mpegaudiodec_fixed.c

@@ -25,13 +25,13 @@
 
 #include "mpegaudio.h"
 
-#define SHR(a,b)       ((a)>>(b))
+#define SHR(a,b)       (((int)(a))>>(b))
 /* WARNING: only correct for positive numbers */
 #define FIXR_OLD(a)    ((int)((a) * FRAC_ONE + 0.5))
 #define FIXR(a)        ((int)((a) * FRAC_ONE + 0.5))
 #define FIXHR(a)       ((int)((a) * (1LL<<32) + 0.5))
 #define MULH3(x, y, s) MULH((s)*(x), y)
-#define MULLx(x, y, s) MULL(x,y,s)
+#define MULLx(x, y, s) MULL((int)(x),(y),s)
 #define RENAME(a)      a ## _fixed
 #define OUT_FMT   AV_SAMPLE_FMT_S16
 #define OUT_FMT_P AV_SAMPLE_FMT_S16P

libavcodec/mpegaudiodec_template.c

@@ -1182,9 +1182,9 @@ found2:
     } while (0)
 #else
 #define AA(j) do {                                              \
-        int tmp0 = ptr[-1-j];                                   \
+        SUINT tmp0 = ptr[-1-j];                                   \
-        int tmp1 = ptr[   j];                                   \
+        SUINT tmp1 = ptr[   j];                                   \
-        int tmp2 = MULH(tmp0 + tmp1, csa_table[j][0]);          \
+        SUINT tmp2 = MULH(tmp0 + tmp1, csa_table[j][0]);          \
         ptr[-1-j] = 4 * (tmp2 - MULH(tmp1, csa_table[j][2]));   \
         ptr[   j] = 4 * (tmp2 + MULH(tmp0, csa_table[j][3]));   \
     } while (0)

libavcodec/mpegaudiodsp_template.c

@@ -63,8 +63,8 @@ static inline int round_sample(int64_t *sum)
 #   define MACS(rt, ra, rb) MAC64(rt, ra, rb)
 #   define MLSS(rt, ra, rb) MLS64(rt, ra, rb)
 #   define MULH3(x, y, s) MULH((s)*(x), y)
-#   define MULLx(x, y, s) MULL(x,y,s)
+#   define MULLx(x, y, s) MULL((int)(x),(y),s)
-#   define SHR(a,b)       ((a)>>(b))
+#   define SHR(a,b)       (((int)(a))>>(b))
 #   define FIXR(a)        ((int)((a) * FRAC_ONE + 0.5))
 #   define FIXHR(a)       ((int)((a) * (1LL<<32) + 0.5))
 #endif
@@ -300,11 +300,11 @@ static const INTFLOAT icos36h[9] = {
 };
 
 /* using Lee like decomposition followed by hand coded 9 points DCT */
-static void imdct36(INTFLOAT *out, INTFLOAT *buf, INTFLOAT *in, INTFLOAT *win)
+static void imdct36(INTFLOAT *out, INTFLOAT *buf, SUINTFLOAT *in, INTFLOAT *win)
 {
     int i, j;
-    INTFLOAT t0, t1, t2, t3, s0, s1, s2, s3;
+    SUINTFLOAT t0, t1, t2, t3, s0, s1, s2, s3;
-    INTFLOAT tmp[18], *tmp1, *in1;
+    SUINTFLOAT tmp[18], *tmp1, *in1;
 
     for (i = 17; i >= 1; i--)
         in[i] += in[i-1];