avcodec/flac: Fix several integer overflows

Fixes: 686513-media
Found-by: Matt Wolenetz <wolenetz@google.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/flacdec.c

@@ -326,7 +326,7 @@ static int decode_subframe_fixed(FLACContext *s, int32_t *decoded,
     return 0;
 }
 
-static void lpc_analyze_remodulate(int32_t *decoded, const int coeffs[32],
+static void lpc_analyze_remodulate(SUINT32 *decoded, const int coeffs[32],
                                    int order, int qlevel, int len, int bps)
 {
     int i, j;
@@ -342,7 +342,7 @@ static void lpc_analyze_remodulate(int32_t *decoded, const int coeffs[32],
     for (i = len - 1; i >= order; i--) {
         int64_t p = 0;
         for (j = 0; j < order; j++)
-            p += coeffs[j] * (int64_t)decoded[i-order+j];
+            p += coeffs[j] * (int64_t)(int32_t)decoded[i-order+j];
         decoded[i] -= p >> qlevel;
     }
     for (i = order; i < len; i++, decoded++) {

libavcodec/flacdsp.c

@@ -43,14 +43,6 @@
 #define PLANAR 1
 #include "flacdsp_template.c"
 
-// For debuging we use signed operations so overflows can be detected (by ubsan)
-// For production we use unsigned so there are no undefined operations
-#ifdef CHECKED
-#define SUINT   int
-#else
-#define SUINT   unsigned
-#endif
-
 static void flac_lpc_16_c(int32_t *decoded, const int coeffs[32],
                           int pred_order, int qlevel, int len)
 {
@@ -67,9 +59,9 @@ static void flac_lpc_16_c(int32_t *decoded, const int coeffs[32],
             c = coeffs[j];
         }
         s0 += c*d;
-        d = decoded[j] += s0 >> qlevel;
+        d = decoded[j] += (SUINT)(s0 >> qlevel);
         s1 += c*d;
-        decoded[j + 1] += s1 >> qlevel;
+        decoded[j + 1] += (SUINT)(s1 >> qlevel);
     }
     if (i < len) {
         int sum = 0;

libavcodec/flacdsp.h

@@ -20,8 +20,19 @@
 #define AVCODEC_FLACDSP_H
 
 #include <stdint.h>
+#include "libavutil/internal.h"
 #include "libavutil/samplefmt.h"
 
+// For debuging we use signed operations so overflows can be detected (by ubsan)
+// For production we use unsigned so there are no undefined operations
+#ifdef CHECKED
+#define SUINT   int
+#define SUINT32 int32_t
+#else
+#define SUINT   unsigned
+#define SUINT32 uint32_t
+#endif
+
 typedef struct FLACDSPContext {
     void (*decorrelate[4])(uint8_t **out, int32_t **in, int channels,
                            int len, int shift);