From 3e1028c625e11d9d19376f5c88267de1cee8fa70 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Sun, 19 Feb 2017 14:34:55 +0100
Subject: [PATCH] avcodec/flac: Fix several integer overflows

Fixes: 686513-media
Found-by: Matt Wolenetz <wolenetz@google.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/flacdec.c |  4 ++--
 libavcodec/flacdsp.c | 12 ++----------
 libavcodec/flacdsp.h | 11 +++++++++++
 3 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/libavcodec/flacdec.c b/libavcodec/flacdec.c
index 0fffc2dd94..6ea86d4eb1 100644
--- a/libavcodec/flacdec.c
+++ b/libavcodec/flacdec.c
@@ -326,7 +326,7 @@ static int decode_subframe_fixed(FLACContext *s, int32_t *decoded,
     return 0;
 }
 
-static void lpc_analyze_remodulate(int32_t *decoded, const int coeffs[32],
+static void lpc_analyze_remodulate(SUINT32 *decoded, const int coeffs[32],
                                    int order, int qlevel, int len, int bps)
 {
     int i, j;
@@ -342,7 +342,7 @@ static void lpc_analyze_remodulate(int32_t *decoded, const int coeffs[32],
     for (i = len - 1; i >= order; i--) {
         int64_t p = 0;
         for (j = 0; j < order; j++)
-            p += coeffs[j] * (int64_t)decoded[i-order+j];
+            p += coeffs[j] * (int64_t)(int32_t)decoded[i-order+j];
         decoded[i] -= p >> qlevel;
     }
     for (i = order; i < len; i++, decoded++) {
diff --git a/libavcodec/flacdsp.c b/libavcodec/flacdsp.c
index 52c3e32ed0..560091f73a 100644
--- a/libavcodec/flacdsp.c
+++ b/libavcodec/flacdsp.c
@@ -43,14 +43,6 @@
 #define PLANAR 1
 #include "flacdsp_template.c"
 
-// For debuging we use signed operations so overflows can be detected (by ubsan)
-// For production we use unsigned so there are no undefined operations
-#ifdef CHECKED
-#define SUINT   int
-#else
-#define SUINT   unsigned
-#endif
-
 static void flac_lpc_16_c(int32_t *decoded, const int coeffs[32],
                           int pred_order, int qlevel, int len)
 {
@@ -67,9 +59,9 @@ static void flac_lpc_16_c(int32_t *decoded, const int coeffs[32],
             c = coeffs[j];
         }
         s0 += c*d;
-        d = decoded[j] += s0 >> qlevel;
+        d = decoded[j] += (SUINT)(s0 >> qlevel);
         s1 += c*d;
-        decoded[j + 1] += s1 >> qlevel;
+        decoded[j + 1] += (SUINT)(s1 >> qlevel);
     }
     if (i < len) {
         int sum = 0;
diff --git a/libavcodec/flacdsp.h b/libavcodec/flacdsp.h
index f5cbd94724..31417f8414 100644
--- a/libavcodec/flacdsp.h
+++ b/libavcodec/flacdsp.h
@@ -20,8 +20,19 @@
 #define AVCODEC_FLACDSP_H
 
 #include <stdint.h>
+#include "libavutil/internal.h"
 #include "libavutil/samplefmt.h"
 
+// For debuging we use signed operations so overflows can be detected (by ubsan)
+// For production we use unsigned so there are no undefined operations
+#ifdef CHECKED
+#define SUINT   int
+#define SUINT32 int32_t
+#else
+#define SUINT   unsigned
+#define SUINT32 uint32_t
+#endif
+
 typedef struct FLACDSPContext {
     void (*decorrelate[4])(uint8_t **out, int32_t **in, int channels,
                            int len, int shift);