* src/base/ftcalc.c (FT_Matrix_Check): Fix integer overflow.

Reported as

  https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=9811

ChangeLog

@@ -1,3 +1,11 @@
+2018-08-11  Werner Lemberg  <wl@gnu.org>
+
+	* src/base/ftcalc.c (FT_Matrix_Check): Fix integer overflow.
+
+	Reported as
+
+	  https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=9811
+
 2018-08-10  Alexei Podtelezhnikov  <apodtele@gmail.com>
 
 	* src/sfnt/ttsbit.c (tt_sbit_decoder_load_compound): Follow specs.

include/freetype/freetype.h

@@ -4669,6 +4669,11 @@ FT_BEGIN_HEADER
    *   This section contains various functions used to perform
    *   computations on 16.16 fixed-float numbers or 2d vectors.
    *
+   *   *Attention*: Most arithmetic functions take `FT_Long' as arguments.
+   *   For historical reasons, FreeType was designed under the assumption
+   *   that `FT_Long' is a 32-bit integer; results can thus be undefined
+   *   if the arguments don't fit into 32 bits.
+   *
    * @order:
    *   FT_MulDiv
    *   FT_MulFix

src/base/ftcalc.c

@@ -701,8 +701,8 @@
     if ( !delta )
       return FT_THROW( Invalid_Argument );  /* matrix can't be inverted */
 
-    matrix->xy = - FT_DivFix( matrix->xy, delta );
-    matrix->yx = - FT_DivFix( matrix->yx, delta );
+    matrix->xy = -FT_DivFix( matrix->xy, delta );
+    matrix->yx = -FT_DivFix( matrix->yx, delta );
 
     xx = matrix->xx;
     yy = matrix->yy;
@@ -784,6 +784,10 @@
         nonzero_minval = val[i];
     }
 
+    /* we only handle 32bit values */
+    if ( maxval > 0x7FFFFFFFL )
+      return 0;
+
     if ( maxval > 23170 )
     {
       FT_Fixed  scale = FT_DivFix( maxval, 23170 );