[smooth] Fix complex rendering at high ppem.

We used to split large glyphs into horizontal bands and continue
bisecting them still horizontally if that was not enough.  This is
guaranteed to fail when a single scanline cannot fit into the
rendering memory pool.  Now we bisect the bands vertically so that
the smallest unit is a column of the band height, which is guranteed
to fit into memory.

* src/smooth/ftgrays.c (gray_convert_glyph): Implement it.

ChangeLog

@@ -1,3 +1,16 @@
+2017-11-01  Alexei Podtelezhnikov  <apodtele@gmail.com>
+
+	[smooth] Fix complex rendering at high ppem.
+
+	We used to split large glyphs into horizontal bands and continue
+	bisecting them still horizontally if that was not enough.  This is
+	guaranteed to fail when a single scanline cannot fit into the
+	rendering memory pool.  Now we bisect the bands vertically so that
+	the smallest unit is a column of the band height, which is guranteed
+	to fit into memory.
+
+	* src/smooth/ftgrays.c (gray_convert_glyph): Implement it.
+
 2017-10-20  Alexei Podtelezhnikov  <apodtele@gmail.com>
 
 	[smooth] Improve complex rendering at high ppem.

src/smooth/ftgrays.c

@@ -1736,35 +1736,43 @@ typedef ptrdiff_t  FT_PtrDist;
   static int
   gray_convert_glyph( RAS_ARG )
   {
+    const TCoord  yMin = ras.min_ey;
+    const TCoord  yMax = ras.max_ey;
+    const TCoord  xMin = ras.min_ex;
+    const TCoord  xMax = ras.max_ex;
+
     TCell    buffer[FT_MAX_GRAY_POOL];
-    TCoord   band_size = FT_MAX_GRAY_POOL / 8;
-    TCoord   count = ras.max_ey - ras.min_ey;
-    int      num_bands;
-    TCoord   min, max, max_y;
+    size_t   height = (size_t)( yMax - yMin );
+    size_t   n = FT_MAX_GRAY_POOL / 8;
+    TCoord   y;
     TCoord   bands[32];  /* enough to accommodate bisections */
     TCoord*  band;
 
 
     /* set up vertical bands */
-    if ( count > band_size )
+    if ( height > n )
     {
       /* two divisions rounded up */
-      num_bands = (int)( ( count + band_size - 1) / band_size );
-      band_size = ( count + num_bands - 1 ) / num_bands;
+      n       = ( height + n - 1 ) / n;
+      height  = ( height + n - 1 ) / n;
     }
 
-    min   = ras.min_ey;
-    max_y = ras.max_ey;
+    /* memory management */
+    n = ( height * sizeof ( PCell ) + sizeof ( TCell ) - 1 ) / sizeof ( TCell );
+
+    ras.cells     = buffer + n;
+    ras.max_cells = (FT_PtrDist)( FT_MAX_GRAY_POOL - n );
+    ras.ycells    = (PCell*)buffer;
 
-    for ( ; min < max_y; min = max )
+    for ( y = yMin; y < yMax; )
     {
-      max = min + band_size;
-      if ( max > max_y )
-        max = max_y;
+      ras.min_ey = y;
+      y         += height;
+      ras.max_ey = FT_MIN( y, yMax );
 
       band    = bands;
-      band[1] = min;
-      band[0] = max;
+      band[1] = xMin;
+      band[0] = xMax;
 
       do
       {
@@ -1772,27 +1780,12 @@ typedef ptrdiff_t  FT_PtrDist;
         int     error;
 
 
-        /* memory management */
-        {
-          size_t  ycount = (size_t)width;
-          size_t  cell_start;
-
-
-          cell_start = ( ycount * sizeof ( PCell ) + sizeof ( TCell ) - 1 ) /
-                       sizeof ( TCell );
-
-          ras.cells     = buffer + cell_start;
-          ras.max_cells = (FT_PtrDist)( FT_MAX_GRAY_POOL - cell_start );
-          ras.num_cells = 0;
-
-          ras.ycells = (PCell*)buffer;
-          while ( ycount )
-            ras.ycells[--ycount] = NULL;
-        }
+        FT_MEM_ZERO( ras.ycells, height * sizeof ( PCell ) );
 
+        ras.num_cells = 0;
         ras.invalid   = 1;
-        ras.min_ey    = band[1];
-        ras.max_ey    = band[0];
+        ras.min_ex    = band[1];
+        ras.max_ex    = band[0];
 
         error = gray_convert_glyph_inner( RAS_VAR );
 
@@ -1808,8 +1801,7 @@ typedef ptrdiff_t  FT_PtrDist;
         /* render pool overflow; we will reduce the render band by half */
         width >>= 1;
 
-        /* This is too complex for a single scanline; there must */
-        /* be some problems.                                     */
+        /* this should never happen even with tiny rendering pool */
         if ( width == 0 )
         {
           FT_TRACE7(( "gray_convert_glyph: rotten glyph\n" ));