Make get_leading_bearing return bool

Fixes https://github.com/harfbuzz/harfbuzz/issues/3496

Part of supporting >64k glyphs correctly.

src/OT/glyf/Glyph.hh

@@ -105,11 +105,13 @@ struct Glyph
     if (unlikely (!points.resize (points.length + PHANTOM_COUNT))) return false;
     hb_array_t<contour_point_t> phantoms = points.sub_array (points.length - PHANTOM_COUNT, PHANTOM_COUNT);
     {
-      int h_delta = (int) header->xMin -
-		    glyf_accelerator.hmtx->get_leading_bearing_without_var_unscaled (gid);
+      int lsb = 0;
+      int h_delta = glyf_accelerator.hmtx->get_leading_bearing_without_var_unscaled (gid, &lsb) ?
+		    (int) header->xMin - lsb : 0;
+      int tsb = 0;
       int v_orig  = (int) header->yMax +
 #ifndef HB_NO_VERTICAL
-		    glyf_accelerator.vmtx->get_leading_bearing_without_var_unscaled (gid)
+		    ((void) glyf_accelerator.vmtx->get_leading_bearing_without_var_unscaled (gid, &tsb), tsb)
 #else
 		    0
 #endif

src/OT/glyf/GlyphHeader.hh

@@ -19,7 +19,9 @@ struct GlyphHeader
   {
     /* Undocumented rasterizer behavior: shift glyph to the left by (lsb - xMin), i.e., xMin = lsb */
     /* extents->x_bearing = hb_min (glyph_header.xMin, glyph_header.xMax); */
-    extents->x_bearing = font->em_scale_x (glyf_accelerator.hmtx->get_leading_bearing_without_var_unscaled (gid));
+    int lsb = hb_min (xMin, xMax);
+    (void) glyf_accelerator.hmtx->get_leading_bearing_without_var_unscaled (gid, &lsb);
+    extents->x_bearing = font->em_scale_x (lsb);
     extents->y_bearing = font->em_scale_y (hb_max (yMin, yMax));
     extents->width     = font->em_scale_x (hb_max (xMin, xMax) - hb_min (xMin, xMax));
     extents->height    = font->em_scale_y (hb_min (yMin, yMax) - hb_max (yMin, yMax));

src/OT/glyf/glyf.hh

@@ -281,23 +281,20 @@ struct glyf_accelerator_t
     return hb_clamp (roundf (result), 0.f, (float) UINT_MAX / 2);
   }
 
-  int get_leading_bearing_with_var_unscaled (hb_font_t *font, hb_codepoint_t gid, bool is_vertical) const
+  bool get_leading_bearing_with_var_unscaled (hb_font_t *font, hb_codepoint_t gid, bool is_vertical, int *lsb) const
   {
-    if (unlikely (gid >= num_glyphs)) return 0;
+    if (unlikely (gid >= num_glyphs)) return false;
 
     hb_glyph_extents_t extents;
 
     contour_point_t phantoms[glyf_impl::PHANTOM_COUNT];
     if (unlikely (!get_points (font, gid, points_aggregator_t (font, &extents, phantoms, false))))
-      return
-#ifndef HB_NO_VERTICAL
-	is_vertical ? vmtx->get_leading_bearing_without_var_unscaled (gid) :
-#endif
-	hmtx->get_leading_bearing_without_var_unscaled (gid);
+      return false;
 
-    return is_vertical
+    *lsb = is_vertical
 	 ? roundf (phantoms[glyf_impl::PHANTOM_TOP].y) - extents.y_bearing
 	 : roundf (phantoms[glyf_impl::PHANTOM_LEFT].x);
+    return true;
   }
 #endif
 

src/hb-ot-font.cc

@@ -311,10 +311,10 @@ hb_ot_get_glyph_v_origin (hb_font_t *font,
   hb_glyph_extents_t extents = {0};
   if (ot_face->glyf->get_extents (font, glyph, &extents))
   {
-    if (ot_face->vmtx->has_data ())
+    const OT::vmtx_accelerator_t &vmtx = *ot_face->vmtx;
+    int tsb = 0;
+    if (vmtx.get_leading_bearing_with_var_unscaled (font, glyph, &tsb))
     {
-      const OT::vmtx_accelerator_t &vmtx = *ot_face->vmtx;
-      hb_position_t tsb = vmtx.get_leading_bearing_with_var_unscaled (font, glyph);
       *y = extents.y_bearing + font->em_scale_y (tsb);
       return true;
     }
@@ -514,10 +514,11 @@ hb_ot_font_set_funcs (hb_font_t *font)
 }
 
 #ifndef HB_NO_VAR
-int
-_glyf_get_leading_bearing_with_var_unscaled (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical)
+bool
+_glyf_get_leading_bearing_with_var_unscaled (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical,
+					     int *lsb)
 {
-  return font->face->table.glyf->get_leading_bearing_with_var_unscaled (font, glyph, is_vertical);
+  return font->face->table.glyf->get_leading_bearing_with_var_unscaled (font, glyph, is_vertical, lsb);
 }
 
 unsigned

src/hb-ot-hmtx-table.hh

@@ -43,8 +43,8 @@
 #define HB_OT_TAG_vmtx HB_TAG('v','m','t','x')
 
 
-HB_INTERNAL int
-_glyf_get_leading_bearing_with_var_unscaled (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical);
+HB_INTERNAL bool
+_glyf_get_leading_bearing_with_var_unscaled (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical, int *lsb);
 
 HB_INTERNAL unsigned
 _glyf_get_advance_with_var_unscaled (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical);
@@ -150,7 +150,9 @@ struct hmtxvmtx
 		hb_codepoint_t old_gid;
 		if (!c->plan->old_gid_for_new_gid (_, &old_gid))
 		  return hb_pair (0u, 0);
-		return hb_pair (_mtx.get_advance_without_var_unscaled (old_gid), _mtx.get_leading_bearing_without_var_unscaled (old_gid));
+		int lsb = 0;
+		(void) _mtx.get_leading_bearing_without_var_unscaled (old_gid, &lsb);
+		return hb_pair (_mtx.get_advance_without_var_unscaled (old_gid), +lsb);
 	      })
     ;
 
@@ -221,33 +223,42 @@ struct hmtxvmtx
 
     bool has_data () const { return (bool) num_bearings; }
 
-    int get_leading_bearing_without_var_unscaled (hb_codepoint_t glyph) const
+    bool get_leading_bearing_without_var_unscaled (hb_codepoint_t glyph,
+						   int *lsb) const
     {
       if (glyph < num_long_metrics)
-	return table->longMetricZ[glyph].sb;
+      {
+	*lsb = table->longMetricZ[glyph].sb;
+	return true;
+      }
 
       if (unlikely (glyph >= num_bearings))
-	return 0;
+	return false;
 
       const FWORD *bearings = (const FWORD *) &table->longMetricZ[num_long_metrics];
-      return bearings[glyph - num_long_metrics];
+      *lsb = bearings[glyph - num_long_metrics];
+      return true;
     }
 
-    int get_leading_bearing_with_var_unscaled (hb_font_t *font, hb_codepoint_t glyph) const
+    bool get_leading_bearing_with_var_unscaled (hb_font_t *font,
+						hb_codepoint_t glyph,
+						int *lsb) const
     {
-      int side_bearing = get_leading_bearing_without_var_unscaled (glyph);
+      if (!font->num_coords)
+	return get_leading_bearing_without_var_unscaled (glyph, lsb);
 
 #ifndef HB_NO_VAR
-      if (unlikely (glyph >= num_bearings) || !font->num_coords)
-	return side_bearing;
-
-      float lsb;
-      if (var_table->get_lsb_delta_unscaled (glyph, font->coords, font->num_coords, &lsb))
-	return side_bearing + roundf (lsb);
+      float delta;
+      if (var_table->get_lsb_delta_unscaled (glyph, font->coords, font->num_coords, &delta) &&
+	  get_leading_bearing_without_var_unscaled (glyph, lsb))
+      {
+	*lsb += roundf (delta);
+	return true;
+      }
 
-      return _glyf_get_leading_bearing_with_var_unscaled (font, glyph, T::tableTag == HB_OT_TAG_vmtx);
+      return _glyf_get_leading_bearing_with_var_unscaled (font, glyph, T::tableTag == HB_OT_TAG_vmtx, lsb);
 #else
-      return side_bearing;
+      return false;
 #endif
     }