[instancer] update variable device flags in ValueFormat

src/OT/Layout/GPOS/Common.hh

@@ -23,7 +23,7 @@ static void SinglePos_serialize (hb_serialize_context_t *c,
                                  const SrcLookup *src,
                                  Iterator it,
                                  const hb_hashmap_t<unsigned, hb_pair_t<unsigned, int>> *layout_variation_idx_delta_map,
-                                 bool all_axes_pinned);
+                                 bool update_var_device_flags);
 
 
 }

src/OT/Layout/GPOS/PairPosFormat1.hh

@@ -131,20 +131,17 @@ struct PairPosFormat1_3
     auto *out = c->serializer->start_embed (*this);
     if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
     out->format = format;
-    out->valueFormat[0] = valueFormat[0];
-    out->valueFormat[1] = valueFormat[1];
+
+    hb_pair_t<unsigned, unsigned> newFormats = hb_pair (valueFormat[0], valueFormat[1]);
+
+    if (c->plan->normalized_coords)
+      newFormats = compute_effective_value_formats (glyphset, &c->plan->layout_variation_idx_delta_map);
+
     if (c->plan->flags & HB_SUBSET_FLAGS_NO_HINTING)
-    {
-      hb_pair_t<unsigned, unsigned> newFormats = compute_effective_value_formats (glyphset);
-      out->valueFormat[0] = newFormats.first;
-      out->valueFormat[1] = newFormats.second;
-    }
+      newFormats = compute_effective_value_formats (glyphset);
     
-    if (c->plan->all_axes_pinned)
-    {
-      out->valueFormat[0] = out->valueFormat[0].drop_device_table_flags ();
-      out->valueFormat[1] = out->valueFormat[1].drop_device_table_flags ();
-    }
+    out->valueFormat[0] = newFormats.first;
+    out->valueFormat[1] = newFormats.second;
 
     hb_sorted_vector_t<hb_codepoint_t> new_coverage;
 
@@ -175,7 +172,8 @@ struct PairPosFormat1_3
   }
 
 
-  hb_pair_t<unsigned, unsigned> compute_effective_value_formats (const hb_set_t& glyphset) const
+  hb_pair_t<unsigned, unsigned> compute_effective_value_formats (const hb_set_t& glyphset,
+                                                                 const hb_hashmap_t<unsigned, hb_pair_t<unsigned, int>> *varidx_delta_map = nullptr) const
   {
     unsigned record_size = PairSet::get_size (valueFormat);
 
@@ -195,8 +193,16 @@ struct PairPosFormat1_3
       {
         if (record->intersects (glyphset))
         {
-          format1 = format1 | valueFormat[0].get_effective_format (record->get_values_1 ());
-          format2 = format2 | valueFormat[1].get_effective_format (record->get_values_2 (valueFormat[0]));
+          if (!varidx_delta_map)
+          {
+            format1 = format1 | valueFormat[0].get_effective_format (record->get_values_1 ());
+            format2 = format2 | valueFormat[1].get_effective_format (record->get_values_2 (valueFormat[0]));
+          }
+          else
+          {
+            format1 = format1 | valueFormat[0].update_var_device_table_flags (record->get_values_1 (), &set, varidx_delta_map);
+            format2 = format2 | valueFormat[1].update_var_device_table_flags (record->get_values_2 (valueFormat[0]), &set, varidx_delta_map);
+          }
         }
         record = &StructAtOffset<const PairValueRecord> (record, record_size);
       }

src/OT/Layout/GPOS/PairPosFormat2.hh

@@ -287,18 +287,15 @@ struct PairPosFormat2_4
     unsigned len2 = valueFormat2.get_len ();
 
     hb_pair_t<unsigned, unsigned> newFormats = hb_pair (valueFormat1, valueFormat2);
+    if (c->plan->normalized_coords)
+      newFormats = compute_effective_value_formats (klass1_map, klass2_map, &c->plan->layout_variation_idx_delta_map);
+
     if (c->plan->flags & HB_SUBSET_FLAGS_NO_HINTING)
       newFormats = compute_effective_value_formats (klass1_map, klass2_map);
 
     out->valueFormat1 = newFormats.first;
     out->valueFormat2 = newFormats.second;
 
-    if (c->plan->all_axes_pinned)
-    {
-      out->valueFormat1 = out->valueFormat1.drop_device_table_flags ();
-      out->valueFormat2 = out->valueFormat2.drop_device_table_flags ();
-    }
-
     unsigned total_len = len1 + len2;
     hb_vector_t<unsigned> class2_idxs (+ hb_range ((unsigned) class2Count) | hb_filter (klass2_map));
     for (unsigned class1_idx : + hb_range ((unsigned) class1Count) | hb_filter (klass1_map))
@@ -326,7 +323,8 @@ struct PairPosFormat2_4
 
 
   hb_pair_t<unsigned, unsigned> compute_effective_value_formats (const hb_map_t& klass1_map,
-                                                                 const hb_map_t& klass2_map) const
+                                                                 const hb_map_t& klass2_map,
+                                                                 const hb_hashmap_t<unsigned, hb_pair_t<unsigned, int>> *varidx_delta_map = nullptr) const
   {
     unsigned len1 = valueFormat1.get_len ();
     unsigned len2 = valueFormat2.get_len ();
@@ -340,8 +338,16 @@ struct PairPosFormat2_4
       for (unsigned class2_idx : + hb_range ((unsigned) class2Count) | hb_filter (klass2_map))
       {
         unsigned idx = (class1_idx * (unsigned) class2Count + class2_idx) * record_size;
-        format1 = format1 | valueFormat1.get_effective_format (&values[idx]);
-        format2 = format2 | valueFormat2.get_effective_format (&values[idx + len1]);
+        if (!varidx_delta_map)
+        {
+          format1 = format1 | valueFormat1.get_effective_format (&values[idx]);
+          format2 = format2 | valueFormat2.get_effective_format (&values[idx + len1]);
+        }
+        else
+        {
+          format1 = format1 | valueFormat1.update_var_device_table_flags (&values[idx], this, varidx_delta_map);
+          format2 = format2 | valueFormat2.update_var_device_table_flags (&values[idx + len1], this, varidx_delta_map);
+        }
       }
 
       if (format1 == valueFormat1 && format2 == valueFormat2)

src/OT/Layout/GPOS/SinglePos.hh

@@ -39,14 +39,15 @@ struct SinglePos
                   const SrcLookup* src,
                   Iterator glyph_val_iter_pairs,
                   const hb_hashmap_t<unsigned, hb_pair_t<unsigned, int>> *layout_variation_idx_delta_map,
-                  bool all_axes_pinned)
+                  bool update_var_device_flags)
   {
     if (unlikely (!c->extend_min (u.format))) return;
     unsigned format = 2;
     ValueFormat new_format = src->get_value_format ();
 
-    if (all_axes_pinned)
-      new_format = new_format.drop_device_table_flags ();
+    if (update_var_device_flags)
+      new_format = new_format.update_var_device_table_flags (+ glyph_val_iter_pairs | hb_map (hb_second),
+                                                             src, layout_variation_idx_delta_map);
 
     if (glyph_val_iter_pairs)
       format = get_format (glyph_val_iter_pairs);
@@ -89,8 +90,8 @@ SinglePos_serialize (hb_serialize_context_t *c,
                      const SrcLookup *src,
                      Iterator it,
                      const hb_hashmap_t<unsigned, hb_pair_t<unsigned, int>> *layout_variation_idx_delta_map,
-                     bool all_axes_pinned)
-{ c->start_embed<SinglePos> ()->serialize (c, src, it, layout_variation_idx_delta_map, all_axes_pinned); }
+                     bool update_var_device_flags)
+{ c->start_embed<SinglePos> ()->serialize (c, src, it, layout_variation_idx_delta_map, update_var_device_flags); }
 
 
 }

src/OT/Layout/GPOS/SinglePosFormat1.hh

@@ -153,7 +153,7 @@ struct SinglePosFormat1
     ;
 
     bool ret = bool (it);
-    SinglePos_serialize (c->serializer, this, it, &c->plan->layout_variation_idx_delta_map, c->plan->all_axes_pinned);
+    SinglePos_serialize (c->serializer, this, it, &c->plan->layout_variation_idx_delta_map, bool (c->plan->normalized_coords));
     return_trace (ret);
   }
 };

src/OT/Layout/GPOS/SinglePosFormat2.hh

@@ -164,7 +164,7 @@ struct SinglePosFormat2
     ;
 
     bool ret = bool (it);
-    SinglePos_serialize (c->serializer, this, it, &c->plan->layout_variation_idx_delta_map, c->plan->all_axes_pinned);
+    SinglePos_serialize (c->serializer, this, it, &c->plan->layout_variation_idx_delta_map, bool (c->plan->normalized_coords));
     return_trace (ret);
   }
 };

src/OT/Layout/GPOS/ValueFormat.hh

@@ -251,15 +251,36 @@ struct ValueFormat : HBUINT16
     }
   }
 
-  unsigned drop_device_table_flags () const
+  unsigned update_var_device_table_flags (const Value *values,
+                                          const void *base,
+                                          const hb_hashmap_t<unsigned, hb_pair_t<unsigned, int>> *varidx_delta_map) const
   {
     unsigned format = *this;
-    for (unsigned flag = xPlaDevice; flag <= yAdvDevice; flag = flag << 1)
-      format = format & ~flag;
+    for (unsigned flag = xPlacement; flag <= yAdvDevice; flag = flag << 1)
+    {
+      if (format & flag)
+      {
+        if (flag >= xPlaDevice) update_var_flag (values, (Flags) flag, &format, base, varidx_delta_map);
+        values++;
+      }
+    }
 
     return format;
   }
 
+  template<typename Iterator,
+      hb_requires (hb_is_iterator (Iterator))>
+  unsigned update_var_device_table_flags (Iterator it,
+                                          const void* base,
+                                          const hb_hashmap_t<unsigned, hb_pair_t<unsigned, int>> *varidx_delta_map) const
+  {
+    unsigned new_format = 0;
+    for (const hb_array_t<const Value>& values : it)
+      new_format = new_format | update_var_device_table_flags (&values, base, varidx_delta_map);
+
+    return new_format;
+  }
+
   private:
   bool sanitize_value_devices (hb_sanitize_context_t *c, const void *base, const Value *values) const
   {
@@ -403,6 +424,20 @@ struct ValueFormat : HBUINT16
     *format = *format & ~flag;
   }
 
+  void update_var_flag (const Value* value, Flags flag,
+                        unsigned int* format, const void *base,
+                        const hb_hashmap_t<unsigned, hb_pair_t<unsigned, int>> *varidx_delta_map) const
+  {
+    if (*value)
+    {
+      unsigned varidx = (base + get_device (value)).get_variation_index ();
+      hb_pair_t<unsigned, int> *varidx_delta;
+      if (varidx_delta_map->has (varidx, &varidx_delta) &&
+          varidx_delta->first != HB_OT_LAYOUT_NO_VARIATIONS_INDEX)
+        return;
+    }
+    *format = *format & ~flag;
+  }
 };
 
 }