Disallow sanitize recursing into Extension subtables multiple times

Fixes https://oss-fuzz.com/v2/testcase-detail/5216838347653120
which is a stack overflow, not by way of infinite recursion, just
being deep.  That's disallowed anyway, so catch it as it happens,
not afterwards.

src/hb-ot-layout-gsub-table.hh

@@ -1272,10 +1272,9 @@ struct SubstLookup : Lookup
     {
       /* The spec says all subtables of an Extension lookup should
        * have the same type, which shall not be the Extension type
-       * itself. This is specially important if one has a reverse type! */
+       * itself (but we already checked for that).
+       * This is specially important if one has a reverse type! */
       unsigned int type = get_subtable (0).u.extension.get_type ();
-      if (unlikely (type == SubstLookupSubTable::Extension))
-	return_trace (false);
       unsigned int count = get_subtable_count ();
       for (unsigned int i = 1; i < count; i++)
         if (get_subtable (i).u.extension.get_type () != type)

src/hb-ot-layout-gsubgpos-private.hh

@@ -2230,7 +2230,9 @@ struct ExtensionFormat1
   inline bool sanitize (hb_sanitize_context_t *c) const
   {
     TRACE_SANITIZE (this);
-    return_trace (c->check_struct (this) && extensionOffset != 0);
+    return_trace (c->check_struct (this) &&
+		  extensionOffset != 0 &&
+		  extensionLookupType != T::LookupSubTable::Extension);
   }
 
   protected: