[CFF] oss-fuzz issue 11690 ASSERT: substr.offset >= opStart (#1461)

* fix oss-fuzz 11690: substr.offset >= opStart

detect recursive subroutine call & handle as error

* fix build failure

* add minimized test case for oss-fuzz 11690

* removed asserts

src/hb-cff-interp-common.hh

@@ -332,8 +332,10 @@ struct ByteStr
   inline bool sanitize (hb_sanitize_context_t *c) const { return str->sanitize (c, len); }
 
   inline const HBUINT8& operator [] (unsigned int i) const {
-    assert (str && (i < len));
-    return (*str)[i];
+    if (likely (str && (i < len)))
+      return (*str)[i];
+    else
+      return Null(HBUINT8);
   }
 
   inline bool serialize (hb_serialize_context_t *c, const ByteStr &src)
@@ -628,7 +630,6 @@ struct ParsedValues
   {
     VAL *val = values.push ();
     val->op = op;
-    assert (substr.offset >= opStart);
     val->str = ByteStr (substr.str, opStart, substr.offset - opStart);
     opStart = substr.offset;
   }
@@ -637,7 +638,6 @@ struct ParsedValues
   {
     VAL *val = values.push (v);
     val->op = op;
-    assert (substr.offset >= opStart);
     val->str = ByteStr (substr.str, opStart, substr.offset - opStart);
     opStart = substr.offset;
   }
@@ -707,7 +707,6 @@ struct InterpEnv
 
   inline void pop_n_args (unsigned int n)
   {
-    assert (n <= argStack.get_count ());
     argStack.pop (n);
   }
 

src/hb-subset-cff-common.hh

@@ -526,11 +526,19 @@ struct SubrSubsetParam
   }
 
   template <typename ENV>
-  inline void set_current_str (ENV &env)
+  inline void set_current_str (ENV &env, bool calling)
   {
     ParsedCStr  *parsed_str = get_parsed_str_for_context (env.context);
     if (likely (parsed_str != nullptr))
-      current_parsed_str = parsed_str;
+    {
+      /* If the called subroutine is parsed partially but not completely yet,
+       * it must be because we are calling it recursively.
+       * Handle it as an error. */
+      if (unlikely (calling && !parsed_str->is_parsed () && (parsed_str->values.len > 0)))
+      	env.set_error ();
+      else
+      	current_parsed_str = parsed_str;
+    }
     else
       env.set_error ();
   }

src/hb-subset-cff1.cc

@@ -362,7 +362,7 @@ struct CFF1CSOpSet_SubrSubset : CFF1CSOpSet<CFF1CSOpSet_SubrSubset, SubrSubsetPa
 	param.current_parsed_str->add_op (op, env.substr);
 	param.current_parsed_str->set_parsed ();
 	env.returnFromSubr ();
-	param.set_current_str (env);
+	param.set_current_str (env, false);
 	break;
 
       case OpCode_endchar:
@@ -395,7 +395,7 @@ struct CFF1CSOpSet_SubrSubset : CFF1CSOpSet<CFF1CSOpSet_SubrSubset, SubrSubsetPa
     env.callSubr (subrs, type);
     param.current_parsed_str->add_call_op (op, substr, env.context.subr_num);
     hb_set_add (closure, env.context.subr_num);
-    param.set_current_str (env);
+    param.set_current_str (env, true);
   }
 
   private:

src/hb-subset-cff2.cc

@@ -184,7 +184,7 @@ struct CFF2CSOpSet_SubrSubset : CFF2CSOpSet<CFF2CSOpSet_SubrSubset, SubrSubsetPa
       case OpCode_return:
 	param.current_parsed_str->set_parsed ();
 	env.returnFromSubr ();
-	param.set_current_str (env);
+	param.set_current_str (env, false);
 	break;
 
       case OpCode_endchar:
@@ -216,7 +216,7 @@ struct CFF2CSOpSet_SubrSubset : CFF2CSOpSet<CFF2CSOpSet_SubrSubset, SubrSubsetPa
     env.callSubr (subrs, type);
     param.current_parsed_str->add_call_op (op, substr, env.context.subr_num);
     hb_set_add (closure, env.context.subr_num);
-    param.set_current_str (env);
+    param.set_current_str (env, true);
   }
 
   private: