Merge pull request #19498 from arjunroy/hpack_parser_on_hdr_v2

Codegen optimizations for hpack_parser on_hdr().

src/core/ext/transport/chttp2/transport/hpack_parser.cc

@@ -622,33 +622,37 @@ static const uint8_t inverse_base64[256] = {
     255,
 };
 
+static void GPR_ATTRIBUTE_NOINLINE on_hdr_log(grpc_mdelem md) {
+  char* k = grpc_slice_to_c_string(GRPC_MDKEY(md));
+  char* v = nullptr;
+  if (grpc_is_binary_header_internal(GRPC_MDKEY(md))) {
+    v = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX);
+  } else {
+    v = grpc_slice_to_c_string(GRPC_MDVALUE(md));
+  }
+  gpr_log(
+      GPR_INFO,
+      "Decode: '%s: %s', elem_interned=%d [%d], k_interned=%d, v_interned=%d",
+      k, v, GRPC_MDELEM_IS_INTERNED(md), GRPC_MDELEM_STORAGE(md),
+      grpc_slice_is_interned(GRPC_MDKEY(md)),
+      grpc_slice_is_interned(GRPC_MDVALUE(md)));
+  gpr_free(k);
+  gpr_free(v);
+}
+
 /* emission helpers */
-static grpc_error* on_hdr(grpc_chttp2_hpack_parser* p, grpc_mdelem md,
+template <bool do_add>
-                          int add_to_table) {
+static grpc_error* on_hdr(grpc_chttp2_hpack_parser* p, grpc_mdelem md) {
   if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
-    char* k = grpc_slice_to_c_string(GRPC_MDKEY(md));
+    on_hdr_log(md);
-    char* v = nullptr;
-    if (grpc_is_binary_header_internal(GRPC_MDKEY(md))) {
-      v = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX);
-    } else {
-      v = grpc_slice_to_c_string(GRPC_MDVALUE(md));
-    }
-    gpr_log(
-        GPR_INFO,
-        "Decode: '%s: %s', elem_interned=%d [%d], k_interned=%d, v_interned=%d",
-        k, v, GRPC_MDELEM_IS_INTERNED(md), GRPC_MDELEM_STORAGE(md),
-        grpc_slice_is_interned(GRPC_MDKEY(md)),
-        grpc_slice_is_interned(GRPC_MDVALUE(md)));
-    gpr_free(k);
-    gpr_free(v);
   }
-  if (add_to_table) {
+  if (do_add) {
-    GPR_ASSERT(GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_INTERNED ||
+    GPR_DEBUG_ASSERT(GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_INTERNED ||
-               GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC);
+                     GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC);
     grpc_error* err = grpc_chttp2_hptbl_add(&p->table, md);
-    if (err != GRPC_ERROR_NONE) return err;
+    if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) return err;
   }
-  if (p->on_header == nullptr) {
+  if (GPR_UNLIKELY(p->on_header == nullptr)) {
     GRPC_MDELEM_UNREF(md);
     return GRPC_ERROR_CREATE_FROM_STATIC_STRING("on_header callback not set");
   }
@@ -765,7 +769,7 @@ static grpc_error* finish_indexed_field(grpc_chttp2_hpack_parser* p,
   }
   GRPC_MDELEM_REF(md);
   GRPC_STATS_INC_HPACK_RECV_INDEXED();
-  grpc_error* err = on_hdr(p, md, 0);
+  grpc_error* err = on_hdr<false>(p, md);
   if (err != GRPC_ERROR_NONE) return err;
   return parse_begin(p, cur, end);
 }
@@ -798,11 +802,9 @@ static grpc_error* finish_lithdr_incidx(grpc_chttp2_hpack_parser* p,
   grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */
   GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX();
-  grpc_error* err =
+  grpc_error* err = on_hdr<true>(
-      on_hdr(p,
+      p, grpc_mdelem_from_slices(grpc_slice_ref_internal(GRPC_MDKEY(md)),
-             grpc_mdelem_from_slices(grpc_slice_ref_internal(GRPC_MDKEY(md)),
+                                 take_string(p, &p->value, true)));
-                                     take_string(p, &p->value, true)),
-             1);
   if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
   return parse_begin(p, cur, end);
 }
@@ -813,10 +815,8 @@ static grpc_error* finish_lithdr_incidx_v(grpc_chttp2_hpack_parser* p,
                                           const uint8_t* end) {
   GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX_V();
   grpc_error* err =
-      on_hdr(p,
+      on_hdr<true>(p, grpc_mdelem_from_slices(take_string(p, &p->key, true),
-             grpc_mdelem_from_slices(take_string(p, &p->key, true),
+                                              take_string(p, &p->value, true)));
-                                     take_string(p, &p->value, true)),
-             1);
   if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
   return parse_begin(p, cur, end);
 }
@@ -865,11 +865,9 @@ static grpc_error* finish_lithdr_notidx(grpc_chttp2_hpack_parser* p,
   grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */
   GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX();
-  grpc_error* err =
+  grpc_error* err = on_hdr<false>(
-      on_hdr(p,
+      p, grpc_mdelem_from_slices(grpc_slice_ref_internal(GRPC_MDKEY(md)),
-             grpc_mdelem_from_slices(grpc_slice_ref_internal(GRPC_MDKEY(md)),
+                                 take_string(p, &p->value, false)));
-                                     take_string(p, &p->value, false)),
-             0);
   if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
   return parse_begin(p, cur, end);
 }
@@ -879,11 +877,9 @@ static grpc_error* finish_lithdr_notidx_v(grpc_chttp2_hpack_parser* p,
                                           const uint8_t* cur,
                                           const uint8_t* end) {
   GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX_V();
-  grpc_error* err =
+  grpc_error* err = on_hdr<false>(
-      on_hdr(p,
+      p, grpc_mdelem_from_slices(take_string(p, &p->key, true),
-             grpc_mdelem_from_slices(take_string(p, &p->key, true),
+                                 take_string(p, &p->value, false)));
-                                     take_string(p, &p->value, false)),
-             0);
   if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
   return parse_begin(p, cur, end);
 }
@@ -932,11 +928,9 @@ static grpc_error* finish_lithdr_nvridx(grpc_chttp2_hpack_parser* p,
   grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index);
   GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */
   GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX();
-  grpc_error* err =
+  grpc_error* err = on_hdr<false>(
-      on_hdr(p,
+      p, grpc_mdelem_from_slices(grpc_slice_ref_internal(GRPC_MDKEY(md)),
-             grpc_mdelem_from_slices(grpc_slice_ref_internal(GRPC_MDKEY(md)),
+                                 take_string(p, &p->value, false)));
-                                     take_string(p, &p->value, false)),
-             0);
   if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
   return parse_begin(p, cur, end);
 }
@@ -946,11 +940,9 @@ static grpc_error* finish_lithdr_nvridx_v(grpc_chttp2_hpack_parser* p,
                                           const uint8_t* cur,
                                           const uint8_t* end) {
   GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX_V();
-  grpc_error* err =
+  grpc_error* err = on_hdr<false>(
-      on_hdr(p,
+      p, grpc_mdelem_from_slices(take_string(p, &p->key, true),
-             grpc_mdelem_from_slices(take_string(p, &p->key, true),
+                                 take_string(p, &p->value, false)));
-                                     take_string(p, &p->value, false)),
-             0);
   if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err);
   return parse_begin(p, cur, end);
 }