Preallocate for incoming metadatas.

There is usually 9 metadata in incoming headers. We are calling
arena_alloc for all of them and that accounts for 75% of the
calls to arena_alloc.

Simply preallocate 10 of them in the structure so that we can
avoid the atomic op, per header.

src/core/ext/transport/chttp2/transport/incoming_metadata.cc

@@ -30,11 +30,15 @@
 grpc_error* grpc_chttp2_incoming_metadata_buffer_add(
     grpc_chttp2_incoming_metadata_buffer* buffer, grpc_mdelem elem) {
   buffer->size += GRPC_MDELEM_LENGTH(elem);
-  return grpc_metadata_batch_add_tail(
-      &buffer->batch,
-      static_cast<grpc_linked_mdelem*>(
-          gpr_arena_alloc(buffer->arena, sizeof(grpc_linked_mdelem))),
-      elem);
+  grpc_linked_mdelem* storage;
+  if (buffer->count < buffer->kPreallocatedMDElem) {
+    storage = &buffer->preallocated_mdelems[buffer->count];
+    buffer->count++;
+  } else {
+    storage = static_cast<grpc_linked_mdelem*>(
+        gpr_arena_alloc(buffer->arena, sizeof(grpc_linked_mdelem)));
+  }
+  return grpc_metadata_batch_add_tail(&buffer->batch, storage, elem);
 }
 
 grpc_error* grpc_chttp2_incoming_metadata_buffer_replace_or_add(

src/core/ext/transport/chttp2/transport/incoming_metadata.h

@@ -32,9 +32,14 @@ struct grpc_chttp2_incoming_metadata_buffer {
     grpc_metadata_batch_destroy(&batch);
   }
 
+  static constexpr size_t kPreallocatedMDElem = 10;
+
   gpr_arena* arena;
+  size_t size = 0;   // total size of metadata.
+  size_t count = 0;  // minimum of count of metadata and kPreallocatedMDElem.
+  // These preallocated mdelems are used while count < kPreallocatedMDElem.
+  grpc_linked_mdelem preallocated_mdelems[kPreallocatedMDElem];
   grpc_metadata_batch batch;
-  size_t size = 0;  // total size of metadata
 };
 
 void grpc_chttp2_incoming_metadata_buffer_publish(