From e0778b2c1817f3b7aff5af4b1b15493d4a8a207d Mon Sep 17 00:00:00 2001
From: "Mark D. Roth" <roth@google.com>
Date: Fri, 21 Jul 2017 15:42:00 -0700
Subject: [PATCH] Add cancellation to asynchronous security APIs.

---
 include/grpc/impl/codegen/atm.h               |   1 +
 .../composite/composite_credentials.c         | 122 ++--
 .../lib/security/credentials/credentials.c    |  26 +-
 .../lib/security/credentials/credentials.h    |  74 +--
 .../credentials/credentials_metadata.c        |  77 +--
 .../credentials/fake/fake_credentials.c       |  47 +-
 .../credentials/fake/fake_credentials.h       |   4 +-
 .../credentials/iam/iam_credentials.c         |  45 +-
 .../credentials/iam/iam_credentials.h         |   2 +-
 .../credentials/jwt/jwt_credentials.c         |  51 +-
 .../credentials/jwt/jwt_credentials.h         |   2 +-
 .../credentials/oauth2/oauth2_credentials.c   | 193 +++++--
 .../credentials/oauth2/oauth2_credentials.h   |  15 +-
 .../credentials/plugin/plugin_credentials.c   | 165 ++++--
 .../credentials/plugin/plugin_credentials.h   |  16 +-
 .../security/transport/client_auth_filter.c   | 186 ++++--
 .../security/transport/security_connector.c   |  57 +-
 .../security/transport/security_connector.h   |  29 +-
 src/node/test/credentials_test.js             |   8 +-
 test/core/end2end/fixtures/h2_oauth2.c        |  12 +-
 test/core/security/credentials_test.c         | 530 ++++++++----------
 test/core/security/oauth2_utils.c             |  43 +-
 .../print_google_default_creds_token.c        |  42 +-
 test/cpp/end2end/end2end_test.cc              |   8 +-
 24 files changed, 1006 insertions(+), 749 deletions(-)

diff --git a/include/grpc/impl/codegen/atm.h b/include/grpc/impl/codegen/atm.h
index c3b528be4d4..2cfd22ab637 100644
--- a/include/grpc/impl/codegen/atm.h
+++ b/include/grpc/impl/codegen/atm.h
@@ -60,6 +60,7 @@
    int gpr_atm_no_barrier_cas(gpr_atm *p, gpr_atm o, gpr_atm n);
    int gpr_atm_acq_cas(gpr_atm *p, gpr_atm o, gpr_atm n);
    int gpr_atm_rel_cas(gpr_atm *p, gpr_atm o, gpr_atm n);
+   int gpr_atm_full_cas(gpr_atm *p, gpr_atm o, gpr_atm n);
 
    // Atomically, set *p=n and return the old value of *p
    gpr_atm gpr_atm_full_xchg(gpr_atm *p, gpr_atm n);
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
index 77d7b04627b..09fd60a12c2 100644
--- a/src/core/lib/security/credentials/composite/composite_credentials.c
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -32,88 +32,98 @@
 typedef struct {
   grpc_composite_call_credentials *composite_creds;
   size_t creds_index;
-  grpc_credentials_md_store *md_elems;
-  grpc_auth_metadata_context auth_md_context;
-  void *user_data;
   grpc_polling_entity *pollent;
-  grpc_credentials_metadata_cb cb;
+  grpc_auth_metadata_context auth_md_context;
+  grpc_credentials_mdelem_array *md_array;
+  grpc_closure *on_request_metadata;
+  grpc_closure internal_on_request_metadata;
 } grpc_composite_call_credentials_metadata_context;
 
 static void composite_call_destruct(grpc_exec_ctx *exec_ctx,
                                     grpc_call_credentials *creds) {
   grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->inner.num_creds; i++) {
+  for (size_t i = 0; i < c->inner.num_creds; i++) {
     grpc_call_credentials_unref(exec_ctx, c->inner.creds_array[i]);
   }
   gpr_free(c->inner.creds_array);
 }
 
-static void composite_call_md_context_destroy(
-    grpc_exec_ctx *exec_ctx,
-    grpc_composite_call_credentials_metadata_context *ctx) {
-  grpc_credentials_md_store_unref(exec_ctx, ctx->md_elems);
-  gpr_free(ctx);
-}
-
-static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
-                                       grpc_credentials_md *md_elems,
-                                       size_t num_md,
-                                       grpc_credentials_status status,
-                                       const char *error_details) {
+static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *arg,
+                                       grpc_error *error) {
   grpc_composite_call_credentials_metadata_context *ctx =
-      (grpc_composite_call_credentials_metadata_context *)user_data;
-  if (status != GRPC_CREDENTIALS_OK) {
-    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status, error_details);
-    return;
-  }
-
-  /* Copy the metadata in the context. */
-  if (num_md > 0) {
-    size_t i;
-    for (i = 0; i < num_md; i++) {
-      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
-                                    md_elems[i].value);
+      (grpc_composite_call_credentials_metadata_context *)arg;
+  if (error == GRPC_ERROR_NONE) {
+    /* See if we need to get some more metadata. */
+    if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+      grpc_call_credentials *inner_creds =
+          ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+      if (grpc_call_credentials_get_request_metadata(
+              exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
+              ctx->md_array, &ctx->internal_on_request_metadata, &error)) {
+        // Synchronous response, so call ourselves recursively.
+        composite_call_metadata_cb(exec_ctx, arg, error);
+        GRPC_ERROR_UNREF(error);
+      }
+      return;
     }
+    // We're done!
   }
-
-  /* See if we need to get some more metadata. */
-  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
-    grpc_call_credentials *inner_creds =
-        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
-    grpc_call_credentials_get_request_metadata(
-        exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
-        composite_call_metadata_cb, ctx);
-    return;
-  }
-
-  /* We're done!. */
-  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
-          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK, NULL);
-  composite_call_md_context_destroy(exec_ctx, ctx);
+  GRPC_CLOSURE_SCHED(exec_ctx, ctx->on_request_metadata, GRPC_ERROR_REF(error));
+  gpr_free(ctx);
 }
 
-static void composite_call_get_request_metadata(
+static bool composite_call_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context auth_md_context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
   grpc_composite_call_credentials_metadata_context *ctx;
-
   ctx = gpr_zalloc(sizeof(grpc_composite_call_credentials_metadata_context));
-  ctx->auth_md_context = auth_md_context;
-  ctx->user_data = user_data;
-  ctx->cb = cb;
   ctx->composite_creds = c;
   ctx->pollent = pollent;
-  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, c->inner.creds_array[ctx->creds_index++], ctx->pollent,
-      auth_md_context, composite_call_metadata_cb, ctx);
+  ctx->auth_md_context = auth_md_context;
+  ctx->md_array = md_array;
+  ctx->on_request_metadata = on_request_metadata;
+  GRPC_CLOSURE_INIT(&ctx->internal_on_request_metadata,
+                    composite_call_metadata_cb, ctx, grpc_schedule_on_exec_ctx);
+  while (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+    grpc_call_credentials *inner_creds =
+        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+    if (grpc_call_credentials_get_request_metadata(
+            exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
+            ctx->md_array, &ctx->internal_on_request_metadata, error)) {
+      if (*error != GRPC_ERROR_NONE) break;
+    } else {
+      break;
+    }
+  }
+  // If we got through all creds synchronously or we got a synchronous
+  // error on one of them, return synchronously.
+  if (ctx->creds_index == ctx->composite_creds->inner.num_creds ||
+      *error != GRPC_ERROR_NONE) {
+    gpr_free(ctx);
+    return true;
+  }
+  // At least one inner cred is returning asynchronously, so we'll
+  // return asynchronously as well.
+  return false;
+}
+
+static void composite_call_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  for (size_t i = 0; i < c->inner.num_creds; ++i) {
+    grpc_call_credentials_cancel_get_request_metadata(
+        exec_ctx, c->inner.creds_array[i], md_array, GRPC_ERROR_REF(error));
+  }
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable composite_call_credentials_vtable = {
-    composite_call_destruct, composite_call_get_request_metadata};
+    composite_call_destruct, composite_call_get_request_metadata,
+    composite_call_cancel_get_request_metadata};
 
 static grpc_call_credentials_array get_creds_array(
     grpc_call_credentials **creds_addr) {
diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c
index b1f1e82076c..8a67c9865be 100644
--- a/src/core/lib/security/credentials/credentials.c
+++ b/src/core/lib/security/credentials/credentials.c
@@ -38,13 +38,10 @@
 /* -- Common. -- */
 
 grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
-    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
-    void *user_data) {
+    grpc_call_credentials *creds) {
   grpc_credentials_metadata_request *r =
       gpr_zalloc(sizeof(grpc_credentials_metadata_request));
   r->creds = grpc_call_credentials_ref(creds);
-  r->cb = cb;
-  r->user_data = user_data;
   return r;
 }
 
@@ -104,18 +101,25 @@ void grpc_call_credentials_release(grpc_call_credentials *creds) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-void grpc_call_credentials_get_request_metadata(
+bool grpc_call_credentials_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
-    if (cb != NULL) {
-      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
-    }
+    return true;
+  }
+  return creds->vtable->get_request_metadata(
+      exec_ctx, creds, pollent, context, md_array, on_request_metadata, error);
+}
+
+void grpc_call_credentials_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  if (creds == NULL || creds->vtable->cancel_get_request_metadata == NULL) {
     return;
   }
-  creds->vtable->get_request_metadata(exec_ctx, creds, pollent, context, cb,
-                                      user_data);
+  creds->vtable->cancel_get_request_metadata(exec_ctx, creds, md_array, error);
 }
 
 grpc_security_status grpc_channel_credentials_create_security_connector(
diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h
index c45c2e957cc..04a54b0ca83 100644
--- a/src/core/lib/security/credentials/credentials.h
+++ b/src/core/lib/security/credentials/credentials.h
@@ -138,48 +138,39 @@ grpc_channel_credentials *grpc_channel_credentials_from_arg(
 grpc_channel_credentials *grpc_channel_credentials_find_in_args(
     const grpc_channel_args *args);
 
-/* --- grpc_credentials_md. --- */
+/* --- grpc_credentials_mdelem_array. --- */
 
 typedef struct {
-  grpc_slice key;
-  grpc_slice value;
-} grpc_credentials_md;
+  grpc_mdelem *md;
+  size_t size;
+} grpc_credentials_mdelem_array;
 
-typedef struct {
-  grpc_credentials_md *entries;
-  size_t num_entries;
-  size_t allocated;
-  gpr_refcount refcount;
-} grpc_credentials_md_store;
+/// Takes a new ref to \a md.
+void grpc_credentials_mdelem_array_add(grpc_credentials_mdelem_array *list,
+                                       grpc_mdelem md);
 
-grpc_credentials_md_store *grpc_credentials_md_store_create(
-    size_t initial_capacity);
+/// Appends all elements from \a src to \a dst, taking a new ref to each one.
+void grpc_credentials_mdelem_array_append(grpc_credentials_mdelem_array *dst,
+                                          grpc_credentials_mdelem_array *src);
 
-/* Will ref key and value. */
-void grpc_credentials_md_store_add(grpc_credentials_md_store *store,
-                                   grpc_slice key, grpc_slice value);
-void grpc_credentials_md_store_add_cstrings(grpc_credentials_md_store *store,
-                                            const char *key, const char *value);
-grpc_credentials_md_store *grpc_credentials_md_store_ref(
-    grpc_credentials_md_store *store);
-void grpc_credentials_md_store_unref(grpc_exec_ctx *exec_ctx,
-                                     grpc_credentials_md_store *store);
+void grpc_credentials_mdelem_array_destroy(grpc_exec_ctx *exec_ctx,
+                                           grpc_credentials_mdelem_array *list);
 
 /* --- grpc_call_credentials. --- */
 
-/* error_details must be NULL if status is GRPC_CREDENTIALS_OK. */
-typedef void (*grpc_credentials_metadata_cb)(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details);
-
 typedef struct {
   void (*destruct)(grpc_exec_ctx *exec_ctx, grpc_call_credentials *c);
-  void (*get_request_metadata)(grpc_exec_ctx *exec_ctx,
+  bool (*get_request_metadata)(grpc_exec_ctx *exec_ctx,
                                grpc_call_credentials *c,
                                grpc_polling_entity *pollent,
                                grpc_auth_metadata_context context,
-                               grpc_credentials_metadata_cb cb,
-                               void *user_data);
+                               grpc_credentials_mdelem_array *md_array,
+                               grpc_closure *on_request_metadata,
+                               grpc_error **error);
+  void (*cancel_get_request_metadata)(grpc_exec_ctx *exec_ctx,
+                                      grpc_call_credentials *c,
+                                      grpc_credentials_mdelem_array *md_array,
+                                      grpc_error *error);
 } grpc_call_credentials_vtable;
 
 struct grpc_call_credentials {
@@ -191,15 +182,29 @@ struct grpc_call_credentials {
 grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds);
 void grpc_call_credentials_unref(grpc_exec_ctx *exec_ctx,
                                  grpc_call_credentials *creds);
-void grpc_call_credentials_get_request_metadata(
+
+/// Returns true if completed synchronously, in which case \a error will
+/// be set to indicate the result.  Otherwise, \a on_request_metadata will
+/// be invoked asynchronously when complete.  \a md_array will be populated
+/// with the resulting metadata once complete.
+bool grpc_call_credentials_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data);
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error);
+
+/// Cancels a pending asynchronous operation started by
+/// grpc_call_credentials_get_request_metadata() with the corresponding
+/// value of \a md_array.
+void grpc_call_credentials_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error);
 
 /* Metadata-only credentials with the specified key and value where
    asynchronicity can be simulated for testing. */
 grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async);
+    grpc_exec_ctx *exec_ctx, const char *md_key, const char *md_value,
+    bool is_async);
 
 /* --- grpc_server_credentials. --- */
 
@@ -238,14 +243,11 @@ grpc_server_credentials *grpc_find_server_credentials_in_args(
 
 typedef struct {
   grpc_call_credentials *creds;
-  grpc_credentials_metadata_cb cb;
   grpc_http_response response;
-  void *user_data;
 } grpc_credentials_metadata_request;
 
 grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
-    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
-    void *user_data);
+    grpc_call_credentials *creds);
 
 void grpc_credentials_metadata_request_destroy(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *r);
diff --git a/src/core/lib/security/credentials/credentials_metadata.c b/src/core/lib/security/credentials/credentials_metadata.c
index fcfdd52d20b..ccd39e610f3 100644
--- a/src/core/lib/security/credentials/credentials_metadata.c
+++ b/src/core/lib/security/credentials/credentials_metadata.c
@@ -24,65 +24,36 @@
 
 #include "src/core/lib/slice/slice_internal.h"
 
-static void store_ensure_capacity(grpc_credentials_md_store *store) {
-  if (store->num_entries == store->allocated) {
-    store->allocated = (store->allocated == 0) ? 1 : store->allocated * 2;
-    store->entries = gpr_realloc(
-        store->entries, store->allocated * sizeof(grpc_credentials_md));
+static void mdelem_list_ensure_capacity(grpc_credentials_mdelem_array *list,
+                                        size_t additional_space_needed) {
+  size_t target_size = list->size + additional_space_needed;
+  // Find the next power of two greater than the target size (i.e.,
+  // whenever we add more space, we double what we already have).
+  size_t new_size = 2;
+  while (new_size < target_size) {
+    new_size *= 2;
   }
+  list->md = gpr_realloc(list->md, sizeof(grpc_mdelem) * new_size);
 }
 
-grpc_credentials_md_store *grpc_credentials_md_store_create(
-    size_t initial_capacity) {
-  grpc_credentials_md_store *store =
-      gpr_zalloc(sizeof(grpc_credentials_md_store));
-  if (initial_capacity > 0) {
-    store->entries = gpr_malloc(initial_capacity * sizeof(grpc_credentials_md));
-    store->allocated = initial_capacity;
-  }
-  gpr_ref_init(&store->refcount, 1);
-  return store;
-}
-
-void grpc_credentials_md_store_add(grpc_credentials_md_store *store,
-                                   grpc_slice key, grpc_slice value) {
-  if (store == NULL) return;
-  store_ensure_capacity(store);
-  store->entries[store->num_entries].key = grpc_slice_ref_internal(key);
-  store->entries[store->num_entries].value = grpc_slice_ref_internal(value);
-  store->num_entries++;
+void grpc_credentials_mdelem_array_add(grpc_credentials_mdelem_array *list,
+                                       grpc_mdelem md) {
+  mdelem_list_ensure_capacity(list, 1);
+  list->md[list->size++] = GRPC_MDELEM_REF(md);
 }
 
-void grpc_credentials_md_store_add_cstrings(grpc_credentials_md_store *store,
-                                            const char *key,
-                                            const char *value) {
-  if (store == NULL) return;
-  store_ensure_capacity(store);
-  store->entries[store->num_entries].key = grpc_slice_from_copied_string(key);
-  store->entries[store->num_entries].value =
-      grpc_slice_from_copied_string(value);
-  store->num_entries++;
-}
-
-grpc_credentials_md_store *grpc_credentials_md_store_ref(
-    grpc_credentials_md_store *store) {
-  if (store == NULL) return NULL;
-  gpr_ref(&store->refcount);
-  return store;
+void grpc_credentials_mdelem_array_append(grpc_credentials_mdelem_array *dst,
+                                          grpc_credentials_mdelem_array *src) {
+  mdelem_list_ensure_capacity(dst, src->size);
+  for (size_t i = 0; i < src->size; ++i) {
+    dst->md[dst->size++] = GRPC_MDELEM_REF(src->md[i]);
+  }
 }
 
-void grpc_credentials_md_store_unref(grpc_exec_ctx *exec_ctx,
-                                     grpc_credentials_md_store *store) {
-  if (store == NULL) return;
-  if (gpr_unref(&store->refcount)) {
-    if (store->entries != NULL) {
-      size_t i;
-      for (i = 0; i < store->num_entries; i++) {
-        grpc_slice_unref_internal(exec_ctx, store->entries[i].key);
-        grpc_slice_unref_internal(exec_ctx, store->entries[i].value);
-      }
-      gpr_free(store->entries);
-    }
-    gpr_free(store);
+void grpc_credentials_mdelem_array_destroy(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_mdelem_array *list) {
+  for (size_t i = 0; i < list->size; ++i) {
+    GRPC_MDELEM_UNREF(exec_ctx, list->md[i]);
   }
+  gpr_free(list->md);
 }
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c
index 67e74f7b921..ac9017850fe 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.c
+++ b/src/core/lib/security/credentials/fake/fake_credentials.c
@@ -98,49 +98,44 @@ const char *grpc_fake_transport_get_expected_targets(
 static void md_only_test_destruct(grpc_exec_ctx *exec_ctx,
                                   grpc_call_credentials *creds) {
   grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->md_store);
+  GRPC_MDELEM_UNREF(exec_ctx, c->md);
 }
 
-static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
-                                          void *user_data, grpc_error *error) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
-  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
-        GRPC_CREDENTIALS_OK, NULL);
-  grpc_credentials_metadata_request_destroy(exec_ctx, r);
-}
-
-static void md_only_test_get_request_metadata(
+static bool md_only_test_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-
+  grpc_credentials_mdelem_array_add(md_array, c->md);
   if (c->is_async) {
-    grpc_credentials_metadata_request *cb_arg =
-        grpc_credentials_metadata_request_create(creds, cb, user_data);
-    GRPC_CLOSURE_SCHED(exec_ctx,
-                       GRPC_CLOSURE_CREATE(on_simulated_token_fetch_done,
-                                           cb_arg, grpc_executor_scheduler),
-                       GRPC_ERROR_NONE);
-  } else {
-    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK, NULL);
+    GRPC_CLOSURE_SCHED(exec_ctx, on_request_metadata, GRPC_ERROR_NONE);
+    return false;
   }
+  return true;
+}
+
+static void md_only_test_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable md_only_test_vtable = {
-    md_only_test_destruct, md_only_test_get_request_metadata};
+    md_only_test_destruct, md_only_test_get_request_metadata,
+    md_only_test_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async) {
+    grpc_exec_ctx *exec_ctx, const char *md_key, const char *md_value,
+    bool is_async) {
   grpc_md_only_test_credentials *c =
       gpr_zalloc(sizeof(grpc_md_only_test_credentials));
   c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
   c->base.vtable = &md_only_test_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->md_store = grpc_credentials_md_store_create(1);
-  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
+  c->md =
+      grpc_mdelem_from_slices(exec_ctx, grpc_slice_from_copied_string(md_key),
+                              grpc_slice_from_copied_string(md_value));
   c->is_async = is_async;
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h
index fae7a6a9c45..aa0f3b6e207 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.h
+++ b/src/core/lib/security/credentials/fake/fake_credentials.h
@@ -52,8 +52,8 @@ const char *grpc_fake_transport_get_expected_targets(
 
 typedef struct {
   grpc_call_credentials base;
-  grpc_credentials_md_store *md_store;
-  int is_async;
+  grpc_mdelem md;
+  bool is_async;
 } grpc_md_only_test_credentials;
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.c b/src/core/lib/security/credentials/iam/iam_credentials.c
index 4b32c5a047c..3de8319d98a 100644
--- a/src/core/lib/security/credentials/iam/iam_credentials.c
+++ b/src/core/lib/security/credentials/iam/iam_credentials.c
@@ -30,26 +30,33 @@
 static void iam_destruct(grpc_exec_ctx *exec_ctx,
                          grpc_call_credentials *creds) {
   grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->iam_md);
+  grpc_credentials_mdelem_array_destroy(exec_ctx, &c->md_array);
 }
 
-static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
+static bool iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                      grpc_call_credentials *creds,
                                      grpc_polling_entity *pollent,
                                      grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
+                                     grpc_credentials_mdelem_array *md_array,
+                                     grpc_closure *on_request_metadata,
+                                     grpc_error **error) {
   grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
-     GRPC_CREDENTIALS_OK, NULL);
+  grpc_credentials_mdelem_array_append(md_array, &c->md_array);
+  return true;
 }
 
-static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
-                                                  iam_get_request_metadata};
+static void iam_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
+}
+
+static grpc_call_credentials_vtable iam_vtable = {
+    iam_destruct, iam_get_request_metadata, iam_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_google_iam_credentials_create(
     const char *token, const char *authority_selector, void *reserved) {
-  grpc_google_iam_credentials *c;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   GRPC_API_TRACE(
       "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
       "reserved=%p)",
@@ -57,14 +64,22 @@ grpc_call_credentials *grpc_google_iam_credentials_create(
   GPR_ASSERT(reserved == NULL);
   GPR_ASSERT(token != NULL);
   GPR_ASSERT(authority_selector != NULL);
-  c = gpr_zalloc(sizeof(grpc_google_iam_credentials));
+  grpc_google_iam_credentials *c = gpr_zalloc(sizeof(*c));
   c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
   c->base.vtable = &iam_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->iam_md = grpc_credentials_md_store_create(2);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
+  grpc_mdelem md = grpc_mdelem_from_slices(
+      &exec_ctx,
+      grpc_slice_from_static_string(GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY),
+      grpc_slice_from_copied_string(token));
+  grpc_credentials_mdelem_array_add(&c->md_array, md);
+  GRPC_MDELEM_UNREF(&exec_ctx, md);
+  md = grpc_mdelem_from_slices(
+      &exec_ctx,
+      grpc_slice_from_static_string(GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY),
+      grpc_slice_from_copied_string(authority_selector));
+  grpc_credentials_mdelem_array_add(&c->md_array, md);
+  GRPC_MDELEM_UNREF(&exec_ctx, md);
+  grpc_exec_ctx_finish(&exec_ctx);
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.h b/src/core/lib/security/credentials/iam/iam_credentials.h
index fff3c6d2ca1..5e3cf65bad4 100644
--- a/src/core/lib/security/credentials/iam/iam_credentials.h
+++ b/src/core/lib/security/credentials/iam/iam_credentials.h
@@ -23,7 +23,7 @@
 
 typedef struct {
   grpc_call_credentials base;
-  grpc_credentials_md_store *iam_md;
+  grpc_credentials_mdelem_array md_array;
 } grpc_google_iam_credentials;
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c
index 4357657defc..02c82e99ba8 100644
--- a/src/core/lib/security/credentials/jwt/jwt_credentials.c
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c
@@ -29,10 +29,8 @@
 
 static void jwt_reset_cache(grpc_exec_ctx *exec_ctx,
                             grpc_service_account_jwt_access_credentials *c) {
-  if (c->cached.jwt_md != NULL) {
-    grpc_credentials_md_store_unref(exec_ctx, c->cached.jwt_md);
-    c->cached.jwt_md = NULL;
-  }
+  GRPC_MDELEM_UNREF(exec_ctx, c->cached.jwt_md);
+  c->cached.jwt_md = GRPC_MDNULL;
   if (c->cached.service_url != NULL) {
     gpr_free(c->cached.service_url);
     c->cached.service_url = NULL;
@@ -49,33 +47,34 @@ static void jwt_destruct(grpc_exec_ctx *exec_ctx,
   gpr_mu_destroy(&c->cache_mu);
 }
 
-static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
+static bool jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                      grpc_call_credentials *creds,
                                      grpc_polling_entity *pollent,
                                      grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
+                                     grpc_credentials_mdelem_array *md_array,
+                                     grpc_closure *on_request_metadata,
+                                     grpc_error **error) {
   grpc_service_account_jwt_access_credentials *c =
       (grpc_service_account_jwt_access_credentials *)creds;
   gpr_timespec refresh_threshold = gpr_time_from_seconds(
       GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
 
   /* See if we can return a cached jwt. */
-  grpc_credentials_md_store *jwt_md = NULL;
+  grpc_mdelem jwt_md = GRPC_MDNULL;
   {
     gpr_mu_lock(&c->cache_mu);
     if (c->cached.service_url != NULL &&
         strcmp(c->cached.service_url, context.service_url) == 0 &&
-        c->cached.jwt_md != NULL &&
+        !GRPC_MDISNULL(c->cached.jwt_md) &&
         (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
                                    gpr_now(GPR_CLOCK_REALTIME)),
                       refresh_threshold) > 0)) {
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+      jwt_md = GRPC_MDELEM_REF(c->cached.jwt_md);
     }
     gpr_mu_unlock(&c->cache_mu);
   }
 
-  if (jwt_md == NULL) {
+  if (GRPC_MDISNULL(jwt_md)) {
     char *jwt = NULL;
     /* Generate a new jwt. */
     gpr_mu_lock(&c->cache_mu);
@@ -89,27 +88,33 @@ static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
       c->cached.jwt_expiration =
           gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
       c->cached.service_url = gpr_strdup(context.service_url);
-      c->cached.jwt_md = grpc_credentials_md_store_create(1);
-      grpc_credentials_md_store_add_cstrings(
-          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
+      c->cached.jwt_md = grpc_mdelem_from_slices(
+          exec_ctx,
+          grpc_slice_from_static_string(GRPC_AUTHORIZATION_METADATA_KEY),
+          grpc_slice_from_copied_string(md_value));
       gpr_free(md_value);
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+      jwt_md = GRPC_MDELEM_REF(c->cached.jwt_md);
     }
     gpr_mu_unlock(&c->cache_mu);
   }
 
-  if (jwt_md != NULL) {
-    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
-       GRPC_CREDENTIALS_OK, NULL);
-    grpc_credentials_md_store_unref(exec_ctx, jwt_md);
+  if (!GRPC_MDISNULL(jwt_md)) {
+    grpc_credentials_mdelem_array_add(md_array, jwt_md);
+    GRPC_MDELEM_UNREF(exec_ctx, jwt_md);
   } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
-       "Could not generate JWT.");
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Could not generate JWT.");
   }
+  return true;
+}
+
+static void jwt_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
-static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
-                                                  jwt_get_request_metadata};
+static grpc_call_credentials_vtable jwt_vtable = {
+    jwt_destruct, jwt_get_request_metadata, jwt_cancel_get_request_metadata};
 
 grpc_call_credentials *
 grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.h b/src/core/lib/security/credentials/jwt/jwt_credentials.h
index 6e461f17156..07f4022669d 100644
--- a/src/core/lib/security/credentials/jwt/jwt_credentials.h
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h
@@ -29,7 +29,7 @@ typedef struct {
   // the service_url for a more sophisticated one.
   gpr_mu cache_mu;
   struct {
-    grpc_credentials_md_store *jwt_md;
+    grpc_mdelem jwt_md;
     char *service_url;
     gpr_timespec jwt_expiration;
   } cached;
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
index 9de561b3101..ffa941bb9e9 100644
--- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
@@ -107,7 +107,7 @@ static void oauth2_token_fetcher_destruct(grpc_exec_ctx *exec_ctx,
                                           grpc_call_credentials *creds) {
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->access_token_md);
+  GRPC_MDELEM_UNREF(exec_ctx, c->access_token_md);
   gpr_mu_destroy(&c->mu);
   grpc_httpcli_context_destroy(exec_ctx, &c->httpcli_context);
 }
@@ -115,7 +115,7 @@ static void oauth2_token_fetcher_destruct(grpc_exec_ctx *exec_ctx,
 grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(
     grpc_exec_ctx *exec_ctx, const grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime) {
+    grpc_mdelem *token_md, gpr_timespec *token_lifetime) {
   char *null_terminated_body = NULL;
   char *new_access_token = NULL;
   grpc_credentials_status status = GRPC_CREDENTIALS_OK;
@@ -184,17 +184,18 @@ grpc_oauth2_token_fetcher_credentials_parse_server_response(
     token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
     token_lifetime->tv_nsec = 0;
     token_lifetime->clock_type = GPR_TIMESPAN;
-    if (*token_md != NULL) grpc_credentials_md_store_unref(exec_ctx, *token_md);
-    *token_md = grpc_credentials_md_store_create(1);
-    grpc_credentials_md_store_add_cstrings(
-        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
+    if (!GRPC_MDISNULL(*token_md)) GRPC_MDELEM_UNREF(exec_ctx, *token_md);
+    *token_md = grpc_mdelem_from_slices(
+        exec_ctx,
+        grpc_slice_from_static_string(GRPC_AUTHORIZATION_METADATA_KEY),
+        grpc_slice_from_copied_string(new_access_token));
     status = GRPC_CREDENTIALS_OK;
   }
 
 end:
-  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
-    grpc_credentials_md_store_unref(exec_ctx, *token_md);
-    *token_md = NULL;
+  if (status != GRPC_CREDENTIALS_OK && !GRPC_MDISNULL(*token_md)) {
+    GRPC_MDELEM_UNREF(exec_ctx, *token_md);
+    *token_md = GRPC_MDNULL;
   }
   if (null_terminated_body != NULL) gpr_free(null_terminated_body);
   if (new_access_token != NULL) gpr_free(new_access_token);
@@ -205,63 +206,124 @@ end:
 static void on_oauth2_token_fetcher_http_response(grpc_exec_ctx *exec_ctx,
                                                   void *user_data,
                                                   grpc_error *error) {
+  GRPC_LOG_IF_ERROR("oauth_fetch", GRPC_ERROR_REF(error));
   grpc_credentials_metadata_request *r =
       (grpc_credentials_metadata_request *)user_data;
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)r->creds;
+  grpc_mdelem access_token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
-  grpc_credentials_status status;
-
-  GRPC_LOG_IF_ERROR("oauth_fetch", GRPC_ERROR_REF(error));
-
+  grpc_credentials_status status =
+      grpc_oauth2_token_fetcher_credentials_parse_server_response(
+          exec_ctx, &r->response, &access_token_md, &token_lifetime);
+  // Update cache and grab list of pending requests.
   gpr_mu_lock(&c->mu);
-  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
-      exec_ctx, &r->response, &c->access_token_md, &token_lifetime);
-  if (status == GRPC_CREDENTIALS_OK) {
-    c->token_expiration =
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
-    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
-          c->access_token_md->num_entries, GRPC_CREDENTIALS_OK, NULL);
-  } else {
-    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-    r->cb(exec_ctx, r->user_data, NULL, 0, status,
-          "Error occured when fetching oauth2 token.");
-  }
+  c->token_fetch_pending = false;
+  c->access_token_md = GRPC_MDELEM_REF(access_token_md);
+  c->token_expiration =
+      status == GRPC_CREDENTIALS_OK
+          ? gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime)
+          : gpr_inf_past(GPR_CLOCK_REALTIME);
+  grpc_oauth2_pending_get_request_metadata *pending_request =
+      c->pending_requests;
+  c->pending_requests = NULL;
   gpr_mu_unlock(&c->mu);
+  // Invoke callbacks for all pending requests.
+  while (pending_request != NULL) {
+    if (status == GRPC_CREDENTIALS_OK) {
+      grpc_credentials_mdelem_array_add(pending_request->md_array,
+                                        access_token_md);
+    } else {
+      error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+          "Error occured when fetching oauth2 token.", &error, 1);
+    }
+    GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata, error);
+    grpc_oauth2_pending_get_request_metadata *prev = pending_request;
+    pending_request = pending_request->next;
+    gpr_free(prev);
+  }
+  GRPC_MDELEM_UNREF(exec_ctx, access_token_md);
+  grpc_call_credentials_unref(exec_ctx, r->creds);
   grpc_credentials_metadata_request_destroy(exec_ctx, r);
 }
 
-static void oauth2_token_fetcher_get_request_metadata(
+static bool oauth2_token_fetcher_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)creds;
+  // Check if we can use the cached token.
   gpr_timespec refresh_threshold = gpr_time_from_seconds(
       GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-  grpc_credentials_md_store *cached_access_token_md = NULL;
-  {
-    gpr_mu_lock(&c->mu);
-    if (c->access_token_md != NULL &&
-        (gpr_time_cmp(
-             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
-             refresh_threshold) > 0)) {
-      cached_access_token_md =
-          grpc_credentials_md_store_ref(c->access_token_md);
-    }
+  grpc_mdelem cached_access_token_md = GRPC_MDNULL;
+  gpr_mu_lock(&c->mu);
+  if (!GRPC_MDISNULL(c->access_token_md) &&
+      (gpr_time_cmp(
+           gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
+           refresh_threshold) > 0)) {
+    cached_access_token_md = GRPC_MDELEM_REF(c->access_token_md);
+  }
+  if (!GRPC_MDISNULL(cached_access_token_md)) {
     gpr_mu_unlock(&c->mu);
+    grpc_credentials_mdelem_array_add(md_array, cached_access_token_md);
+    GRPC_MDELEM_UNREF(exec_ctx, cached_access_token_md);
+    return true;
   }
-  if (cached_access_token_md != NULL) {
-    cb(exec_ctx, user_data, cached_access_token_md->entries,
-       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK, NULL);
-    grpc_credentials_md_store_unref(exec_ctx, cached_access_token_md);
-  } else {
-    c->fetch_func(
-        exec_ctx,
-        grpc_credentials_metadata_request_create(creds, cb, user_data),
-        &c->httpcli_context, pollent, on_oauth2_token_fetcher_http_response,
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
+  // Couldn't get the token from the cache.
+  // Add request to c->pending_requests and start a new fetch if needed.
+  grpc_oauth2_pending_get_request_metadata *pending_request =
+      (grpc_oauth2_pending_get_request_metadata *)gpr_malloc(
+          sizeof(*pending_request));
+  pending_request->md_array = md_array;
+  pending_request->on_request_metadata = on_request_metadata;
+  pending_request->next = c->pending_requests;
+  c->pending_requests = pending_request;
+  bool start_fetch = false;
+  if (!c->token_fetch_pending) {
+    c->token_fetch_pending = true;
+    start_fetch = true;
+  }
+  gpr_mu_unlock(&c->mu);
+  if (start_fetch) {
+    grpc_call_credentials_ref(creds);
+    c->fetch_func(exec_ctx, grpc_credentials_metadata_request_create(creds),
+                  &c->httpcli_context, pollent,
+                  on_oauth2_token_fetcher_http_response,
+                  gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
   }
+  return false;
+}
+
+static void oauth2_token_fetcher_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  gpr_mu_lock(&c->mu);
+  grpc_oauth2_pending_get_request_metadata *prev = NULL;
+  grpc_oauth2_pending_get_request_metadata *pending_request =
+      c->pending_requests;
+  while (pending_request != NULL) {
+    if (pending_request->md_array == md_array) {
+      // Remove matching pending request from the list.
+      if (prev != NULL) {
+        prev->next = pending_request->next;
+      } else {
+        c->pending_requests = pending_request->next;
+      }
+      // Invoke the callback immediately with an error.
+      GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata,
+                         GRPC_ERROR_REF(error));
+      gpr_free(pending_request);
+      break;
+    }
+    prev = pending_request;
+    pending_request = pending_request->next;
+  }
+  gpr_mu_unlock(&c->mu);
+  GRPC_ERROR_UNREF(error);
 }
 
 static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
@@ -280,7 +342,8 @@ static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
 //
 
 static grpc_call_credentials_vtable compute_engine_vtable = {
-    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
+    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata,
+    oauth2_token_fetcher_cancel_get_request_metadata};
 
 static void compute_engine_fetch_oauth2(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
@@ -301,7 +364,6 @@ static void compute_engine_fetch_oauth2(
   grpc_httpcli_get(
       exec_ctx, httpcli_context, pollent, resource_quota, &request, deadline,
       GRPC_CLOSURE_CREATE(response_cb, metadata_req, grpc_schedule_on_exec_ctx),
-
       &metadata_req->response);
   grpc_resource_quota_unref_internal(exec_ctx, resource_quota);
 }
@@ -331,7 +393,8 @@ static void refresh_token_destruct(grpc_exec_ctx *exec_ctx,
 }
 
 static grpc_call_credentials_vtable refresh_token_vtable = {
-    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
+    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata,
+    oauth2_token_fetcher_cancel_get_request_metadata};
 
 static void refresh_token_fetch_oauth2(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
@@ -416,26 +479,33 @@ grpc_call_credentials *grpc_google_refresh_token_credentials_create(
 static void access_token_destruct(grpc_exec_ctx *exec_ctx,
                                   grpc_call_credentials *creds) {
   grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->access_token_md);
+  GRPC_MDELEM_UNREF(exec_ctx, c->access_token_md);
 }
 
-static void access_token_get_request_metadata(
+static bool access_token_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK,
-     NULL);
+  grpc_credentials_mdelem_array_add(md_array, c->access_token_md);
+  return true;
+}
+
+static void access_token_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable access_token_vtable = {
-    access_token_destruct, access_token_get_request_metadata};
+    access_token_destruct, access_token_get_request_metadata,
+    access_token_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_access_token_credentials_create(
     const char *access_token, void *reserved) {
   grpc_access_token_credentials *c =
       gpr_zalloc(sizeof(grpc_access_token_credentials));
-  char *token_md_value;
   GRPC_API_TRACE(
       "grpc_access_token_credentials_create(access_token=<redacted>, "
       "reserved=%p)",
@@ -444,10 +514,13 @@ grpc_call_credentials *grpc_access_token_credentials_create(
   c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
   c->base.vtable = &access_token_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->access_token_md = grpc_credentials_md_store_create(1);
+  char *token_md_value;
   gpr_asprintf(&token_md_value, "Bearer %s", access_token);
-  grpc_credentials_md_store_add_cstrings(
-      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  c->access_token_md = grpc_mdelem_from_slices(
+      &exec_ctx, grpc_slice_from_static_string(GRPC_AUTHORIZATION_METADATA_KEY),
+      grpc_slice_from_copied_string(token_md_value));
+  grpc_exec_ctx_finish(&exec_ctx);
   gpr_free(token_md_value);
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
index 72093afe9e7..9d041a20eaf 100644
--- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
@@ -58,11 +58,20 @@ typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
                                        grpc_polling_entity *pollent,
                                        grpc_iomgr_cb_func cb,
                                        gpr_timespec deadline);
+
+typedef struct grpc_oauth2_pending_get_request_metadata {
+  grpc_credentials_mdelem_array *md_array;
+  grpc_closure *on_request_metadata;
+  struct grpc_oauth2_pending_get_request_metadata *next;
+} grpc_oauth2_pending_get_request_metadata;
+
 typedef struct {
   grpc_call_credentials base;
   gpr_mu mu;
-  grpc_credentials_md_store *access_token_md;
+  grpc_mdelem access_token_md;
   gpr_timespec token_expiration;
+  bool token_fetch_pending;
+  grpc_oauth2_pending_get_request_metadata *pending_requests;
   grpc_httpcli_context httpcli_context;
   grpc_fetch_oauth2_func fetch_func;
 } grpc_oauth2_token_fetcher_credentials;
@@ -76,7 +85,7 @@ typedef struct {
 // Access token credentials.
 typedef struct {
   grpc_call_credentials base;
-  grpc_credentials_md_store *access_token_md;
+  grpc_mdelem access_token_md;
 } grpc_access_token_credentials;
 
 // Private constructor for refresh token credentials from an already parsed
@@ -89,6 +98,6 @@ grpc_refresh_token_credentials_create_from_auth_refresh_token(
 grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(
     grpc_exec_ctx *exec_ctx, const struct grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
+    grpc_mdelem *token_md, gpr_timespec *token_lifetime);
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c
index 96ebfb4d0d4..73e0c23e0f7 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.c
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c
@@ -31,19 +31,28 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/validate_metadata.h"
 
-typedef struct {
-  void *user_data;
-  grpc_credentials_metadata_cb cb;
-} grpc_metadata_plugin_request;
-
 static void plugin_destruct(grpc_exec_ctx *exec_ctx,
                             grpc_call_credentials *creds) {
   grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  gpr_mu_destroy(&c->mu);
   if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
     c->plugin.destroy(c->plugin.state);
   }
 }
 
+static void pending_request_remove_locked(
+    grpc_plugin_credentials *c,
+    grpc_plugin_credentials_pending_request *pending_request) {
+  if (pending_request->prev == NULL) {
+    c->pending_requests = pending_request->next;
+  } else {
+    pending_request->prev->next = pending_request->next;
+  }
+  if (pending_request->next != NULL) {
+    pending_request->next->prev = pending_request->prev;
+  }
+}
+
 static void plugin_md_request_metadata_ready(void *request,
                                              const grpc_metadata *md,
                                              size_t num_md,
@@ -53,76 +62,117 @@ static void plugin_md_request_metadata_ready(void *request,
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(
       GRPC_EXEC_CTX_FLAG_IS_FINISHED | GRPC_EXEC_CTX_FLAG_THREAD_RESOURCE_LOOP,
       NULL, NULL);
-  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
-  if (status != GRPC_STATUS_OK) {
-    if (error_details != NULL) {
-      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
-              error_details);
-    }
-    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
-          error_details);
-  } else {
-    size_t i;
-    bool seen_illegal_header = false;
-    grpc_credentials_md *md_array = NULL;
-    for (i = 0; i < num_md; i++) {
-      if (!GRPC_LOG_IF_ERROR("validate_metadata_from_plugin",
-                             grpc_validate_header_key_is_legal(md[i].key))) {
-        seen_illegal_header = true;
-        break;
-      } else if (!grpc_is_binary_header(md[i].key) &&
-                 !GRPC_LOG_IF_ERROR(
-                     "validate_metadata_from_plugin",
-                     grpc_validate_header_nonbin_value_is_legal(md[i].value))) {
-        gpr_log(GPR_ERROR, "Plugin added invalid metadata value.");
-        seen_illegal_header = true;
-        break;
+  grpc_plugin_credentials_pending_request *r =
+      (grpc_plugin_credentials_pending_request *)request;
+  // Check if the request has been cancelled.
+  // If not, remove it from the pending list, so that it cannot be
+  // cancelled out from under us.
+  gpr_mu_lock(&r->creds->mu);
+  if (!r->cancelled) pending_request_remove_locked(r->creds, r);
+  gpr_mu_unlock(&r->creds->mu);
+  grpc_call_credentials_unref(&exec_ctx, &r->creds->base);
+  // If it has not been cancelled, process it.
+  if (!r->cancelled) {
+    if (status != GRPC_STATUS_OK) {
+      char *msg;
+      gpr_asprintf(&msg, "Getting metadata from plugin failed with error: %s",
+                   error_details);
+      GRPC_CLOSURE_SCHED(&exec_ctx, r->on_request_metadata,
+                         GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg));
+      gpr_free(msg);
+    } else {
+      bool seen_illegal_header = false;
+      for (size_t i = 0; i < num_md; ++i) {
+        if (!GRPC_LOG_IF_ERROR("validate_metadata_from_plugin",
+                               grpc_validate_header_key_is_legal(md[i].key))) {
+          seen_illegal_header = true;
+          break;
+        } else if (!grpc_is_binary_header(md[i].key) &&
+                   !GRPC_LOG_IF_ERROR(
+                       "validate_metadata_from_plugin",
+                       grpc_validate_header_nonbin_value_is_legal(
+                           md[i].value))) {
+          gpr_log(GPR_ERROR, "Plugin added invalid metadata value.");
+          seen_illegal_header = true;
+          break;
+        }
       }
-    }
-    if (seen_illegal_header) {
-      r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
-            "Illegal metadata");
-    } else if (num_md > 0) {
-      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
-      for (i = 0; i < num_md; i++) {
-        md_array[i].key = grpc_slice_ref_internal(md[i].key);
-        md_array[i].value = grpc_slice_ref_internal(md[i].value);
+      if (seen_illegal_header) {
+        GRPC_CLOSURE_SCHED(
+            &exec_ctx, r->on_request_metadata,
+            GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal metadata"));
+      } else {
+        for (size_t i = 0; i < num_md; ++i) {
+          grpc_mdelem mdelem = grpc_mdelem_from_slices(
+              &exec_ctx, grpc_slice_ref_internal(md[i].key),
+              grpc_slice_ref_internal(md[i].value));
+          grpc_credentials_mdelem_array_add(r->md_array, mdelem);
+          GRPC_MDELEM_UNREF(&exec_ctx, mdelem);
+        }
+        GRPC_CLOSURE_SCHED(&exec_ctx, r->on_request_metadata, GRPC_ERROR_NONE);
       }
-      r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK,
-            NULL);
-      for (i = 0; i < num_md; i++) {
-        grpc_slice_unref_internal(&exec_ctx, md_array[i].key);
-        grpc_slice_unref_internal(&exec_ctx, md_array[i].value);
-      }
-      gpr_free(md_array);
-    } else if (num_md == 0) {
-      r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
     }
   }
   gpr_free(r);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
+static bool plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                         grpc_call_credentials *creds,
                                         grpc_polling_entity *pollent,
                                         grpc_auth_metadata_context context,
-                                        grpc_credentials_metadata_cb cb,
-                                        void *user_data) {
+                                        grpc_credentials_mdelem_array *md_array,
+                                        grpc_closure *on_request_metadata,
+                                        grpc_error **error) {
   grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
   if (c->plugin.get_metadata != NULL) {
-    grpc_metadata_plugin_request *request = gpr_zalloc(sizeof(*request));
-    request->user_data = user_data;
-    request->cb = cb;
+    // Create pending_request object.
+    grpc_plugin_credentials_pending_request *pending_request =
+        (grpc_plugin_credentials_pending_request *)gpr_zalloc(
+            sizeof(*pending_request));
+    pending_request->creds = c;
+    pending_request->md_array = md_array;
+    pending_request->on_request_metadata = on_request_metadata;
+    // Add it to the pending list.
+    gpr_mu_lock(&c->mu);
+    if (c->pending_requests != NULL) {
+      c->pending_requests->prev = pending_request;
+    }
+    pending_request->next = c->pending_requests;
+    c->pending_requests = pending_request;
+    gpr_mu_unlock(&c->mu);
+    // Invoke the plugin.  The callback holds a ref to us.
+    grpc_call_credentials_ref(creds);
     c->plugin.get_metadata(c->plugin.state, context,
-                           plugin_md_request_metadata_ready, request);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
+                           plugin_md_request_metadata_ready, pending_request);
+    return false;
+  }
+  return true;
+}
+
+static void plugin_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  gpr_mu_lock(&c->mu);
+  for (grpc_plugin_credentials_pending_request *pending_request =
+           c->pending_requests;
+       pending_request != NULL; pending_request = pending_request->next) {
+    if (pending_request->md_array == md_array) {
+      pending_request->cancelled = true;
+      GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata,
+                         GRPC_ERROR_REF(error));
+      pending_request_remove_locked(c, pending_request);
+      break;
+    }
   }
+  gpr_mu_unlock(&c->mu);
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable plugin_vtable = {
-    plugin_destruct, plugin_get_request_metadata};
+    plugin_destruct, plugin_get_request_metadata,
+    plugin_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
     grpc_metadata_credentials_plugin plugin, void *reserved) {
@@ -134,5 +184,6 @@ grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
   c->base.vtable = &plugin_vtable;
   gpr_ref_init(&c->base.refcount, 1);
   c->plugin = plugin;
+  gpr_mu_init(&c->mu);
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h
index ba3dd76859e..57266d589a4 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.h
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h
@@ -21,10 +21,22 @@
 
 #include "src/core/lib/security/credentials/credentials.h"
 
-typedef struct {
+struct grpc_plugin_credentials;
+
+typedef struct grpc_plugin_credentials_pending_request {
+  bool cancelled;
+  struct grpc_plugin_credentials *creds;
+  grpc_credentials_mdelem_array *md_array;
+  grpc_closure *on_request_metadata;
+  struct grpc_plugin_credentials_pending_request *prev;
+  struct grpc_plugin_credentials_pending_request *next;
+} grpc_plugin_credentials_pending_request;
+
+typedef struct grpc_plugin_credentials {
   grpc_call_credentials base;
   grpc_metadata_credentials_plugin plugin;
-  grpc_credentials_md_store *plugin_md;
+  gpr_mu mu;
+  grpc_plugin_credentials_pending_request *pending_requests;
 } grpc_plugin_credentials;
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H */
diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c
index 50a51b31cdb..531a88434f3 100644
--- a/src/core/lib/security/transport/client_auth_filter.c
+++ b/src/core/lib/security/transport/client_auth_filter.c
@@ -51,8 +51,15 @@ typedef struct {
   grpc_polling_entity *pollent;
   gpr_atm security_context_set;
   gpr_mu security_context_mu;
+  grpc_credentials_mdelem_array md_array;
   grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT];
   grpc_auth_metadata_context auth_md_context;
+  grpc_closure closure;
+  // Either 0 (no cancellation and no async operation in flight),
+  // a grpc_closure* (if the lowest bit is 0),
+  // or a grpc_error* (if the lowest bit is 1).
+  gpr_atm cancellation_state;
+  grpc_closure cancel_closure;
 } call_data;
 
 /* We can have a per-channel credentials. */
@@ -61,6 +68,43 @@ typedef struct {
   grpc_auth_context *auth_context;
 } channel_data;
 
+static void decode_cancel_state(gpr_atm cancel_state, grpc_closure **func,
+                                grpc_error **error) {
+  // If the lowest bit is 1, the value is a grpc_error*.
+  // Otherwise, if non-zdero, the value is a grpc_closure*.
+  if (cancel_state & 1) {
+    *error = (grpc_error *)(cancel_state & ~(gpr_atm)1);
+  } else if (cancel_state != 0) {
+    *func = (grpc_closure *)cancel_state;
+  }
+}
+
+static gpr_atm encode_cancel_state_error(grpc_error *error) {
+  // Set the lowest bit to 1 to indicate that it's an error.
+  return (gpr_atm)1 | (gpr_atm)error;
+}
+
+// Returns an error if the call has been cancelled.  Otherwise, sets the
+// cancellation function to be called upon cancellation.
+static grpc_error *set_cancel_func(grpc_call_element *elem,
+                                   grpc_iomgr_cb_func func) {
+  call_data *calld = (call_data *)elem->call_data;
+  // Decode original state.
+  gpr_atm original_state = gpr_atm_acq_load(&calld->cancellation_state);
+  grpc_error *original_error = GRPC_ERROR_NONE;
+  grpc_closure *original_func = NULL;
+  decode_cancel_state(original_state, &original_func, &original_error);
+  // If error is set, return it.
+  if (original_error != GRPC_ERROR_NONE) return GRPC_ERROR_REF(original_error);
+  // Otherwise, store func.
+  GRPC_CLOSURE_INIT(&calld->cancel_closure, func, elem,
+                    grpc_schedule_on_exec_ctx);
+  GPR_ASSERT(((gpr_atm)&calld->cancel_closure & (gpr_atm)1) == 0);
+  gpr_atm_rel_store(&calld->cancellation_state,
+                    (gpr_atm)&calld->cancel_closure);
+  return GRPC_ERROR_NONE;
+}
+
 static void reset_auth_metadata_context(
     grpc_auth_metadata_context *auth_md_context) {
   if (auth_md_context->service_url != NULL) {
@@ -86,41 +130,29 @@ static void add_error(grpc_error **combined, grpc_error *error) {
   *combined = grpc_error_add_child(*combined, error);
 }
 
-static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data,
-                                    grpc_credentials_md *md_elems,
-                                    size_t num_md,
-                                    grpc_credentials_status status,
-                                    const char *error_details) {
-  grpc_transport_stream_op_batch *batch =
-      (grpc_transport_stream_op_batch *)user_data;
+static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *arg,
+                                    grpc_error *input_error) {
+  grpc_transport_stream_op_batch *batch = (grpc_transport_stream_op_batch *)arg;
   grpc_call_element *elem = batch->handler_private.extra_arg;
   call_data *calld = elem->call_data;
   reset_auth_metadata_context(&calld->auth_md_context);
-  grpc_error *error = GRPC_ERROR_NONE;
-  if (status != GRPC_CREDENTIALS_OK) {
-    error = grpc_error_set_int(
-        GRPC_ERROR_CREATE_FROM_COPIED_STRING(
-            error_details != NULL && strlen(error_details) > 0
-                ? error_details
-                : "Credentials failed to get metadata."),
-        GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED);
-  } else {
-    GPR_ASSERT(num_md <= MAX_CREDENTIALS_METADATA_COUNT);
+  grpc_error *error = GRPC_ERROR_REF(input_error);
+  if (error == GRPC_ERROR_NONE) {
+    GPR_ASSERT(calld->md_array.size <= MAX_CREDENTIALS_METADATA_COUNT);
     GPR_ASSERT(batch->send_initial_metadata);
     grpc_metadata_batch *mdb =
         batch->payload->send_initial_metadata.send_initial_metadata;
-    for (size_t i = 0; i < num_md; i++) {
-      add_error(&error,
-                grpc_metadata_batch_add_tail(
-                    exec_ctx, mdb, &calld->md_links[i],
-                    grpc_mdelem_from_slices(
-                        exec_ctx, grpc_slice_ref_internal(md_elems[i].key),
-                        grpc_slice_ref_internal(md_elems[i].value))));
+    for (size_t i = 0; i < calld->md_array.size; ++i) {
+      add_error(&error, grpc_metadata_batch_add_tail(
+                            exec_ctx, mdb, &calld->md_links[i],
+                            GRPC_MDELEM_REF(calld->md_array.md[i])));
     }
   }
   if (error == GRPC_ERROR_NONE) {
     grpc_call_next_op(exec_ctx, elem, batch);
   } else {
+    error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS,
+                               GRPC_STATUS_UNAUTHENTICATED);
     grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error);
   }
 }
@@ -155,6 +187,14 @@ void build_auth_metadata_context(grpc_security_connector *sc,
   gpr_free(host);
 }
 
+static void cancel_get_request_metadata(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  grpc_call_credentials_cancel_get_request_metadata(
+      exec_ctx, calld->creds, &calld->md_array, GRPC_ERROR_REF(error));
+}
+
 static void send_security_metadata(grpc_exec_ctx *exec_ctx,
                                    grpc_call_element *elem,
                                    grpc_transport_stream_op_batch *batch) {
@@ -193,20 +233,33 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
 
   build_auth_metadata_context(&chand->security_connector->base,
                               chand->auth_context, calld);
+
+  grpc_error *cancel_error = set_cancel_func(elem, cancel_get_request_metadata);
+  if (cancel_error != GRPC_ERROR_NONE) {
+    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch,
+                                                       cancel_error);
+    return;
+  }
   GPR_ASSERT(calld->pollent != NULL);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, calld->creds, calld->pollent, calld->auth_md_context,
-      on_credentials_metadata, batch);
+  GRPC_CLOSURE_INIT(&calld->closure, on_credentials_metadata, batch,
+                    grpc_schedule_on_exec_ctx);
+  grpc_error *error = GRPC_ERROR_NONE;
+  if (grpc_call_credentials_get_request_metadata(
+          exec_ctx, calld->creds, calld->pollent, calld->auth_md_context,
+          &calld->md_array, &calld->closure, &error)) {
+    // Synchronous return; invoke on_credentials_metadata() directly.
+    on_credentials_metadata(exec_ctx, batch, error);
+    GRPC_ERROR_UNREF(error);
+  }
 }
 
-static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data,
-                            grpc_security_status status) {
-  grpc_transport_stream_op_batch *batch =
-      (grpc_transport_stream_op_batch *)user_data;
+static void on_host_checked(grpc_exec_ctx *exec_ctx, void *arg,
+                            grpc_error *error) {
+  grpc_transport_stream_op_batch *batch = (grpc_transport_stream_op_batch *)arg;
   grpc_call_element *elem = batch->handler_private.extra_arg;
   call_data *calld = elem->call_data;
 
-  if (status == GRPC_SECURITY_OK) {
+  if (error == GRPC_ERROR_NONE) {
     send_security_metadata(exec_ctx, elem, batch);
   } else {
     char *error_msg;
@@ -223,6 +276,16 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data,
   }
 }
 
+static void cancel_check_call_host(grpc_exec_ctx *exec_ctx, void *arg,
+                                   grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  channel_data *chand = (channel_data *)elem->channel_data;
+  grpc_channel_security_connector_cancel_check_call_host(
+      exec_ctx, chand->security_connector, &calld->closure,
+      GRPC_ERROR_REF(error));
+}
+
 static void auth_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *batch) {
@@ -232,7 +295,32 @@ static void auth_start_transport_stream_op_batch(
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
 
-  if (!batch->cancel_stream) {
+  if (batch->cancel_stream) {
+    while (true) {
+      // Decode the original cancellation state.
+      gpr_atm original_state = gpr_atm_acq_load(&calld->cancellation_state);
+      grpc_error *cancel_error = GRPC_ERROR_NONE;
+      grpc_closure *func = NULL;
+      decode_cancel_state(original_state, &func, &cancel_error);
+      // If we had already set a cancellation error, there's nothing
+      // more to do.
+      if (cancel_error != GRPC_ERROR_NONE) break;
+      // If there's a cancel func, call it.
+      // Note that even if the cancel func has been changed by some
+      // other thread between when we decoded it and now, it will just
+      // be a no-op.
+      cancel_error = GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error);
+      if (func != NULL) {
+        GRPC_CLOSURE_SCHED(exec_ctx, func, GRPC_ERROR_REF(cancel_error));
+      }
+      // Encode the new error into cancellation state.
+      if (gpr_atm_full_cas(&calld->cancellation_state, original_state,
+                           encode_cancel_state_error(cancel_error))) {
+        break;  // Success.
+      }
+      // The cas failed, so try again.
+    }
+  } else {
     /* double checked lock over security context to ensure it's set once */
     if (gpr_atm_acq_load(&calld->security_context_set) == 0) {
       gpr_mu_lock(&calld->security_context_mu);
@@ -277,12 +365,26 @@ static void auth_start_transport_stream_op_batch(
       }
     }
     if (calld->have_host) {
-      char *call_host = grpc_slice_to_c_string(calld->host);
-      batch->handler_private.extra_arg = elem;
-      grpc_channel_security_connector_check_call_host(
-          exec_ctx, chand->security_connector, call_host, chand->auth_context,
-          on_host_checked, batch);
-      gpr_free(call_host);
+      grpc_error *cancel_error = set_cancel_func(elem, cancel_check_call_host);
+      if (cancel_error != GRPC_ERROR_NONE) {
+        grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch,
+                                                           cancel_error);
+      } else {
+        char *call_host = grpc_slice_to_c_string(calld->host);
+        batch->handler_private.extra_arg = elem;
+        grpc_error *error = GRPC_ERROR_NONE;
+        if (grpc_channel_security_connector_check_call_host(
+                exec_ctx, chand->security_connector, call_host,
+                chand->auth_context,
+                GRPC_CLOSURE_INIT(&calld->closure, on_host_checked, batch,
+                                  grpc_schedule_on_exec_ctx),
+                &error)) {
+          // Synchronous return; invoke on_host_checked() directly.
+          on_host_checked(exec_ctx, batch, error);
+          GRPC_ERROR_UNREF(error);
+        }
+        gpr_free(call_host);
+      }
       GPR_TIMER_END("auth_start_transport_stream_op_batch", 0);
       return; /* early exit */
     }
@@ -315,6 +417,7 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
                               grpc_closure *ignored) {
   call_data *calld = elem->call_data;
+  grpc_credentials_mdelem_array_destroy(exec_ctx, &calld->md_array);
   grpc_call_credentials_unref(exec_ctx, calld->creds);
   if (calld->have_host) {
     grpc_slice_unref_internal(exec_ctx, calld->host);
@@ -324,6 +427,11 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   }
   reset_auth_metadata_context(&calld->auth_md_context);
   gpr_mu_destroy(&calld->security_context_mu);
+  gpr_atm cancel_state = gpr_atm_acq_load(&calld->cancellation_state);
+  grpc_error *cancel_error = GRPC_ERROR_NONE;
+  grpc_closure *cancel_func = NULL;
+  decode_cancel_state(cancel_state, &cancel_func, &cancel_error);
+  GRPC_ERROR_UNREF(cancel_error);
 }
 
 /* Constructor for channel_data */
diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c
index 6788126769a..a7568b995f5 100644
--- a/src/core/lib/security/transport/security_connector.c
+++ b/src/core/lib/security/transport/security_connector.c
@@ -136,15 +136,27 @@ void grpc_security_connector_check_peer(grpc_exec_ctx *exec_ctx,
   }
 }
 
-void grpc_channel_security_connector_check_call_host(
+bool grpc_channel_security_connector_check_call_host(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
     const char *host, grpc_auth_context *auth_context,
-    grpc_security_call_host_check_cb cb, void *user_data) {
+    grpc_closure *on_call_host_checked, grpc_error **error) {
   if (sc == NULL || sc->check_call_host == NULL) {
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR);
-  } else {
-    sc->check_call_host(exec_ctx, sc, host, auth_context, cb, user_data);
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "cannot check call host -- no security connector");
+    return true;
   }
+  return sc->check_call_host(exec_ctx, sc, host, auth_context,
+                             on_call_host_checked, error);
+}
+
+void grpc_channel_security_connector_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error) {
+  if (sc == NULL || sc->cancel_check_call_host == NULL) {
+    GRPC_ERROR_UNREF(error);
+    return;
+  }
+  sc->cancel_check_call_host(exec_ctx, sc, on_call_host_checked, error);
 }
 
 #ifndef NDEBUG
@@ -368,13 +380,19 @@ static void fake_server_check_peer(grpc_exec_ctx *exec_ctx,
   fake_check_peer(exec_ctx, sc, peer, auth_context, on_peer_checked);
 }
 
-static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
+static bool fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
                                          grpc_channel_security_connector *sc,
                                          const char *host,
                                          grpc_auth_context *auth_context,
-                                         grpc_security_call_host_check_cb cb,
-                                         void *user_data) {
-  cb(exec_ctx, user_data, GRPC_SECURITY_OK);
+                                         grpc_closure *on_call_host_checked,
+                                         grpc_error **error) {
+  return true;
+}
+
+static void fake_channel_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static void fake_channel_add_handshakers(
@@ -413,6 +431,7 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create(
   c->base.request_metadata_creds =
       grpc_call_credentials_ref(request_metadata_creds);
   c->base.check_call_host = fake_channel_check_call_host;
+  c->base.cancel_check_call_host = fake_channel_cancel_check_call_host;
   c->base.add_handshakers = fake_channel_add_handshakers;
   c->target = gpr_strdup(target);
   const char *expected_targets = grpc_fake_transport_get_expected_targets(args);
@@ -663,26 +682,35 @@ void tsi_shallow_peer_destruct(tsi_peer *peer) {
   if (peer->properties != NULL) gpr_free(peer->properties);
 }
 
-static void ssl_channel_check_call_host(grpc_exec_ctx *exec_ctx,
+static bool ssl_channel_check_call_host(grpc_exec_ctx *exec_ctx,
                                         grpc_channel_security_connector *sc,
                                         const char *host,
                                         grpc_auth_context *auth_context,
-                                        grpc_security_call_host_check_cb cb,
-                                        void *user_data) {
+                                        grpc_closure *on_call_host_checked,
+                                        grpc_error **error) {
   grpc_ssl_channel_security_connector *c =
       (grpc_ssl_channel_security_connector *)sc;
   grpc_security_status status = GRPC_SECURITY_ERROR;
   tsi_peer peer = tsi_shallow_peer_from_ssl_auth_context(auth_context);
   if (ssl_host_matches_name(&peer, host)) status = GRPC_SECURITY_OK;
-
   /* If the target name was overridden, then the original target_name was
      'checked' transitively during the previous peer check at the end of the
      handshake. */
   if (c->overridden_target_name != NULL && strcmp(host, c->target_name) == 0) {
     status = GRPC_SECURITY_OK;
   }
-  cb(exec_ctx, user_data, status);
+  if (status != GRPC_SECURITY_OK) {
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "call host does not match SSL server name");
+  }
   tsi_shallow_peer_destruct(&peer);
+  return true;
+}
+
+static void ssl_channel_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_security_connector_vtable ssl_channel_vtable = {
@@ -811,6 +839,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
   c->base.request_metadata_creds =
       grpc_call_credentials_ref(request_metadata_creds);
   c->base.check_call_host = ssl_channel_check_call_host;
+  c->base.cancel_check_call_host = ssl_channel_cancel_check_call_host;
   c->base.add_handshakers = ssl_channel_add_handshakers;
   gpr_split_host_port(target_name, &c->target_name, &port);
   gpr_free(port);
diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h
index 1c0fe40045d..4f9b63ad202 100644
--- a/src/core/lib/security/transport/security_connector.h
+++ b/src/core/lib/security/transport/security_connector.h
@@ -117,27 +117,38 @@ grpc_security_connector *grpc_security_connector_find_in_args(
 
 typedef struct grpc_channel_security_connector grpc_channel_security_connector;
 
-typedef void (*grpc_security_call_host_check_cb)(grpc_exec_ctx *exec_ctx,
-                                                 void *user_data,
-                                                 grpc_security_status status);
-
 struct grpc_channel_security_connector {
   grpc_security_connector base;
   grpc_call_credentials *request_metadata_creds;
-  void (*check_call_host)(grpc_exec_ctx *exec_ctx,
+  bool (*check_call_host)(grpc_exec_ctx *exec_ctx,
                           grpc_channel_security_connector *sc, const char *host,
                           grpc_auth_context *auth_context,
-                          grpc_security_call_host_check_cb cb, void *user_data);
+                          grpc_closure *on_call_host_checked,
+                          grpc_error **error);
+  void (*cancel_check_call_host)(grpc_exec_ctx *exec_ctx,
+                                 grpc_channel_security_connector *sc,
+                                 grpc_closure *on_call_host_checked,
+                                 grpc_error *error);
   void (*add_handshakers)(grpc_exec_ctx *exec_ctx,
                           grpc_channel_security_connector *sc,
                           grpc_handshake_manager *handshake_mgr);
 };
 
-/* Checks that the host that will be set for a call is acceptable. */
-void grpc_channel_security_connector_check_call_host(
+/// Checks that the host that will be set for a call is acceptable.
+/// Returns true if completed synchronously, in which case \a error will
+/// be set to indicate the result.  Otherwise, \a on_call_host_checked
+/// will be invoked when complete.
+bool grpc_channel_security_connector_check_call_host(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
     const char *host, grpc_auth_context *auth_context,
-    grpc_security_call_host_check_cb cb, void *user_data);
+    grpc_closure *on_call_host_checked, grpc_error **error);
+
+/// Cancels a pending asychronous call to
+/// grpc_channel_security_connector_check_call_host() with
+/// \a on_call_host_checked as its callback.
+void grpc_channel_security_connector_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error);
 
 /* Registers handshakers with \a handshake_mgr. */
 void grpc_channel_security_connector_add_handshakers(
diff --git a/src/node/test/credentials_test.js b/src/node/test/credentials_test.js
index 3688f03512f..0ff838e7d05 100644
--- a/src/node/test/credentials_test.js
+++ b/src/node/test/credentials_test.js
@@ -319,7 +319,9 @@ describe('client credentials', function() {
                             client_options);
     client.unary({}, function(err, data) {
       assert(err);
-      assert.strictEqual(err.message, 'Authentication error');
+      assert.strictEqual(err.message,
+                         'Getting metadata from plugin failed with error: ' +
+                         'Authentication error');
       assert.strictEqual(err.code, grpc.status.UNAUTHENTICATED);
       done();
     });
@@ -367,7 +369,9 @@ describe('client credentials', function() {
                             client_options);
     client.unary({}, function(err, data) {
       assert(err);
-      assert.strictEqual(err.message, 'Authentication failure');
+      assert.strictEqual(err.message,
+                         'Getting metadata from plugin failed with error: ' +
+                         'Authentication failure');
       done();
     });
   });
diff --git a/test/core/end2end/fixtures/h2_oauth2.c b/test/core/end2end/fixtures/h2_oauth2.c
index 9cbaaa08117..ee1d0b14169 100644
--- a/test/core/end2end/fixtures/h2_oauth2.c
+++ b/test/core/end2end/fixtures/h2_oauth2.c
@@ -137,10 +137,11 @@ void chttp2_tear_down_secure_fullstack(grpc_end2end_test_fixture *f) {
 
 static void chttp2_init_client_simple_ssl_with_oauth2_secure_fullstack(
     grpc_end2end_test_fixture *f, grpc_channel_args *client_args) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_channel_credentials *ssl_creds =
       grpc_ssl_credentials_create(test_root_cert, NULL, NULL);
-  grpc_call_credentials *oauth2_creds =
-      grpc_md_only_test_credentials_create("authorization", oauth2_md, 1);
+  grpc_call_credentials *oauth2_creds = grpc_md_only_test_credentials_create(
+      &exec_ctx, "authorization", oauth2_md, true /* is_async */);
   grpc_channel_credentials *ssl_oauth2_creds =
       grpc_composite_channel_credentials_create(ssl_creds, oauth2_creds, NULL);
   grpc_arg ssl_name_override = {GRPC_ARG_STRING,
@@ -149,13 +150,10 @@ static void chttp2_init_client_simple_ssl_with_oauth2_secure_fullstack(
   grpc_channel_args *new_client_args =
       grpc_channel_args_copy_and_add(client_args, &ssl_name_override, 1);
   chttp2_init_client_secure_fullstack(f, new_client_args, ssl_oauth2_creds);
-  {
-    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-    grpc_channel_args_destroy(&exec_ctx, new_client_args);
-    grpc_exec_ctx_finish(&exec_ctx);
-  }
+  grpc_channel_args_destroy(&exec_ctx, new_client_args);
   grpc_channel_credentials_release(ssl_creds);
   grpc_call_credentials_release(oauth2_creds);
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static int fail_server_auth_check(grpc_channel_args *server_args) {
diff --git a/test/core/security/credentials_test.c b/test/core/security/credentials_test.c
index a76cb0499d8..e60e398767e 100644
--- a/test/core/security/credentials_test.c
+++ b/test/core/security/credentials_test.c
@@ -105,8 +105,6 @@ static const char valid_oauth2_json_response[] =
     " \"expires_in\":3599, "
     " \"token_type\":\"Bearer\"}";
 
-static const char test_user_data[] = "user data";
-
 static const char test_scope[] = "perm1 perm2";
 
 static const char test_signed_jwt[] =
@@ -134,11 +132,6 @@ static char *test_json_key_str(void) {
   return result;
 }
 
-typedef struct {
-  const char *key;
-  const char *value;
-} expected_md;
-
 static grpc_httpcli_response http_response(int status, const char *body) {
   grpc_httpcli_response response;
   memset(&response, 0, sizeof(grpc_httpcli_response));
@@ -150,89 +143,57 @@ static grpc_httpcli_response http_response(int status, const char *body) {
 
 /* -- Tests. -- */
 
-static void test_empty_md_store(void) {
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *store = grpc_credentials_md_store_create(0);
-  GPR_ASSERT(store->num_entries == 0);
-  GPR_ASSERT(store->allocated == 0);
-  grpc_credentials_md_store_unref(&exec_ctx, store);
-  grpc_exec_ctx_finish(&exec_ctx);
-}
-
-static void test_ref_unref_empty_md_store(void) {
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *store = grpc_credentials_md_store_create(0);
-  grpc_credentials_md_store_ref(store);
-  grpc_credentials_md_store_ref(store);
-  GPR_ASSERT(store->num_entries == 0);
-  GPR_ASSERT(store->allocated == 0);
-  grpc_credentials_md_store_unref(&exec_ctx, store);
-  grpc_credentials_md_store_unref(&exec_ctx, store);
-  grpc_credentials_md_store_unref(&exec_ctx, store);
-  grpc_exec_ctx_finish(&exec_ctx);
-}
-
-static void test_add_to_empty_md_store(void) {
+static void test_empty_md_array(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *store = grpc_credentials_md_store_create(0);
-  const char *key_str = "hello";
-  const char *value_str = "there blah blah blah blah blah blah blah";
-  grpc_slice key = grpc_slice_from_copied_string(key_str);
-  grpc_slice value = grpc_slice_from_copied_string(value_str);
-  grpc_credentials_md_store_add(store, key, value);
-  GPR_ASSERT(store->num_entries == 1);
-  GPR_ASSERT(grpc_slice_eq(key, store->entries[0].key));
-  GPR_ASSERT(grpc_slice_eq(value, store->entries[0].value));
-  grpc_slice_unref(key);
-  grpc_slice_unref(value);
-  grpc_credentials_md_store_unref(&exec_ctx, store);
+  grpc_credentials_mdelem_array md_array;
+  memset(&md_array, 0, sizeof(md_array));
+  GPR_ASSERT(md_array.md == NULL);
+  GPR_ASSERT(md_array.size == 0);
+  grpc_credentials_mdelem_array_destroy(&exec_ctx, &md_array);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void test_add_cstrings_to_empty_md_store(void) {
+static void test_add_to_empty_md_array(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *store = grpc_credentials_md_store_create(0);
-  const char *key_str = "hello";
-  const char *value_str = "there blah blah blah blah blah blah blah";
-  grpc_credentials_md_store_add_cstrings(store, key_str, value_str);
-  GPR_ASSERT(store->num_entries == 1);
-  GPR_ASSERT(grpc_slice_str_cmp(store->entries[0].key, key_str) == 0);
-  GPR_ASSERT(grpc_slice_str_cmp(store->entries[0].value, value_str) == 0);
-  grpc_credentials_md_store_unref(&exec_ctx, store);
+  grpc_credentials_mdelem_array md_array;
+  memset(&md_array, 0, sizeof(md_array));
+  const char *key = "hello";
+  const char *value = "there blah blah blah blah blah blah blah";
+  grpc_mdelem md =
+      grpc_mdelem_from_slices(&exec_ctx, grpc_slice_from_copied_string(key),
+                              grpc_slice_from_copied_string(value));
+  grpc_credentials_mdelem_array_add(&md_array, md);
+  GPR_ASSERT(md_array.size == 1);
+  GPR_ASSERT(grpc_mdelem_eq(md, md_array.md[0]));
+  GRPC_MDELEM_UNREF(&exec_ctx, md);
+  grpc_credentials_mdelem_array_destroy(&exec_ctx, &md_array);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void test_empty_preallocated_md_store(void) {
+static void test_add_abunch_to_md_array(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *store = grpc_credentials_md_store_create(4);
-  GPR_ASSERT(store->num_entries == 0);
-  GPR_ASSERT(store->allocated == 4);
-  GPR_ASSERT(store->entries != NULL);
-  grpc_credentials_md_store_unref(&exec_ctx, store);
-  grpc_exec_ctx_finish(&exec_ctx);
-}
-
-static void test_add_abunch_to_md_store(void) {
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *store = grpc_credentials_md_store_create(4);
+  grpc_credentials_mdelem_array md_array;
+  memset(&md_array, 0, sizeof(md_array));
+  const char *key = "hello";
+  const char *value = "there blah blah blah blah blah blah blah";
+  grpc_mdelem md =
+      grpc_mdelem_from_slices(&exec_ctx, grpc_slice_from_copied_string(key),
+                              grpc_slice_from_copied_string(value));
   size_t num_entries = 1000;
-  const char *key_str = "hello";
-  const char *value_str = "there blah blah blah blah blah blah blah";
-  size_t i;
-  for (i = 0; i < num_entries; i++) {
-    grpc_credentials_md_store_add_cstrings(store, key_str, value_str);
+  for (size_t i = 0; i < num_entries; ++i) {
+    grpc_credentials_mdelem_array_add(&md_array, md);
   }
-  for (i = 0; i < num_entries; i++) {
-    GPR_ASSERT(grpc_slice_str_cmp(store->entries[i].key, key_str) == 0);
-    GPR_ASSERT(grpc_slice_str_cmp(store->entries[i].value, value_str) == 0);
+  for (size_t i = 0; i < num_entries; ++i) {
+    GPR_ASSERT(grpc_mdelem_eq(md_array.md[i], md));
   }
-  grpc_credentials_md_store_unref(&exec_ctx, store);
+  GRPC_MDELEM_UNREF(&exec_ctx, md);
+  grpc_credentials_mdelem_array_destroy(&exec_ctx, &md_array);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void test_oauth2_token_fetcher_creds_parsing_ok(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *token_md = NULL;
+  grpc_mdelem token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
   grpc_httpcli_response response =
       http_response(200, valid_oauth2_json_response);
@@ -241,20 +202,18 @@ static void test_oauth2_token_fetcher_creds_parsing_ok(void) {
              GRPC_CREDENTIALS_OK);
   GPR_ASSERT(token_lifetime.tv_sec == 3599);
   GPR_ASSERT(token_lifetime.tv_nsec == 0);
-  GPR_ASSERT(token_md->num_entries == 1);
-  GPR_ASSERT(grpc_slice_str_cmp(token_md->entries[0].key, "authorization") ==
-             0);
-  GPR_ASSERT(grpc_slice_str_cmp(token_md->entries[0].value,
+  GPR_ASSERT(grpc_slice_str_cmp(GRPC_MDKEY(token_md), "authorization") == 0);
+  GPR_ASSERT(grpc_slice_str_cmp(GRPC_MDVALUE(token_md),
                                 "Bearer ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_") ==
              0);
-  grpc_credentials_md_store_unref(&exec_ctx, token_md);
+  GRPC_MDELEM_UNREF(&exec_ctx, token_md);
   grpc_http_response_destroy(&response);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void test_oauth2_token_fetcher_creds_parsing_bad_http_status(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *token_md = NULL;
+  grpc_mdelem token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
   grpc_httpcli_response response =
       http_response(401, valid_oauth2_json_response);
@@ -267,7 +226,7 @@ static void test_oauth2_token_fetcher_creds_parsing_bad_http_status(void) {
 
 static void test_oauth2_token_fetcher_creds_parsing_empty_http_body(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *token_md = NULL;
+  grpc_mdelem token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
   grpc_httpcli_response response = http_response(200, "");
   GPR_ASSERT(grpc_oauth2_token_fetcher_credentials_parse_server_response(
@@ -279,7 +238,7 @@ static void test_oauth2_token_fetcher_creds_parsing_empty_http_body(void) {
 
 static void test_oauth2_token_fetcher_creds_parsing_invalid_json(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *token_md = NULL;
+  grpc_mdelem token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
   grpc_httpcli_response response =
       http_response(200,
@@ -295,7 +254,7 @@ static void test_oauth2_token_fetcher_creds_parsing_invalid_json(void) {
 
 static void test_oauth2_token_fetcher_creds_parsing_missing_token(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *token_md = NULL;
+  grpc_mdelem token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
   grpc_httpcli_response response = http_response(200,
                                                  "{"
@@ -310,7 +269,7 @@ static void test_oauth2_token_fetcher_creds_parsing_missing_token(void) {
 
 static void test_oauth2_token_fetcher_creds_parsing_missing_token_type(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *token_md = NULL;
+  grpc_mdelem token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
   grpc_httpcli_response response =
       http_response(200,
@@ -327,7 +286,7 @@ static void test_oauth2_token_fetcher_creds_parsing_missing_token_type(void) {
 static void test_oauth2_token_fetcher_creds_parsing_missing_token_lifetime(
     void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_credentials_md_store *token_md = NULL;
+  grpc_mdelem token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
   grpc_httpcli_response response =
       http_response(200,
@@ -340,75 +299,121 @@ static void test_oauth2_token_fetcher_creds_parsing_missing_token_lifetime(
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void check_metadata(expected_md *expected, grpc_credentials_md *md_elems,
-                           size_t num_md) {
-  size_t i;
-  for (i = 0; i < num_md; i++) {
+typedef struct {
+  const char *key;
+  const char *value;
+} expected_md;
+
+typedef struct {
+  grpc_error *expected_error;
+  const expected_md *expected;
+  size_t expected_size;
+  grpc_credentials_mdelem_array md_array;
+  grpc_closure on_request_metadata;
+  grpc_call_credentials *creds;
+} request_metadata_state;
+
+static void check_metadata(const expected_md *expected,
+                           grpc_credentials_mdelem_array *md_array) {
+  for (size_t i = 0; i < md_array->size; ++i) {
     size_t j;
-    for (j = 0; j < num_md; j++) {
-      if (0 == grpc_slice_str_cmp(md_elems[j].key, expected[i].key)) {
-        GPR_ASSERT(grpc_slice_str_cmp(md_elems[j].value, expected[i].value) ==
-                   0);
+    for (j = 0; j < md_array->size; ++j) {
+      if (0 ==
+          grpc_slice_str_cmp(GRPC_MDKEY(md_array->md[j]), expected[i].key)) {
+        GPR_ASSERT(grpc_slice_str_cmp(GRPC_MDVALUE(md_array->md[j]),
+                                      expected[i].value) == 0);
         break;
       }
     }
-    if (j == num_md) {
+    if (j == md_array->size) {
       gpr_log(GPR_ERROR, "key %s not found", expected[i].key);
       GPR_ASSERT(0);
     }
   }
 }
 
-static void check_google_iam_metadata(grpc_exec_ctx *exec_ctx, void *user_data,
-                                      grpc_credentials_md *md_elems,
-                                      size_t num_md,
-                                      grpc_credentials_status status,
-                                      const char *error_details) {
-  grpc_call_credentials *c = (grpc_call_credentials *)user_data;
-  expected_md emd[] = {{GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY,
-                        test_google_iam_authorization_token},
-                       {GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY,
-                        test_google_iam_authority_selector}};
-  GPR_ASSERT(status == GRPC_CREDENTIALS_OK);
-  GPR_ASSERT(error_details == NULL);
-  GPR_ASSERT(num_md == 2);
-  check_metadata(emd, md_elems, num_md);
-  grpc_call_credentials_unref(exec_ctx, c);
+static void check_request_metadata(grpc_exec_ctx *exec_ctx, void *arg,
+                                   grpc_error *error) {
+  request_metadata_state *state = (request_metadata_state *)arg;
+  gpr_log(GPR_INFO, "expected_error: %s",
+          grpc_error_string(state->expected_error));
+  gpr_log(GPR_INFO, "actual_error: %s", grpc_error_string(error));
+  if (state->expected_error == GRPC_ERROR_NONE) {
+    GPR_ASSERT(error == GRPC_ERROR_NONE);
+  } else {
+    grpc_slice expected_error;
+    GPR_ASSERT(grpc_error_get_str(state->expected_error,
+                                  GRPC_ERROR_STR_DESCRIPTION, &expected_error));
+    grpc_slice actual_error;
+    GPR_ASSERT(
+        grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION, &actual_error));
+    GPR_ASSERT(grpc_slice_cmp(expected_error, actual_error) == 0);
+    GRPC_ERROR_UNREF(state->expected_error);
+  }
+  gpr_log(GPR_INFO, "expected_size=%" PRIdPTR " actual_size=%" PRIdPTR,
+          state->expected_size, state->md_array.size);
+  GPR_ASSERT(state->md_array.size == state->expected_size);
+  check_metadata(state->expected, &state->md_array);
+  grpc_credentials_mdelem_array_destroy(exec_ctx, &state->md_array);
+  gpr_free(state);
+}
+
+static request_metadata_state *make_request_metadata_state(
+    grpc_error *expected_error, const expected_md *expected,
+    size_t expected_size) {
+  request_metadata_state *state = gpr_zalloc(sizeof(*state));
+  state->expected_error = expected_error;
+  state->expected = expected;
+  state->expected_size = expected_size;
+  GRPC_CLOSURE_INIT(&state->on_request_metadata, check_request_metadata, state,
+                    grpc_schedule_on_exec_ctx);
+  return state;
+}
+
+static void run_request_metadata_test(grpc_exec_ctx *exec_ctx,
+                                      grpc_call_credentials *creds,
+                                      grpc_auth_metadata_context auth_md_ctx,
+                                      request_metadata_state *state) {
+  grpc_error *error = GRPC_ERROR_NONE;
+  if (grpc_call_credentials_get_request_metadata(
+          exec_ctx, creds, NULL, auth_md_ctx, &state->md_array,
+          &state->on_request_metadata, &error)) {
+    // Synchronous result.  Invoke the callback directly.
+    check_request_metadata(exec_ctx, state, error);
+    GRPC_ERROR_UNREF(error);
+  }
 }
 
 static void test_google_iam_creds(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  expected_md emd[] = {{GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY,
+                        test_google_iam_authorization_token},
+                       {GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY,
+                        test_google_iam_authority_selector}};
+  request_metadata_state *state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_call_credentials *creds = grpc_google_iam_credentials_create(
       test_google_iam_authorization_token, test_google_iam_authority_selector,
       NULL);
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, creds, NULL, auth_md_ctx, check_google_iam_metadata, creds);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
+  grpc_call_credentials_unref(&exec_ctx, creds);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void check_access_token_metadata(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details) {
-  grpc_call_credentials *c = (grpc_call_credentials *)user_data;
-  expected_md emd[] = {{GRPC_AUTHORIZATION_METADATA_KEY, "Bearer blah"}};
-  GPR_ASSERT(status == GRPC_CREDENTIALS_OK);
-  GPR_ASSERT(error_details == NULL);
-  GPR_ASSERT(num_md == 1);
-  check_metadata(emd, md_elems, num_md);
-  grpc_call_credentials_unref(exec_ctx, c);
-}
-
 static void test_access_token_creds(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  expected_md emd[] = {{GRPC_AUTHORIZATION_METADATA_KEY, "Bearer blah"}};
+  request_metadata_state *state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_call_credentials *creds =
       grpc_access_token_credentials_create("blah", NULL);
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
   GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_OAUTH2) == 0);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, creds, NULL, auth_md_ctx, check_access_token_metadata, creds);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
+  grpc_call_credentials_unref(&exec_ctx, creds);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
@@ -444,30 +449,20 @@ static void test_channel_oauth2_composite_creds(void) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void check_oauth2_google_iam_composite_metadata(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details) {
-  grpc_call_credentials *c = (grpc_call_credentials *)user_data;
+static void test_oauth2_google_iam_composite_creds(void) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   expected_md emd[] = {
       {GRPC_AUTHORIZATION_METADATA_KEY, test_oauth2_bearer_token},
       {GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY,
        test_google_iam_authorization_token},
       {GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY,
        test_google_iam_authority_selector}};
-  GPR_ASSERT(status == GRPC_CREDENTIALS_OK);
-  GPR_ASSERT(error_details == NULL);
-  GPR_ASSERT(num_md == 3);
-  check_metadata(emd, md_elems, num_md);
-  grpc_call_credentials_unref(exec_ctx, c);
-}
-
-static void test_oauth2_google_iam_composite_creds(void) {
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  const grpc_call_credentials_array *creds_array;
+  request_metadata_state *state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
   grpc_call_credentials *oauth2_creds = grpc_md_only_test_credentials_create(
-      "authorization", test_oauth2_bearer_token, 0);
+      &exec_ctx, "authorization", test_oauth2_bearer_token, 0);
   grpc_call_credentials *google_iam_creds = grpc_google_iam_credentials_create(
       test_google_iam_authorization_token, test_google_iam_authority_selector,
       NULL);
@@ -478,16 +473,15 @@ static void test_oauth2_google_iam_composite_creds(void) {
   grpc_call_credentials_unref(&exec_ctx, google_iam_creds);
   GPR_ASSERT(
       strcmp(composite_creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0);
-  creds_array =
+  const grpc_call_credentials_array *creds_array =
       grpc_composite_call_credentials_get_credentials(composite_creds);
   GPR_ASSERT(creds_array->num_creds == 2);
   GPR_ASSERT(strcmp(creds_array->creds_array[0]->type,
                     GRPC_CALL_CREDENTIALS_TYPE_OAUTH2) == 0);
   GPR_ASSERT(strcmp(creds_array->creds_array[1]->type,
                     GRPC_CALL_CREDENTIALS_TYPE_IAM) == 0);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, composite_creds, NULL, auth_md_ctx,
-      check_oauth2_google_iam_composite_metadata, composite_creds);
+  run_request_metadata_test(&exec_ctx, composite_creds, auth_md_ctx, state);
+  grpc_call_credentials_unref(&exec_ctx, composite_creds);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
@@ -541,29 +535,6 @@ static void test_channel_oauth2_google_iam_composite_creds(void) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void on_oauth2_creds_get_metadata_success(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details) {
-  GPR_ASSERT(status == GRPC_CREDENTIALS_OK);
-  GPR_ASSERT(error_details == NULL);
-  GPR_ASSERT(num_md == 1);
-  GPR_ASSERT(grpc_slice_str_cmp(md_elems[0].key, "authorization") == 0);
-  GPR_ASSERT(grpc_slice_str_cmp(md_elems[0].value,
-                                "Bearer ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_") ==
-             0);
-  GPR_ASSERT(user_data != NULL);
-  GPR_ASSERT(strcmp((const char *)user_data, test_user_data) == 0);
-}
-
-static void on_oauth2_creds_get_metadata_failure(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details) {
-  GPR_ASSERT(status == GRPC_CREDENTIALS_ERROR);
-  GPR_ASSERT(num_md == 0);
-  GPR_ASSERT(user_data != NULL);
-  GPR_ASSERT(strcmp((const char *)user_data, test_user_data) == 0);
-}
-
 static void validate_compute_engine_http_request(
     const grpc_httpcli_request *request) {
   GPR_ASSERT(request->handshaker != &grpc_httpcli_ssl);
@@ -616,43 +587,48 @@ static int httpcli_get_should_not_be_called(grpc_exec_ctx *exec_ctx,
 
 static void test_compute_engine_creds_success(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_call_credentials *compute_engine_creds =
+  expected_md emd[] = {
+      {"authorization", "Bearer ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_"}};
+  grpc_call_credentials *creds =
       grpc_google_compute_engine_credentials_create(NULL);
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
 
   /* First request: http get should be called. */
+  request_metadata_state *state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_httpcli_set_override(compute_engine_httpcli_get_success_override,
                             httpcli_post_should_not_be_called);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, compute_engine_creds, NULL, auth_md_ctx,
-      on_oauth2_creds_get_metadata_success, (void *)test_user_data);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
   grpc_exec_ctx_flush(&exec_ctx);
 
   /* Second request: the cached token should be served directly. */
+  state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_httpcli_set_override(httpcli_get_should_not_be_called,
                             httpcli_post_should_not_be_called);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, compute_engine_creds, NULL, auth_md_ctx,
-      on_oauth2_creds_get_metadata_success, (void *)test_user_data);
-  grpc_exec_ctx_finish(&exec_ctx);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
+  grpc_exec_ctx_flush(&exec_ctx);
 
-  grpc_call_credentials_unref(&exec_ctx, compute_engine_creds);
+  grpc_call_credentials_unref(&exec_ctx, creds);
   grpc_httpcli_set_override(NULL, NULL);
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void test_compute_engine_creds_failure(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  request_metadata_state *state = make_request_metadata_state(
+      GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "Error occured when fetching oauth2 token."),
+      NULL, 0);
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
-  grpc_call_credentials *compute_engine_creds =
+  grpc_call_credentials *creds =
       grpc_google_compute_engine_credentials_create(NULL);
   grpc_httpcli_set_override(compute_engine_httpcli_get_failure_override,
                             httpcli_post_should_not_be_called);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, compute_engine_creds, NULL, auth_md_ctx,
-      on_oauth2_creds_get_metadata_failure, (void *)test_user_data);
-  grpc_call_credentials_unref(&exec_ctx, compute_engine_creds);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
+  grpc_call_credentials_unref(&exec_ctx, creds);
   grpc_httpcli_set_override(NULL, NULL);
   grpc_exec_ctx_finish(&exec_ctx);
 }
@@ -702,46 +678,48 @@ static int refresh_token_httpcli_post_failure(
 
 static void test_refresh_token_creds_success(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  expected_md emd[] = {
+      {"authorization", "Bearer ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_"}};
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
-  grpc_call_credentials *refresh_token_creds =
-      grpc_google_refresh_token_credentials_create(test_refresh_token_str,
-                                                   NULL);
+  grpc_call_credentials *creds = grpc_google_refresh_token_credentials_create(
+      test_refresh_token_str, NULL);
 
   /* First request: http get should be called. */
+  request_metadata_state *state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_httpcli_set_override(httpcli_get_should_not_be_called,
                             refresh_token_httpcli_post_success);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, refresh_token_creds, NULL, auth_md_ctx,
-      on_oauth2_creds_get_metadata_success, (void *)test_user_data);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
   grpc_exec_ctx_flush(&exec_ctx);
 
   /* Second request: the cached token should be served directly. */
+  state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_httpcli_set_override(httpcli_get_should_not_be_called,
                             httpcli_post_should_not_be_called);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, refresh_token_creds, NULL, auth_md_ctx,
-      on_oauth2_creds_get_metadata_success, (void *)test_user_data);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
   grpc_exec_ctx_flush(&exec_ctx);
 
-  grpc_call_credentials_unref(&exec_ctx, refresh_token_creds);
+  grpc_call_credentials_unref(&exec_ctx, creds);
   grpc_httpcli_set_override(NULL, NULL);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void test_refresh_token_creds_failure(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  request_metadata_state *state = make_request_metadata_state(
+      GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "Error occured when fetching oauth2 token."),
+      NULL, 0);
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
-  grpc_call_credentials *refresh_token_creds =
-      grpc_google_refresh_token_credentials_create(test_refresh_token_str,
-                                                   NULL);
+  grpc_call_credentials *creds = grpc_google_refresh_token_credentials_create(
+      test_refresh_token_str, NULL);
   grpc_httpcli_set_override(httpcli_get_should_not_be_called,
                             refresh_token_httpcli_post_failure);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, refresh_token_creds, NULL, auth_md_ctx,
-      on_oauth2_creds_get_metadata_failure, (void *)test_user_data);
-  grpc_call_credentials_unref(&exec_ctx, refresh_token_creds);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
+  grpc_call_credentials_unref(&exec_ctx, creds);
   grpc_httpcli_set_override(NULL, NULL);
   grpc_exec_ctx_finish(&exec_ctx);
 }
@@ -792,30 +770,6 @@ static char *encode_and_sign_jwt_should_not_be_called(
   return NULL;
 }
 
-static void on_jwt_creds_get_metadata_success(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details) {
-  char *expected_md_value;
-  gpr_asprintf(&expected_md_value, "Bearer %s", test_signed_jwt);
-  GPR_ASSERT(status == GRPC_CREDENTIALS_OK);
-  GPR_ASSERT(error_details == NULL);
-  GPR_ASSERT(num_md == 1);
-  GPR_ASSERT(grpc_slice_str_cmp(md_elems[0].key, "authorization") == 0);
-  GPR_ASSERT(grpc_slice_str_cmp(md_elems[0].value, expected_md_value) == 0);
-  GPR_ASSERT(user_data != NULL);
-  GPR_ASSERT(strcmp((const char *)user_data, test_user_data) == 0);
-  gpr_free(expected_md_value);
-}
-
-static void on_jwt_creds_get_metadata_failure(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details) {
-  GPR_ASSERT(status == GRPC_CREDENTIALS_ERROR);
-  GPR_ASSERT(num_md == 0);
-  GPR_ASSERT(user_data != NULL);
-  GPR_ASSERT(strcmp((const char *)user_data, test_user_data) == 0);
-}
-
 static grpc_service_account_jwt_access_credentials *creds_as_jwt(
     grpc_call_credentials *creds) {
   GPR_ASSERT(creds != NULL);
@@ -860,37 +814,42 @@ static void test_jwt_creds_success(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
-  grpc_call_credentials *jwt_creds =
+  char *expected_md_value;
+  gpr_asprintf(&expected_md_value, "Bearer %s", test_signed_jwt);
+  expected_md emd[] = {{"authorization", expected_md_value}};
+  grpc_call_credentials *creds =
       grpc_service_account_jwt_access_credentials_create(
           json_key_string, grpc_max_auth_token_lifetime(), NULL);
 
   /* First request: jwt_encode_and_sign should be called. */
+  request_metadata_state *state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_jwt_encode_and_sign_set_override(encode_and_sign_jwt_success);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, jwt_creds, NULL, auth_md_ctx,
-      on_jwt_creds_get_metadata_success, (void *)test_user_data);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
   grpc_exec_ctx_flush(&exec_ctx);
 
   /* Second request: the cached token should be served directly. */
+  state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_jwt_encode_and_sign_set_override(
       encode_and_sign_jwt_should_not_be_called);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, jwt_creds, NULL, auth_md_ctx,
-      on_jwt_creds_get_metadata_success, (void *)test_user_data);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
   grpc_exec_ctx_flush(&exec_ctx);
 
   /* Third request: Different service url so jwt_encode_and_sign should be
      called again (no caching). */
+  state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   auth_md_ctx.service_url = other_test_service_url;
   grpc_jwt_encode_and_sign_set_override(encode_and_sign_jwt_success);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, jwt_creds, NULL, auth_md_ctx,
-      on_jwt_creds_get_metadata_success, (void *)test_user_data);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
   grpc_exec_ctx_flush(&exec_ctx);
 
+  grpc_call_credentials_unref(&exec_ctx, creds);
   gpr_free(json_key_string);
-  grpc_call_credentials_unref(&exec_ctx, jwt_creds);
+  gpr_free(expected_md_value);
   grpc_jwt_encode_and_sign_set_override(NULL);
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void test_jwt_creds_signing_failure(void) {
@@ -898,17 +857,17 @@ static void test_jwt_creds_signing_failure(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
-  grpc_call_credentials *jwt_creds =
+  request_metadata_state *state = make_request_metadata_state(
+      GRPC_ERROR_CREATE_FROM_STATIC_STRING("Could not generate JWT."), NULL, 0);
+  grpc_call_credentials *creds =
       grpc_service_account_jwt_access_credentials_create(
           json_key_string, grpc_max_auth_token_lifetime(), NULL);
 
   grpc_jwt_encode_and_sign_set_override(encode_and_sign_jwt_failure);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, jwt_creds, NULL, auth_md_ctx,
-      on_jwt_creds_get_metadata_failure, (void *)test_user_data);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, state);
 
   gpr_free(json_key_string);
-  grpc_call_credentials_unref(&exec_ctx, jwt_creds);
+  grpc_call_credentials_unref(&exec_ctx, creds);
   grpc_jwt_encode_and_sign_set_override(NULL);
   grpc_exec_ctx_finish(&exec_ctx);
 }
@@ -986,8 +945,10 @@ static char *null_well_known_creds_path_getter(void) { return NULL; }
 
 static void test_google_default_creds_gce(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_composite_channel_credentials *creds;
-  grpc_channel_credentials *cached_creds;
+  expected_md emd[] = {
+      {"authorization", "Bearer ya29.AHES6ZRN3-HlhAPya30GnW_bHSb_"}};
+  request_metadata_state *state =
+      make_request_metadata_state(GRPC_ERROR_NONE, emd, GPR_ARRAY_SIZE(emd));
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
   grpc_flush_cached_google_default_credentials();
@@ -999,33 +960,33 @@ static void test_google_default_creds_gce(void) {
   grpc_httpcli_set_override(
       default_creds_gce_detection_httpcli_get_success_override,
       httpcli_post_should_not_be_called);
-  creds = (grpc_composite_channel_credentials *)
-      grpc_google_default_credentials_create();
+  grpc_composite_channel_credentials *creds =
+      (grpc_composite_channel_credentials *)
+          grpc_google_default_credentials_create();
 
   /* Verify that the default creds actually embeds a GCE creds. */
   GPR_ASSERT(creds != NULL);
   GPR_ASSERT(creds->call_creds != NULL);
   grpc_httpcli_set_override(compute_engine_httpcli_get_success_override,
                             httpcli_post_should_not_be_called);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, creds->call_creds, NULL, auth_md_ctx,
-      on_oauth2_creds_get_metadata_success, (void *)test_user_data);
+  run_request_metadata_test(&exec_ctx, creds->call_creds, auth_md_ctx, state);
   grpc_exec_ctx_flush(&exec_ctx);
-  grpc_exec_ctx_finish(&exec_ctx);
 
   /* Check that we get a cached creds if we call
      grpc_google_default_credentials_create again.
      GCE detection should not occur anymore either. */
   grpc_httpcli_set_override(httpcli_get_should_not_be_called,
                             httpcli_post_should_not_be_called);
-  cached_creds = grpc_google_default_credentials_create();
+  grpc_channel_credentials *cached_creds =
+      grpc_google_default_credentials_create();
   GPR_ASSERT(cached_creds == &creds->base);
 
   /* Cleanup. */
-  grpc_channel_credentials_release(cached_creds);
-  grpc_channel_credentials_release(&creds->base);
+  grpc_channel_credentials_unref(&exec_ctx, cached_creds);
+  grpc_channel_credentials_unref(&exec_ctx, &creds->base);
   grpc_httpcli_set_override(NULL, NULL);
   grpc_override_well_known_credentials_path_getter(NULL);
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static int default_creds_gce_detection_httpcli_get_failure_override(
@@ -1068,12 +1029,7 @@ typedef enum {
   PLUGIN_DESTROY_CALLED_STATE
 } plugin_state;
 
-typedef struct {
-  const char *key;
-  const char *value;
-} plugin_metadata;
-
-static const plugin_metadata plugin_md[] = {{"foo", "bar"}, {"hi", "there"}};
+static const expected_md plugin_md[] = {{"foo", "bar"}, {"hi", "there"}};
 
 static void plugin_get_metadata_success(void *state,
                                         grpc_auth_metadata_context context,
@@ -1110,79 +1066,60 @@ static void plugin_get_metadata_failure(void *state,
   cb(user_data, NULL, 0, GRPC_STATUS_UNAUTHENTICATED, plugin_error_details);
 }
 
-static void on_plugin_metadata_received_success(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details) {
-  size_t i = 0;
-  GPR_ASSERT(user_data == NULL);
-  GPR_ASSERT(md_elems != NULL);
-  GPR_ASSERT(num_md == GPR_ARRAY_SIZE(plugin_md));
-  for (i = 0; i < num_md; i++) {
-    GPR_ASSERT(grpc_slice_str_cmp(md_elems[i].key, plugin_md[i].key) == 0);
-    GPR_ASSERT(grpc_slice_str_cmp(md_elems[i].value, plugin_md[i].value) == 0);
-  }
-}
-
-static void on_plugin_metadata_received_failure(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details) {
-  GPR_ASSERT(user_data == NULL);
-  GPR_ASSERT(md_elems == NULL);
-  GPR_ASSERT(num_md == 0);
-  GPR_ASSERT(status == GRPC_CREDENTIALS_ERROR);
-  GPR_ASSERT(error_details != NULL);
-  GPR_ASSERT(strcmp(error_details, plugin_error_details) == 0);
-}
-
 static void plugin_destroy(void *state) {
   plugin_state *s = (plugin_state *)state;
   *s = PLUGIN_DESTROY_CALLED_STATE;
 }
 
 static void test_metadata_plugin_success(void) {
-  grpc_call_credentials *creds;
   plugin_state state = PLUGIN_INITIAL_STATE;
   grpc_metadata_credentials_plugin plugin;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
+  request_metadata_state *md_state = make_request_metadata_state(
+      GRPC_ERROR_NONE, plugin_md, GPR_ARRAY_SIZE(plugin_md));
 
   plugin.state = &state;
   plugin.get_metadata = plugin_get_metadata_success;
   plugin.destroy = plugin_destroy;
 
-  creds = grpc_metadata_credentials_create_from_plugin(plugin, NULL);
+  grpc_call_credentials *creds =
+      grpc_metadata_credentials_create_from_plugin(plugin, NULL);
   GPR_ASSERT(state == PLUGIN_INITIAL_STATE);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, creds, NULL, auth_md_ctx, on_plugin_metadata_received_success,
-      NULL);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, md_state);
   GPR_ASSERT(state == PLUGIN_GET_METADATA_CALLED_STATE);
-  grpc_call_credentials_release(creds);
-  GPR_ASSERT(state == PLUGIN_DESTROY_CALLED_STATE);
+  grpc_call_credentials_unref(&exec_ctx, creds);
   grpc_exec_ctx_finish(&exec_ctx);
+  GPR_ASSERT(state == PLUGIN_DESTROY_CALLED_STATE);
 }
 
 static void test_metadata_plugin_failure(void) {
-  grpc_call_credentials *creds;
   plugin_state state = PLUGIN_INITIAL_STATE;
   grpc_metadata_credentials_plugin plugin;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_auth_metadata_context auth_md_ctx = {test_service_url, test_method, NULL,
                                             NULL};
+  char *expected_error;
+  gpr_asprintf(&expected_error,
+               "Getting metadata from plugin failed with error: %s",
+               plugin_error_details);
+  request_metadata_state *md_state = make_request_metadata_state(
+      GRPC_ERROR_CREATE_FROM_COPIED_STRING(expected_error), NULL, 0);
+  gpr_free(expected_error);
 
   plugin.state = &state;
   plugin.get_metadata = plugin_get_metadata_failure;
   plugin.destroy = plugin_destroy;
 
-  creds = grpc_metadata_credentials_create_from_plugin(plugin, NULL);
+  grpc_call_credentials *creds =
+      grpc_metadata_credentials_create_from_plugin(plugin, NULL);
   GPR_ASSERT(state == PLUGIN_INITIAL_STATE);
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, creds, NULL, auth_md_ctx, on_plugin_metadata_received_failure,
-      NULL);
+  run_request_metadata_test(&exec_ctx, creds, auth_md_ctx, md_state);
   GPR_ASSERT(state == PLUGIN_GET_METADATA_CALLED_STATE);
-  grpc_call_credentials_release(creds);
-  GPR_ASSERT(state == PLUGIN_DESTROY_CALLED_STATE);
+  grpc_call_credentials_unref(&exec_ctx, creds);
   grpc_exec_ctx_finish(&exec_ctx);
+  GPR_ASSERT(state == PLUGIN_DESTROY_CALLED_STATE);
 }
 
 static void test_get_well_known_google_credentials_file_path(void) {
@@ -1233,12 +1170,9 @@ static void test_channel_creds_duplicate_without_call_creds(void) {
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
   grpc_init();
-  test_empty_md_store();
-  test_ref_unref_empty_md_store();
-  test_add_to_empty_md_store();
-  test_add_cstrings_to_empty_md_store();
-  test_empty_preallocated_md_store();
-  test_add_abunch_to_md_store();
+  test_empty_md_array();
+  test_add_to_empty_md_array();
+  test_add_abunch_to_md_array();
   test_oauth2_token_fetcher_creds_parsing_ok();
   test_oauth2_token_fetcher_creds_parsing_bad_http_status();
   test_oauth2_token_fetcher_creds_parsing_empty_http_body();
diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c
index e2331fbd97c..fdbc6ea7417 100644
--- a/test/core/security/oauth2_utils.c
+++ b/test/core/security/oauth2_utils.c
@@ -32,29 +32,31 @@
 typedef struct {
   gpr_mu *mu;
   grpc_polling_entity pops;
-  int is_done;
+  bool is_done;
   char *token;
+
+  grpc_credentials_mdelem_array md_array;
+  grpc_closure closure;
 } oauth2_request;
 
-static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data,
-                               grpc_credentials_md *md_elems, size_t num_md,
-                               grpc_credentials_status status,
-                               const char *error_details) {
-  oauth2_request *request = (oauth2_request *)user_data;
+static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *arg,
+                               grpc_error *error) {
+  oauth2_request *request = (oauth2_request *)arg;
   char *token = NULL;
   grpc_slice token_slice;
-  if (status == GRPC_CREDENTIALS_ERROR) {
-    gpr_log(GPR_ERROR, "Fetching token failed.");
+  if (error != GRPC_ERROR_NONE) {
+    gpr_log(GPR_ERROR, "Fetching token failed: %s", grpc_error_string(error));
   } else {
-    GPR_ASSERT(num_md == 1);
-    token_slice = md_elems[0].value;
+    GPR_ASSERT(request->md_array.size == 1);
+    token_slice = GRPC_MDVALUE(request->md_array.md[0]);
     token = (char *)gpr_malloc(GRPC_SLICE_LENGTH(token_slice) + 1);
     memcpy(token, GRPC_SLICE_START_PTR(token_slice),
            GRPC_SLICE_LENGTH(token_slice));
     token[GRPC_SLICE_LENGTH(token_slice)] = '\0';
   }
+  grpc_credentials_mdelem_array_destroy(exec_ctx, &request->md_array);
   gpr_mu_lock(request->mu);
-  request->is_done = 1;
+  request->is_done = true;
   request->token = token;
   GRPC_LOG_IF_ERROR(
       "pollset_kick",
@@ -68,6 +70,7 @@ static void do_nothing(grpc_exec_ctx *exec_ctx, void *unused,
 char *grpc_test_fetch_oauth2_token_with_credentials(
     grpc_call_credentials *creds) {
   oauth2_request request;
+  memset(&request, 0, sizeof(request));
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_closure do_nothing_closure;
   grpc_auth_metadata_context null_ctx = {"", "", NULL, NULL};
@@ -75,15 +78,23 @@ char *grpc_test_fetch_oauth2_token_with_credentials(
   grpc_pollset *pollset = (grpc_pollset *)gpr_zalloc(grpc_pollset_size());
   grpc_pollset_init(pollset, &request.mu);
   request.pops = grpc_polling_entity_create_from_pollset(pollset);
-  request.is_done = 0;
+  request.is_done = false;
 
   GRPC_CLOSURE_INIT(&do_nothing_closure, do_nothing, NULL,
                     grpc_schedule_on_exec_ctx);
 
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, creds, &request.pops, null_ctx, on_oauth2_response, &request);
+  GRPC_CLOSURE_INIT(&request.closure, on_oauth2_response, &request,
+                    grpc_schedule_on_exec_ctx);
 
-  grpc_exec_ctx_finish(&exec_ctx);
+  grpc_error *error = GRPC_ERROR_NONE;
+  if (grpc_call_credentials_get_request_metadata(
+          &exec_ctx, creds, &request.pops, null_ctx, &request.md_array,
+          &request.closure, &error)) {
+    // Synchronous result; invoke callback directly.
+    on_oauth2_response(&exec_ctx, &request, error);
+    GRPC_ERROR_UNREF(error);
+  }
+  grpc_exec_ctx_flush(&exec_ctx);
 
   gpr_mu_lock(request.mu);
   while (!request.is_done) {
@@ -94,7 +105,7 @@ char *grpc_test_fetch_oauth2_token_with_credentials(
                               grpc_polling_entity_pollset(&request.pops),
                               &worker, gpr_now(GPR_CLOCK_MONOTONIC),
                               gpr_inf_future(GPR_CLOCK_MONOTONIC)))) {
-      request.is_done = 1;
+      request.is_done = true;
     }
   }
   gpr_mu_unlock(request.mu);
diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c
index 3c3d3a7c105..e1385a80caa 100644
--- a/test/core/security/print_google_default_creds_token.c
+++ b/test/core/security/print_google_default_creds_token.c
@@ -35,25 +35,26 @@
 typedef struct {
   gpr_mu *mu;
   grpc_polling_entity pops;
-  int is_done;
+  bool is_done;
+
+  grpc_credentials_mdelem_array md_array;
+  grpc_closure on_request_metadata;
 } synchronizer;
 
-static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *user_data,
-                                 grpc_credentials_md *md_elems, size_t num_md,
-                                 grpc_credentials_status status,
-                                 const char *error_details) {
-  synchronizer *sync = user_data;
-  if (status == GRPC_CREDENTIALS_ERROR) {
-    fprintf(stderr, "Fetching token failed.\n");
+static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *arg,
+                                 grpc_error *error) {
+  synchronizer *sync = arg;
+  if (error != GRPC_ERROR_NONE) {
+    fprintf(stderr, "Fetching token failed: %s\n", grpc_error_string(error));
   } else {
     char *token;
-    GPR_ASSERT(num_md == 1);
-    token = grpc_slice_to_c_string(md_elems[0].value);
+    GPR_ASSERT(sync->md_array.size == 1);
+    token = grpc_slice_to_c_string(GRPC_MDVALUE(sync->md_array.md[0]));
     printf("\nGot token: %s\n\n", token);
     gpr_free(token);
   }
   gpr_mu_lock(sync->mu);
-  sync->is_done = 1;
+  sync->is_done = true;
   GRPC_LOG_IF_ERROR(
       "pollset_kick",
       grpc_pollset_kick(grpc_polling_entity_pollset(&sync->pops), NULL));
@@ -83,14 +84,23 @@ int main(int argc, char **argv) {
     goto end;
   }
 
+  memset(&sync, 0, sizeof(sync));
   grpc_pollset *pollset = gpr_zalloc(grpc_pollset_size());
   grpc_pollset_init(pollset, &sync.mu);
   sync.pops = grpc_polling_entity_create_from_pollset(pollset);
-  sync.is_done = 0;
+  sync.is_done = false;
+  GRPC_CLOSURE_INIT(&sync.on_request_metadata, on_metadata_response, &sync,
+                    grpc_schedule_on_exec_ctx);
 
-  grpc_call_credentials_get_request_metadata(
-      &exec_ctx, ((grpc_composite_channel_credentials *)creds)->call_creds,
-      &sync.pops, context, on_metadata_response, &sync);
+  grpc_error *error = GRPC_ERROR_NONE;
+  if (grpc_call_credentials_get_request_metadata(
+          &exec_ctx, ((grpc_composite_channel_credentials *)creds)->call_creds,
+          &sync.pops, context, &sync.md_array, &sync.on_request_metadata,
+          &error)) {
+    // Synchronous response.  Invoke callback directly.
+    on_metadata_response(&exec_ctx, &sync, error);
+    GRPC_ERROR_UNREF(error);
+  }
 
   gpr_mu_lock(sync.mu);
   while (!sync.is_done) {
@@ -101,7 +111,7 @@ int main(int argc, char **argv) {
                               grpc_polling_entity_pollset(&sync.pops), &worker,
                               gpr_now(GPR_CLOCK_MONOTONIC),
                               gpr_inf_future(GPR_CLOCK_MONOTONIC))))
-      sync.is_done = 1;
+      sync.is_done = true;
     gpr_mu_unlock(sync.mu);
     grpc_exec_ctx_flush(&exec_ctx);
     gpr_mu_lock(sync.mu);
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index 8d12971bc1b..8bada48a2b1 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -1565,7 +1565,9 @@ TEST_P(SecureEnd2endTest, NonBlockingAuthMetadataPluginFailure) {
   Status s = stub_->Echo(&context, request, &response);
   EXPECT_FALSE(s.ok());
   EXPECT_EQ(s.error_code(), StatusCode::UNAUTHENTICATED);
-  EXPECT_EQ(s.error_message(), kTestCredsPluginErrorMsg);
+  EXPECT_EQ(s.error_message(),
+            grpc::string("Getting metadata from plugin failed with error: ") +
+                kTestCredsPluginErrorMsg);
 }
 
 TEST_P(SecureEnd2endTest, NonBlockingAuthMetadataPluginAndProcessorSuccess) {
@@ -1624,7 +1626,9 @@ TEST_P(SecureEnd2endTest, BlockingAuthMetadataPluginFailure) {
   Status s = stub_->Echo(&context, request, &response);
   EXPECT_FALSE(s.ok());
   EXPECT_EQ(s.error_code(), StatusCode::UNAUTHENTICATED);
-  EXPECT_EQ(s.error_message(), kTestCredsPluginErrorMsg);
+  EXPECT_EQ(s.error_message(),
+            grpc::string("Getting metadata from plugin failed with error: ") +
+                kTestCredsPluginErrorMsg);
 }
 
 TEST_P(SecureEnd2endTest, ClientAuthContext) {