Merge pull request #3176 from jboeuf/core_creds_plugin

Design and implementation of the core credentials plugin API.

include/grpc++/security/credentials.h

@@ -34,10 +34,13 @@
 #ifndef GRPCXX_CREDENTIALS_H
 #define GRPCXX_CREDENTIALS_H
 
+#include <map>
 #include <memory>
 
 #include <grpc++/impl/grpc_library.h>
 #include <grpc++/support/config.h>
+#include <grpc++/support/status.h>
+#include <grpc++/support/string_ref.h>
 
 namespace grpc {
 class ChannelArguments;
@@ -165,6 +168,24 @@ std::shared_ptr<Credentials> CompositeCredentials(
 /// Credentials for an unencrypted, unauthenticated channel
 std::shared_ptr<Credentials> InsecureCredentials();
 
+// User defined metadata credentials.
+class MetadataCredentialsPlugin {
+ public:
+  virtual ~MetadataCredentialsPlugin() {}
+
+  // If this method returns true, the Process function will be scheduled in
+  // a different thread from the one processing the call.
+  virtual bool IsBlocking() const { return true; }
+
+  // Gets the auth metatada produced by this plugin. */
+  virtual Status GetMetadata(
+      grpc::string_ref service_url,
+      std::multimap<grpc::string, grpc::string_ref>* metadata) = 0;
+};
+
+std::shared_ptr<Credentials> MetadataCredentialsFromPlugin(
+    std::unique_ptr<MetadataCredentialsPlugin> plugin);
+
 }  // namespace grpc
 
 #endif  // GRPCXX_CREDENTIALS_H

include/grpc/grpc_security.h

@@ -131,6 +131,46 @@ grpc_credentials *grpc_google_iam_credentials_create(
     const char *authorization_token, const char *authority_selector,
     void *reserved);
 
+/* Callback function to be called by the metadata credentials plugin
+   implementation when the metadata is ready.
+   - user_data is the opaque pointer that was passed in the get_metadata method
+     of the grpc_metadata_credentials_plugin (see below).
+   - creds_md is an array of credentials metadata produced by the plugin. It
+     may be set to NULL in case of an error.
+   - num_creds_md is the number of items in the creds_md array.
+   - status must be GRPC_STATUS_OK in case of success or another specific error
+     code otherwise.
+   - error_details contains details about the error if any. In case of success
+     it should be NULL and will be otherwise ignored. */
+typedef void (*grpc_credentials_plugin_metadata_cb)(
+    void *user_data, const grpc_metadata *creds_md, size_t num_creds_md,
+    grpc_status_code status, const char *error_details);
+
+/* grpc_metadata_credentials plugin is an API user provided structure used to
+   create grpc_credentials objects that can be set on a channel (composed) or
+   a call. See grpc_credentials_metadata_create_from_plugin below.
+   The grpc client stack will call the get_metadata method of the plugin for
+   every call in scope for the credentials created from it. */
+typedef struct {
+  /* The implementation of this method has to be non-blocking.
+     - service_url is the fully qualified URL that the client stack is
+       connecting to.
+     - cb is the callback that needs to be called when the metadata is ready.
+     - user_data needs to be passed as the first parameter of the callback. */
+  void (*get_metadata)(void *state, const char *service_url,
+                       grpc_credentials_plugin_metadata_cb cb, void *user_data);
+
+  /* Destroys the plugin state. */
+  void (*destroy)(void *state);
+
+  /* State that will be set as the first parameter of the methods above. */
+  void *state;
+} grpc_metadata_credentials_plugin;
+
+/* Creates a credentials object from a plugin. */
+grpc_credentials *grpc_metadata_credentials_create_from_plugin(
+    grpc_metadata_credentials_plugin plugin, void *reserved);
+
 /* --- Secure channel creation. --- */
 
 /* Creates a secure channel using the passed-in credentials. */

src/core/security/client_auth_filter.c

@@ -63,6 +63,7 @@ typedef struct {
   int sent_initial_metadata;
   gpr_uint8 security_context_set;
   grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT];
+  char *service_url;
 } call_data;
 
 /* We can have a per-channel credentials. */
@@ -75,6 +76,13 @@ typedef struct {
   grpc_mdstr *status_key;
 } channel_data;
 
+static void reset_service_url(call_data *calld) {
+  if (calld->service_url != NULL) {
+    gpr_free(calld->service_url);
+    calld->service_url = NULL;
+  }
+}
+
 static void bubble_up_error(grpc_call_element *elem, grpc_status_code status,
                             const char *error_msg) {
   call_data *calld = elem->call_data;
@@ -93,6 +101,7 @@ static void on_credentials_metadata(void *user_data,
   grpc_transport_stream_op *op = &calld->op;
   grpc_metadata_batch *mdb;
   size_t i;
+  reset_service_url(calld);
   if (status != GRPC_CREDENTIALS_OK) {
     bubble_up_error(elem, GRPC_STATUS_UNAUTHENTICATED,
                     "Credentials failed to get metadata.");
@@ -111,8 +120,7 @@ static void on_credentials_metadata(void *user_data,
   grpc_call_next_op(elem, op);
 }
 
-static char *build_service_url(const char *url_scheme, call_data *calld) {
-  char *service_url;
+void build_service_url(const char *url_scheme, call_data *calld) {
   char *service = gpr_strdup(grpc_mdstr_as_c_string(calld->method));
   char *last_slash = strrchr(service, '/');
   if (last_slash == NULL) {
@@ -125,10 +133,10 @@ static char *build_service_url(const char *url_scheme, call_data *calld) {
     *last_slash = '\0';
   }
   if (url_scheme == NULL) url_scheme = "";
-  gpr_asprintf(&service_url, "%s://%s%s", url_scheme,
+  reset_service_url(calld);
+  gpr_asprintf(&calld->service_url, "%s://%s%s", url_scheme,
                grpc_mdstr_as_c_string(calld->host), service);
   gpr_free(service);
-  return service_url;
 }
 
 static void send_security_metadata(grpc_call_element *elem,
@@ -137,7 +145,6 @@ static void send_security_metadata(grpc_call_element *elem,
   channel_data *chand = elem->channel_data;
   grpc_client_security_context *ctx =
       (grpc_client_security_context *)op->context[GRPC_CONTEXT_SECURITY].value;
-  char *service_url = NULL;
   grpc_credentials *channel_creds =
       chand->security_connector->request_metadata_creds;
   int channel_creds_has_md =
@@ -165,13 +172,12 @@ static void send_security_metadata(grpc_call_element *elem,
         grpc_credentials_ref(call_creds_has_md ? ctx->creds : channel_creds);
   }
 
-  service_url =
   build_service_url(chand->security_connector->base.url_scheme, calld);
   calld->op = *op; /* Copy op (originates from the caller's stack). */
   GPR_ASSERT(calld->pollset);
-  grpc_credentials_get_request_metadata(
-      calld->creds, calld->pollset, service_url, on_credentials_metadata, elem);
-  gpr_free(service_url);
+  grpc_credentials_get_request_metadata(calld->creds, calld->pollset,
+                                        calld->service_url,
+                                        on_credentials_metadata, elem);
 }
 
 static void on_host_checked(void *user_data, grpc_security_status status) {
@@ -274,13 +280,7 @@ static void init_call_elem(grpc_call_element *elem,
                            const void *server_transport_data,
                            grpc_transport_stream_op *initial_op) {
   call_data *calld = elem->call_data;
-  calld->creds = NULL;
-  calld->host = NULL;
-  calld->method = NULL;
-  calld->pollset = NULL;
-  calld->sent_initial_metadata = 0;
-  calld->security_context_set = 0;
-
+  memset(calld, 0, sizeof(*calld));
   GPR_ASSERT(!initial_op || !initial_op->send_ops);
 }
 
@@ -294,6 +294,7 @@ static void destroy_call_elem(grpc_call_element *elem) {
   if (calld->method != NULL) {
     GRPC_MDSTR_UNREF(calld->method);
   }
+  reset_service_url(calld);
 }
 
 /* Constructor for channel_data */

src/core/security/credentials.c

@@ -1185,3 +1185,95 @@ grpc_credentials *grpc_google_iam_credentials_create(
       c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
   return &c->base;
 }
+
+/* -- Plugin credentials. -- */
+
+typedef struct {
+  void *user_data;
+  grpc_credentials_metadata_cb cb;
+} grpc_metadata_plugin_request;
+
+static void plugin_destruct(grpc_credentials *creds) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
+    c->plugin.destroy(c->plugin.state);
+  }
+}
+
+static int plugin_has_request_metadata(const grpc_credentials *creds) {
+  return 1;
+}
+
+static int plugin_has_request_metadata_only(const grpc_credentials *creds) {
+  return 1;
+}
+
+static void plugin_md_request_metadata_ready(void *request,
+                                             const grpc_metadata *md,
+                                             size_t num_md,
+                                             grpc_status_code status,
+                                             const char *error_details) {
+  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
+  if (status != GRPC_STATUS_OK) {
+    if (error_details != NULL) {
+      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
+              error_details);
+    }
+    r->cb(r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
+  } else {
+    size_t i;
+    grpc_credentials_md *md_array = NULL;
+    if (num_md > 0) {
+      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
+      for (i = 0; i < num_md; i++) {
+        md_array[i].key = gpr_slice_from_copied_string(md[i].key);
+        md_array[i].value =
+            gpr_slice_from_copied_buffer(md[i].value, md[i].value_length);
+      }
+    }
+    r->cb(r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK);
+    if (md_array != NULL) {
+      for (i = 0; i < num_md; i++) {
+        gpr_slice_unref(md_array[i].key);
+        gpr_slice_unref(md_array[i].value);
+      }
+      gpr_free(md_array);
+    }
+  }
+  gpr_free(r);
+}
+
+static void plugin_get_request_metadata(grpc_credentials *creds,
+                                        grpc_pollset *pollset,
+                                        const char *service_url,
+                                        grpc_credentials_metadata_cb cb,
+                                        void *user_data) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.get_metadata != NULL) {
+    grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request));
+    memset(request, 0, sizeof(*request));
+    request->user_data = user_data;
+    request->cb = cb;
+    c->plugin.get_metadata(c->plugin.state, service_url,
+                           plugin_md_request_metadata_ready, request);
+  } else {
+    cb(user_data, NULL, 0, GRPC_CREDENTIALS_OK);
+  }
+}
+
+static grpc_credentials_vtable plugin_vtable = {
+    plugin_destruct, plugin_has_request_metadata,
+    plugin_has_request_metadata_only, plugin_get_request_metadata, NULL};
+
+grpc_credentials *grpc_metadata_credentials_create_from_plugin(
+    grpc_metadata_credentials_plugin plugin, void *reserved) {
+  grpc_plugin_credentials *c = gpr_malloc(sizeof(*c));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(*c));
+  c->base.type = GRPC_CREDENTIALS_TYPE_METADATA_PLUGIN;
+  c->base.vtable = &plugin_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->plugin = plugin;
+  return &c->base;
+}
+

src/core/security/credentials.h

@@ -56,6 +56,7 @@ typedef enum {
 
 #define GRPC_CREDENTIALS_TYPE_SSL "Ssl"
 #define GRPC_CREDENTIALS_TYPE_OAUTH2 "Oauth2"
+#define GRPC_CREDENTIALS_TYPE_METADATA_PLUGIN "Plugin"
 #define GRPC_CREDENTIALS_TYPE_JWT "Jwt"
 #define GRPC_CREDENTIALS_TYPE_IAM "Iam"
 #define GRPC_CREDENTIALS_TYPE_COMPOSITE "Composite"
@@ -322,4 +323,12 @@ typedef struct {
   grpc_credentials *connector_creds;
 } grpc_composite_credentials;
 
+/* -- Plugin credentials. -- */
+
+typedef struct {
+  grpc_credentials base;
+  grpc_metadata_credentials_plugin plugin;
+  grpc_credentials_md_store *plugin_md;
+} grpc_plugin_credentials;
+
 #endif /* GRPC_INTERNAL_CORE_SECURITY_CREDENTIALS_H */

src/cpp/client/secure_credentials.cc

@@ -144,4 +144,64 @@ std::shared_ptr<Credentials> CompositeCredentials(
   return nullptr;
 }
 
+void MetadataCredentialsPluginWrapper::Destroy(void* wrapper) {
+  if (wrapper == nullptr) return;
+  MetadataCredentialsPluginWrapper* w =
+      reinterpret_cast<MetadataCredentialsPluginWrapper*>(wrapper);
+  delete w;
+}
+
+void MetadataCredentialsPluginWrapper::GetMetadata(
+    void* wrapper, const char* service_url,
+    grpc_credentials_plugin_metadata_cb cb, void* user_data) {
+  GPR_ASSERT(wrapper != nullptr);
+  MetadataCredentialsPluginWrapper* w =
+      reinterpret_cast<MetadataCredentialsPluginWrapper*>(wrapper);
+  if (w->plugin_ == nullptr) {
+    cb(user_data, NULL, 0, GRPC_STATUS_OK, NULL);
+    return;
+  }
+  if (w->plugin_->IsBlocking()) {
+    w->thread_pool_->Add(
+        std::bind(&MetadataCredentialsPluginWrapper::InvokePlugin, w,
+                  service_url, cb, user_data));
+  } else {
+    w->InvokePlugin(service_url, cb, user_data);
+  }
+}
+
+void MetadataCredentialsPluginWrapper::InvokePlugin(
+    const char* service_url, grpc_credentials_plugin_metadata_cb cb,
+    void* user_data) {
+  std::multimap<grpc::string, grpc::string_ref> metadata;
+  Status status = plugin_->GetMetadata(service_url, &metadata);
+  std::vector<grpc_metadata> md;
+  for (auto it = metadata.begin(); it != metadata.end(); ++it) {
+    md.push_back({it->first.c_str(),
+                  it->second.data(),
+                  it->second.size(),
+                  0,
+                  {{nullptr, nullptr, nullptr, nullptr}}});
+  }
+  cb(user_data, &md[0], md.size(),
+     static_cast<grpc_status_code>(status.error_code()),
+     status.error_message().c_str());
+}
+
+MetadataCredentialsPluginWrapper::MetadataCredentialsPluginWrapper(
+    std::unique_ptr<MetadataCredentialsPlugin> plugin)
+    : thread_pool_(CreateDefaultThreadPool()), plugin_(std::move(plugin)) {}
+
+std::shared_ptr<Credentials> MetadataCredentialsFromPlugin(
+    std::unique_ptr<MetadataCredentialsPlugin> plugin) {
+  GrpcLibrary init;  // To call grpc_init().
+  MetadataCredentialsPluginWrapper* wrapper =
+      new MetadataCredentialsPluginWrapper(std::move(plugin));
+  grpc_metadata_credentials_plugin c_plugin = {
+      MetadataCredentialsPluginWrapper::GetMetadata,
+      MetadataCredentialsPluginWrapper::Destroy, wrapper};
+  return WrapCredentials(
+      grpc_metadata_credentials_create_from_plugin(c_plugin, nullptr));
+}
+
 }  // namespace grpc

src/cpp/client/secure_credentials.h

@@ -39,6 +39,8 @@
 #include <grpc++/support/config.h>
 #include <grpc++/security/credentials.h>
 
+#include "src/cpp/server/thread_pool_interface.h"
+
 namespace grpc {
 
 class SecureCredentials GRPC_FINAL : public Credentials {
@@ -56,6 +58,23 @@ class SecureCredentials GRPC_FINAL : public Credentials {
   grpc_credentials* const c_creds_;
 };
 
+class MetadataCredentialsPluginWrapper GRPC_FINAL {
+ public:
+  static void Destroy(void* wrapper);
+  static void GetMetadata(void* wrapper, const char* service_url,
+                          grpc_credentials_plugin_metadata_cb cb,
+                          void* user_data);
+
+  explicit MetadataCredentialsPluginWrapper(
+      std::unique_ptr<MetadataCredentialsPlugin> plugin);
+
+ private:
+  void InvokePlugin(const char* service_url,
+                    grpc_credentials_plugin_metadata_cb cb, void* user_data);
+  std::unique_ptr<ThreadPoolInterface> thread_pool_;
+  std::unique_ptr<MetadataCredentialsPlugin> plugin_;
+};
+
 }  // namespace grpc
 
 #endif  // GRPC_INTERNAL_CPP_CLIENT_SECURE_CREDENTIALS_H

test/core/security/credentials_test.c

@@ -833,6 +833,109 @@ static void test_google_default_creds_access_token(void) {
   gpr_setenv(GRPC_GOOGLE_CREDENTIALS_ENV_VAR, ""); /* Reset. */
 }
 
+typedef enum {
+  PLUGIN_INITIAL_STATE,
+  PLUGIN_GET_METADATA_CALLED_STATE,
+  PLUGIN_DESTROY_CALLED_STATE
+} plugin_state;
+
+typedef struct {
+  const char *key;
+  const char *value;
+} plugin_metadata;
+
+static const plugin_metadata plugin_md[] = {{"foo", "bar"}, {"hi", "there"}};
+
+static void plugin_get_metadata_success(void *state, const char *service_url,
+                                        grpc_credentials_plugin_metadata_cb cb,
+                                        void *user_data) {
+  size_t i;
+  grpc_metadata md[GPR_ARRAY_SIZE(plugin_md)];
+  plugin_state *s = (plugin_state *)state;
+  GPR_ASSERT(strcmp(service_url, test_service_url) == 0);
+  *s = PLUGIN_GET_METADATA_CALLED_STATE;
+  for (i = 0; i < GPR_ARRAY_SIZE(plugin_md); i++) {
+    memset(&md[i], 0, sizeof(grpc_metadata));
+    md[i].key = plugin_md[i].key;
+    md[i].value = plugin_md[i].value;
+    md[i].value_length = strlen(plugin_md[i].value);
+  }
+  cb(user_data, md, GPR_ARRAY_SIZE(md), GRPC_STATUS_OK, NULL);
+}
+
+static void plugin_get_metadata_failure(void *state, const char *service_url,
+                                        grpc_credentials_plugin_metadata_cb cb,
+                                        void *user_data) {
+  plugin_state *s = (plugin_state *)state;
+  GPR_ASSERT(strcmp(service_url, test_service_url) == 0);
+  *s = PLUGIN_GET_METADATA_CALLED_STATE;
+  cb(user_data, NULL, 0, GRPC_STATUS_UNAUTHENTICATED,
+     "Could not get metadata for plugin.");
+}
+
+static void on_plugin_metadata_received_success(
+    void *user_data, grpc_credentials_md *md_elems, size_t num_md,
+    grpc_credentials_status status) {
+  size_t i = 0;
+  GPR_ASSERT(user_data == NULL);
+  GPR_ASSERT(md_elems != NULL);
+  GPR_ASSERT(num_md == GPR_ARRAY_SIZE(plugin_md));
+  for (i = 0; i < num_md; i++) {
+    GPR_ASSERT(gpr_slice_str_cmp(md_elems[i].key, plugin_md[i].key) == 0);
+    GPR_ASSERT(gpr_slice_str_cmp(md_elems[i].value, plugin_md[i].value) == 0);
+  }
+}
+
+static void on_plugin_metadata_received_failure(
+    void *user_data, grpc_credentials_md *md_elems, size_t num_md,
+    grpc_credentials_status status) {
+  GPR_ASSERT(user_data == NULL);
+  GPR_ASSERT(md_elems == NULL);
+  GPR_ASSERT(num_md == 0);
+  GPR_ASSERT(status == GRPC_CREDENTIALS_ERROR);
+}
+
+static void plugin_destroy(void *state) {
+  plugin_state *s = (plugin_state *)state;
+  *s = PLUGIN_DESTROY_CALLED_STATE;
+}
+
+static void test_metadata_plugin_success(void) {
+  grpc_credentials *creds;
+  plugin_state state = PLUGIN_INITIAL_STATE;
+  grpc_metadata_credentials_plugin plugin;
+
+  plugin.state = &state;
+  plugin.get_metadata = plugin_get_metadata_success;
+  plugin.destroy = plugin_destroy;
+
+  creds = grpc_metadata_credentials_create_from_plugin(plugin, NULL);
+  GPR_ASSERT(state == PLUGIN_INITIAL_STATE);
+  grpc_credentials_get_request_metadata(
+      creds, NULL, test_service_url, on_plugin_metadata_received_success, NULL);
+  GPR_ASSERT(state == PLUGIN_GET_METADATA_CALLED_STATE);
+  grpc_credentials_release(creds);
+  GPR_ASSERT(state == PLUGIN_DESTROY_CALLED_STATE);
+}
+
+static void test_metadata_plugin_failure(void) {
+  grpc_credentials *creds;
+  plugin_state state = PLUGIN_INITIAL_STATE;
+  grpc_metadata_credentials_plugin plugin;
+
+  plugin.state = &state;
+  plugin.get_metadata = plugin_get_metadata_failure;
+  plugin.destroy = plugin_destroy;
+
+  creds = grpc_metadata_credentials_create_from_plugin(plugin, NULL);
+  GPR_ASSERT(state == PLUGIN_INITIAL_STATE);
+  grpc_credentials_get_request_metadata(
+      creds, NULL, test_service_url, on_plugin_metadata_received_failure, NULL);
+  GPR_ASSERT(state == PLUGIN_GET_METADATA_CALLED_STATE);
+  grpc_credentials_release(creds);
+  GPR_ASSERT(state == PLUGIN_DESTROY_CALLED_STATE);
+}
+
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
   test_empty_md_store();
@@ -860,5 +963,7 @@ int main(int argc, char **argv) {
   test_jwt_creds_signing_failure();
   test_google_default_creds_auth_key();
   test_google_default_creds_access_token();
+  test_metadata_plugin_success();
+  test_metadata_plugin_failure();
   return 0;
 }

test/cpp/end2end/end2end_test.cc

@@ -108,6 +108,39 @@ bool CheckIsLocalhost(const grpc::string& addr) {
          addr.substr(0, kIpv6.size()) == kIpv6;
 }
 
+class TestMetadataCredentialsPlugin : public MetadataCredentialsPlugin {
+ public:
+  static const char kMetadataKey[];
+
+  TestMetadataCredentialsPlugin(grpc::string_ref metadata_value,
+                                bool is_blocking, bool is_successful)
+      : metadata_value_(metadata_value.data(), metadata_value.length()),
+        is_blocking_(is_blocking),
+        is_successful_(is_successful) {}
+
+  bool IsBlocking() const GRPC_OVERRIDE { return is_blocking_; }
+
+  Status GetMetadata(grpc::string_ref service_url,
+                     std::multimap<grpc::string, grpc::string_ref>* metadata)
+      GRPC_OVERRIDE {
+    EXPECT_GT(service_url.length(), 0UL);
+    EXPECT_TRUE(metadata != nullptr);
+    if (is_successful_) {
+      metadata->insert(std::make_pair(kMetadataKey, metadata_value_));
+      return Status::OK;
+    } else {
+      return Status(StatusCode::NOT_FOUND, "Could not find plugin metadata.");
+    }
+  }
+
+ private:
+  grpc::string metadata_value_;
+  bool is_blocking_;
+  bool is_successful_;
+};
+
+const char TestMetadataCredentialsPlugin::kMetadataKey[] = "TestPluginMetadata";
+
 class TestAuthMetadataProcessor : public AuthMetadataProcessor {
  public:
   static const char kGoodGuy[];
@@ -115,10 +148,15 @@ class TestAuthMetadataProcessor : public AuthMetadataProcessor {
   TestAuthMetadataProcessor(bool is_blocking) : is_blocking_(is_blocking) {}
 
   std::shared_ptr<Credentials> GetCompatibleClientCreds() {
-    return AccessTokenCredentials(kGoodGuy);
+    return MetadataCredentialsFromPlugin(
+        std::unique_ptr<MetadataCredentialsPlugin>(
+            new TestMetadataCredentialsPlugin(kGoodGuy, is_blocking_, true)));
   }
+
   std::shared_ptr<Credentials> GetIncompatibleClientCreds() {
-    return AccessTokenCredentials("Mr Hyde");
+    return MetadataCredentialsFromPlugin(
+        std::unique_ptr<MetadataCredentialsPlugin>(
+            new TestMetadataCredentialsPlugin("Mr Hyde", is_blocking_, true)));
   }
 
   // Interface implementation
@@ -130,10 +168,11 @@ class TestAuthMetadataProcessor : public AuthMetadataProcessor {
     EXPECT_TRUE(consumed_auth_metadata != nullptr);
     EXPECT_TRUE(context != nullptr);
     EXPECT_TRUE(response_metadata != nullptr);
-    auto auth_md = auth_metadata.find(GRPC_AUTHORIZATION_METADATA_KEY);
+    auto auth_md =
+        auth_metadata.find(TestMetadataCredentialsPlugin::kMetadataKey);
     EXPECT_NE(auth_md, auth_metadata.end());
     string_ref auth_md_value = auth_md->second;
-    if (auth_md_value.ends_with(kGoodGuy)) {
+    if (auth_md_value == kGoodGuy) {
       context->AddProperty(kIdentityPropName, kGoodGuy);
       context->SetPeerIdentityPropertyName(kIdentityPropName);
       consumed_auth_metadata->insert(
@@ -147,7 +186,7 @@ class TestAuthMetadataProcessor : public AuthMetadataProcessor {
     }
   }
 
- protected:
+ private:
   static const char kIdentityPropName[];
   bool is_blocking_;
 };
@@ -876,7 +915,24 @@ TEST_F(End2endTest, OverridePerCallCredentials) {
   EXPECT_TRUE(s.ok());
 }
 
-TEST_F(End2endTest, NonBlockingAuthMetadataProcessorSuccess) {
+TEST_F(End2endTest, NonBlockingAuthMetadataPluginFailure) {
+  ResetStub(false);
+  EchoRequest request;
+  EchoResponse response;
+  ClientContext context;
+  context.set_credentials(
+      MetadataCredentialsFromPlugin(std::unique_ptr<MetadataCredentialsPlugin>(
+          new TestMetadataCredentialsPlugin(
+              "Does not matter, will fail anyway (see 3rd param)", false,
+              false))));
+  request.set_message("Hello");
+
+  Status s = stub_->Echo(&context, request, &response);
+  EXPECT_FALSE(s.ok());
+  EXPECT_EQ(s.error_code(), StatusCode::UNAUTHENTICATED);
+}
+
+TEST_F(End2endTest, NonBlockingAuthMetadataPluginAndProcessorSuccess) {
   auto* processor = new TestAuthMetadataProcessor(false);
   StartServer(std::shared_ptr<AuthMetadataProcessor>(processor));
   ResetStub(false);
@@ -899,7 +955,7 @@ TEST_F(End2endTest, NonBlockingAuthMetadataProcessorSuccess) {
       grpc::string("Bearer ") + TestAuthMetadataProcessor::kGoodGuy));
 }
 
-TEST_F(End2endTest, NonBlockingAuthMetadataProcessorFailure) {
+TEST_F(End2endTest, NonBlockingAuthMetadataPluginAndProcessorFailure) {
   auto* processor = new TestAuthMetadataProcessor(false);
   StartServer(std::shared_ptr<AuthMetadataProcessor>(processor));
   ResetStub(false);
@@ -914,7 +970,24 @@ TEST_F(End2endTest, NonBlockingAuthMetadataProcessorFailure) {
   EXPECT_EQ(s.error_code(), StatusCode::UNAUTHENTICATED);
 }
 
-TEST_F(End2endTest, BlockingAuthMetadataProcessorSuccess) {
+TEST_F(End2endTest, BlockingAuthMetadataPluginFailure) {
+  ResetStub(false);
+  EchoRequest request;
+  EchoResponse response;
+  ClientContext context;
+  context.set_credentials(
+      MetadataCredentialsFromPlugin(std::unique_ptr<MetadataCredentialsPlugin>(
+          new TestMetadataCredentialsPlugin(
+              "Does not matter, will fail anyway (see 3rd param)", true,
+              false))));
+  request.set_message("Hello");
+
+  Status s = stub_->Echo(&context, request, &response);
+  EXPECT_FALSE(s.ok());
+  EXPECT_EQ(s.error_code(), StatusCode::UNAUTHENTICATED);
+}
+
+TEST_F(End2endTest, BlockingAuthMetadataPluginAndProcessorSuccess) {
   auto* processor = new TestAuthMetadataProcessor(true);
   StartServer(std::shared_ptr<AuthMetadataProcessor>(processor));
   ResetStub(false);
@@ -937,7 +1010,7 @@ TEST_F(End2endTest, BlockingAuthMetadataProcessorSuccess) {
       grpc::string("Bearer ") + TestAuthMetadataProcessor::kGoodGuy));
 }
 
-TEST_F(End2endTest, BlockingAuthMetadataProcessorFailure) {
+TEST_F(End2endTest, BlockingAuthMetadataPluginAndProcessorFailure) {
   auto* processor = new TestAuthMetadataProcessor(true);
   StartServer(std::shared_ptr<AuthMetadataProcessor>(processor));
   ResetStub(false);