Merge branch 'master' of https://github.com/grpc/grpc into no-more-extern-c

BUILD

@@ -480,6 +480,7 @@ grpc_cc_library(
     ],
     hdrs = [
         "src/core/lib/profiling/timers.h",
+        "src/core/lib/support/abstract.h",
         "src/core/lib/support/arena.h",
         "src/core/lib/support/atomic.h",
         "src/core/lib/support/atomic_with_atm.h",

CMakeLists.txt

@@ -430,6 +430,7 @@ add_dependencies(buildtests_c gpr_env_test)
 add_dependencies(buildtests_c gpr_histogram_test)
 add_dependencies(buildtests_c gpr_host_port_test)
 add_dependencies(buildtests_c gpr_log_test)
+add_dependencies(buildtests_c gpr_manual_constructor_test)
 add_dependencies(buildtests_c gpr_mpscq_test)
 add_dependencies(buildtests_c gpr_spinlock_test)
 add_dependencies(buildtests_c gpr_stack_lockfree_test)
@@ -6390,6 +6391,34 @@ target_link_libraries(gpr_log_test
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
+add_executable(gpr_manual_constructor_test
+  test/core/support/manual_constructor_test.cc
+)
+
+
+target_include_directories(gpr_manual_constructor_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${BORINGSSL_ROOT_DIR}/include
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp
+)
+
+target_link_libraries(gpr_manual_constructor_test
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  gpr_test_util
+  gpr
+)
+
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
 add_executable(gpr_mpscq_test
   test/core/support/mpscq_test.cc
 )

Makefile

@@ -328,6 +328,7 @@ ifeq ($(SYSTEM),Darwin)
 CXXFLAGS += -stdlib=libc++
 endif
 CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 -Ithird_party/abseil-cpp
+COREFLAGS += -fno-rtti -fno-exceptions
 LDFLAGS += -g
 
 CPPFLAGS += $(CPPFLAGS_$(CONFIG))
@@ -990,6 +991,7 @@ gpr_env_test: $(BINDIR)/$(CONFIG)/gpr_env_test
 gpr_histogram_test: $(BINDIR)/$(CONFIG)/gpr_histogram_test
 gpr_host_port_test: $(BINDIR)/$(CONFIG)/gpr_host_port_test
 gpr_log_test: $(BINDIR)/$(CONFIG)/gpr_log_test
+gpr_manual_constructor_test: $(BINDIR)/$(CONFIG)/gpr_manual_constructor_test
 gpr_mpscq_test: $(BINDIR)/$(CONFIG)/gpr_mpscq_test
 gpr_spinlock_test: $(BINDIR)/$(CONFIG)/gpr_spinlock_test
 gpr_stack_lockfree_test: $(BINDIR)/$(CONFIG)/gpr_stack_lockfree_test
@@ -1384,6 +1386,7 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/gpr_histogram_test \
   $(BINDIR)/$(CONFIG)/gpr_host_port_test \
   $(BINDIR)/$(CONFIG)/gpr_log_test \
+  $(BINDIR)/$(CONFIG)/gpr_manual_constructor_test \
   $(BINDIR)/$(CONFIG)/gpr_mpscq_test \
   $(BINDIR)/$(CONFIG)/gpr_spinlock_test \
   $(BINDIR)/$(CONFIG)/gpr_stack_lockfree_test \
@@ -1831,6 +1834,8 @@ test_c: buildtests_c
 	$(Q) $(BINDIR)/$(CONFIG)/gpr_host_port_test || ( echo test gpr_host_port_test failed ; exit 1 )
 	$(E) "[RUN]     Testing gpr_log_test"
 	$(Q) $(BINDIR)/$(CONFIG)/gpr_log_test || ( echo test gpr_log_test failed ; exit 1 )
+	$(E) "[RUN]     Testing gpr_manual_constructor_test"
+	$(Q) $(BINDIR)/$(CONFIG)/gpr_manual_constructor_test || ( echo test gpr_manual_constructor_test failed ; exit 1 )
 	$(E) "[RUN]     Testing gpr_mpscq_test"
 	$(Q) $(BINDIR)/$(CONFIG)/gpr_mpscq_test || ( echo test gpr_mpscq_test failed ; exit 1 )
 	$(E) "[RUN]     Testing gpr_spinlock_test"
@@ -2579,6 +2584,16 @@ $(OBJDIR)/$(CONFIG)/src/compiler/%.o : src/compiler/%.cc
 	$(Q) mkdir -p `dirname $@`
 	$(Q) $(HOST_CXX) $(HOST_CXXFLAGS) $(HOST_CPPFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $<
 
+$(OBJDIR)/$(CONFIG)/src/core/%.o : src/core/%.cc
+	$(E) "[CXX]     Compiling $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(COREFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $<
+
+$(OBJDIR)/$(CONFIG)/test/core/%.o : test/core/%.cc
+	$(E) "[CXX]     Compiling $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(COREFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $<
+
 $(OBJDIR)/$(CONFIG)/%.o : %.cc
 	$(E) "[CXX]     Compiling $<"
 	$(Q) mkdir -p `dirname $@`
@@ -10147,6 +10162,38 @@ endif
 endif
 
 
+GPR_MANUAL_CONSTRUCTOR_TEST_SRC = \
+    test/core/support/manual_constructor_test.cc \
+
+GPR_MANUAL_CONSTRUCTOR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_MANUAL_CONSTRUCTOR_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/gpr_manual_constructor_test: openssl_dep_error
+
+else
+
+
+
+$(BINDIR)/$(CONFIG)/gpr_manual_constructor_test: $(GPR_MANUAL_CONSTRUCTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LD) $(LDFLAGS) $(GPR_MANUAL_CONSTRUCTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/gpr_manual_constructor_test
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/support/manual_constructor_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_gpr_manual_constructor_test: $(GPR_MANUAL_CONSTRUCTOR_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(GPR_MANUAL_CONSTRUCTOR_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 GPR_MPSCQ_TEST_SRC = \
     test/core/support/mpscq_test.cc \
 

WORKSPACE

@@ -10,7 +10,7 @@ bind(
 
 bind(
     name = "zlib",
-    actual = "@submodule_zlib//:z",
+    actual = "@com_github_madler_zlib//:z",
 )
 
 bind(
@@ -35,22 +35,22 @@ bind(
 
 bind(
     name = "cares",
-    actual = "@submodule_cares//:ares",
+    actual = "@com_github_cares_cares//:ares",
 )
 
 bind(
     name = "gtest",
-    actual = "@submodule_gtest//:gtest",
+    actual = "@com_github_google_googletest//:gtest",
 )
 
 bind(
     name = "gmock",
-    actual = "@submodule_gtest//:gmock",
+    actual = "@com_github_google_googletest//:gmock",
 )
 
 bind(
     name = "benchmark",
-    actual = "@submodule_benchmark//:benchmark",
+    actual = "@com_github_google_benchmark//:benchmark",
 )
 
 bind(
@@ -58,47 +58,60 @@ bind(
     actual = "@com_github_gflags_gflags//:gflags",
 )
 
-local_repository(
+http_archive(
     name = "boringssl",
-    path = "third_party/boringssl-with-bazel",
+    # on the master-with-bazel branch
+    url = "https://boringssl.googlesource.com/boringssl/+archive/886e7d75368e3f4fab3f4d0d3584e4abfc557755.tar.gz",
 )
 
-new_local_repository(
+new_http_archive(
-    name = "submodule_zlib",
+    name = "com_github_madler_zlib",
     build_file = "third_party/zlib.BUILD",
-    path = "third_party/zlib",
+    strip_prefix = "zlib-cacf7f1d4e3d44d871b605da3b647f07d718623f",
+    url = "https://github.com/madler/zlib/archive/cacf7f1d4e3d44d871b605da3b647f07d718623f.tar.gz",
 )
 
-new_local_repository(
+http_archive(
     name = "com_google_protobuf",
-    build_file = "third_party/protobuf/BUILD",
+    strip_prefix = "protobuf-80a37e0782d2d702d52234b62dd4b9ec74fd2c95",
-    path = "third_party/protobuf",
+    url = "https://github.com/google/protobuf/archive/80a37e0782d2d702d52234b62dd4b9ec74fd2c95.tar.gz",
 )
 
-new_local_repository(
+new_http_archive(
-    name = "submodule_gtest",
+    name = "com_github_google_googletest",
     build_file = "third_party/gtest.BUILD",
-    path = "third_party/googletest",
+    strip_prefix = "googletest-ec44c6c1675c25b9827aacd08c02433cccde7780",
+    url = "https://github.com/google/googletest/archive/ec44c6c1675c25b9827aacd08c02433cccde7780.tar.gz",
 )
 
-local_repository(
+http_archive(
     name = "com_github_gflags_gflags",
-    path = "third_party/gflags",
+    strip_prefix = "gflags-30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e",
+    url = "https://github.com/gflags/gflags/archive/30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e.tar.gz",
 )
 
-new_local_repository(
+new_http_archive(
-    name = "submodule_benchmark",
+    name = "com_github_google_benchmark",
-    path = "third_party/benchmark",
     build_file = "third_party/benchmark.BUILD",
+    strip_prefix = "benchmark-5b7683f49e1e9223cf9927b24f6fd3d6bd82e3f8",
+    url = "https://github.com/google/benchmark/archive/5b7683f49e1e9223cf9927b24f6fd3d6bd82e3f8.tar.gz",
 )
 
 new_local_repository(
-    name = "submodule_cares",
+    name = "cares_local_files",
+    build_file = "third_party/cares/cares_local_files.BUILD",
     path = "third_party/cares",
+)
+
+new_http_archive(
+    name = "com_github_cares_cares",
     build_file = "third_party/cares/cares.BUILD",
+    strip_prefix = "c-ares-3be1924221e1326df520f8498d704a5c4c8d0cce",
+    url = "https://github.com/c-ares/c-ares/archive/3be1924221e1326df520f8498d704a5c4c8d0cce.tar.gz",
 )
 
-local_repository(
+http_archive(
     name = "com_google_absl",
-    path = "third_party/abseil-cpp",
+    strip_prefix = "abseil-cpp-cc4bed2d74f7c8717e31f9579214ab52a9c9c610",
+    url = "https://github.com/abseil/abseil-cpp/archive/cc4bed2d74f7c8717e31f9579214ab52a9c9c610.tar.gz",
 )

build.yaml

@@ -104,6 +104,7 @@ filegroups:
   - include/grpc/support/useful.h
   headers:
   - src/core/lib/profiling/timers.h
+  - src/core/lib/support/abstract.h
   - src/core/lib/support/arena.h
   - src/core/lib/support/atomic.h
   - src/core/lib/support/atomic_with_atm.h
@@ -2215,6 +2216,16 @@ targets:
   - gpr_test_util
   - gpr
   uses_polling: false
+- name: gpr_manual_constructor_test
+  cpu_cost: 3
+  build: test
+  language: c
+  src:
+  - test/core/support/manual_constructor_test.cc
+  deps:
+  - gpr_test_util
+  - gpr
+  uses_polling: false
 - name: gpr_mpscq_test
   cpu_cost: 30
   build: test
@@ -4965,6 +4976,7 @@ defaults:
     CPPFLAGS: -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM
       -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX
   global:
+    COREFLAGS: -fno-rtti -fno-exceptions
     CPPFLAGS: -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1
       -Ithird_party/abseil-cpp
     LDFLAGS: -g

gRPC-Core.podspec

@@ -187,6 +187,7 @@ Pod::Spec.new do |s|
 
     # To save you from scrolling, this is the last part of the podspec.
     ss.source_files = 'src/core/lib/profiling/timers.h',
+                      'src/core/lib/support/abstract.h',
                       'src/core/lib/support/arena.h',
                       'src/core/lib/support/atomic.h',
                       'src/core/lib/support/atomic_with_atm.h',
@@ -706,6 +707,7 @@ Pod::Spec.new do |s|
                       'src/core/plugin_registry/grpc_plugin_registry.cc'
 
     ss.private_header_files = 'src/core/lib/profiling/timers.h',
+                              'src/core/lib/support/abstract.h',
                               'src/core/lib/support/arena.h',
                               'src/core/lib/support/atomic.h',
                               'src/core/lib/support/atomic_with_atm.h',

grpc.gemspec

@@ -84,6 +84,7 @@ Gem::Specification.new do |s|
   s.files += %w( include/grpc/impl/codegen/sync_posix.h )
   s.files += %w( include/grpc/impl/codegen/sync_windows.h )
   s.files += %w( src/core/lib/profiling/timers.h )
+  s.files += %w( src/core/lib/support/abstract.h )
   s.files += %w( src/core/lib/support/arena.h )
   s.files += %w( src/core/lib/support/atomic.h )
   s.files += %w( src/core/lib/support/atomic_with_atm.h )

include/grpc++/impl/codegen/call.h

@@ -579,6 +579,7 @@ class CallOpClientRecvStatus {
     op->data.recv_status_on_client.trailing_metadata = metadata_map_->arr();
     op->data.recv_status_on_client.status = &status_code_;
     op->data.recv_status_on_client.status_details = &error_message_;
+    op->data.recv_status_on_client.error_string = nullptr;
     op->flags = 0;
     op->reserved = NULL;
   }

include/grpc/impl/codegen/grpc_types.h

@@ -558,6 +558,10 @@ typedef struct grpc_op {
       grpc_metadata_array* trailing_metadata;
       grpc_status_code* status;
       grpc_slice* status_details;
+      /** If this is not nullptr, it will be populated with the full fidelity
+       * error string for debugging purposes. The application is responsible
+       * for freeing the data. */
+      const char** error_string;
     } recv_status_on_client;
     struct grpc_op_recv_close_on_server {
       /** out argument, set to 1 if the call failed in any way (seen as a

include/grpc/impl/codegen/port_platform.h

@@ -297,6 +297,27 @@
 #endif
 #endif /* GPR_NO_AUTODETECT_PLATFORM */
 
+/*
+ *  There are platforms for which TLS should not be used even though the
+ * compiler makes it seem like it's supported (Android NDK < r12b for example).
+ * This is primarily because of linker problems and toolchain misconfiguration:
+ * TLS isn't supported until NDK r12b per
+ * https://developer.android.com/ndk/downloads/revision_history.html
+ * Since NDK r16, `__NDK_MAJOR__` and `__NDK_MINOR__` are defined in
+ * <android/ndk-version.h>. For NDK < r16, users should define these macros,
+ * e.g. `-D__NDK_MAJOR__=11 -D__NKD_MINOR__=0` for NDK r11. */
+#if defined(__ANDROID__) && defined(__clang__) && defined(GPR_GCC_TLS)
+#if __has_include(<android/ndk-version.h>)
+#include <android/ndk-version.h>
+#endif /* __has_include(<android/ndk-version.h>) */
+#if defined(__ANDROID__) && defined(__clang__) && defined(__NDK_MAJOR__) && \
+    defined(__NDK_MINOR__) &&                                               \
+    ((__NDK_MAJOR__ < 12) || ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1)))
+#undef GPR_GCC_TLS
+#define GPR_PTHREAD_TLS 1
+#endif
+#endif /*defined(__ANDROID__) && defined(__clang__) && defined(GPR_GCC_TLS) */
+
 #if defined(__has_include)
 #if __has_include(<atomic>)
 #define GRPC_HAS_CXX11_ATOMIC

package.xml

@@ -96,6 +96,7 @@
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_posix.h" role="src" />
     <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_windows.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/profiling/timers.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/support/abstract.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/arena.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/atomic.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/atomic_with_atm.h" role="src" />

src/core/ext/transport/chttp2/transport/chttp2_transport.cc

@@ -1758,7 +1758,7 @@ static void send_goaway(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
   grpc_http2_error_code http_error;
   grpc_slice slice;
   grpc_error_get_status(exec_ctx, error, GRPC_MILLIS_INF_FUTURE, nullptr,
-                        &slice, &http_error);
+                        &slice, &http_error, nullptr);
   grpc_chttp2_goaway_append(t->last_new_stream_id, (uint32_t)http_error,
                             grpc_slice_ref_internal(slice), &t->qbuf);
   grpc_chttp2_initiate_write(exec_ctx, t,
@@ -2061,7 +2061,7 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx* exec_ctx,
     if (s->id != 0) {
       grpc_http2_error_code http_error;
       grpc_error_get_status(exec_ctx, due_to_error, s->deadline, nullptr,
-                            nullptr, &http_error);
+                            nullptr, &http_error, nullptr);
       grpc_slice_buffer_add(
           &t->qbuf, grpc_chttp2_rst_stream_create(s->id, (uint32_t)http_error,
                                                   &s->stats.outgoing));
@@ -2079,7 +2079,8 @@ void grpc_chttp2_fake_status(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
                              grpc_chttp2_stream* s, grpc_error* error) {
   grpc_status_code status;
   grpc_slice slice;
-  grpc_error_get_status(exec_ctx, error, s->deadline, &status, &slice, nullptr);
+  grpc_error_get_status(exec_ctx, error, s->deadline, &status, &slice, nullptr,
+                        nullptr);
   if (status != GRPC_STATUS_OK) {
     s->seen_error = true;
   }
@@ -2244,7 +2245,7 @@ static void close_from_api(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
   grpc_status_code grpc_status;
   grpc_slice slice;
   grpc_error_get_status(exec_ctx, error, s->deadline, &grpc_status, &slice,
-                        nullptr);
+                        nullptr, nullptr);
 
   GPR_ASSERT(grpc_status >= 0 && (int)grpc_status < 100);
 

src/core/ext/transport/chttp2/transport/flow_control.h

@@ -19,6 +19,7 @@
 #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FLOW_CONTROL_H
 #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FLOW_CONTROL_H
 
+#include <grpc/support/port_platform.h>
 #include <stdint.h>
 
 #include <grpc/support/useful.h>

src/core/lib/channel/channel_stack.h

@@ -80,6 +80,7 @@ typedef struct {
 typedef struct {
   grpc_call_stats stats;
   grpc_status_code final_status;
+  const char** error_string;
 } grpc_call_final_info;
 
 /* Channel filters specify:

src/core/lib/iomgr/ev_epoll1_linux.cc

@@ -263,11 +263,13 @@ static grpc_fd* fd_create(int fd, const char* name) {
 
   if (new_fd == nullptr) {
     new_fd = (grpc_fd*)gpr_malloc(sizeof(grpc_fd));
+    new_fd->read_closure.Init();
+    new_fd->write_closure.Init();
   }
 
   new_fd->fd = fd;
-  new_fd->read_closure.Init();
+  new_fd->read_closure->InitEvent();
-  new_fd->write_closure.Init();
+  new_fd->write_closure->InitEvent();
   gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL);
 
   new_fd->freelist_next = nullptr;
@@ -336,8 +338,8 @@ static void fd_orphan(grpc_exec_ctx* exec_ctx, grpc_fd* fd,
   GRPC_CLOSURE_SCHED(exec_ctx, on_done, GRPC_ERROR_REF(error));
 
   grpc_iomgr_unregister_object(&fd->iomgr_object);
-  fd->read_closure.Destroy();
+  fd->read_closure->DestroyEvent();
-  fd->write_closure.Destroy();
+  fd->write_closure->DestroyEvent();
 
   gpr_mu_lock(&fd_freelist_mu);
   fd->freelist_next = fd_freelist;

src/core/lib/iomgr/ev_epollex_linux.cc

@@ -286,8 +286,8 @@ static void fd_destroy(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {
   fd->freelist_next = fd_freelist;
   fd_freelist = fd;
 
-  fd->read_closure.Destroy();
+  fd->read_closure->DestroyEvent();
-  fd->write_closure.Destroy();
+  fd->write_closure->DestroyEvent();
 
   gpr_mu_unlock(&fd_freelist_mu);
 }
@@ -340,6 +340,8 @@ static grpc_fd* fd_create(int fd, const char* name) {
 
   if (new_fd == nullptr) {
     new_fd = (grpc_fd*)gpr_malloc(sizeof(grpc_fd));
+    new_fd->read_closure.Init();
+    new_fd->write_closure.Init();
   }
 
   gpr_mu_init(&new_fd->pollable_mu);
@@ -347,8 +349,8 @@ static grpc_fd* fd_create(int fd, const char* name) {
   new_fd->pollable_obj = nullptr;
   gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1);
   new_fd->fd = fd;
-  new_fd->read_closure.Init();
+  new_fd->read_closure->InitEvent();
-  new_fd->write_closure.Init();
+  new_fd->write_closure->InitEvent();
   gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL);
 
   new_fd->freelist_next = nullptr;

src/core/lib/iomgr/ev_epollsig_linux.cc

@@ -767,8 +767,8 @@ static void unref_by(grpc_fd* fd, int n) {
     fd_freelist = fd;
     grpc_iomgr_unregister_object(&fd->iomgr_object);
 
-    fd->read_closure.Destroy();
+    fd->read_closure->DestroyEvent();
-    fd->write_closure.Destroy();
+    fd->write_closure->DestroyEvent();
 
     gpr_mu_unlock(&fd_freelist_mu);
   } else {
@@ -819,6 +819,8 @@ static grpc_fd* fd_create(int fd, const char* name) {
   if (new_fd == nullptr) {
     new_fd = (grpc_fd*)gpr_malloc(sizeof(grpc_fd));
     gpr_mu_init(&new_fd->po.mu);
+    new_fd->read_closure.Init();
+    new_fd->write_closure.Init();
   }
 
   /* Note: It is not really needed to get the new_fd->po.mu lock here. If this
@@ -833,8 +835,8 @@ static grpc_fd* fd_create(int fd, const char* name) {
   gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1);
   new_fd->fd = fd;
   new_fd->orphaned = false;
-  new_fd->read_closure.Init();
+  new_fd->read_closure->InitEvent();
-  new_fd->write_closure.Init();
+  new_fd->write_closure->InitEvent();
   gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL);
 
   new_fd->freelist_next = nullptr;

src/core/lib/iomgr/lockfree_event.cc

@@ -57,7 +57,9 @@ extern grpc_core::TraceFlag grpc_polling_trace;
 
 namespace grpc_core {
 
-LockfreeEvent::LockfreeEvent() {
+LockfreeEvent::LockfreeEvent() { InitEvent(); }
+
+void LockfreeEvent::InitEvent() {
   /* Perform an atomic store to start the state machine.
 
      Note carefully that LockfreeEvent *MAY* be used whilst in a destroyed
@@ -67,7 +69,7 @@ LockfreeEvent::LockfreeEvent() {
   gpr_atm_no_barrier_store(&state_, kClosureNotReady);
 }
 
-LockfreeEvent::~LockfreeEvent() {
+void LockfreeEvent::DestroyEvent() {
   gpr_atm curr;
   do {
     curr = gpr_atm_no_barrier_load(&state_);

src/core/lib/iomgr/lockfree_event.h

@@ -30,11 +30,16 @@ namespace grpc_core {
 class LockfreeEvent {
  public:
   LockfreeEvent();
-  ~LockfreeEvent();
 
   LockfreeEvent(const LockfreeEvent&) = delete;
   LockfreeEvent& operator=(const LockfreeEvent&) = delete;
 
+  // These methods are used to initialize and destroy the internal state. These
+  // cannot be done in constructor and destructor because SetReady may be called
+  // when the event is destroyed and put in a freelist.
+  void InitEvent();
+  void DestroyEvent();
+
   bool IsShutdown() const {
     return (gpr_atm_no_barrier_load(&state_) & kShutdownBit) != 0;
   }

src/core/lib/iomgr/tcp_posix.cc

@@ -81,9 +81,7 @@ typedef struct {
 
   grpc_slice_buffer* incoming_buffer;
   grpc_slice_buffer* outgoing_buffer;
-  /** slice within outgoing_buffer to write next */
+  /** byte within outgoing_buffer->slices[0] to write next */
-  size_t outgoing_slice_idx;
-  /** byte within outgoing_buffer->slices[outgoing_slice_idx] to write next */
   size_t outgoing_byte_idx;
 
   grpc_closure* read_cb;
@@ -532,23 +530,26 @@ static bool tcp_flush(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp,
   size_t unwind_slice_idx;
   size_t unwind_byte_idx;
 
+  // We always start at zero, because we eagerly unref and trim the slice
+  // buffer as we write
+  size_t outgoing_slice_idx = 0;
+
   for (;;) {
     sending_length = 0;
-    unwind_slice_idx = tcp->outgoing_slice_idx;
+    unwind_slice_idx = outgoing_slice_idx;
     unwind_byte_idx = tcp->outgoing_byte_idx;
-    for (iov_size = 0; tcp->outgoing_slice_idx != tcp->outgoing_buffer->count &&
+    for (iov_size = 0; outgoing_slice_idx != tcp->outgoing_buffer->count &&
                        iov_size != MAX_WRITE_IOVEC;
          iov_size++) {
       iov[iov_size].iov_base =
           GRPC_SLICE_START_PTR(
-              tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]) +
+              tcp->outgoing_buffer->slices[outgoing_slice_idx]) +
           tcp->outgoing_byte_idx;
       iov[iov_size].iov_len =
-          GRPC_SLICE_LENGTH(
+          GRPC_SLICE_LENGTH(tcp->outgoing_buffer->slices[outgoing_slice_idx]) -
-              tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]) -
           tcp->outgoing_byte_idx;
       sending_length += iov[iov_size].iov_len;
-      tcp->outgoing_slice_idx++;
+      outgoing_slice_idx++;
       tcp->outgoing_byte_idx = 0;
     }
     GPR_ASSERT(iov_size > 0);
@@ -574,16 +575,25 @@ static bool tcp_flush(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp,
 
     if (sent_length < 0) {
       if (errno == EAGAIN) {
-        tcp->outgoing_slice_idx = unwind_slice_idx;
         tcp->outgoing_byte_idx = unwind_byte_idx;
+        // unref all and forget about all slices that have been written to this
+        // point
+        for (size_t idx = 0; idx < unwind_slice_idx; ++idx) {
+          grpc_slice_unref_internal(
+              exec_ctx, grpc_slice_buffer_take_first(tcp->outgoing_buffer));
+        }
         return false;
       } else if (errno == EPIPE) {
         *error = grpc_error_set_int(GRPC_OS_ERROR(errno, "sendmsg"),
                                     GRPC_ERROR_INT_GRPC_STATUS,
                                     GRPC_STATUS_UNAVAILABLE);
+        grpc_slice_buffer_reset_and_unref_internal(exec_ctx,
+                                                   tcp->outgoing_buffer);
         return true;
       } else {
         *error = tcp_annotate_error(GRPC_OS_ERROR(errno, "sendmsg"), tcp);
+        grpc_slice_buffer_reset_and_unref_internal(exec_ctx,
+                                                   tcp->outgoing_buffer);
         return true;
       }
     }
@@ -593,9 +603,9 @@ static bool tcp_flush(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp,
     while (trailing > 0) {
       size_t slice_length;
 
-      tcp->outgoing_slice_idx--;
+      outgoing_slice_idx--;
-      slice_length = GRPC_SLICE_LENGTH(
+      slice_length =
-          tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]);
+          GRPC_SLICE_LENGTH(tcp->outgoing_buffer->slices[outgoing_slice_idx]);
       if (slice_length > trailing) {
         tcp->outgoing_byte_idx = slice_length - trailing;
         break;
@@ -604,11 +614,13 @@ static bool tcp_flush(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp,
       }
     }
 
-    if (tcp->outgoing_slice_idx == tcp->outgoing_buffer->count) {
+    if (outgoing_slice_idx == tcp->outgoing_buffer->count) {
       *error = GRPC_ERROR_NONE;
+      grpc_slice_buffer_reset_and_unref_internal(exec_ctx,
+                                                 tcp->outgoing_buffer);
       return true;
     }
-  };
+  }
 }
 
 static void tcp_handle_write(grpc_exec_ctx* exec_ctx, void* arg /* grpc_tcp */,
@@ -672,7 +684,6 @@ static void tcp_write(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep,
     return;
   }
   tcp->outgoing_buffer = buf;
-  tcp->outgoing_slice_idx = 0;
   tcp->outgoing_byte_idx = 0;
 
   if (!tcp_flush(exec_ctx, tcp, &error)) {

src/core/lib/slice/slice_internal.h

@@ -28,6 +28,9 @@ grpc_slice grpc_slice_ref_internal(grpc_slice slice);
 void grpc_slice_unref_internal(grpc_exec_ctx* exec_ctx, grpc_slice slice);
 void grpc_slice_buffer_reset_and_unref_internal(grpc_exec_ctx* exec_ctx,
                                                 grpc_slice_buffer* sb);
+void grpc_slice_buffer_partial_unref_internal(grpc_exec_ctx* exec_ctx,
+                                              grpc_slice_buffer* sb,
+                                              size_t idx);
 void grpc_slice_buffer_destroy_internal(grpc_exec_ctx* exec_ctx,
                                         grpc_slice_buffer* sb);
 

src/core/lib/support/abstract.h

@@ -0,0 +1,29 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SUPPORT_ABSTRACT_H
+#define GRPC_CORE_LIB_SUPPORT_ABSTRACT_H
+
+// This is needed to support abstract base classes in the c core. Since gRPC
+// doesn't have a c++ runtime, it will hit a linker error on delete unless
+// we define a virtual operator delete. See this blog for more info:
+// https://eli.thegreenplace.net/2015/c-deleting-destructors-and-virtual-operator-delete/
+#define GRPC_ABSTRACT_BASE_CLASS \
+  static void operator delete(void* p) { abort(); }
+
+#endif /* GRPC_CORE_LIB_SUPPORT_ABSTRACT_H */

src/core/lib/support/cpu_posix.cc

@@ -18,21 +18,23 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_CPU_POSIX
+#if defined(GPR_CPU_POSIX)
 
 #include <errno.h>
+#include <pthread.h>
 #include <string.h>
 #include <unistd.h>
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
 
-static __thread char magic_thread_local;
-
 static long ncpus = 0;
 
+static pthread_key_t thread_id_key;
+
 static void init_ncpus() {
   ncpus = sysconf(_SC_NPROCESSORS_ONLN);
   if (ncpus < 1 || ncpus > INT32_MAX) {
@@ -47,12 +49,32 @@ unsigned gpr_cpu_num_cores(void) {
   return (unsigned)ncpus;
 }
 
+static void delete_thread_id(void* value) {
+  if (value) {
+    gpr_free(value);
+  }
+}
+
+static void init_thread_id_key(void) {
+  pthread_key_create(&thread_id_key, delete_thread_id);
+}
+
 unsigned gpr_cpu_current_cpu(void) {
   /* NOTE: there's no way I know to return the actual cpu index portably...
      most code that's using this is using it to shard across work queues though,
      so here we use thread identity instead to achieve a similar though not
      identical effect */
-  return (unsigned)GPR_HASH_POINTER(&magic_thread_local, gpr_cpu_num_cores());
+  static gpr_once once = GPR_ONCE_INIT;
+  gpr_once_init(&once, init_thread_id_key);
+
+  unsigned int* thread_id =
+      static_cast<unsigned int*>(pthread_getspecific(thread_id_key));
+  if (thread_id == nullptr) {
+    thread_id = static_cast<unsigned int*>(gpr_malloc(sizeof(unsigned int)));
+    pthread_setspecific(thread_id_key, thread_id);
+  }
+
+  return (unsigned)GPR_HASH_POINTER(thread_id, gpr_cpu_num_cores());
 }
 
 #endif /* GPR_CPU_POSIX */

src/core/lib/support/manual_constructor.h

@@ -22,12 +22,147 @@
 // manually construct a region of memory with some type
 
 #include <stddef.h>
+#include <stdlib.h>
 #include <new>
 #include <type_traits>
 #include <utility>
 
+#include <grpc/support/log.h>
+
 namespace grpc_core {
 
+// this contains templated helpers needed to implement the ManualConstructors
+// in this file.
+namespace manual_ctor_impl {
+
+// is_one_of returns true it a class, Member, is present in a variadic list of
+// classes, List.
+template <class Member, class... List>
+class is_one_of;
+
+template <class Member, class... List>
+class is_one_of<Member, Member, List...> {
+ public:
+  static constexpr const bool value = true;
+};
+
+template <class Member, class A, class... List>
+class is_one_of<Member, A, List...> {
+ public:
+  static constexpr const bool value = is_one_of<Member, List...>::value;
+};
+
+template <class Member>
+class is_one_of<Member> {
+ public:
+  static constexpr const bool value = false;
+};
+
+// max_size_of returns sizeof(Type) for the largest type in the variadic list
+// of classes, Types.
+template <class... Types>
+class max_size_of;
+
+template <class A>
+class max_size_of<A> {
+ public:
+  static constexpr const size_t value = sizeof(A);
+};
+
+template <class A, class... B>
+class max_size_of<A, B...> {
+ public:
+  static constexpr const size_t value = sizeof(A) > max_size_of<B...>::value
+                                            ? sizeof(A)
+                                            : max_size_of<B...>::value;
+};
+
+// max_size_of returns alignof(Type) for the largest type in the variadic list
+// of classes, Types.
+template <class... Types>
+class max_align_of;
+
+template <class A>
+class max_align_of<A> {
+ public:
+  static constexpr const size_t value = alignof(A);
+};
+
+template <class A, class... B>
+class max_align_of<A, B...> {
+ public:
+  static constexpr const size_t value = alignof(A) > max_align_of<B...>::value
+                                            ? alignof(A)
+                                            : max_align_of<B...>::value;
+};
+
+}  // namespace manual_ctor_impl
+
+template <class BaseType, class... DerivedTypes>
+class PolymorphicManualConstructor {
+ public:
+  // No constructor or destructor because one of the most useful uses of
+  // this class is as part of a union, and members of a union could not have
+  // constructors or destructors till C++11.  And, anyway, the whole point of
+  // this class is to bypass constructor and destructor.
+
+  BaseType* get() { return reinterpret_cast<BaseType*>(&space_); }
+  const BaseType* get() const {
+    return reinterpret_cast<const BaseType*>(&space_);
+  }
+
+  BaseType* operator->() { return get(); }
+  const BaseType* operator->() const { return get(); }
+
+  BaseType& operator*() { return *get(); }
+  const BaseType& operator*() const { return *get(); }
+
+  template <class DerivedType>
+  void Init() {
+    FinishInit(new (&space_) DerivedType);
+  }
+
+  // Init() constructs the Type instance using the given arguments
+  // (which are forwarded to Type's constructor).
+  //
+  // Note that Init() with no arguments performs default-initialization,
+  // not zero-initialization (i.e it behaves the same as "new Type;", not
+  // "new Type();"), so it will leave non-class types uninitialized.
+  template <class DerivedType, typename... Ts>
+  void Init(Ts&&... args) {
+    FinishInit(new (&space_) DerivedType(std::forward<Ts>(args)...));
+  }
+
+  // Init() that is equivalent to copy and move construction.
+  // Enables usage like this:
+  //   ManualConstructor<std::vector<int>> v;
+  //   v.Init({1, 2, 3});
+  template <class DerivedType>
+  void Init(const DerivedType& x) {
+    FinishInit(new (&space_) DerivedType(x));
+  }
+  template <class DerivedType>
+  void Init(DerivedType&& x) {
+    FinishInit(new (&space_) DerivedType(std::move(x)));
+  }
+
+  void Destroy() { get()->~BaseType(); }
+
+ private:
+  template <class DerivedType>
+  void FinishInit(DerivedType* p) {
+    static_assert(
+        manual_ctor_impl::is_one_of<DerivedType, DerivedTypes...>::value,
+        "DerivedType must be one of the predeclared DerivedTypes");
+    GPR_ASSERT(reinterpret_cast<BaseType*>(static_cast<DerivedType*>(p)) == p);
+  }
+
+  typename std::aligned_storage<
+      grpc_core::manual_ctor_impl::max_size_of<DerivedTypes...>::value,
+      grpc_core::manual_ctor_impl::max_align_of<DerivedTypes...>::value>::type
+      space_;
+};
+
 template <typename Type>
 class ManualConstructor {
  public:

src/core/lib/surface/call.cc

@@ -234,6 +234,7 @@ struct grpc_call {
     struct {
       grpc_status_code* status;
       grpc_slice* status_details;
+      const char** error_string;
     } client;
     struct {
       int* cancelled;
@@ -284,7 +285,8 @@ static void receiving_slice_ready(grpc_exec_ctx* exec_ctx, void* bctlp,
 static void get_final_status(grpc_exec_ctx* exec_ctx, grpc_call* call,
                              void (*set_value)(grpc_status_code code,
                                                void* user_data),
-                             void* set_value_user_data, grpc_slice* details);
+                             void* set_value_user_data, grpc_slice* details,
+                             const char** error_string);
 static void set_status_value_directly(grpc_status_code status, void* dest);
 static void set_status_from_error(grpc_exec_ctx* exec_ctx, grpc_call* call,
                                   status_source source, grpc_error* error);
@@ -546,7 +548,8 @@ static void destroy_call(grpc_exec_ctx* exec_ctx, void* call,
   }
 
   get_final_status(exec_ctx, c, set_status_value_directly,
-                   &c->final_info.final_status, nullptr);
+                   &c->final_info.final_status, nullptr,
+                   c->final_info.error_string);
   c->final_info.stats.latency =
       gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time);
 
@@ -733,16 +736,15 @@ static void cancel_with_status(grpc_exec_ctx* exec_ctx, grpc_call* c,
  * FINAL STATUS CODE MANIPULATION
  */
 
-static bool get_final_status_from(grpc_exec_ctx* exec_ctx, grpc_call* call,
+static bool get_final_status_from(
-                                  grpc_error* error, bool allow_ok_status,
+    grpc_exec_ctx* exec_ctx, grpc_call* call, grpc_error* error,
-                                  void (*set_value)(grpc_status_code code,
+    bool allow_ok_status,
-                                                    void* user_data),
+    void (*set_value)(grpc_status_code code, void* user_data),
-                                  void* set_value_user_data,
+    void* set_value_user_data, grpc_slice* details, const char** error_string) {
-                                  grpc_slice* details) {
   grpc_status_code code;
   grpc_slice slice = grpc_empty_slice();
   grpc_error_get_status(exec_ctx, error, call->send_deadline, &code, &slice,
-                        nullptr);
+                        nullptr, error_string);
   if (code == GRPC_STATUS_OK && !allow_ok_status) {
     return false;
   }
@@ -757,7 +759,8 @@ static bool get_final_status_from(grpc_exec_ctx* exec_ctx, grpc_call* call,
 static void get_final_status(grpc_exec_ctx* exec_ctx, grpc_call* call,
                              void (*set_value)(grpc_status_code code,
                                                void* user_data),
-                             void* set_value_user_data, grpc_slice* details) {
+                             void* set_value_user_data, grpc_slice* details,
+                             const char** error_string) {
   int i;
   received_status status[STATUS_SOURCE_COUNT];
   for (i = 0; i < STATUS_SOURCE_COUNT; i++) {
@@ -781,7 +784,7 @@ static void get_final_status(grpc_exec_ctx* exec_ctx, grpc_call* call,
           grpc_error_has_clear_grpc_status(status[i].error)) {
         if (get_final_status_from(exec_ctx, call, status[i].error,
                                   allow_ok_status != 0, set_value,
-                                  set_value_user_data, details)) {
+                                  set_value_user_data, details, error_string)) {
           return;
         }
       }
@@ -791,7 +794,7 @@ static void get_final_status(grpc_exec_ctx* exec_ctx, grpc_call* call,
       if (status[i].is_set) {
         if (get_final_status_from(exec_ctx, call, status[i].error,
                                   allow_ok_status != 0, set_value,
-                                  set_value_user_data, details)) {
+                                  set_value_user_data, details, error_string)) {
           return;
         }
       }
@@ -1332,10 +1335,11 @@ static void post_batch_completion(grpc_exec_ctx* exec_ctx,
     if (call->is_client) {
       get_final_status(exec_ctx, call, set_status_value_directly,
                        call->final_op.client.status,
-                       call->final_op.client.status_details);
+                       call->final_op.client.status_details,
+                       call->final_op.client.error_string);
     } else {
       get_final_status(exec_ctx, call, set_cancelled_value,
-                       call->final_op.server.cancelled, nullptr);
+                       call->final_op.server.cancelled, nullptr, nullptr);
     }
 
     GRPC_ERROR_UNREF(error);
@@ -1992,6 +1996,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx* exec_ctx,
         call->final_op.client.status = op->data.recv_status_on_client.status;
         call->final_op.client.status_details =
             op->data.recv_status_on_client.status_details;
+        call->final_op.client.error_string =
+            op->data.recv_status_on_client.error_string;
         stream_op->recv_trailing_metadata = true;
         stream_op->collect_stats = true;
         stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =

src/core/lib/transport/error_utils.cc

@@ -18,6 +18,7 @@
 
 #include "src/core/lib/transport/error_utils.h"
 
+#include <grpc/support/string_util.h>
 #include "src/core/lib/iomgr/error_internal.h"
 #include "src/core/lib/transport/status_conversion.h"
 
@@ -41,8 +42,8 @@ static grpc_error* recursively_find_error_with_field(grpc_error* error,
 
 void grpc_error_get_status(grpc_exec_ctx* exec_ctx, grpc_error* error,
                            grpc_millis deadline, grpc_status_code* code,
-                           grpc_slice* slice,
+                           grpc_slice* slice, grpc_http2_error_code* http_error,
-                           grpc_http2_error_code* http_error) {
+                           const char** error_string) {
   // Start with the parent error and recurse through the tree of children
   // until we find the first one that has a status code.
   grpc_error* found_error =
@@ -69,6 +70,10 @@ void grpc_error_get_status(grpc_exec_ctx* exec_ctx, grpc_error* error,
   }
   if (code != nullptr) *code = status;
 
+  if (error_string != NULL && status != GRPC_STATUS_OK) {
+    *error_string = gpr_strdup(grpc_error_string(error));
+  }
+
   if (http_error != nullptr) {
     if (grpc_error_get_int(found_error, GRPC_ERROR_INT_HTTP2_ERROR, &integer)) {
       *http_error = (grpc_http2_error_code)integer;

src/core/lib/transport/error_utils.h

@@ -26,13 +26,15 @@
 /// A utility function to get the status code and message to be returned
 /// to the application.  If not set in the top-level message, looks
 /// through child errors until it finds the first one with these attributes.
-/// All attributes are pulled from the same child error. If any of the
+/// All attributes are pulled from the same child error. error_string will
-/// attributes (code, msg, http_status) are unneeded, they can be passed as
+/// be populated with the entire error string. If any of the attributes (code,
+/// msg, http_status, error_string) are unneeded, they can be passed as
 /// NULL.
 void grpc_error_get_status(grpc_exec_ctx* exec_ctx, grpc_error* error,
                            grpc_millis deadline, grpc_status_code* code,
                            grpc_slice* slice,
-                           grpc_http2_error_code* http_status);
+                           grpc_http2_error_code* http_status,
+                           const char** error_string);
 
 /// A utility function to check whether there is a clear status code that
 /// doesn't need to be guessed in \a error. This means that \a error or some

src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs

@@ -64,7 +64,7 @@ namespace Grpc.Core.Internal.Tests
         public void CancelNotificationAfterStartDisposes()
         {
             var finishedTask = asyncCallServer.ServerSideCallAsync();
-            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+            fakeCall.ReceivedCloseOnServerCallback.OnReceivedCloseOnServer(true, cancelled: true);
             AssertFinished(asyncCallServer, fakeCall, finishedTask);
         }
 
@@ -76,8 +76,8 @@ namespace Grpc.Core.Internal.Tests
 
             var moveNextTask = requestStream.MoveNext();
 
-            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+            fakeCall.ReceivedCloseOnServerCallback.OnReceivedCloseOnServer(true, cancelled: true);
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
             Assert.IsFalse(moveNextTask.Result);
 
             AssertFinished(asyncCallServer, fakeCall, finishedTask);
@@ -89,7 +89,7 @@ namespace Grpc.Core.Internal.Tests
             var finishedTask = asyncCallServer.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<string, string>(asyncCallServer);
 
-            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+            fakeCall.ReceivedCloseOnServerCallback.OnReceivedCloseOnServer(true, cancelled: true);
 
             // Check that starting a read after cancel notification has been processed is legal.
             var moveNextTask = requestStream.MoveNext();
@@ -107,10 +107,10 @@ namespace Grpc.Core.Internal.Tests
             // if a read completion's success==false, the request stream will silently finish
             // and we rely on C core cancelling the call.
             var moveNextTask = requestStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(false, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(false, null);
             Assert.IsFalse(moveNextTask.Result);
 
-            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+            fakeCall.ReceivedCloseOnServerCallback.OnReceivedCloseOnServer(true, cancelled: true);
             AssertFinished(asyncCallServer, fakeCall, finishedTask);
         }
 
@@ -120,7 +120,7 @@ namespace Grpc.Core.Internal.Tests
             var finishedTask = asyncCallServer.ServerSideCallAsync();
             var responseStream = new ServerResponseStream<string, string>(asyncCallServer);
 
-            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+            fakeCall.ReceivedCloseOnServerCallback.OnReceivedCloseOnServer(true, cancelled: true);
 
             // TODO(jtattermusch): should we throw a different exception type instead?
             Assert.Throws(typeof(InvalidOperationException), () => responseStream.WriteAsync("request1"));
@@ -134,10 +134,10 @@ namespace Grpc.Core.Internal.Tests
             var responseStream = new ServerResponseStream<string, string>(asyncCallServer);
 
             var writeTask = responseStream.WriteAsync("request1");
-            fakeCall.SendCompletionHandler(false);
+            fakeCall.SendCompletionCallback.OnSendCompletion(false);
             Assert.ThrowsAsync(typeof(IOException), async () => await writeTask);
 
-            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+            fakeCall.ReceivedCloseOnServerCallback.OnReceivedCloseOnServer(true, cancelled: true);
             AssertFinished(asyncCallServer, fakeCall, finishedTask);
         }
 
@@ -150,13 +150,13 @@ namespace Grpc.Core.Internal.Tests
             var writeTask = responseStream.WriteAsync("request1");
             var writeStatusTask = asyncCallServer.SendStatusFromServerAsync(Status.DefaultSuccess, new Metadata(), null);
 
-            fakeCall.SendCompletionHandler(true);
+            fakeCall.SendCompletionCallback.OnSendCompletion(true);
-            fakeCall.SendStatusFromServerHandler(true);
+            fakeCall.SendStatusFromServerCallback.OnSendStatusFromServerCompletion(true);
 
             Assert.DoesNotThrowAsync(async () => await writeTask);
             Assert.DoesNotThrowAsync(async () => await writeStatusTask);
 
-            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+            fakeCall.ReceivedCloseOnServerCallback.OnReceivedCloseOnServer(true, cancelled: true);
 
             AssertFinished(asyncCallServer, fakeCall, finishedTask);
         }
@@ -170,8 +170,8 @@ namespace Grpc.Core.Internal.Tests
             asyncCallServer.SendStatusFromServerAsync(Status.DefaultSuccess, new Metadata(), null);
             Assert.ThrowsAsync(typeof(InvalidOperationException), async () => await responseStream.WriteAsync("request1"));
 
-            fakeCall.SendStatusFromServerHandler(true);
+            fakeCall.SendStatusFromServerCallback.OnSendStatusFromServerCompletion(true);
-            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+            fakeCall.ReceivedCloseOnServerCallback.OnReceivedCloseOnServer(true, cancelled: true);
 
             AssertFinished(asyncCallServer, fakeCall, finishedTask);
         }

src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs

@@ -73,7 +73,7 @@ namespace Grpc.Core.Internal.Tests
         public void AsyncUnary_Success()
         {
             var resultTask = asyncCall.UnaryCallAsync("request1");
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 new ClientSideStatus(Status.DefaultSuccess, new Metadata()),
                 CreateResponsePayload(),
                 new Metadata());
@@ -85,7 +85,7 @@ namespace Grpc.Core.Internal.Tests
         public void AsyncUnary_NonSuccessStatusCode()
         {
             var resultTask = asyncCall.UnaryCallAsync("request1");
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 CreateClientSideStatus(StatusCode.InvalidArgument),
                 null,
                 new Metadata());
@@ -97,7 +97,7 @@ namespace Grpc.Core.Internal.Tests
         public void AsyncUnary_NullResponsePayload()
         {
             var resultTask = asyncCall.UnaryCallAsync("request1");
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 new ClientSideStatus(Status.DefaultSuccess, new Metadata()),
                 null,
                 new Metadata());
@@ -118,7 +118,7 @@ namespace Grpc.Core.Internal.Tests
         public void ClientStreaming_NoRequest_Success()
         {
             var resultTask = asyncCall.ClientStreamingCallAsync();
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 new ClientSideStatus(Status.DefaultSuccess, new Metadata()),
                 CreateResponsePayload(),
                 new Metadata());
@@ -130,7 +130,7 @@ namespace Grpc.Core.Internal.Tests
         public void ClientStreaming_NoRequest_NonSuccessStatusCode()
         {
             var resultTask = asyncCall.ClientStreamingCallAsync();
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 CreateClientSideStatus(StatusCode.InvalidArgument),
                 null,
                 new Metadata());
@@ -145,18 +145,18 @@ namespace Grpc.Core.Internal.Tests
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
 
             var writeTask = requestStream.WriteAsync("request1");
-            fakeCall.SendCompletionHandler(true);
+            fakeCall.SendCompletionCallback.OnSendCompletion(true);
             writeTask.Wait();
 
             var writeTask2 = requestStream.WriteAsync("request2");
-            fakeCall.SendCompletionHandler(true);
+            fakeCall.SendCompletionCallback.OnSendCompletion(true);
             writeTask2.Wait();
 
             var completeTask = requestStream.CompleteAsync();
-            fakeCall.SendCompletionHandler(true);
+            fakeCall.SendCompletionCallback.OnSendCompletion(true);
             completeTask.Wait();
 
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 new ClientSideStatus(Status.DefaultSuccess, new Metadata()),
                 CreateResponsePayload(),
                 new Metadata());
@@ -171,12 +171,12 @@ namespace Grpc.Core.Internal.Tests
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
 
             var writeTask = requestStream.WriteAsync("request1");
-            fakeCall.SendCompletionHandler(false);
+            fakeCall.SendCompletionCallback.OnSendCompletion(false);
 
             // The write will wait for call to finish to receive the status code.
             Assert.IsFalse(writeTask.IsCompleted);
 
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 CreateClientSideStatus(StatusCode.Internal),
                 null,
                 new Metadata());
@@ -195,12 +195,12 @@ namespace Grpc.Core.Internal.Tests
 
             var writeTask = requestStream.WriteAsync("request1");
 
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 CreateClientSideStatus(StatusCode.Internal),
                 null,
                 new Metadata());
 
-            fakeCall.SendCompletionHandler(false);
+            fakeCall.SendCompletionCallback.OnSendCompletion(false);
 
             var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(StatusCode.Internal, ex.Status.StatusCode);
@@ -215,13 +215,13 @@ namespace Grpc.Core.Internal.Tests
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
 
             var writeTask = requestStream.WriteAsync("request1");
-            fakeCall.SendCompletionHandler(false);
+            fakeCall.SendCompletionCallback.OnSendCompletion(false);
 
             // Until the delayed write completion has been triggered,
             // we still act as if there was an active write.
             Assert.Throws(typeof(InvalidOperationException), () => requestStream.WriteAsync("request2"));
 
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 CreateClientSideStatus(StatusCode.Internal),
                 null,
                 new Metadata());
@@ -242,7 +242,7 @@ namespace Grpc.Core.Internal.Tests
             var resultTask = asyncCall.ClientStreamingCallAsync();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
 
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 new ClientSideStatus(Status.DefaultSuccess, new Metadata()),
                 CreateResponsePayload(),
                 new Metadata());
@@ -260,7 +260,7 @@ namespace Grpc.Core.Internal.Tests
             var resultTask = asyncCall.ClientStreamingCallAsync();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
 
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 new ClientSideStatus(new Status(StatusCode.OutOfRange, ""), new Metadata()),
                 CreateResponsePayload(),
                 new Metadata());
@@ -282,9 +282,9 @@ namespace Grpc.Core.Internal.Tests
 
             Assert.Throws(typeof(InvalidOperationException), () => requestStream.WriteAsync("request1"));
 
-            fakeCall.SendCompletionHandler(true);
+            fakeCall.SendCompletionCallback.OnSendCompletion(true);
 
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 new ClientSideStatus(Status.DefaultSuccess, new Metadata()),
                 CreateResponsePayload(),
                 new Metadata());
@@ -298,7 +298,7 @@ namespace Grpc.Core.Internal.Tests
             var resultTask = asyncCall.ClientStreamingCallAsync();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
 
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 new ClientSideStatus(Status.DefaultSuccess, new Metadata()),
                 CreateResponsePayload(),
                 new Metadata());
@@ -319,7 +319,7 @@ namespace Grpc.Core.Internal.Tests
             var writeTask = requestStream.WriteAsync("request1");
             Assert.ThrowsAsync(typeof(TaskCanceledException), async () => await writeTask);
 
-            fakeCall.UnaryResponseClientHandler(true,
+            fakeCall.UnaryResponseClientCallback.OnUnaryResponseClient(true,
                 CreateClientSideStatus(StatusCode.Cancelled),
                 null,
                 new Metadata());
@@ -342,11 +342,11 @@ namespace Grpc.Core.Internal.Tests
             var responseStream = new ClientResponseStream<string, string>(asyncCall);
             var readTask = responseStream.MoveNext();
 
-            fakeCall.ReceivedResponseHeadersHandler(true, new Metadata());
+            fakeCall.ReceivedResponseHeadersCallback.OnReceivedResponseHeaders(true, new Metadata());
             Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count);
 
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
-            fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
 
             AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask);
         }
@@ -359,8 +359,8 @@ namespace Grpc.Core.Internal.Tests
             var readTask = responseStream.MoveNext();
 
             // try alternative order of completions
-            fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
 
             AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask);
         }
@@ -372,8 +372,8 @@ namespace Grpc.Core.Internal.Tests
             var responseStream = new ClientResponseStream<string, string>(asyncCall);
             var readTask = responseStream.MoveNext();
 
-            fakeCall.ReceivedMessageHandler(false, null);  // after a failed read, we rely on C core to deliver appropriate status code.
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(false, null);  // after a failed read, we rely on C core to deliver appropriate status code.
-            fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.Internal));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, CreateClientSideStatus(StatusCode.Internal));
 
             AssertStreamingResponseError(asyncCall, fakeCall, readTask, StatusCode.Internal);
         }
@@ -385,18 +385,18 @@ namespace Grpc.Core.Internal.Tests
             var responseStream = new ClientResponseStream<string, string>(asyncCall);
 
             var readTask1 = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, CreateResponsePayload());
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, CreateResponsePayload());
             Assert.IsTrue(readTask1.Result);
             Assert.AreEqual("response1", responseStream.Current);
 
             var readTask2 = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, CreateResponsePayload());
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, CreateResponsePayload());
             Assert.IsTrue(readTask2.Result);
             Assert.AreEqual("response1", responseStream.Current);
 
             var readTask3 = responseStream.MoveNext();
-            fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
 
             AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask3);
         }
@@ -409,12 +409,12 @@ namespace Grpc.Core.Internal.Tests
             var responseStream = new ClientResponseStream<string, string>(asyncCall);
 
             var writeTask1 = requestStream.CompleteAsync();
-            fakeCall.SendCompletionHandler(true);
+            fakeCall.SendCompletionCallback.OnSendCompletion(true);
             Assert.DoesNotThrowAsync(async () => await writeTask1);
 
             var readTask = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
-            fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
 
             AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask);
         }
@@ -427,8 +427,8 @@ namespace Grpc.Core.Internal.Tests
             var responseStream = new ClientResponseStream<string, string>(asyncCall);
 
             var readTask = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
-            fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
 
             AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask);
 
@@ -445,8 +445,8 @@ namespace Grpc.Core.Internal.Tests
             var responseStream = new ClientResponseStream<string, string>(asyncCall);
 
             var readTask = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
-            fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata()));
 
             AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask);
 
@@ -461,14 +461,14 @@ namespace Grpc.Core.Internal.Tests
             var responseStream = new ClientResponseStream<string, string>(asyncCall);
 
             var writeTask = requestStream.WriteAsync("request1");
-            fakeCall.SendCompletionHandler(false);
+            fakeCall.SendCompletionCallback.OnSendCompletion(false);
 
             // The write will wait for call to finish to receive the status code.
             Assert.IsFalse(writeTask.IsCompleted);
 
             var readTask = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
-            fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.PermissionDenied));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, CreateClientSideStatus(StatusCode.PermissionDenied));
 
             var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(StatusCode.PermissionDenied, ex.Status.StatusCode);
@@ -486,9 +486,9 @@ namespace Grpc.Core.Internal.Tests
             var writeTask = requestStream.WriteAsync("request1");
 
             var readTask = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
-            fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.PermissionDenied));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, CreateClientSideStatus(StatusCode.PermissionDenied));
-            fakeCall.SendCompletionHandler(false);
+            fakeCall.SendCompletionCallback.OnSendCompletion(false);
 
             var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(StatusCode.PermissionDenied, ex.Status.StatusCode);
@@ -510,8 +510,8 @@ namespace Grpc.Core.Internal.Tests
             Assert.ThrowsAsync(typeof(TaskCanceledException), async () => await writeTask);
 
             var readTask = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
-            fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.Cancelled));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, CreateClientSideStatus(StatusCode.Cancelled));
 
             AssertStreamingResponseError(asyncCall, fakeCall, readTask, StatusCode.Cancelled);
         }
@@ -526,13 +526,13 @@ namespace Grpc.Core.Internal.Tests
             Assert.IsTrue(fakeCall.IsCancelled);
 
             var readTask1 = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, CreateResponsePayload());
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, CreateResponsePayload());
             Assert.IsTrue(readTask1.Result);
             Assert.AreEqual("response1", responseStream.Current);
 
             var readTask2 = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
-            fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.Cancelled));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, CreateClientSideStatus(StatusCode.Cancelled));
 
             AssertStreamingResponseError(asyncCall, fakeCall, readTask2, StatusCode.Cancelled);
         }
@@ -547,13 +547,13 @@ namespace Grpc.Core.Internal.Tests
             asyncCall.Cancel();
             Assert.IsTrue(fakeCall.IsCancelled);
 
-            fakeCall.ReceivedMessageHandler(true, CreateResponsePayload());
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, CreateResponsePayload());
             Assert.IsTrue(readTask1.Result);
             Assert.AreEqual("response1", responseStream.Current);
 
             var readTask2 = responseStream.MoveNext();
-            fakeCall.ReceivedMessageHandler(true, null);
+            fakeCall.ReceivedMessageCallback.OnReceivedMessage(true, null);
-            fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.Cancelled));
+            fakeCall.ReceivedStatusOnClientCallback.OnReceivedStatusOnClient(true, CreateClientSideStatus(StatusCode.Cancelled));
 
             AssertStreamingResponseError(asyncCall, fakeCall, readTask2, StatusCode.Cancelled);
         }

src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs

@@ -31,43 +31,43 @@ namespace Grpc.Core.Internal.Tests
     /// </summary>
     internal class FakeNativeCall : INativeCall
     {
-        public UnaryResponseClientHandler UnaryResponseClientHandler
+        public IUnaryResponseClientCallback UnaryResponseClientCallback
         {
             get;
             set;
         }
 
-        public ReceivedStatusOnClientHandler ReceivedStatusOnClientHandler
+        public IReceivedStatusOnClientCallback ReceivedStatusOnClientCallback
         {
             get;
             set;
         }
 
-        public ReceivedMessageHandler ReceivedMessageHandler
+        public IReceivedMessageCallback ReceivedMessageCallback
         {
             get;
             set;
         }
 
-        public ReceivedResponseHeadersHandler ReceivedResponseHeadersHandler
+        public IReceivedResponseHeadersCallback ReceivedResponseHeadersCallback
         {
             get;
             set;
         }
 
-        public SendCompletionHandler SendCompletionHandler
+        public ISendCompletionCallback SendCompletionCallback
         {
             get;
             set;
         }
 
-        public SendCompletionHandler SendStatusFromServerHandler
+        public ISendStatusFromServerCompletionCallback SendStatusFromServerCallback
         {
             get;
             set;
         }
 
-        public ReceivedCloseOnServerHandler ReceivedCloseOnServerHandler
+        public IReceivedCloseOnServerCallback ReceivedCloseOnServerCallback
         {
             get;
             set;
@@ -100,9 +100,9 @@ namespace Grpc.Core.Internal.Tests
             return "PEER";
         }
 
-        public void StartUnary(UnaryResponseClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
+        public void StartUnary(IUnaryResponseClientCallback callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
-            UnaryResponseClientHandler = callback;
+            UnaryResponseClientCallback = callback;
         }
 
         public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
@@ -110,55 +110,55 @@ namespace Grpc.Core.Internal.Tests
             throw new NotImplementedException();
         }
 
-        public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
+        public void StartClientStreaming(IUnaryResponseClientCallback callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
-            UnaryResponseClientHandler = callback;
+            UnaryResponseClientCallback = callback;
         }
 
-        public void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
+        public void StartServerStreaming(IReceivedStatusOnClientCallback callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
-            ReceivedStatusOnClientHandler = callback;
+            ReceivedStatusOnClientCallback = callback;
         }
 
-        public void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
+        public void StartDuplexStreaming(IReceivedStatusOnClientCallback callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
-            ReceivedStatusOnClientHandler = callback;
+            ReceivedStatusOnClientCallback = callback;
         }
 
-        public void StartReceiveMessage(ReceivedMessageHandler callback)
+        public void StartReceiveMessage(IReceivedMessageCallback callback)
         {
-            ReceivedMessageHandler = callback;
+            ReceivedMessageCallback = callback;
         }
 
-        public void StartReceiveInitialMetadata(ReceivedResponseHeadersHandler callback)
+        public void StartReceiveInitialMetadata(IReceivedResponseHeadersCallback callback)
         {
-            ReceivedResponseHeadersHandler = callback;
+            ReceivedResponseHeadersCallback = callback;
         }
 
-        public void StartSendInitialMetadata(SendCompletionHandler callback, MetadataArraySafeHandle metadataArray)
+        public void StartSendInitialMetadata(ISendCompletionCallback callback, MetadataArraySafeHandle metadataArray)
         {
-            SendCompletionHandler = callback;
+            SendCompletionCallback = callback;
         }
 
-        public void StartSendMessage(SendCompletionHandler callback, byte[] payload, WriteFlags writeFlags, bool sendEmptyInitialMetadata)
+        public void StartSendMessage(ISendCompletionCallback callback, byte[] payload, WriteFlags writeFlags, bool sendEmptyInitialMetadata)
         {
-            SendCompletionHandler = callback;
+            SendCompletionCallback = callback;
         }
 
-        public void StartSendCloseFromClient(SendCompletionHandler callback)
+        public void StartSendCloseFromClient(ISendCompletionCallback callback)
         {
-            SendCompletionHandler = callback;
+            SendCompletionCallback = callback;
         }
 
-        public void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
+        public void StartSendStatusFromServer(ISendStatusFromServerCompletionCallback callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
             byte[] optionalPayload, WriteFlags writeFlags)
         {
-            SendStatusFromServerHandler = callback;
+            SendStatusFromServerCallback = callback;
         }
 
-        public void StartServerSide(ReceivedCloseOnServerHandler callback)
+        public void StartServerSide(IReceivedCloseOnServerCallback callback)
         {
-            ReceivedCloseOnServerHandler = callback;
+            ReceivedCloseOnServerCallback = callback;
         }
 
         public void Dispose()

src/csharp/Grpc.Core.Tests/PInvokeTest.cs

@@ -63,7 +63,7 @@ namespace Grpc.Core.Tests
         [Ignore("Prevent running on Jenkins")]
         public void NativeCallbackBenchmark()
         {
-            OpCompletionDelegate handler = Handler;
+            NativeCallbackTestDelegate handler = Handler;
 
             counter = 0;
             BenchmarkUtil.RunBenchmark(
@@ -91,7 +91,7 @@ namespace Grpc.Core.Tests
                 10000, 10000,
                 () =>
                 {
-                    Native.grpcsharp_test_callback(new OpCompletionDelegate(Handler));
+                    Native.grpcsharp_test_callback(new NativeCallbackTestDelegate(Handler));
                 });
             Assert.AreNotEqual(0, counter);
         }

src/csharp/Grpc.Core/Channel.cs

@@ -127,6 +127,20 @@ namespace Grpc.Core
             }
         }
 
+        // cached handler for watch connectivity state
+        static readonly BatchCompletionDelegate WatchConnectivityStateHandler = (success, ctx, state) =>
+        {
+            var tcs = (TaskCompletionSource<object>) state;
+            if (success)
+            {
+                tcs.SetResult(null);
+            }
+            else
+            {
+                tcs.SetCanceled();
+            }
+        };
+
         /// <summary>
         /// Returned tasks completes once channel state has become different from 
         /// given lastObservedState. 
@@ -138,18 +152,8 @@ namespace Grpc.Core
                 "Shutdown is a terminal state. No further state changes can occur.");
             var tcs = new TaskCompletionSource<object>();
             var deadlineTimespec = deadline.HasValue ? Timespec.FromDateTime(deadline.Value) : Timespec.InfFuture;
-            var handler = new BatchCompletionDelegate((success, ctx) =>
+            // pass "tcs" as "state" for WatchConnectivityStateHandler.
-            {
+            handle.WatchConnectivityState(lastObservedState, deadlineTimespec, completionQueue, WatchConnectivityStateHandler, tcs);
-                if (success)
-                {
-                    tcs.SetResult(null);
-                }
-                else
-                {
-                    tcs.SetCanceled();
-                }
-            });
-            handle.WatchConnectivityState(lastObservedState, deadlineTimespec, completionQueue, handler);
             return tcs.Task;
         }
 

src/csharp/Grpc.Core/Internal/AsyncCall.cs

@@ -27,7 +27,7 @@ namespace Grpc.Core.Internal
     /// <summary>
     /// Manages client side native call lifecycle.
     /// </summary>
-    internal class AsyncCall<TRequest, TResponse> : AsyncCallBase<TRequest, TResponse>
+    internal class AsyncCall<TRequest, TResponse> : AsyncCallBase<TRequest, TResponse>, IUnaryResponseClientCallback, IReceivedStatusOnClientCallback, IReceivedResponseHeadersCallback
     {
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<AsyncCall<TRequest, TResponse>>();
 
@@ -138,7 +138,7 @@ namespace Grpc.Core.Internal
                 unaryResponseTcs = new TaskCompletionSource<TResponse>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
-                    call.StartUnary(HandleUnaryResponse, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags);
+                    call.StartUnary(UnaryResponseClientCallback, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags);
                 }
                 return unaryResponseTcs.Task;
             }
@@ -162,7 +162,7 @@ namespace Grpc.Core.Internal
                 unaryResponseTcs = new TaskCompletionSource<TResponse>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
-                    call.StartClientStreaming(HandleUnaryResponse, metadataArray, details.Options.Flags);
+                    call.StartClientStreaming(UnaryResponseClientCallback, metadataArray, details.Options.Flags);
                 }
 
                 return unaryResponseTcs.Task;
@@ -188,9 +188,9 @@ namespace Grpc.Core.Internal
                 streamingResponseCallFinishedTcs = new TaskCompletionSource<object>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
-                    call.StartServerStreaming(HandleFinished, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags);
+                    call.StartServerStreaming(ReceivedStatusOnClientCallback, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags);
                 }
-                call.StartReceiveInitialMetadata(HandleReceivedResponseHeaders);
+                call.StartReceiveInitialMetadata(ReceivedResponseHeadersCallback);
             }
         }
 
@@ -210,9 +210,9 @@ namespace Grpc.Core.Internal
                 streamingResponseCallFinishedTcs = new TaskCompletionSource<object>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
-                    call.StartDuplexStreaming(HandleFinished, metadataArray, details.Options.Flags);
+                    call.StartDuplexStreaming(ReceivedStatusOnClientCallback, metadataArray, details.Options.Flags);
                 }
-                call.StartReceiveInitialMetadata(HandleReceivedResponseHeaders);
+                call.StartReceiveInitialMetadata(ReceivedResponseHeadersCallback);
             }
         }
 
@@ -256,7 +256,7 @@ namespace Grpc.Core.Internal
                     halfcloseRequested = true;
                     return TaskUtils.CompletedTask;
                 }
-                call.StartSendCloseFromClient(HandleSendFinished);
+                call.StartSendCloseFromClient(SendCompletionCallback);
 
                 halfcloseRequested = true;
                 streamingWriteTcs = new TaskCompletionSource<object>();
@@ -516,5 +516,26 @@ namespace Grpc.Core.Internal
 
             streamingResponseCallFinishedTcs.SetResult(null);
         }
+
+        IUnaryResponseClientCallback UnaryResponseClientCallback => this;
+
+        void IUnaryResponseClientCallback.OnUnaryResponseClient(bool success, ClientSideStatus receivedStatus, byte[] receivedMessage, Metadata responseHeaders)
+        {
+            HandleUnaryResponse(success, receivedStatus, receivedMessage, responseHeaders);
+        }
+
+        IReceivedStatusOnClientCallback ReceivedStatusOnClientCallback => this;
+
+        void IReceivedStatusOnClientCallback.OnReceivedStatusOnClient(bool success, ClientSideStatus receivedStatus)
+        {
+            HandleFinished(success, receivedStatus);
+        }
+
+        IReceivedResponseHeadersCallback ReceivedResponseHeadersCallback => this;
+
+        void IReceivedResponseHeadersCallback.OnReceivedResponseHeaders(bool success, Metadata responseHeaders)
+        {
+            HandleReceivedResponseHeaders(success, responseHeaders);
+        }
     }
 }

src/csharp/Grpc.Core/Internal/AsyncCallBase.cs

@@ -35,7 +35,7 @@ namespace Grpc.Core.Internal
     /// Base for handling both client side and server side calls.
     /// Manages native call lifecycle and provides convenience methods.
     /// </summary>
-    internal abstract class AsyncCallBase<TWrite, TRead>
+    internal abstract class AsyncCallBase<TWrite, TRead> : IReceivedMessageCallback, ISendCompletionCallback
     {
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<AsyncCallBase<TWrite, TRead>>();
         protected static readonly Status DeserializeResponseFailureStatus = new Status(StatusCode.Internal, "Failed to deserialize response message.");
@@ -126,7 +126,7 @@ namespace Grpc.Core.Internal
                     return earlyResult;
                 }
 
-                call.StartSendMessage(HandleSendFinished, payload, writeFlags, !initialMetadataSent);
+                call.StartSendMessage(SendCompletionCallback, payload, writeFlags, !initialMetadataSent);
 
                 initialMetadataSent = true;
                 streamingWritesCounter++;
@@ -154,7 +154,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(streamingReadTcs == null, "Only one read can be pending at a time");
                 GrpcPreconditions.CheckState(!disposed);
 
-                call.StartReceiveMessage(HandleReadFinished);
+                call.StartReceiveMessage(ReceivedMessageCallback);
                 streamingReadTcs = new TaskCompletionSource<TRead>();
                 return streamingReadTcs.Task;
             }
@@ -342,5 +342,19 @@ namespace Grpc.Core.Internal
             }
             origTcs.SetResult(msg);
         }
+
+        protected ISendCompletionCallback SendCompletionCallback => this;
+
+        void ISendCompletionCallback.OnSendCompletion(bool success)
+        {
+            HandleSendFinished(success);
+        }
+
+        IReceivedMessageCallback ReceivedMessageCallback => this;
+
+        void IReceivedMessageCallback.OnReceivedMessage(bool success, byte[] receivedMessage)
+        {
+            HandleReadFinished(success, receivedMessage);
+        }
     }
 }

src/csharp/Grpc.Core/Internal/AsyncCallServer.cs

@@ -31,7 +31,7 @@ namespace Grpc.Core.Internal
     /// <summary>
     /// Manages server side native call lifecycle.
     /// </summary>
-    internal class AsyncCallServer<TRequest, TResponse> : AsyncCallBase<TResponse, TRequest>
+    internal class AsyncCallServer<TRequest, TResponse> : AsyncCallBase<TResponse, TRequest>, IReceivedCloseOnServerCallback, ISendStatusFromServerCompletionCallback
     {
         readonly TaskCompletionSource<object> finishedServersideTcs = new TaskCompletionSource<object>();
         readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
@@ -70,7 +70,7 @@ namespace Grpc.Core.Internal
 
                 started = true;
 
-                call.StartServerSide(HandleFinishedServerside);
+                call.StartServerSide(ReceiveCloseOnServerCallback);
                 return finishedServersideTcs.Task;
             }
         }
@@ -114,7 +114,7 @@ namespace Grpc.Core.Internal
 
                 using (var metadataArray = MetadataArraySafeHandle.Create(headers))
                 {
-                    call.StartSendInitialMetadata(HandleSendFinished, metadataArray);
+                    call.StartSendInitialMetadata(SendCompletionCallback, metadataArray);
                 }
 
                 this.initialMetadataSent = true;
@@ -127,10 +127,10 @@ namespace Grpc.Core.Internal
         /// Sends call result status, indicating we are done with writes.
         /// Sending a status different from StatusCode.OK will also implicitly cancel the call.
         /// </summary>
-        public Task SendStatusFromServerAsync(Status status, Metadata trailers, Tuple<TResponse, WriteFlags> optionalWrite)
+        public Task SendStatusFromServerAsync(Status status, Metadata trailers, ResponseWithFlags? optionalWrite)
         {
-            byte[] payload = optionalWrite != null ? UnsafeSerialize(optionalWrite.Item1) : null;
+            byte[] payload = optionalWrite.HasValue ? UnsafeSerialize(optionalWrite.Value.Response) : null;
-            var writeFlags = optionalWrite != null ? optionalWrite.Item2 : default(WriteFlags);
+            var writeFlags = optionalWrite.HasValue ? optionalWrite.Value.WriteFlags : default(WriteFlags);
 
             lock (myLock)
             {
@@ -140,13 +140,13 @@ namespace Grpc.Core.Internal
 
                 using (var metadataArray = MetadataArraySafeHandle.Create(trailers))
                 {
-                    call.StartSendStatusFromServer(HandleSendStatusFromServerFinished, status, metadataArray, !initialMetadataSent,
+                    call.StartSendStatusFromServer(SendStatusFromServerCompletionCallback, status, metadataArray, !initialMetadataSent,
                         payload, writeFlags);
                 }
                 halfcloseRequested = true;
                 initialMetadataSent = true;
                 sendStatusFromServerTcs = new TaskCompletionSource<object>();
-                if (optionalWrite != null)
+                if (optionalWrite.HasValue)
                 {
                     streamingWritesCounter++;
                 }
@@ -227,5 +227,31 @@ namespace Grpc.Core.Internal
 
             finishedServersideTcs.SetResult(null);
         }
+
+        IReceivedCloseOnServerCallback ReceiveCloseOnServerCallback => this;
+
+        void IReceivedCloseOnServerCallback.OnReceivedCloseOnServer(bool success, bool cancelled)
+        {
+            HandleFinishedServerside(success, cancelled);
+        }
+
+        ISendStatusFromServerCompletionCallback SendStatusFromServerCompletionCallback => this;
+
+        void ISendStatusFromServerCompletionCallback.OnSendStatusFromServerCompletion(bool success)
+        {
+            HandleSendStatusFromServerFinished(success);
+        }
+
+        public struct ResponseWithFlags
+        {
+            public ResponseWithFlags(TResponse response, WriteFlags writeFlags)
+            {
+                this.Response = response;
+                this.WriteFlags = writeFlags;
+            }
+
+            public TResponse Response { get; }
+            public WriteFlags WriteFlags { get; }
+        }
     }
 }

src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs

@@ -20,15 +20,25 @@ using System;
 using System.Runtime.InteropServices;
 using System.Text;
 using Grpc.Core;
+using Grpc.Core.Logging;
+using Grpc.Core.Utils;
 
 namespace Grpc.Core.Internal
 {
+    internal interface IOpCompletionCallback
+    {
+        void OnComplete(bool success);
+    }
+
     /// <summary>
     /// grpcsharp_batch_context
     /// </summary>
-    internal class BatchContextSafeHandle : SafeHandleZeroIsInvalid
+    internal class BatchContextSafeHandle : SafeHandleZeroIsInvalid, IOpCompletionCallback
     {
         static readonly NativeMethods Native = NativeMethods.Get();
+        static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<BatchContextSafeHandle>();
+
+        CompletionCallbackData completionCallbackData;
 
         private BatchContextSafeHandle()
         {
@@ -47,19 +57,26 @@ namespace Grpc.Core.Internal
             }
         }
 
+        public void SetCompletionCallback(BatchCompletionDelegate callback, object state)
+        {
+            GrpcPreconditions.CheckState(completionCallbackData.Callback == null);
+            GrpcPreconditions.CheckNotNull(callback, nameof(callback));
+            completionCallbackData = new CompletionCallbackData(callback, state);
+        }
+
         // Gets data of recv_initial_metadata completion.
         public Metadata GetReceivedInitialMetadata()
         {
             IntPtr metadataArrayPtr = Native.grpcsharp_batch_context_recv_initial_metadata(this);
             return MetadataArraySafeHandle.ReadMetadataFromPtrUnsafe(metadataArrayPtr);
         }
-            
+
         // Gets data of recv_status_on_client completion.
         public ClientSideStatus GetReceivedStatusOnClient()
         {
             UIntPtr detailsLength;
             IntPtr detailsPtr = Native.grpcsharp_batch_context_recv_status_on_client_details(this, out detailsLength);
-            string details = MarshalUtils.PtrToStringUTF8(detailsPtr, (int) detailsLength.ToUInt32());
+            string details = MarshalUtils.PtrToStringUTF8(detailsPtr, (int)detailsLength.ToUInt32());
             var status = new Status(Native.grpcsharp_batch_context_recv_status_on_client_status(this), details);
 
             IntPtr metadataArrayPtr = Native.grpcsharp_batch_context_recv_status_on_client_trailing_metadata(this);
@@ -86,11 +103,40 @@ namespace Grpc.Core.Internal
         {
             return Native.grpcsharp_batch_context_recv_close_on_server_cancelled(this) != 0;
         }
-            
+
         protected override bool ReleaseHandle()
         {
             Native.grpcsharp_batch_context_destroy(handle);
             return true;
         }
+
+        void IOpCompletionCallback.OnComplete(bool success)
+        {
+            try
+            {
+                completionCallbackData.Callback(success, this, completionCallbackData.State);
+            }
+            catch (Exception e)
+            {
+                Logger.Error(e, "Exception occured while invoking batch completion delegate.");
+            }
+            finally
+            {
+                completionCallbackData = default(CompletionCallbackData);
+                Dispose();
+            }
+        }
+
+        struct CompletionCallbackData
+        {
+            public CompletionCallbackData(BatchCompletionDelegate callback, object state)
+            {
+                this.Callback = callback;
+                this.State = state;
+            }
+
+            public BatchCompletionDelegate Callback { get; }
+            public object State { get; }
+        }
     }
 }

src/csharp/Grpc.Core/Internal/CallSafeHandle.cs

@@ -32,6 +32,23 @@ namespace Grpc.Core.Internal
         public static readonly CallSafeHandle NullInstance = new CallSafeHandle();
         static readonly NativeMethods Native = NativeMethods.Get();
 
+        // Completion handlers are pre-allocated to avoid unneccessary delegate allocations.
+        // The "state" field is used to store the actual callback to invoke.
+        static readonly BatchCompletionDelegate CompletionHandler_IUnaryResponseClientCallback =
+            (success, context, state) => ((IUnaryResponseClientCallback)state).OnUnaryResponseClient(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata());
+        static readonly BatchCompletionDelegate CompletionHandler_IReceivedStatusOnClientCallback =
+            (success, context, state) => ((IReceivedStatusOnClientCallback)state).OnReceivedStatusOnClient(success, context.GetReceivedStatusOnClient());
+        static readonly BatchCompletionDelegate CompletionHandler_IReceivedMessageCallback =
+            (success, context, state) => ((IReceivedMessageCallback)state).OnReceivedMessage(success, context.GetReceivedMessage());
+        static readonly BatchCompletionDelegate CompletionHandler_IReceivedResponseHeadersCallback =
+            (success, context, state) => ((IReceivedResponseHeadersCallback)state).OnReceivedResponseHeaders(success, context.GetReceivedInitialMetadata());
+        static readonly BatchCompletionDelegate CompletionHandler_ISendCompletionCallback =
+            (success, context, state) => ((ISendCompletionCallback)state).OnSendCompletion(success);
+        static readonly BatchCompletionDelegate CompletionHandler_ISendStatusFromServerCompletionCallback =
+            (success, context, state) => ((ISendStatusFromServerCompletionCallback)state).OnSendStatusFromServerCompletion(success);
+        static readonly BatchCompletionDelegate CompletionHandler_IReceivedCloseOnServerCallback =
+            (success, context, state) => ((IReceivedCloseOnServerCallback)state).OnReceivedCloseOnServer(success, context.GetReceivedCloseOnServerCancelled());
+
         const uint GRPC_WRITE_BUFFER_HINT = 1;
         CompletionQueueSafeHandle completionQueue;
 
@@ -49,12 +66,12 @@ namespace Grpc.Core.Internal
             Native.grpcsharp_call_set_credentials(this, credentials).CheckOk();
         }
 
-        public void StartUnary(UnaryResponseClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
+        public void StartUnary(IUnaryResponseClientCallback callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_IUnaryResponseClientCallback, callback);
                 Native.grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), writeFlags, metadataArray, callFlags)
                     .CheckOk();
             }
@@ -66,106 +83,106 @@ namespace Grpc.Core.Internal
                 .CheckOk();
         }
 
-        public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
+        public void StartClientStreaming(IUnaryResponseClientCallback callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_IUnaryResponseClientCallback, callback);
                 Native.grpcsharp_call_start_client_streaming(this, ctx, metadataArray, callFlags).CheckOk();
             }
         }
 
-        public void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
+        public void StartServerStreaming(IReceivedStatusOnClientCallback callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_IReceivedStatusOnClientCallback, callback);
                 Native.grpcsharp_call_start_server_streaming(this, ctx, payload, new UIntPtr((ulong)payload.Length), writeFlags, metadataArray, callFlags).CheckOk();
             }
         }
 
-        public void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
+        public void StartDuplexStreaming(IReceivedStatusOnClientCallback callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_IReceivedStatusOnClientCallback, callback);
                 Native.grpcsharp_call_start_duplex_streaming(this, ctx, metadataArray, callFlags).CheckOk();
             }
         }
 
-        public void StartSendMessage(SendCompletionHandler callback, byte[] payload, WriteFlags writeFlags, bool sendEmptyInitialMetadata)
+        public void StartSendMessage(ISendCompletionCallback callback, byte[] payload, WriteFlags writeFlags, bool sendEmptyInitialMetadata)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_ISendCompletionCallback, callback);
                 Native.grpcsharp_call_send_message(this, ctx, payload, new UIntPtr((ulong)payload.Length), writeFlags, sendEmptyInitialMetadata ? 1 : 0).CheckOk();
             }
         }
 
-        public void StartSendCloseFromClient(SendCompletionHandler callback)
+        public void StartSendCloseFromClient(ISendCompletionCallback callback)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_ISendCompletionCallback, callback);
                 Native.grpcsharp_call_send_close_from_client(this, ctx).CheckOk();
             }
         }
 
-        public void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
+        public void StartSendStatusFromServer(ISendStatusFromServerCompletionCallback callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
             byte[] optionalPayload, WriteFlags writeFlags)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
                 var optionalPayloadLength = optionalPayload != null ? new UIntPtr((ulong)optionalPayload.Length) : UIntPtr.Zero;
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_ISendStatusFromServerCompletionCallback, callback);
                 var statusDetailBytes = MarshalUtils.GetBytesUTF8(status.Detail);
                 Native.grpcsharp_call_send_status_from_server(this, ctx, status.StatusCode, statusDetailBytes, new UIntPtr((ulong)statusDetailBytes.Length), metadataArray, sendEmptyInitialMetadata ? 1 : 0,
                     optionalPayload, optionalPayloadLength, writeFlags).CheckOk();
             }
         }
 
-        public void StartReceiveMessage(ReceivedMessageHandler callback)
+        public void StartReceiveMessage(IReceivedMessageCallback callback)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedMessage()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_IReceivedMessageCallback, callback);
                 Native.grpcsharp_call_recv_message(this, ctx).CheckOk();
             }
         }
 
-        public void StartReceiveInitialMetadata(ReceivedResponseHeadersHandler callback)
+        public void StartReceiveInitialMetadata(IReceivedResponseHeadersCallback callback)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_IReceivedResponseHeadersCallback, callback);
                 Native.grpcsharp_call_recv_initial_metadata(this, ctx).CheckOk();
             }
         }
 
-        public void StartServerSide(ReceivedCloseOnServerHandler callback)
+        public void StartServerSide(IReceivedCloseOnServerCallback callback)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedCloseOnServerCancelled()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_IReceivedCloseOnServerCallback, callback);
                 Native.grpcsharp_call_start_serverside(this, ctx).CheckOk();
             }
         }
 
-        public void StartSendInitialMetadata(SendCompletionHandler callback, MetadataArraySafeHandle metadataArray)
+        public void StartSendInitialMetadata(ISendCompletionCallback callback, MetadataArraySafeHandle metadataArray)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, CompletionHandler_ISendCompletionCallback, callback);
                 Native.grpcsharp_call_send_initial_metadata(this, ctx, metadataArray).CheckOk();
             }
         }

src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs

@@ -64,10 +64,10 @@ namespace Grpc.Core.Internal
             return Native.grpcsharp_channel_check_connectivity_state(this, tryToConnect ? 1 : 0);
         }
 
-        public void WatchConnectivityState(ChannelState lastObservedState, Timespec deadline, CompletionQueueSafeHandle cq, BatchCompletionDelegate callback)
+        public void WatchConnectivityState(ChannelState lastObservedState, Timespec deadline, CompletionQueueSafeHandle cq, BatchCompletionDelegate callback, object callbackState)
         {
             var ctx = BatchContextSafeHandle.Create();
-            cq.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
+            cq.CompletionRegistry.RegisterBatchCompletion(ctx, callback, callbackState);
             Native.grpcsharp_channel_watch_connectivity_state(this, lastObservedState, deadline, cq, ctx);
         }
 

src/csharp/Grpc.Core/Internal/CompletionRegistry.cs

@@ -19,15 +19,15 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Runtime.InteropServices;
+using System.Threading;
 using Grpc.Core.Logging;
 using Grpc.Core.Utils;
 
 namespace Grpc.Core.Internal
 {
-    internal delegate void OpCompletionDelegate(bool success);
+    internal delegate void BatchCompletionDelegate(bool success, BatchContextSafeHandle ctx, object state);
-
-    internal delegate void BatchCompletionDelegate(bool success, BatchContextSafeHandle ctx);
 
     internal delegate void RequestCallCompletionDelegate(bool success, RequestCallContextSafeHandle ctx);
 
@@ -36,8 +36,8 @@ namespace Grpc.Core.Internal
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<CompletionRegistry>();
 
         readonly GrpcEnvironment environment;
-        readonly Dictionary<IntPtr, OpCompletionDelegate> dict = new Dictionary<IntPtr, OpCompletionDelegate>(new IntPtrComparer());
+        readonly Dictionary<IntPtr, IOpCompletionCallback> dict = new Dictionary<IntPtr, IOpCompletionCallback>(new IntPtrComparer());
-        readonly object myLock = new object();
+        SpinLock spinLock = new SpinLock(Debugger.IsAttached);
         IntPtr lastRegisteredKey;  // only for testing
 
         public CompletionRegistry(GrpcEnvironment environment)
@@ -45,38 +45,51 @@ namespace Grpc.Core.Internal
             this.environment = environment;
         }
 
-        public void Register(IntPtr key, OpCompletionDelegate callback)
+        public void Register(IntPtr key, IOpCompletionCallback callback)
         {
             environment.DebugStats.PendingBatchCompletions.Increment();
-            lock (myLock)
+
+            bool lockTaken = false;
+            try
             {
+                spinLock.Enter(ref lockTaken);
+
                 dict.Add(key, callback);
                 this.lastRegisteredKey = key;
             }
+            finally
+            {
+                if (lockTaken) spinLock.Exit();
+            }
         }
 
-        public void RegisterBatchCompletion(BatchContextSafeHandle ctx, BatchCompletionDelegate callback)
+        public void RegisterBatchCompletion(BatchContextSafeHandle ctx, BatchCompletionDelegate callback, object state)
         {
-            // TODO(jtattermusch): get rid of new delegate creation here
+            ctx.SetCompletionCallback(callback, state);
-            OpCompletionDelegate opCallback = ((success) => HandleBatchCompletion(success, ctx, callback));
+            Register(ctx.Handle, ctx);
-            Register(ctx.Handle, opCallback);
         }
 
         public void RegisterRequestCallCompletion(RequestCallContextSafeHandle ctx, RequestCallCompletionDelegate callback)
         {
-            // TODO(jtattermusch): get rid of new delegate creation here
+            ctx.CompletionCallback = callback;
-            OpCompletionDelegate opCallback = ((success) => HandleRequestCallCompletion(success, ctx, callback));
+            Register(ctx.Handle, ctx);
-            Register(ctx.Handle, opCallback);
         }
 
-        public OpCompletionDelegate Extract(IntPtr key)
+        public IOpCompletionCallback Extract(IntPtr key)
         {
-            OpCompletionDelegate value = null;
+            IOpCompletionCallback value = null;
-            lock (myLock)
+            bool lockTaken = false;
+            try
             {
+                spinLock.Enter(ref lockTaken);
+
                 value = dict[key];
                 dict.Remove(key);
             }
+            finally
+            {
+                if (lockTaken) spinLock.Exit();
+            }
             environment.DebugStats.PendingBatchCompletions.Decrement();
             return value;
         }
@@ -89,44 +102,6 @@ namespace Grpc.Core.Internal
             get { return this.lastRegisteredKey; }
         }
 
-        private static void HandleBatchCompletion(bool success, BatchContextSafeHandle ctx, BatchCompletionDelegate callback)
-        {
-            try
-            {
-                callback(success, ctx);
-            }
-            catch (Exception e)
-            {
-                Logger.Error(e, "Exception occured while invoking batch completion delegate.");
-            }
-            finally
-            {
-                if (ctx != null)
-                {
-                    ctx.Dispose();
-                }
-            }
-        }
-
-        private static void HandleRequestCallCompletion(bool success, RequestCallContextSafeHandle ctx, RequestCallCompletionDelegate callback)
-        {
-            try
-            {
-                callback(success, ctx);
-            }
-            catch (Exception e)
-            {
-                Logger.Error(e, "Exception occured while invoking request call completion delegate.");
-            }
-            finally
-            {
-                if (ctx != null)
-                {
-                    ctx.Dispose();
-                }
-            }
-        }
-
         /// <summary>
         /// IntPtr doesn't implement <c>IEquatable{IntPtr}</c> so we need to use custom comparer to avoid boxing.
         /// </summary>

src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs

@@ -68,8 +68,8 @@ namespace Grpc.Core.Internal
             GrpcPreconditions.CheckArgument(poolSize >= completionQueueCount,
                 "Thread pool size cannot be smaller than the number of completion queues used.");
 
-            this.runCompletionQueueEventCallbackSuccess = new WaitCallback((callback) => RunCompletionQueueEventCallback((OpCompletionDelegate) callback, true));
+            this.runCompletionQueueEventCallbackSuccess = new WaitCallback((callback) => RunCompletionQueueEventCallback((IOpCompletionCallback) callback, true));
-            this.runCompletionQueueEventCallbackFailure = new WaitCallback((callback) => RunCompletionQueueEventCallback((OpCompletionDelegate) callback, false));
+            this.runCompletionQueueEventCallbackFailure = new WaitCallback((callback) => RunCompletionQueueEventCallback((IOpCompletionCallback) callback, false));
         }
 
         public void Start()
@@ -225,11 +225,11 @@ namespace Grpc.Core.Internal
             return list.AsReadOnly();
         }
 
-        private void RunCompletionQueueEventCallback(OpCompletionDelegate callback, bool success)
+        private void RunCompletionQueueEventCallback(IOpCompletionCallback callback, bool success)
         {
             try
             {
-                callback(success);
+                callback.OnComplete(success);
             }
             catch (Exception e)
             {

src/csharp/Grpc.Core/Internal/INativeCall.cs

@@ -20,18 +20,41 @@ using Grpc.Core;
 
 namespace Grpc.Core.Internal
 {
-    internal delegate void UnaryResponseClientHandler(bool success, ClientSideStatus receivedStatus, byte[] receivedMessage, Metadata responseHeaders);
+    internal interface IUnaryResponseClientCallback
+    {
+        void OnUnaryResponseClient(bool success, ClientSideStatus receivedStatus, byte[] receivedMessage, Metadata responseHeaders);
+    }
 
     // Received status for streaming response calls.
-    internal delegate void ReceivedStatusOnClientHandler(bool success, ClientSideStatus receivedStatus);
+    internal interface IReceivedStatusOnClientCallback
+    {
+        void OnReceivedStatusOnClient(bool success, ClientSideStatus receivedStatus);
+    }
 
-    internal delegate void ReceivedMessageHandler(bool success, byte[] receivedMessage);
+    internal interface IReceivedMessageCallback
+    {
+        void OnReceivedMessage(bool success, byte[] receivedMessage);
+    }
 
-    internal delegate void ReceivedResponseHeadersHandler(bool success, Metadata responseHeaders);
+    internal interface IReceivedResponseHeadersCallback
+    {
+        void OnReceivedResponseHeaders(bool success, Metadata responseHeaders);
+    }
 
-    internal delegate void SendCompletionHandler(bool success);
+    internal interface ISendCompletionCallback
+    {
+        void OnSendCompletion(bool success);
+    }
 
-    internal delegate void ReceivedCloseOnServerHandler(bool success, bool cancelled);
+    internal interface ISendStatusFromServerCompletionCallback
+    {
+        void OnSendStatusFromServerCompletion(bool success);
+    }
+
+    internal interface IReceivedCloseOnServerCallback
+    {
+        void OnReceivedCloseOnServer(bool success, bool cancelled);
+    }
 
     /// <summary>
     /// Abstraction of a native call object.
@@ -44,28 +67,28 @@ namespace Grpc.Core.Internal
 
         string GetPeer();
 
-        void StartUnary(UnaryResponseClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
+        void StartUnary(IUnaryResponseClientCallback callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
         void StartUnary(BatchContextSafeHandle ctx, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
-        void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
+        void StartClientStreaming(IUnaryResponseClientCallback callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
-        void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
+        void StartServerStreaming(IReceivedStatusOnClientCallback callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
-        void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
+        void StartDuplexStreaming(IReceivedStatusOnClientCallback callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
-        void StartReceiveMessage(ReceivedMessageHandler callback);
+        void StartReceiveMessage(IReceivedMessageCallback callback);
 
-        void StartReceiveInitialMetadata(ReceivedResponseHeadersHandler callback);
+        void StartReceiveInitialMetadata(IReceivedResponseHeadersCallback callback);
 
-        void StartSendInitialMetadata(SendCompletionHandler callback, MetadataArraySafeHandle metadataArray);
+        void StartSendInitialMetadata(ISendCompletionCallback callback, MetadataArraySafeHandle metadataArray);
 
-        void StartSendMessage(SendCompletionHandler callback, byte[] payload, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
+        void StartSendMessage(ISendCompletionCallback callback, byte[] payload, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
 
-        void StartSendCloseFromClient(SendCompletionHandler callback);
+        void StartSendCloseFromClient(ISendCompletionCallback callback);
 
-        void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata, byte[] optionalPayload, WriteFlags writeFlags);
+        void StartSendStatusFromServer(ISendStatusFromServerCompletionCallback callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata, byte[] optionalPayload, WriteFlags writeFlags);
 
-        void StartServerSide(ReceivedCloseOnServerHandler callback);
+        void StartServerSide(IReceivedCloseOnServerCallback callback);
     }
 }

src/csharp/Grpc.Core/Internal/NativeMethods.cs

@@ -29,6 +29,8 @@ using Grpc.Core.Utils;
 
 namespace Grpc.Core.Internal
 {
+    internal delegate void NativeCallbackTestDelegate(bool success);
+
     /// <summary>
     /// Provides access to all native methods provided by <c>NativeExtension</c>.
     /// An extra level of indirection is added to P/Invoke calls to allow intelligent loading
@@ -420,7 +422,7 @@ namespace Grpc.Core.Internal
             public delegate Timespec gprsharp_convert_clock_type_delegate(Timespec t, ClockType targetClock);
             public delegate int gprsharp_sizeof_timespec_delegate();
 
-            public delegate CallError grpcsharp_test_callback_delegate([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
+            public delegate CallError grpcsharp_test_callback_delegate([MarshalAs(UnmanagedType.FunctionPtr)] NativeCallbackTestDelegate callback);
             public delegate IntPtr grpcsharp_test_nop_delegate(IntPtr ptr);
             public delegate void grpcsharp_test_override_method_delegate(string methodName, string variant);
         }

src/csharp/Grpc.Core/Internal/RequestCallContextSafeHandle.cs

@@ -19,15 +19,17 @@
 using System;
 using System.Runtime.InteropServices;
 using Grpc.Core;
+using Grpc.Core.Logging;
 
 namespace Grpc.Core.Internal
 {
     /// <summary>
     /// grpcsharp_request_call_context
     /// </summary>
-    internal class RequestCallContextSafeHandle : SafeHandleZeroIsInvalid
+    internal class RequestCallContextSafeHandle : SafeHandleZeroIsInvalid, IOpCompletionCallback
     {
         static readonly NativeMethods Native = NativeMethods.Get();
+        static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<RequestCallContextSafeHandle>();
 
         private RequestCallContextSafeHandle()
         {
@@ -46,6 +48,8 @@ namespace Grpc.Core.Internal
             }
         }
 
+        public RequestCallCompletionDelegate CompletionCallback { get; set; }
+
         // Gets data of server_rpc_new completion.
         public ServerRpcNew GetServerRpcNew(Server server)
         {
@@ -72,5 +76,22 @@ namespace Grpc.Core.Internal
             Native.grpcsharp_request_call_context_destroy(handle);
             return true;
         }
+
+        void IOpCompletionCallback.OnComplete(bool success)
+        {
+            try
+            {
+                CompletionCallback(success, this);
+            }
+            catch (Exception e)
+            {
+                Logger.Error(e, "Exception occured while invoking request call completion delegate.");
+            }
+            finally
+            {
+                CompletionCallback = null;
+                Dispose();
+            }
+        }
     }
 }

src/csharp/Grpc.Core/Internal/ServerCallHandler.cs

@@ -60,7 +60,7 @@ namespace Grpc.Core.Internal
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
 
             Status status;
-            Tuple<TResponse,WriteFlags> responseTuple = null;
+            AsyncCallServer<TRequest,TResponse>.ResponseWithFlags? responseWithFlags = null;
             var context = HandlerUtils.NewContext(newRpc, responseStream, asyncCall.CancellationToken);
             try
             {
@@ -68,7 +68,7 @@ namespace Grpc.Core.Internal
                 var request = requestStream.Current;
                 var response = await handler(request, context).ConfigureAwait(false);
                 status = context.Status;
-                responseTuple = Tuple.Create(response, HandlerUtils.GetWriteFlags(context.WriteOptions));
+                responseWithFlags = new AsyncCallServer<TRequest, TResponse>.ResponseWithFlags(response, HandlerUtils.GetWriteFlags(context.WriteOptions));
             } 
             catch (Exception e)
             {
@@ -80,7 +80,7 @@ namespace Grpc.Core.Internal
             }
             try
             {
-                await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, responseTuple).ConfigureAwait(false);
+                await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, responseWithFlags).ConfigureAwait(false);
             }
             catch (Exception)
             {
@@ -177,13 +177,13 @@ namespace Grpc.Core.Internal
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
 
             Status status;
-            Tuple<TResponse,WriteFlags> responseTuple = null;
+            AsyncCallServer<TRequest, TResponse>.ResponseWithFlags? responseWithFlags = null;
             var context = HandlerUtils.NewContext(newRpc, responseStream, asyncCall.CancellationToken);
             try
             {
                 var response = await handler(requestStream, context).ConfigureAwait(false);
                 status = context.Status;
-                responseTuple = Tuple.Create(response, HandlerUtils.GetWriteFlags(context.WriteOptions));
+                responseWithFlags = new AsyncCallServer<TRequest, TResponse>.ResponseWithFlags(response, HandlerUtils.GetWriteFlags(context.WriteOptions));
             }
             catch (Exception e)
             {
@@ -196,7 +196,7 @@ namespace Grpc.Core.Internal
 
             try
             {
-                await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, responseTuple).ConfigureAwait(false);
+                await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, responseWithFlags).ConfigureAwait(false);
             }
             catch (Exception)
             {

src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs

@@ -59,13 +59,15 @@ namespace Grpc.Core.Internal
         {
             Native.grpcsharp_server_start(this);
         }
-    
+
         public void ShutdownAndNotify(BatchCompletionDelegate callback, CompletionQueueSafeHandle completionQueue)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
+                // TODO(jtattermusch): delegate allocation by caller can be avoided by utilizing the "state" object,
+                // but server shutdown isn't worth optimizing right now.
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, callback, null);
                 Native.grpcsharp_server_shutdown_and_notify_callback(this, completionQueue, ctx);
             }
         }

src/csharp/Grpc.Core/Server.cs

@@ -387,7 +387,7 @@ namespace Grpc.Core
         /// <summary>
         /// Handles native callback.
         /// </summary>
-        private void HandleServerShutdown(bool success, BatchContextSafeHandle ctx)
+        private void HandleServerShutdown(bool success, BatchContextSafeHandle ctx, object state)
         {
             shutdownTcs.SetResult(null);
         }

src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs

@@ -0,0 +1,78 @@
+#region Copyright notice and license
+
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#endregion
+
+using System;
+using System.Runtime.InteropServices;
+using System.Threading;
+using Grpc.Core;
+using Grpc.Core.Internal;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace Grpc.Microbenchmarks
+{
+    public class CompletionRegistryBenchmark
+    {
+        GrpcEnvironment environment;
+
+        public void Init()
+        {
+            environment = GrpcEnvironment.AddRef();
+        }
+
+        public void Cleanup()
+        {
+            GrpcEnvironment.ReleaseAsync().Wait();
+        }
+
+        public void Run(int threadCount, int iterations, bool useSharedRegistry)
+        {
+            Console.WriteLine(string.Format("CompletionRegistryBenchmark: threads={0}, iterations={1}, useSharedRegistry={2}", threadCount, iterations, useSharedRegistry));
+            CompletionRegistry sharedRegistry = useSharedRegistry ? new CompletionRegistry(environment) : null;
+            var threadedBenchmark = new ThreadedBenchmark(threadCount, () => ThreadBody(iterations, sharedRegistry));
+            threadedBenchmark.Run();
+            // TODO: parametrize by number of pending completions
+        }
+
+        private void ThreadBody(int iterations, CompletionRegistry optionalSharedRegistry)
+        {
+            var completionRegistry = optionalSharedRegistry ?? new CompletionRegistry(environment);
+            var ctx = BatchContextSafeHandle.Create();
+  
+            var stopwatch = Stopwatch.StartNew();
+            for (int i = 0; i < iterations; i++)
+            {
+                completionRegistry.Register(ctx.Handle, ctx);
+                var callback = completionRegistry.Extract(ctx.Handle);
+                // NOTE: we are not calling the callback to avoid disposing ctx.
+            }
+            stopwatch.Stop();
+            Console.WriteLine("Elapsed millis: " + stopwatch.ElapsedMilliseconds);          
+
+            ctx.Dispose();
+        }
+
+        private class NopCompletionCallback : IOpCompletionCallback
+        {
+            public void OnComplete(bool success)
+            {
+
+            }
+        }
+    }
+}

src/csharp/Grpc.Microbenchmarks/GCStats.cs

@@ -0,0 +1,69 @@
+#region Copyright notice and license
+
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#endregion
+
+using System;
+using Grpc.Core;
+using Grpc.Core.Internal;
+
+namespace Grpc.Microbenchmarks
+{
+    internal class GCStats
+    {
+        readonly object myLock = new object();
+        GCStatsSnapshot lastSnapshot;
+
+        public GCStats()
+        {
+            lastSnapshot = new GCStatsSnapshot(GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2));
+        }
+
+        public GCStatsSnapshot GetSnapshot(bool reset = false)
+        {
+            lock (myLock)
+            {
+                var newSnapshot = new GCStatsSnapshot(GC.CollectionCount(0) - lastSnapshot.Gen0,
+                    GC.CollectionCount(1) - lastSnapshot.Gen1,
+                    GC.CollectionCount(2) - lastSnapshot.Gen2);
+                if (reset)
+                {
+                    lastSnapshot = newSnapshot;
+                }
+                return newSnapshot;
+            }
+        }
+    }
+
+    public class GCStatsSnapshot
+    {
+        public GCStatsSnapshot(int gen0, int gen1, int gen2)
+        {
+            this.Gen0 = gen0;
+            this.Gen1 = gen1;
+            this.Gen2 = gen2;
+        }
+
+        public int Gen0 { get; }
+        public int Gen1 { get; }
+        public int Gen2 { get; }
+
+        public override string ToString()
+        {
+            return string.Format("[GCCollectionCount: gen0 {0}, gen1 {1}, gen2 {2}]", Gen0, Gen1, Gen2);
+        }
+    }
+}

src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj

@@ -15,6 +15,10 @@
     <ProjectReference Include="../Grpc.Core/Grpc.Core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.1.1-beta" />
+  </ItemGroup>
+
   <ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
     <Reference Include="System" />
     <Reference Include="Microsoft.CSharp" />

src/csharp/Grpc.Microbenchmarks/PInvokeByteArrayBenchmark.cs

@@ -0,0 +1,64 @@
+#region Copyright notice and license
+
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#endregion
+
+using System;
+using System.Runtime.InteropServices;
+using System.Threading;
+using Grpc.Core;
+using Grpc.Core.Internal;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace Grpc.Microbenchmarks
+{
+    public class PInvokeByteArrayBenchmark
+    {
+        static readonly NativeMethods Native = NativeMethods.Get();
+
+        public void Init()
+        {
+        }
+
+        public void Cleanup()
+        {
+        }
+
+        public void Run(int threadCount, int iterations, int payloadSize)
+        {
+            Console.WriteLine(string.Format("PInvokeByteArrayBenchmark: threads={0}, iterations={1}, payloadSize={2}", threadCount, iterations, payloadSize));
+            var threadedBenchmark = new ThreadedBenchmark(threadCount, () => ThreadBody(iterations, payloadSize));
+            threadedBenchmark.Run();
+        }
+
+        private void ThreadBody(int iterations, int payloadSize)
+        {
+            var payload = new byte[payloadSize];
+         
+            var stopwatch = Stopwatch.StartNew();
+            for (int i = 0; i < iterations; i++)
+            {
+                var gcHandle = GCHandle.Alloc(payload, GCHandleType.Pinned);
+                var payloadPtr = gcHandle.AddrOfPinnedObject();
+                Native.grpcsharp_test_nop(payloadPtr);
+                gcHandle.Free();
+            }
+            stopwatch.Stop();
+            Console.WriteLine("Elapsed millis: " + stopwatch.ElapsedMilliseconds);
+        }
+    }
+}

src/csharp/Grpc.Microbenchmarks/Program.cs

@@ -20,14 +20,84 @@ using System;
 using Grpc.Core;
 using Grpc.Core.Internal;
 using Grpc.Core.Logging;
+using CommandLine;
+using CommandLine.Text;
 
 namespace Grpc.Microbenchmarks
 {
     class Program
     {
+        public enum MicrobenchmarkType
+        {
+            CompletionRegistry,
+            PInvokeByteArray,
+            SendMessage
+        }
+
+        private class BenchmarkOptions
+        {
+            [Option("benchmark", Required = true, HelpText = "Benchmark to run")]
+            public MicrobenchmarkType Benchmark { get; set; }
+        }
+
         public static void Main(string[] args)
         {
             GrpcEnvironment.SetLogger(new ConsoleLogger());
+            var parserResult = Parser.Default.ParseArguments<BenchmarkOptions>(args)
+                .WithNotParsed(errors => {
+                    Console.WriteLine("Supported benchmarks:");
+                    foreach (var enumValue in Enum.GetValues(typeof(MicrobenchmarkType)))
+                    {
+                        Console.WriteLine("  " + enumValue);
+                    }
+                    Environment.Exit(1);
+                })
+                .WithParsed(options =>
+                {
+                    switch (options.Benchmark)
+                    {
+                        case MicrobenchmarkType.CompletionRegistry:
+                          RunCompletionRegistryBenchmark();
+                          break;
+                        case MicrobenchmarkType.PInvokeByteArray:
+                          RunPInvokeByteArrayBenchmark();
+                          break;
+                        case MicrobenchmarkType.SendMessage:
+                          RunSendMessageBenchmark();
+                          break;
+                        default:
+                          throw new ArgumentException("Unsupported benchmark.");
+                    }
+                });
+        }
+
+        static void RunCompletionRegistryBenchmark()
+        {
+            var benchmark = new CompletionRegistryBenchmark();
+            benchmark.Init();
+            foreach (int threadCount in new int[] {1, 1, 2, 4, 8, 12})
+            {
+                foreach (bool useSharedRegistry in new bool[] {false, true})
+                {
+                    benchmark.Run(threadCount, 4 * 1000 * 1000, useSharedRegistry);
+                }
+            }
+            benchmark.Cleanup();
+        }
+
+        static void RunPInvokeByteArrayBenchmark()
+        {
+            var benchmark = new PInvokeByteArrayBenchmark();
+            benchmark.Init();
+            foreach (int threadCount in new int[] {1, 1, 2, 4, 8, 12})
+            {
+                benchmark.Run(threadCount, 4 * 1000 * 1000, 0);
+            }
+            benchmark.Cleanup();
+        }
+
+        static void RunSendMessageBenchmark()
+        {
             var benchmark = new SendMessageBenchmark();
             benchmark.Init();
             foreach (int threadCount in new int[] {1, 1, 2, 4, 8, 12})

src/csharp/Grpc.Microbenchmarks/SendMessageBenchmark.cs

@@ -59,16 +59,16 @@ namespace Grpc.Microbenchmarks
             var cq = CompletionQueueSafeHandle.CreateAsync(completionRegistry);
             var call = CreateFakeCall(cq);
 
-            var sendCompletionHandler = new SendCompletionHandler((success) => { });
+            var sendCompletionCallback = new NopSendCompletionCallback();
             var payload = new byte[payloadSize];
             var writeFlags = default(WriteFlags);
 
             var stopwatch = Stopwatch.StartNew();
             for (int i = 0; i < iterations; i++)
             {
-                call.StartSendMessage(sendCompletionHandler, payload, writeFlags, false);
+                call.StartSendMessage(sendCompletionCallback, payload, writeFlags, false);
                 var callback = completionRegistry.Extract(completionRegistry.LastRegisteredKey);
-                callback(true);
+                callback.OnComplete(true);
             }
             stopwatch.Stop();
             Console.WriteLine("Elapsed millis: " + stopwatch.ElapsedMilliseconds);
@@ -87,5 +87,13 @@ namespace Grpc.Microbenchmarks
             }
             return call;
         }
+
+        private class NopSendCompletionCallback : ISendCompletionCallback
+        {
+            public void OnSendCompletion(bool success)
+            {
+                // NOP
+            }
+        }
     }
 }

src/csharp/Grpc.Microbenchmarks/ThreadedBenchmark.cs

@@ -46,6 +46,7 @@ namespace Grpc.Microbenchmarks
         public void Run()
         {
             Console.WriteLine("Running threads.");
+            var gcStats = new GCStats();
             var threads = new List<Thread>();
             for (int i = 0; i < runners.Count; i++)
             {
@@ -58,7 +59,7 @@ namespace Grpc.Microbenchmarks
             {
                 thread.Join();
             }
-            Console.WriteLine("All threads finished.");
+            Console.WriteLine("All threads finished (GC Stats Delta: " + gcStats.GetSnapshot() + ")");
         }
     }
 }

src/python/grpcio/grpc/__init__.py

@@ -424,6 +424,19 @@ class ServerCredentials(object):
         self._credentials = credentials
 
 
+class ServerCertificateConfig(object):
+    """A certificate config for use with an SSL-enabled Server, e.g., can
+    be returned in the certificate config fetching callback.
+
+    This class has no supported interface -- it exists to define the
+    type of its instances and its instances exist to be passed to
+    other functions.
+    """
+
+    def __init__(self, cert_config):
+        self._cert_config = cert_config
+
+
 ########################  Multi-Callable Interfaces  ###########################
 
 
@@ -1252,6 +1265,60 @@ def ssl_server_credentials(private_key_certificate_chain_pairs,
             ], require_client_auth))
 
 
+def ssl_server_certificate_config(private_key_certificate_chain_pairs,
+                                  root_certificates=None):
+    """Creates a ServerCertificateConfig for use with an SSL-enabled Server.
+
+    Args:
+      private_key_certificate_chain_pairs: A collection of pairs of
+        the form [PEM-encoded private key, PEM-encoded certificate
+        chain].
+      root_certificates: An optional byte string of PEM-encoded client root
+        certificates that the server will use to verify client authentication.
+
+    Returns:
+      A ServerCertificateConfig that can be returned in the certificate config
+      fetching callback.
+    """
+    if len(private_key_certificate_chain_pairs) == 0:
+        raise ValueError(
+            'At least one private key-certificate chain pair is required!')
+    else:
+        return ServerCertificateConfig(
+            _cygrpc.server_certificate_config_ssl(root_certificates, [
+                _cygrpc.SslPemKeyCertPair(key, pem)
+                for key, pem in private_key_certificate_chain_pairs
+            ]))
+
+
+def ssl_server_credentials_dynamic_cert_config(initial_cert_config,
+                                               cert_config_fetcher,
+                                               require_client_auth=False):
+    """Creates a ServerCredentials for use with an SSL-enabled Server.
+
+    Args:
+      initial_cert_config (ServerCertificateConfig): the certificate
+        config with which the server will be initialized.
+      cert_config_fetcher (callable): a callable that takes no
+        arguments and should return a ServerCertificateConfig to
+        replace the server's current cert, or None for no change
+        (i.e., the server will continue its current certificate
+        config). The library will call this callback on *every* new
+        client connection before starting the TLS handshake with the
+        client, thus allowing the user application to optionally
+        return a new ServerCertificateConfig that the server will then
+        use for the handshake.
+      require_client_auth: A boolean indicating whether or not to
+        require clients to be authenticated.
+
+    Returns:
+      A ServerCredentials.
+    """
+    return ServerCredentials(
+        _cygrpc.server_credentials_ssl_dynamic_cert_config(
+            initial_cert_config, cert_config_fetcher, require_client_auth))
+
+
 def channel_ready_future(channel):
     """Creates a Future that tracks when a Channel is ready.
 
@@ -1334,18 +1401,20 @@ __all__ = ('FutureTimeoutError', 'FutureCancelledError', 'Future',
            'ChannelConnectivity', 'StatusCode', 'RpcError', 'RpcContext',
            'Call', 'ChannelCredentials', 'CallCredentials',
            'AuthMetadataContext', 'AuthMetadataPluginCallback',
-           'AuthMetadataPlugin', 'ServerCredentials', 'UnaryUnaryMultiCallable',
+           'AuthMetadataPlugin', 'ServerCertificateConfig', 'ServerCredentials',
-           'UnaryStreamMultiCallable', 'StreamUnaryMultiCallable',
+           'UnaryUnaryMultiCallable', 'UnaryStreamMultiCallable',
-           'StreamStreamMultiCallable', 'Channel', 'ServicerContext',
+           'StreamUnaryMultiCallable', 'StreamStreamMultiCallable', 'Channel',
-           'RpcMethodHandler', 'HandlerCallDetails', 'GenericRpcHandler',
+           'ServicerContext', 'RpcMethodHandler', 'HandlerCallDetails',
-           'ServiceRpcHandler', 'Server', 'unary_unary_rpc_method_handler',
+           'GenericRpcHandler', 'ServiceRpcHandler', 'Server',
-           'unary_stream_rpc_method_handler', 'stream_unary_rpc_method_handler',
+           'unary_unary_rpc_method_handler', 'unary_stream_rpc_method_handler',
+           'stream_unary_rpc_method_handler',
            'stream_stream_rpc_method_handler',
            'method_handlers_generic_handler', 'ssl_channel_credentials',
            'metadata_call_credentials', 'access_token_call_credentials',
            'composite_call_credentials', 'composite_channel_credentials',
-           'ssl_server_credentials', 'channel_ready_future', 'insecure_channel',
+           'ssl_server_credentials', 'ssl_server_certificate_config',
-           'secure_channel', 'server',)
+           'ssl_server_credentials_dynamic_cert_config', 'channel_ready_future',
+           'insecure_channel', 'secure_channel', 'server',)
 
 ############################### Extension Shims ################################
 

src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi

@@ -28,12 +28,27 @@ cdef class CallCredentials:
   cdef list references
 
 
+cdef class ServerCertificateConfig:
+
+  cdef grpc_ssl_server_certificate_config *c_cert_config
+  cdef const char *c_pem_root_certs
+  cdef grpc_ssl_pem_key_cert_pair *c_ssl_pem_key_cert_pairs
+  cdef size_t c_ssl_pem_key_cert_pairs_count
+  cdef list references
+
+
 cdef class ServerCredentials:
 
   cdef grpc_server_credentials *c_credentials
   cdef grpc_ssl_pem_key_cert_pair *c_ssl_pem_key_cert_pairs
   cdef size_t c_ssl_pem_key_cert_pairs_count
   cdef list references
+  # the cert config related state is used only if this credentials is
+  # created with cert config/fetcher
+  cdef object initial_cert_config
+  cdef object cert_config_fetcher
+  # whether C-core has asked for the initial_cert_config
+  cdef bint initial_cert_config_fetched
 
 
 cdef class CredentialsMetadataPlugin:

src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi

@@ -14,6 +14,7 @@
 
 cimport cpython
 
+import grpc
 import threading
 import traceback
 
@@ -58,12 +59,30 @@ cdef class CallCredentials:
     grpc_shutdown()
 
 
+cdef class ServerCertificateConfig:
+
+  def __cinit__(self):
+    grpc_init()
+    self.c_cert_config = NULL
+    self.c_pem_root_certs = NULL
+    self.c_ssl_pem_key_cert_pairs = NULL
+    self.references = []
+
+  def __dealloc__(self):
+    grpc_ssl_server_certificate_config_destroy(self.c_cert_config)
+    gpr_free(self.c_ssl_pem_key_cert_pairs)
+    grpc_shutdown()
+
+
 cdef class ServerCredentials:
 
   def __cinit__(self):
     grpc_init()
     self.c_credentials = NULL
     self.references = []
+    self.initial_cert_config = None
+    self.cert_config_fetcher = None
+    self.initial_cert_config_fetched = False
 
   def __dealloc__(self):
     if self.c_credentials != NULL:
@@ -254,34 +273,85 @@ def call_credentials_metadata_plugin(CredentialsMetadataPlugin plugin):
   credentials.references.append(plugin)
   return credentials
 
-def server_credentials_ssl(pem_root_certs, pem_key_cert_pairs,
+cdef const char* _get_c_pem_root_certs(pem_root_certs):
-                           bint force_client_auth):
-  pem_root_certs = str_to_bytes(pem_root_certs)
   cdef char *c_pem_root_certs = NULL
-  if pem_root_certs is not None: 
+  if pem_root_certs is not None:
     c_pem_root_certs = pem_root_certs
-  pem_key_cert_pairs = list(pem_key_cert_pairs)
+  return c_pem_root_certs
+
+cdef grpc_ssl_pem_key_cert_pair* _create_c_ssl_pem_key_cert_pairs(pem_key_cert_pairs):
+  # return a malloc'ed grpc_ssl_pem_key_cert_pair from a _list_ of SslPemKeyCertPair
   for pair in pem_key_cert_pairs:
     if not isinstance(pair, SslPemKeyCertPair):
       raise TypeError("expected pem_key_cert_pairs to be sequence of "
                       "SslPemKeyCertPair")
+  cdef size_t c_ssl_pem_key_cert_pairs_count = len(pem_key_cert_pairs)
+  cdef grpc_ssl_pem_key_cert_pair* c_ssl_pem_key_cert_pairs = NULL
+  with nogil:
+    c_ssl_pem_key_cert_pairs = (
+      <grpc_ssl_pem_key_cert_pair *>gpr_malloc(
+        sizeof(grpc_ssl_pem_key_cert_pair) * c_ssl_pem_key_cert_pairs_count))
+  for i in range(c_ssl_pem_key_cert_pairs_count):
+    c_ssl_pem_key_cert_pairs[i] = (
+      (<SslPemKeyCertPair>pem_key_cert_pairs[i]).c_pair)
+  return c_ssl_pem_key_cert_pairs
+
+def server_credentials_ssl(pem_root_certs, pem_key_cert_pairs,
+                           bint force_client_auth):
+  pem_root_certs = str_to_bytes(pem_root_certs)
+  pem_key_cert_pairs = list(pem_key_cert_pairs)
   cdef ServerCredentials credentials = ServerCredentials()
-  credentials.references.append(pem_key_cert_pairs)
   credentials.references.append(pem_root_certs)
+  credentials.references.append(pem_key_cert_pairs)
+  cdef char * c_pem_root_certs = _get_c_pem_root_certs(pem_root_certs)
   credentials.c_ssl_pem_key_cert_pairs_count = len(pem_key_cert_pairs)
-  with nogil:
+  credentials.c_ssl_pem_key_cert_pairs = _create_c_ssl_pem_key_cert_pairs(pem_key_cert_pairs)
-    credentials.c_ssl_pem_key_cert_pairs = (
+  cdef grpc_ssl_server_certificate_config *c_cert_config = NULL
-        <grpc_ssl_pem_key_cert_pair *>gpr_malloc(
+  c_cert_config = grpc_ssl_server_certificate_config_create(
-            sizeof(grpc_ssl_pem_key_cert_pair) *
+    c_pem_root_certs, credentials.c_ssl_pem_key_cert_pairs,
-                credentials.c_ssl_pem_key_cert_pairs_count
+    credentials.c_ssl_pem_key_cert_pairs_count)
-        ))
+  cdef grpc_ssl_server_credentials_options* c_options = NULL
-  for i in range(credentials.c_ssl_pem_key_cert_pairs_count):
+  # C-core assumes ownership of c_cert_config
-    credentials.c_ssl_pem_key_cert_pairs[i] = (
+  c_options = grpc_ssl_server_credentials_create_options_using_config(
-        (<SslPemKeyCertPair>pem_key_cert_pairs[i]).c_pair)
+    GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
-  credentials.c_credentials = grpc_ssl_server_credentials_create(
+    if force_client_auth else
-      c_pem_root_certs, credentials.c_ssl_pem_key_cert_pairs,
+    GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
-      credentials.c_ssl_pem_key_cert_pairs_count,
+    c_cert_config)
-      GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY if force_client_auth else GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
+  # C-core assumes ownership of c_options
-      NULL)
+  credentials.c_credentials = grpc_ssl_server_credentials_create_with_options(c_options)
   return credentials
 
+def server_certificate_config_ssl(pem_root_certs, pem_key_cert_pairs):
+  pem_root_certs = str_to_bytes(pem_root_certs)
+  pem_key_cert_pairs = list(pem_key_cert_pairs)
+  cdef ServerCertificateConfig cert_config = ServerCertificateConfig()
+  cert_config.references.append(pem_root_certs)
+  cert_config.references.append(pem_key_cert_pairs)
+  cert_config.c_pem_root_certs = _get_c_pem_root_certs(pem_root_certs)
+  cert_config.c_ssl_pem_key_cert_pairs_count = len(pem_key_cert_pairs)
+  cert_config.c_ssl_pem_key_cert_pairs = _create_c_ssl_pem_key_cert_pairs(pem_key_cert_pairs)
+  cert_config.c_cert_config = grpc_ssl_server_certificate_config_create(
+    cert_config.c_pem_root_certs, cert_config.c_ssl_pem_key_cert_pairs,
+    cert_config.c_ssl_pem_key_cert_pairs_count)
+  return cert_config
+
+def server_credentials_ssl_dynamic_cert_config(initial_cert_config,
+                                               cert_config_fetcher,
+                                               bint force_client_auth):
+  if not isinstance(initial_cert_config, grpc.ServerCertificateConfig):
+    raise TypeError('initial_cert_config must be a grpc.ServerCertificateConfig')
+  if not callable(cert_config_fetcher):
+    raise TypeError('cert_config_fetcher must be callable')
+  cdef ServerCredentials credentials = ServerCredentials()
+  credentials.initial_cert_config = initial_cert_config
+  credentials.cert_config_fetcher = cert_config_fetcher
+  cdef grpc_ssl_server_credentials_options* c_options = NULL
+  c_options = grpc_ssl_server_credentials_create_options_using_config_fetcher(
+    GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
+    if force_client_auth else
+    GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
+    _server_cert_config_fetcher_wrapper,
+    <void*>credentials)
+  # C-core assumes ownership of c_options
+  credentials.c_credentials = grpc_ssl_server_credentials_create_with_options(c_options)
+  return credentials

src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi

@@ -391,6 +391,42 @@ cdef extern from "grpc/grpc_security.h":
     GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_BUT_DONT_VERIFY
     GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
 
+  ctypedef enum grpc_ssl_certificate_config_reload_status:
+    GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_UNCHANGED
+    GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_NEW
+    GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_FAIL
+
+  ctypedef struct grpc_ssl_server_certificate_config:
+    # We don't care about the internals
+    pass
+
+  ctypedef struct grpc_ssl_server_credentials_options:
+    # We don't care about the internals
+    pass
+
+  grpc_ssl_server_certificate_config * grpc_ssl_server_certificate_config_create(
+    const char *pem_root_certs,
+    const grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs)
+
+  void grpc_ssl_server_certificate_config_destroy(grpc_ssl_server_certificate_config *config)
+
+  ctypedef grpc_ssl_certificate_config_reload_status (*grpc_ssl_server_certificate_config_callback)(
+    void *user_data,
+    grpc_ssl_server_certificate_config **config)
+
+  grpc_ssl_server_credentials_options *grpc_ssl_server_credentials_create_options_using_config(
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    grpc_ssl_server_certificate_config *certificate_config)
+
+  grpc_ssl_server_credentials_options* grpc_ssl_server_credentials_create_options_using_config_fetcher(
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    grpc_ssl_server_certificate_config_callback cb,
+    void *user_data)
+
+  grpc_server_credentials *grpc_ssl_server_credentials_create_with_options(
+      grpc_ssl_server_credentials_options *options)
+
   ctypedef struct grpc_ssl_pem_key_cert_pair:
     const char *private_key
     const char *certificate_chain "cert_chain"
@@ -440,10 +476,6 @@ cdef extern from "grpc/grpc_security.h":
     # We don't care about the internals (and in fact don't know them)
     pass
 
-  grpc_server_credentials *grpc_ssl_server_credentials_create(
-      const char *pem_root_certs,
-      grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-      size_t num_key_cert_pairs, int force_client_auth, void *reserved)
   void grpc_server_credentials_release(grpc_server_credentials *creds) nogil
 
   int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,

src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi

@@ -14,8 +14,44 @@
 
 cimport cpython
 
+import logging
 import time
+import grpc
 
+cdef grpc_ssl_certificate_config_reload_status _server_cert_config_fetcher_wrapper(
+        void* user_data, grpc_ssl_server_certificate_config **config) with gil:
+  # This is a credentials.ServerCertificateConfig
+  cdef ServerCertificateConfig cert_config = None
+  if not user_data:
+    raise ValueError('internal error: user_data must be specified')
+  credentials = <ServerCredentials>user_data
+  if not credentials.initial_cert_config_fetched:
+    # C-core is asking for the initial cert config
+    credentials.initial_cert_config_fetched = True
+    cert_config = credentials.initial_cert_config._cert_config
+  else:
+    user_cb = credentials.cert_config_fetcher
+    try:
+      cert_config_wrapper = user_cb()
+    except Exception:
+      logging.exception('Error fetching certificate config')
+      return GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_FAIL
+    if cert_config_wrapper is None:
+      return GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_UNCHANGED
+    elif not isinstance(cert_config_wrapper, grpc.ServerCertificateConfig):
+      logging.error('Error fetching certificate config: certificate '
+                    'config must be of type grpc.ServerCertificateConfig, '
+                    'not %s' % type(cert_config_wrapper).__name__)
+      return GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_FAIL
+    else:
+      cert_config = cert_config_wrapper._cert_config
+  config[0] = <grpc_ssl_server_certificate_config*>cert_config.c_cert_config
+  # our caller will assume ownership of memory, so we have to recreate
+  # a copy of c_cert_config here
+  cert_config.c_cert_config = grpc_ssl_server_certificate_config_create(
+      cert_config.c_pem_root_certs, cert_config.c_ssl_pem_key_cert_pairs,
+      cert_config.c_ssl_pem_key_cert_pairs_count)
+  return GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_NEW
 
 cdef class Server:
 

src/python/grpcio_health_checking/setup.py

@@ -60,17 +60,8 @@ INSTALL_REQUIRES = ('protobuf>=3.3.0',
                     'grpcio>={version}'.format(version=grpc_version.VERSION),)
 
 try:
-    # ensure we can load the _pb2_grpc module:
-    from grpc_health.v1 import health_pb2_grpc as _pb2_grpc
-    # if we can find the _pb2_grpc module, the package has already been built.
-    SETUP_REQUIRES = ()
-    COMMAND_CLASS = {
-        # wire up commands to no-op not to break the external dependencies
-        'preprocess': _NoOpCommand,
-        'build_package_protos': _NoOpCommand,
-    }
-except ImportError:  # we are in the build environment
     import health_commands as _health_commands
+    # we are in the build environment, otherwise the above import fails
     SETUP_REQUIRES = (
         'grpcio-tools=={version}'.format(version=grpc_version.VERSION),)
     COMMAND_CLASS = {
@@ -78,6 +69,13 @@ except ImportError:  # we are in the build environment
         'preprocess': _health_commands.CopyProtoModules,
         'build_package_protos': _health_commands.BuildPackageProtos,
     }
+except ImportError:
+    SETUP_REQUIRES = ()
+    COMMAND_CLASS = {
+        # wire up commands to no-op not to break the external dependencies
+        'preprocess': _NoOpCommand,
+        'build_package_protos': _NoOpCommand,
+    }
 
 setuptools.setup(
     name='grpcio-health-checking',

src/python/grpcio_reflection/setup.py

@@ -61,17 +61,8 @@ INSTALL_REQUIRES = ('protobuf>=3.3.0',
                     'grpcio>={version}'.format(version=grpc_version.VERSION),)
 
 try:
-    # ensure we can load the _pb2_grpc module:
-    from grpc_reflection.v1alpha import reflection_pb2_grpc as _pb2_grpc
-    # if we can find the _pb2_grpc module, the package has already been built.
-    SETUP_REQUIRES = ()
-    COMMAND_CLASS = {
-        # wire up commands to no-op not to break the external dependencies
-        'preprocess': _NoOpCommand,
-        'build_package_protos': _NoOpCommand,
-    }
-except ImportError:  # we are in the build environment
     import reflection_commands as _reflection_commands
+    # we are in the build environment, otherwise the above import fails
     SETUP_REQUIRES = (
         'grpcio-tools=={version}'.format(version=grpc_version.VERSION),)
     COMMAND_CLASS = {
@@ -79,6 +70,13 @@ except ImportError:  # we are in the build environment
         'preprocess': _reflection_commands.CopyProtoModules,
         'build_package_protos': _reflection_commands.BuildPackageProtos,
     }
+except ImportError:
+    SETUP_REQUIRES = ()
+    COMMAND_CLASS = {
+        # wire up commands to no-op not to break the external dependencies
+        'preprocess': _NoOpCommand,
+        'build_package_protos': _NoOpCommand,
+    }
 
 setuptools.setup(
     name='grpcio-reflection',

src/python/grpcio_tests/tests/interop/client.py

@@ -29,37 +29,40 @@ def _args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         '--server_host',
-        help='the host to which to connect',
+        default="localhost",
         type=str,
-        default="localhost")
+        help='the host to which to connect')
     parser.add_argument(
-        '--server_port', help='the port to which to connect', type=int)
+        '--server_port',
+        type=int,
+        required=True,
+        help='the port to which to connect')
     parser.add_argument(
         '--test_case',
-        help='the test case to execute',
+        default='large_unary',
         type=str,
-        default="large_unary")
+        help='the test case to execute')
     parser.add_argument(
         '--use_tls',
-        help='require a secure connection',
         default=False,
-        type=resources.parse_bool)
+        type=resources.parse_bool,
+        help='require a secure connection')
     parser.add_argument(
         '--use_test_ca',
-        help='replace platform root CAs with ca.pem',
         default=False,
-        type=resources.parse_bool)
+        type=resources.parse_bool,
+        help='replace platform root CAs with ca.pem')
     parser.add_argument(
         '--server_host_override',
         default="foo.test.google.fr",
-        help='the server host to which to claim to connect',
+        type=str,
-        type=str)
+        help='the server host to which to claim to connect')
     parser.add_argument(
-        '--oauth_scope', help='scope for OAuth tokens', type=str)
+        '--oauth_scope', type=str, help='scope for OAuth tokens')
     parser.add_argument(
         '--default_service_account',
-        help='email address of the default service account',
+        type=str,
-        type=str)
+        help='email address of the default service account')
     return parser.parse_args()
 
 

src/python/grpcio_tests/tests/interop/server.py

@@ -29,12 +29,13 @@ _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 def serve():
     parser = argparse.ArgumentParser()
-    parser.add_argument('--port', help='the port on which to serve', type=int)
+    parser.add_argument(
+        '--port', type=int, required=True, help='the port on which to serve')
     parser.add_argument(
         '--use_tls',
-        help='require a secure connection',
         default=False,
-        type=resources.parse_bool)
+        type=resources.parse_bool,
+        help='require a secure connection')
     args = parser.parse_args()
 
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))

src/python/grpcio_tests/tests/tests.json

@@ -46,6 +46,10 @@
   "unit._reconnect_test.ReconnectTest",
   "unit._resource_exhausted_test.ResourceExhaustedTest",
   "unit._rpc_test.RPCTest",
+  "unit._server_ssl_cert_config_test.ServerSSLCertConfigFetcherParamsChecks",
+  "unit._server_ssl_cert_config_test.ServerSSLCertReloadTestCertConfigReuse",
+  "unit._server_ssl_cert_config_test.ServerSSLCertReloadTestWithClientAuth",
+  "unit._server_ssl_cert_config_test.ServerSSLCertReloadTestWithoutClientAuth",
   "unit._thread_cleanup_test.CleanupThreadTest",
   "unit.beta._beta_features_test.BetaFeaturesTest",
   "unit.beta._beta_features_test.ContextManagementAndLifecycleTest",

src/python/grpcio_tests/tests/unit/_api_test.py

@@ -30,19 +30,22 @@ class AllTest(unittest.TestCase):
             'ChannelConnectivity', 'StatusCode', 'RpcError', 'RpcContext',
             'Call', 'ChannelCredentials', 'CallCredentials',
             'AuthMetadataContext', 'AuthMetadataPluginCallback',
-            'AuthMetadataPlugin', 'ServerCredentials',
+            'AuthMetadataPlugin', 'ServerCertificateConfig',
-            'UnaryUnaryMultiCallable', 'UnaryStreamMultiCallable',
+            'ServerCredentials', 'UnaryUnaryMultiCallable',
-            'StreamUnaryMultiCallable', 'StreamStreamMultiCallable', 'Channel',
+            'UnaryStreamMultiCallable', 'StreamUnaryMultiCallable',
-            'ServicerContext', 'RpcMethodHandler', 'HandlerCallDetails',
+            'StreamStreamMultiCallable', 'Channel', 'ServicerContext',
-            'GenericRpcHandler', 'ServiceRpcHandler', 'Server',
+            'RpcMethodHandler', 'HandlerCallDetails', 'GenericRpcHandler',
-            'unary_unary_rpc_method_handler', 'unary_stream_rpc_method_handler',
+            'ServiceRpcHandler', 'Server', 'unary_unary_rpc_method_handler',
+            'unary_stream_rpc_method_handler',
             'stream_unary_rpc_method_handler',
             'stream_stream_rpc_method_handler',
             'method_handlers_generic_handler', 'ssl_channel_credentials',
             'metadata_call_credentials', 'access_token_call_credentials',
             'composite_call_credentials', 'composite_channel_credentials',
-            'ssl_server_credentials', 'channel_ready_future',
+            'ssl_server_credentials', 'ssl_server_certificate_config',
-            'insecure_channel', 'secure_channel', 'server',)
+            'ssl_server_credentials_dynamic_cert_config',
+            'channel_ready_future', 'insecure_channel', 'secure_channel',
+            'server',)
 
         six.assertCountEqual(self, expected_grpc_code_elements,
                              _from_grpc_import_star.GRPC_ELEMENTS)

src/python/grpcio_tests/tests/unit/_server_ssl_cert_config_test.py

@@ -0,0 +1,520 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This tests server certificate rotation support.
+
+Here we test various aspects of gRPC Python, and in some cases C-core
+by extension, support for server certificate rotation.
+
+* ServerSSLCertReloadTestWithClientAuth: test ability to rotate
+  server's SSL cert for use in future channels with clients while not
+  affecting any existing channel. The server requires client
+  authentication.
+
+* ServerSSLCertReloadTestWithoutClientAuth: like
+  ServerSSLCertReloadTestWithClientAuth except that the server does
+  not authenticate the client.
+
+* ServerSSLCertReloadTestCertConfigReuse: tests gRPC Python's ability
+  to deal with user's reuse of ServerCertificateConfig instances.
+"""
+
+import abc
+import collections
+import os
+import six
+import threading
+import unittest
+
+from concurrent import futures
+
+import grpc
+from tests.unit import resources
+from tests.testing import _application_common
+from tests.testing import _server_application
+from tests.testing.proto import services_pb2_grpc
+
+CA_1_PEM = resources.cert_hier_1_root_ca_cert()
+CA_2_PEM = resources.cert_hier_2_root_ca_cert()
+
+CLIENT_KEY_1_PEM = resources.cert_hier_1_client_1_key()
+CLIENT_CERT_CHAIN_1_PEM = (resources.cert_hier_1_client_1_cert() +
+                           resources.cert_hier_1_intermediate_ca_cert())
+
+CLIENT_KEY_2_PEM = resources.cert_hier_2_client_1_key()
+CLIENT_CERT_CHAIN_2_PEM = (resources.cert_hier_2_client_1_cert() +
+                           resources.cert_hier_2_intermediate_ca_cert())
+
+SERVER_KEY_1_PEM = resources.cert_hier_1_server_1_key()
+SERVER_CERT_CHAIN_1_PEM = (resources.cert_hier_1_server_1_cert() +
+                           resources.cert_hier_1_intermediate_ca_cert())
+
+SERVER_KEY_2_PEM = resources.cert_hier_2_server_1_key()
+SERVER_CERT_CHAIN_2_PEM = (resources.cert_hier_2_server_1_cert() +
+                           resources.cert_hier_2_intermediate_ca_cert())
+
+# for use with the CertConfigFetcher. Roughly a simple custom mock
+# implementation
+Call = collections.namedtuple('Call', ['did_raise', 'returned_cert_config'])
+
+
+def _create_client_stub(
+        port,
+        expect_success,
+        root_certificates=None,
+        private_key=None,
+        certificate_chain=None,):
+    channel = grpc.secure_channel('localhost:{}'.format(port),
+                                  grpc.ssl_channel_credentials(
+                                      root_certificates=root_certificates,
+                                      private_key=private_key,
+                                      certificate_chain=certificate_chain))
+    if expect_success:
+        # per Nathaniel: there's some robustness issue if we start
+        # using a channel without waiting for it to be actually ready
+        grpc.channel_ready_future(channel).result(timeout=10)
+    return services_pb2_grpc.FirstServiceStub(channel)
+
+
+class CertConfigFetcher(object):
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._calls = []
+        self._should_raise = False
+        self._cert_config = None
+
+    def reset(self):
+        with self._lock:
+            self._calls = []
+            self._should_raise = False
+            self._cert_config = None
+
+    def configure(self, should_raise, cert_config):
+        assert not (should_raise and cert_config), (
+            "should not specify both should_raise and a cert_config at the same time"
+        )
+        with self._lock:
+            self._should_raise = should_raise
+            self._cert_config = cert_config
+
+    def getCalls(self):
+        with self._lock:
+            return self._calls
+
+    def __call__(self):
+        with self._lock:
+            if self._should_raise:
+                self._calls.append(Call(True, None))
+                raise ValueError('just for fun, should not affect the test')
+            else:
+                self._calls.append(Call(False, self._cert_config))
+                return self._cert_config
+
+
+class _ServerSSLCertReloadTest(
+        six.with_metaclass(abc.ABCMeta, unittest.TestCase)):
+
+    def __init__(self, *args, **kwargs):
+        super(_ServerSSLCertReloadTest, self).__init__(*args, **kwargs)
+        self.server = None
+        self.port = None
+
+    @abc.abstractmethod
+    def require_client_auth(self):
+        raise NotImplementedError()
+
+    def setUp(self):
+        self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        services_pb2_grpc.add_FirstServiceServicer_to_server(
+            _server_application.FirstServiceServicer(), self.server)
+        switch_cert_on_client_num = 10
+        initial_cert_config = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_1_PEM, SERVER_CERT_CHAIN_1_PEM)],
+            root_certificates=CA_2_PEM)
+        self.cert_config_fetcher = CertConfigFetcher()
+        server_credentials = grpc.ssl_server_credentials_dynamic_cert_config(
+            initial_cert_config,
+            self.cert_config_fetcher,
+            require_client_auth=self.require_client_auth())
+        self.port = self.server.add_secure_port('[::]:0', server_credentials)
+        self.server.start()
+
+    def tearDown(self):
+        if self.server:
+            self.server.stop(None)
+
+    def _perform_rpc(self, client_stub, expect_success):
+        # we don't care about the actual response of the rpc; only
+        # whether we can perform it or not, and if not, the status
+        # code must be UNAVAILABLE
+        request = _application_common.UNARY_UNARY_REQUEST
+        if expect_success:
+            response = client_stub.UnUn(request)
+            self.assertEqual(response, _application_common.UNARY_UNARY_RESPONSE)
+        else:
+            with self.assertRaises(grpc.RpcError) as exception_context:
+                client_stub.UnUn(request)
+            self.assertEqual(exception_context.exception.code(),
+                             grpc.StatusCode.UNAVAILABLE)
+
+    def _do_one_shot_client_rpc(self,
+                                expect_success,
+                                root_certificates=None,
+                                private_key=None,
+                                certificate_chain=None):
+        client_stub = _create_client_stub(
+            self.port,
+            expect_success,
+            root_certificates=root_certificates,
+            private_key=private_key,
+            certificate_chain=certificate_chain)
+        self._perform_rpc(client_stub, expect_success)
+        del client_stub
+
+    def _test(self):
+        # things should work...
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # client should reject server...
+        # fails because client trusts ca2 and so will reject server
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertIsNone(call.returned_cert_config, 'i= {}'.format(i))
+
+        # should work again...
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(True, None)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertTrue(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # if with_client_auth, then client should be rejected by
+        # server because client uses key/cert1, but server trusts ca2,
+        # so server will reject
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            not self.require_client_auth(),
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertIsNone(call.returned_cert_config, 'i= {}'.format(i))
+
+        # should work again...
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # now create the "persistent" clients
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        persistent_client_stub_A = _create_client_stub(
+            self.port,
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        self._perform_rpc(persistent_client_stub_A, True)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        persistent_client_stub_B = _create_client_stub(
+            self.port,
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        self._perform_rpc(persistent_client_stub_B, True)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # moment of truth!! client should reject server because the
+        # server switch cert...
+        cert_config = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_2_PEM, SERVER_CERT_CHAIN_2_PEM)],
+            root_certificates=CA_1_PEM)
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, cert_config)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertEqual(call.returned_cert_config, cert_config,
+                             'i= {}'.format(i))
+
+        # now should work again...
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # client should be rejected by server if with_client_auth
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            not self.require_client_auth(),
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertIsNone(call.returned_cert_config, 'i= {}'.format(i))
+
+        # here client should reject server...
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertIsNone(call.returned_cert_config, 'i= {}'.format(i))
+
+        # persistent clients should continue to work
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._perform_rpc(persistent_client_stub_A, True)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 0)
+
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._perform_rpc(persistent_client_stub_B, True)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 0)
+
+
+class ServerSSLCertConfigFetcherParamsChecks(unittest.TestCase):
+
+    def test_check_on_initial_config(self):
+        with self.assertRaises(TypeError):
+            grpc.ssl_server_credentials_dynamic_cert_config(None, str)
+        with self.assertRaises(TypeError):
+            grpc.ssl_server_credentials_dynamic_cert_config(1, str)
+
+    def test_check_on_config_fetcher(self):
+        cert_config = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_2_PEM, SERVER_CERT_CHAIN_2_PEM)],
+            root_certificates=CA_1_PEM)
+        with self.assertRaises(TypeError):
+            grpc.ssl_server_credentials_dynamic_cert_config(cert_config, None)
+        with self.assertRaises(TypeError):
+            grpc.ssl_server_credentials_dynamic_cert_config(cert_config, 1)
+
+
+class ServerSSLCertReloadTestWithClientAuth(_ServerSSLCertReloadTest):
+
+    def require_client_auth(self):
+        return True
+
+    test = _ServerSSLCertReloadTest._test
+
+
+class ServerSSLCertReloadTestWithoutClientAuth(_ServerSSLCertReloadTest):
+
+    def require_client_auth(self):
+        return False
+
+    test = _ServerSSLCertReloadTest._test
+
+
+class ServerSSLCertReloadTestCertConfigReuse(_ServerSSLCertReloadTest):
+    """Ensures that `ServerCertificateConfig` instances can be reused.
+
+    Because C-core takes ownership of the
+    `grpc_ssl_server_certificate_config` encapsulated by
+    `ServerCertificateConfig`, this test reuses the same
+    `ServerCertificateConfig` instances multiple times to make sure
+    gRPC Python takes care of maintaining the validity of
+    `ServerCertificateConfig` instances, so that such instances can be
+    re-used by user application.
+    """
+
+    def require_client_auth(self):
+        return True
+
+    def setUp(self):
+        self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        services_pb2_grpc.add_FirstServiceServicer_to_server(
+            _server_application.FirstServiceServicer(), self.server)
+        self.cert_config_A = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_1_PEM, SERVER_CERT_CHAIN_1_PEM)],
+            root_certificates=CA_2_PEM)
+        self.cert_config_B = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_2_PEM, SERVER_CERT_CHAIN_2_PEM)],
+            root_certificates=CA_1_PEM)
+        self.cert_config_fetcher = CertConfigFetcher()
+        server_credentials = grpc.ssl_server_credentials_dynamic_cert_config(
+            self.cert_config_A,
+            self.cert_config_fetcher,
+            require_client_auth=True)
+        self.port = self.server.add_secure_port('[::]:0', server_credentials)
+        self.server.start()
+
+    def test_cert_config_reuse(self):
+
+        # succeed with A
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_A)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertEqual(actual_calls[0].returned_cert_config,
+                         self.cert_config_A)
+
+        # fail with A
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_A)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertEqual(call.returned_cert_config, self.cert_config_A,
+                             'i= {}'.format(i))
+
+        # succeed again with A
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_A)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertEqual(actual_calls[0].returned_cert_config,
+                         self.cert_config_A)
+
+        # succeed with B
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_B)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertEqual(actual_calls[0].returned_cert_config,
+                         self.cert_config_B)
+
+        # fail with B
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_B)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertEqual(call.returned_cert_config, self.cert_config_B,
+                             'i= {}'.format(i))
+
+        # succeed again with B
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_B)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertEqual(actual_calls[0].returned_cert_config,
+                         self.cert_config_B)
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)

src/python/grpcio_tests/tests/unit/credentials/README

@@ -1 +0,0 @@
-These are test keys *NOT* to be used in production.

src/python/grpcio_tests/tests/unit/credentials/README.md

@@ -0,0 +1,15 @@
+These are test keys *NOT* to be used in production.
+
+The `certificate_hierarchy_1` and `certificate_hierarchy_2` contain
+two disjoint but similarly organized certificate hierarchies. Each
+contains:
+
+* The respective root CA cert in `certs/ca.cert.pem`
+
+* The intermediate CA cert in
+  `intermediate/certs/intermediate.cert.pem`, signed by the root CA
+
+* A client cert and a server cert--both signed by the intermediate
+  CA--in `intermediate/certs/client.cert.pem` and
+  `intermediate/certs/localhost-1.cert.pem`; the corresponding keys
+  are in `intermediate/private`

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/certs/ca.cert.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFZDCCA0ygAwIBAgIJAKfkDFZ6+Ly/MA0GCSqGSIb3DQEBCwUAMD8xCzAJBgNV
+BAYTAnVzMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxEDAOBgNVBAMM
+B3Jvb3QgY2EwHhcNMTcxMTAyMDAzNzA1WhcNMzcxMDI4MDAzNzA1WjA/MQswCQYD
+VQQGEwJ1czEOMAwGA1UECAwFZHVtbXkxDjAMBgNVBAoMBWR1bW15MRAwDgYDVQQD
+DAdyb290IGNhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxlSUuSbi
+o66tT2ZCqu9wNqSX8VhAJkmrAT5y6m2V0VlQ8Gz7ddynW5UVSmtvDNTebZ15FrvO
+6Ng7QnwXXNs/dEzl6oMe6AKDZpuWScVkiqH1UYWBkMLRygWCTEYpSTWTpZWk1zxj
+DJ2LlIoO1X/ufLyLOfy2a2XEz8ICzJePmqVca6fmfEtCTj1/8FcwCBF6YlUWVzlR
+wewjanQo/lorTYbub+Q6LGxPXZ8W0qoKZzLDSD9cnj4pcJzGGFeu9KkNaW4rldZG
+t7mTGQqIRc98dDRc9Jb7PqL8tMPLidw1KErUi05ofxggc5vqNnj4xBl6aX6b/EYN
+rBLzO2e0FazX6TwNKwwg68vbOanpDq5LVmIUH8bY1zNZ+JPBGO9pXlAA0YwLx86r
+R7YhQ431ZpJ2KGnYjVhYnZ2L3NjV3UYX3x5Z3OrDj9hybhucJB48DMQ1+loEabwK
+fSUJtcSPc8dCIibxVKidBFgaTPXtHy2MPXuhMhR7PCtMpE7RPUoYmdZLr9FNN1ty
+/RAbwBfuhGLbRI2qqJgbOzHJHaOY/FtShfooLz7lt4LIjPTARaNsulG2rbv+m3z9
+mhNjL+peV8gni/xyOYYTbdzZagLrtSHeTWsITvmVt0flMHkjHyv35rw23+hBlSjp
+6+S+0MmwuwxqBBccBSlZ9t3Xh1N+vFkb2UkCAwEAAaNjMGEwHQYDVR0OBBYEFJWE
+tQwTbTCgZWNN08VSxjdNA0oaMB8GA1UdIwQYMBaAFJWEtQwTbTCgZWNN08VSxjdN
+A0oaMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB
+CwUAA4ICAQCNjPv/e3ozX1PuN5Tluf0yOmKCxKVCK3Pp98WkDzH4Rp1urEeYrGJL
+vBNcl17avOJ0e+zTVYPXFviFbsBsU/zaf+TqEujXabsdL+nvvCJ2mMqYn4wyDFjS
+zDNbGH6O0ENZz5NSY0/UGSOHYrYnYB94QRFLbbf0Y3PmBS2eRNjIUnv7ytPZNMi/
+piM+QhPb0Ebyk0rHQZ0RAJaC/wsEtqP8TGV/fx+AzG7zW/zxgPTrgIThy138tLQ+
+xCVDP9H2c17nVP6vjYzKnMZ94uGrGqUzV9vU7EqYl0uZflIf98pLfdKHnQ3heqds
+8KQPNKRxVvcc92qv2pQY951wb1fkhLutjHn7TUvrenyAngz+Vs19NxbqLPys1CTw
+iaL7vZ8VE/aEDm1tjt5SLM474tpATjk1+qMRaWnii8J5rTodYHP+Zu2GxyIrMiGq
+tfNZMYI0tETK1XmEo75E/3s9pmIeQNGKLFp+qL7xrVyN/2ffNv0at8kkqXluunK9
+/Ki0gKYlGFm4Eu8t/nHMqhBx/njYg6pLDuarLW6ftUV7aHd7qKcCWOWqK6gnH/vX
+3Apv31eltZBBVN69p3CFy2oMnjrom2Yn/DUXFwrJLBiNJ1dd1JyDxpqpJ74ZQy+/
+pSRWMTRM5SuC7lYARx5rYPmp6cZJWyWRH/3r7bwS699/W965pa5nug==
+-----END CERTIFICATE-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/client.cert.pem

@@ -0,0 +1,28 @@
+-----BEGIN CERTIFICATE-----
+MIIExzCCAq+gAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwRzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEYMBYGA1UEAwwPaW50ZXJt
+ZWRpYXRlIGNhMB4XDTE3MTEwMjAwMzcwNloXDTI3MTAzMTAwMzcwNlowPjELMAkG
+A1UEBhMCdXMxDjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEPMA0GA1UE
+AwwGY2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwquL6gtP
+R7P9xJK76FTj8fI5TSJa3cAMt1p6CmessjHQq7nQ6DWLGVi4XIt9Sc/1C3rXupOe
+90Ok4L0tsuVZH78Wn0EBmBH7S4IbhU9P+aJ9mcigepj1lnxWqoVblgeJYKMOOwAf
+pAKUNMWDSm+nCfwE+R5d8d8cfA41Awq1jTRjOVpiJq6aoKfs791a1ZkZde3kFrNV
+AVjC06GgA1lZd3sHf94hmLeC+xJztRXVE9e+7dcc7nFDH0t5DIKYBAklsHg77mZa
+3IK4aOZew7Lm6diPoMnAzXh2rWpJU6RrEE29gIkJBsF8CL1Ndg9MzssCg6KBjoai
+Vt5dJ+4TSEGCOwIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQD
+AgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQgQ2Vy
+dGlmaWNhdGUwHQYDVR0OBBYEFPeuKDCswk8jaH9tl6X+uXjo+WM1MB8GA1UdIwQY
+MBaAFCoqYgmKh3CUafVp+paXxfz+He+FMA4GA1UdDwEB/wQEAwIF4DAdBgNVHSUE
+FjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQELBQADggIBADYAp8XS
+UjMEpX/zVjRWpAAT4HNEJylCV1QNyhBIAyx38A6xJYuFIx966Htd6W9/Rw4sUY6y
+F4pOmnLCRxIPqFzzMYcBHSpynlcu5G7zqIod3wYIk7BNlB0AzkZn6yD8bM1y5Plf
+myzQVDEGggrDtaW2EehhNIB+wOmbRGITjIcZUEr8V4BlLXkCqOuxauxl82d5/k2w
+LAhjOb9d1VW6RT8+Lcn6drhHZdvtSCe8Z27BcXhaQLL8366mhbigKYJt5adD0KOx
+pl0MQcoL1Rth5cJEj+1/lgUaxcnvh7TaIIGEx0h3olQXsTxSTypU/nww2Ic41xdG
+xl3xvHsxe20IvOOAMRfS/LPW7MCtQ3k0BqB/rAQvmB0r5YITLlMJuBqg+zjYrG/j
+s5szSGAz9r0leFuPraeuZA41d9UBTAJMoVrrQZ4xVHMXQi1oz9E9KlIdbO9+spvC
+ulfO+D+Z4a9trYSWhnQL2dSHT0+kHqJ/8GipiUNP/yAC76dRpDVR3xtYNr73iw0j
+hyDsVjihTD8JBebs3axnt+Bc+FwoCCd6CVcsggfGUNhu/N5LS78b13PcaRzrUNjU
+Eh+8cJvMLst+UQzePlyazzpn7jjN3KsBzWUkbnXCtUs2qRMn8f2gZqliDo7JSFvy
+WtBSCYpikOivuJSQUlrHQ8NaXeddyWQzLY79
+-----END CERTIFICATE-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/intermediate.cert.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFaDCCA1CgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEQMA4GA1UEAwwHcm9vdCBj
+YTAeFw0xNzExMDIwMDM3MDZaFw0yNzEwMzEwMDM3MDZaMEcxCzAJBgNVBAYTAnVz
+MQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxGDAWBgNVBAMMD2ludGVy
+bWVkaWF0ZSBjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOOxzve7
+CG2P9SvKfXkJVTXkj4y79JSZ77Kud/TiPfDbHTqZWKuLTXkOCkCCxfpuJvWXnnj5
+1AeLCdKx9hEwJQeU23EXDt1K+RsRyl09SXtPNnJnqHD1mUHRQR28vGX5ctrQzK8J
+Sa6/mHW4bX8ol100npbgVMDnM4IDfLYcsv4BXMICGkSHOW6Gn0zJaeHzRVPpmnK/
+0k/GQAcIrU2sZ39kVlVQkWq3HJC28cNL/P04hjh4gAf0evo/k9VrEtxPWYMfiPDt
+kOAKueoPv/VTA/zL5t8lyzfhrhxvsJxFg/klapPXK0gLLbhsHyOhnkbrzvmSR4Rw
+xubYJ2dDK0DKx+BIZqlFznjP9BvOtvtuVVMyqg9cfgc7J/OjvAguO0f93MLSfIWP
+uISqv7Llt/Blvy/xI5owvOKVc/hm3d+5pqjWBC1HkVwo4qugpWmM49dFWl4kc4c7
+ayYUjTmcgoj1ZR89w4Off/bPd1A6gXqSkw2VQfgFF+uOos84fP1V+zPWhp3UDY3P
+bFeJtuTdv1gR5w1jCIq6xVJ+UsyDZBaYP7yBBRiNzS1/yXJpnXrvHmDfUeQHLBPR
+N0nbMjqXJ1dVpZwydiI0Qx9DnJtOaq/spUreXr8+PU2jeQdCCAN21MB1umr2gZBJ
+8MZBStTgE7SDByfGmGfp7B5/s/r4O/rNc4WzAgMBAAGjZjBkMB0GA1UdDgQWBBQq
+KmIJiodwlGn1afqWl8X8/h3vhTAfBgNVHSMEGDAWgBSVhLUME20woGVjTdPFUsY3
+TQNKGjASBgNVHRMBAf8ECDAGAQH/AgEAMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG
+9w0BAQsFAAOCAgEAvzLu/Jc8DlfCltVufC54UZ8DVwUfxdGapNBGv4icrs1wMV3S
+xqdaLO+vSp9NeEufi724+/hj4URapW9kSt2TMD7BNJ61QSATZFJajxTFgGa0Zz95
+RBDw8/b5Arz/2pOF4VX+FJ+wqHvoH/2A0T+fwz8hLORhxZHv/cUN6kif4FKCwryQ
+s89e694kXkEiJfquvu7DR9hYCLOJwzMOOJiTnjz3hlQg4WGu7Z8ZvqzCM+how1hr
+nYbUx6a+HfoUf79AHJB0N1EsEEetJ+omvTdrrayCvy1bHA3QgHlJ28QZIJ7MzX9E
+n11/xQ95iTuSp8iWurzjTjbrm7eHnGUh+5QubYLXOzbqKzNZu72w0uvWv6ptIudU
+usttltiwW8H9kP0ArWTcZDPhhPfS9impFlhiPDk1wUv2/7g+Zz1OaOb7IiSH0s8y
+FG72AB8ucJ5dNa/2q5dJiM8Gm5CbiVw5RXTBjlfTTkNeM6LBI3dRghpPdU7Kbfhn
+xYs9vnRZeRMJHrcodLuwVcpY/gyeJ0k5LD6eIPCJmatkYZ122isYyMX8lL2P5aR+
+7d2hhqcOCproOrtThjp6nW2jWTB+R/O2+s6mhKSPgfbY2cdky1Y9FSJxSNayb9B8
+eQ+A29iOHrGVAA0R/rvw119rLAYxjXzToM28owx7IyXKrBaU4RMv5yokgag=
+-----END CERTIFICATE-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/localhost-1.cert.pem

@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFFzCCAv+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwRzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEYMBYGA1UEAwwPaW50ZXJt
+ZWRpYXRlIGNhMB4XDTE3MTEwMjAwMzcwNloXDTI3MTAzMTAwMzcwNlowTTELMAkG
+A1UEBhMCdXMxDjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEKMAgGA1UE
+CwwBMTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEArRAy0Nim9P883BAisXdFoKmgHGTtcLH/SzwkkPWTFHz0rHU1Klwz
+w8u3OkRyvgoQp7DqkohboNMDwg5VrOOcfKwtM2GZ5jixo+YKvJ25oj8Jfr+40baz
+nyWTmOcfoviKrb7u2T9BPEEz5og+lXRDAsTFATGaQDX2LN3Dd9KIw+7sWY+gc3Zi
+13HHaWYhtmfJjzFbH1vDxHKCdSdgtPyEhqcJ4OC6wbgp/mQ01VlPAr08kRfkC8mT
+TS7atqc410irKViF3sWi4YNPf7LuBrjo75FIIOp+sQgZE6xwOuZ/9bT2Zx/IUtCC
+TqzVgZI0s5NVlINtWR6eyyxQ1uDKTs4xrQIDAQABo4IBBTCCAQEwCQYDVR0TBAIw
+ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu
+ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUDE8pwi7aELJjvyNT
+ed81/KIgGfowaAYDVR0jBGEwX4AUKipiCYqHcJRp9Wn6lpfF/P4d74WhQ6RBMD8x
+CzAJBgNVBAYTAnVzMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxEDAO
+BgNVBAMMB3Jvb3QgY2GCAhAAMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggr
+BgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEA2cvXxJw120Z9oWXyGwR6CH7TcXoy
+1i77B1M5j0Krvkjh2/MkEU+JxpZcrhAgZODK9wMPeIUIpJNw2t6Hg+vigpInu7pY
+MXR4IA5XLnhGV/hueXa0JLia5FG1TISxr4piW6Jd9P2pOt3ECm3Url/F0OeFF/74
+jGaAlWkbhqWJ9M7Gd4QP2wUNm0P4CwAqS9DC6dnMz+JXTakEUirOpmq7U8UKT+5N
+QS1K4WuH671n4MiYye3+UoRYt4zPjOzN+QxzvAMtkUBspPmWD6txmD5tKUYDECqn
+0sSbY6ytD30OTHIbICFp40arOffmEEJSriL+uQNPPmvqMxX1G2kUFGm15NLPs8Xa
+J7ChrAaJzssN5J3myZUbDfCuxmTkWg+hGvGmxLraVNWc3fzKFmdszSkXrGIdf2HR
+gZeFI3w6M4Ktx3KctXlsjwqQTYZI/WwLOEpsrHQBPBLQhISyNw4xjZ4MxK8SFZuQ
+IiGps/do0eEgeQ+o3gD1dIXt8YxFIxrgk0pzJONpXGgv/cZrukbLNTBdkTSkIwtx
+TXKdiJbO17H24MvW+UxFdsIoJXmfQZWdQC3p+Dl0iP+K80aI6WbaysmToHuOi216
+e49nmiM72Izul2zmBi7Cq2nRQbHAETsFfqC34FzJlx0aP8WS953IBD0jNi1BB+AX
+BxwiZ1rPjeMvekI=
+-----END CERTIFICATE-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/private/client.key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAwquL6gtPR7P9xJK76FTj8fI5TSJa3cAMt1p6CmessjHQq7nQ
+6DWLGVi4XIt9Sc/1C3rXupOe90Ok4L0tsuVZH78Wn0EBmBH7S4IbhU9P+aJ9mcig
+epj1lnxWqoVblgeJYKMOOwAfpAKUNMWDSm+nCfwE+R5d8d8cfA41Awq1jTRjOVpi
+Jq6aoKfs791a1ZkZde3kFrNVAVjC06GgA1lZd3sHf94hmLeC+xJztRXVE9e+7dcc
+7nFDH0t5DIKYBAklsHg77mZa3IK4aOZew7Lm6diPoMnAzXh2rWpJU6RrEE29gIkJ
+BsF8CL1Ndg9MzssCg6KBjoaiVt5dJ+4TSEGCOwIDAQABAoIBABECKAFU56JeKYfp
+Qh20fQ4Amd0RaVsCkpnaf9s037PaAl9eptADDZozVDhRv6qZTtGn8/1LNJJqCJfS
+L5H30+egLHvRlDATMh+QyJLHMTegaNTs4IiVoK97QZ84c54SHoCg/ndNNXaA+y35
+K9VvF+sZZ93UN2UQl06Hdz5Cy0YA7L5HIIH3Ezk0ArAw4AarLil5mv4yEz2ApZhm
+Tw4I4yNfxB7tZeP+ekNg0XXRL1quA0tGblp+A5fAFfVMDplqqB2d3/KxPR9FSEOi
+4PzBZ5Mq2wQBPIaNog5um9qkw6VKxjl5sQGhP1GGTA8iZqR9iM2+xh57xdCZm3g3
+jcr+aPECgYEA42mXTsF/4oBQtU6hh/sOCMWHhxAPstKpQHFMKGYLHKEJ/V1qq0Sd
+d0kswAYCmH5G9ookzu5p7pNf0hUUHO5EwelpSZ3FEmtIM+oBwSnDk3vGuadYXN5X
+fPuVUla65B1F9SSwapYNBUAiRgrY69Knca2rkTSdcZQaBuWmo684UQcCgYEA2yRE
+P23I/9N6AVhKB/zTRtil1AxnTW8o+j7AE4q1o+xly7DS7DT34INaLKLiuG6ylV1F
+UoTiqmWqH3A7m3o3Id2AnVf/oDoKV78LCXRF3dJJWvzrPdob2fLlwyjgqXYvmD3O
+UH/OFY2blYcAHOYib1Y1AAhHPlXiHA52BYZtnC0CgYAVjjitWmII0ijURrPA8+cM
+pcyG3NrgFF++n/6cBbAf8pPD1Er8GPDkEaeQPAGa+r03OTjr9GVOG+IFQ8I4S81w
+o/M66x129XxOj2vDJ3ZGUIExr88MXnbkfeRVfasRXET5S5T9RWPOj5mwEe8lyz3b
+5J5SkS4rSeJ9rN7yvPUVmQKBgAvrrB67LRzldxSNpfFLSn7nGBYx2oi2zEbYlQA7
+ImhZWqw64S5iLz2yR3x4G9cmhmZjnXrAqcfVIez14PgzLL6V2wI0ID6qCZf+V25b
+OdW4M69UZMOHks5HTUJRfe8Z87rXWdq9KQu5GUaIAnSP/D2MNfPbf2yfpV4bV0Yz
+qtC9AoGAD3/XXaeGCdV5DPomEmehp84JXU2q/YECRvph46tr4jArG67PCvx2m84B
++W6my4Yi7QJcW4gC0gsdAuxbJl4Y7MCZBnTtNIRCRnHEIciKITJ/+brFln5QUgyn
+WnXEPN8q7VjSVXGrljFuLWkzi2Vh8iZDgourNfW+iYDGCJjx1H0=
+-----END RSA PRIVATE KEY-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/private/localhost-1.key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEArRAy0Nim9P883BAisXdFoKmgHGTtcLH/SzwkkPWTFHz0rHU1
+Klwzw8u3OkRyvgoQp7DqkohboNMDwg5VrOOcfKwtM2GZ5jixo+YKvJ25oj8Jfr+4
+0baznyWTmOcfoviKrb7u2T9BPEEz5og+lXRDAsTFATGaQDX2LN3Dd9KIw+7sWY+g
+c3Zi13HHaWYhtmfJjzFbH1vDxHKCdSdgtPyEhqcJ4OC6wbgp/mQ01VlPAr08kRfk
+C8mTTS7atqc410irKViF3sWi4YNPf7LuBrjo75FIIOp+sQgZE6xwOuZ/9bT2Zx/I
+UtCCTqzVgZI0s5NVlINtWR6eyyxQ1uDKTs4xrQIDAQABAoIBAC56mDswxH4uAmlT
+yA2Da+a/R6n4jTBkDZ1mFKf93Dd3a7rZa6Lpylk+YAI9GdfiGiD/SbB7AKjLo0m9
+0dKx+ngdQbJ39v42obbT9HQ9o/poFaO91+QyvkDytZYuFHgPaidJjRo5e8qz9D1o
+v+4hoFGhCQvOB5BRLcFU+cc3etWr5t61sNL/kKCWEDd+MWWsOCHpdhEoWC+o25pC
+bhD3FG5xoz+8zL7WdNfke/4twfKoBJ/kq89bfIkl8eKpg387WBQY44RJF7/zVr7a
+9dsUuW2y/wVXslCHChjSrxhRlOyy5ssv3EgKh8gPkZ+oeKuONqAGw27nyKyvpjxS
+i62K+WECgYEA4oKpIS2D77RCC6rpYIK6KYfbUcNSOtHFvcbf0RH9Oi8vSRYum2ZA
+/ITdWSFgWkhT6iOSPuvZlu/EvueWDgNgW1ZVsTMFeapz1+Jwk7JRoBKF1dUEwELh
+jdAswdh0MLbgBYs6NXtVVkeK2ocgZtosmt1PUktl566NlyIyhOjH6vkCgYEAw5g0
+cteTpz+noKsfWcbnjyesuQy0caICfZIE01nKv9rKTF8BtCO6Qxj10iM2o00jW7Vl
+tZa/igjuqvozXAHBI3xegtrWV05urkjj3FB/Pyuqsx3wxhAdSNchQjdTjwUBQEzp
+3ztGSlDTRPpijnpW28lg8Kkr3weryaHvl0xM1VUCgYBqnTN8QU8rgT3g/gYw/fcf
+2ylY98V5mAkqBTSN1JjLTTBFh2JSlLOb5/HDpRkUBZ0xxKJuaVaWW67QaHLRj7dH
+5oAZErnOBXPXNmbkrfcLkAxclJJS6Gf/9u9KIla2Iy2YjmrMh4uoO65Yo2eV4bVD
+A031nzWM8jUE4PzEYEjRCQKBgHDdTj6KiQg0Yg0DUabjcNEZasCpRSJhAyDkdmZi
+5OzKWnuxQvFowF1hdM/aQ/f9Vg7gYJ1lLIeBWf9NOv+3f3RzmrHVh2N/vbxSETIb
+PSH9l5WeDEauG8fhY66q8EuR7sPk3ftTX98YPqEJ/n8Ktz5COO8GH2umKInEKNXc
+UGW1AoGAfENy7vInNv0tzFWPSYdFgesvzo7e8mXyVO8hCyWvY3rxW2on7qfLF3Z9
+fHjd7P9gULja0n1kvmxwUC3u20RrvpY59F4hfi+ji2EiubS9Yhszd2e1CLeRMkln
+ojDjnflN32ZbWVHX/i6g3Dabq9JOD0FsOaOlriLMuofdA6jTUFE=
+-----END RSA PRIVATE KEY-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/certs/ca.cert.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFZDCCA0ygAwIBAgIJALhSfZ8i0rWTMA0GCSqGSIb3DQEBCwUAMD8xCzAJBgNV
+BAYTAnVzMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxEDAOBgNVBAMM
+B3Jvb3QgY2EwHhcNMTcxMTAyMDAzNzU4WhcNMzcxMDI4MDAzNzU4WjA/MQswCQYD
+VQQGEwJ1czEOMAwGA1UECAwFZHVtbXkxDjAMBgNVBAoMBWR1bW15MRAwDgYDVQQD
+DAdyb290IGNhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEArHaQ3uyp
+wVaVPZDYvy/EJbnP7KbZNPBvKpQCDEqg9B2TPaC8WVjiqT6I88c+KcGuB8nADJdk
+o10iZC5AwbrXK4ELSCOvBpxYI5sjoFcv3lZ/oe4e650QO2L/ADmtwLaLYK6rZkwW
+Sd90yCGF7ZOTZTJZDwmWEl+ohi+2ow6sRMHKcSKUNfx9G5BB7TOzoqUxqH+moEds
+YpjVMEcKzQi2FmbRd+8Dlg2eGqA2V4faprGQwoYz8NqJZGa/KPpRvXE2VjSTDN6b
+rJ7mmui6eYN53mZEBRYogyoQHdFXhK02FgyoPEgR/wQlLLbQ+xxOcv02YsOljtza
+hl5LjeNUYPMjyhef0QpONp+5NcFhZf38DsSq5EWZLLxPScxwl0lBQkJTjo5ARuFl
+Mrv50RYrLwv4ImsiO2ftE7gAX4vNsgcixnCHd6rNzoGimf1+DSvDVJ9ujWo7HPN3
+7ONuoyjsU4mUJJpYXs8zHx5WSxaYiPJRcmG3LjcU5/A+Fs7bkqSrlEjJsG29xDrO
+vKR7hH+m6MwcIcXSh9wjjAIvHxAALdU9xaYE3hmVkoxew1mRBsYq34h2vpwGOY5r
+0njRQyGGZnVa8qkQd6P3U5fcvLOM8v9QImZqRDS2jAGZXYruo/RIgJpklVX7ZY0+
+CnGdz4YxgLyOBJCDu3aEgL1oON3mg2SsrVMCAwEAAaNjMGEwHQYDVR0OBBYEFOBO
+9R6yEY6KOE+aSClwD2BQtWXKMB8GA1UdIwQYMBaAFOBO9R6yEY6KOE+aSClwD2BQ
+tWXKMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB
+CwUAA4ICAQBElio7lLZ2eNt0tDPVSkChksc0IJ2/DpjYjqJzqxF/jZ2oToaAn2Er
+9iHl8ggTLB5WrTQxO1ev7qOwQsk9hrgvZ+EQCBTbyDyfNwVjgftH5jdQGdbyrnuJ
+yaks1mnd8is5ZZQmmQSd7GVOMTYi/7yH0xI4DHQ386dwnf5eKdoOI7yrVufEMxRx
+tB3Wv8KrX4z47zsIO28H/O0T26oUkrd4PEqFwDa5HQr6iG7QQgoGD/DPLgbBudlO
+kEos9fmXxiX60RLziKCE/DAxI3YWPvG3WhIWnVj22Oz6apz2pYWpOKwlaihNYrhq
+8xc02vIFwKh+t7D+wF4KHfduyMJ/wKVc5dzpNbTgkZePPKSB7QgbsMeRqbdPoXQF
+pMuzfj8VCWzpqBeHqE/adSCZhzeTrnyiYavF4T2vkSC5KJu+MHmbZ3nU9bcnnEy+
+24oEv9cEAiYNkvftbD+5ByEtkcBB2uT47sbiGrAeco+GxFGUVqi1IjObqrkIrPzV
+OjQhTZV6qgYCOuniJiGfoiMeHqdaDybpqo1bIrxSlvGRNcVoOsKt2/KP1DzW4ARZ
+hoRvayU2apHz/T5TAailqAW2MsrjGRaVHQTmeZKag8CKtAcjWzui0J2DnfXxUMn8
+R3ruFu3xJduOT1VghT9L9udvX9YhPCIKVL9+B5eFX9eMV6N7hUnVug==
+-----END CERTIFICATE-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/client.cert.pem

@@ -0,0 +1,28 @@
+-----BEGIN CERTIFICATE-----
+MIIExzCCAq+gAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwRzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEYMBYGA1UEAwwPaW50ZXJt
+ZWRpYXRlIGNhMB4XDTE3MTEwMjAwMzgwMFoXDTI3MTAzMTAwMzgwMFowPjELMAkG
+A1UEBhMCdXMxDjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEPMA0GA1UE
+AwwGY2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyxZFTLqv
+Gd9SpFAykyRyLQgHcR5hgD55mz+9fl1OfnMoAc7yTdPVLksDLmeFUlxcvCtLHysJ
+klIBX62c6LzbsVcfLg/DPJlQxFnkhJCRKen4fp7x9h62qqJkDFVXsiEFza9L1lsN
+4OwqU8i4RRgZ/xggM/s/wVBtynioeW9QADNmKZ1n6HVKkYwdOynbFSggYfFrL3HL
+54bC9roZUETin0G5wZ9QU+srgivT0a/KC3ourBYHXAI40iHuuOBf3syDVJ6xId/r
+3UO3qkiQ5q7pwglg+8Nx7Q3CFtGZY3ewxSSSDo6BOyweGYMsBaxMO3EyTqecyfXn
+3n4XPqwmDalWYQIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQD
+AgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQgQ2Vy
+dGlmaWNhdGUwHQYDVR0OBBYEFP2bodoNQ1tCNEOALPnygGMUfNI+MB8GA1UdIwQY
+MBaAFOWzLd7eBJwSNbzRqNsD7MQDCHg/MA4GA1UdDwEB/wQEAwIF4DAdBgNVHSUE
+FjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQELBQADggIBAHqUuCLt
+olOdR9p/g+KgGPnKuVgMn15Wc2VLCrbbl2P0fuCcNWmnBKqHHgQ1EJEpgnQ2N8m6
+tOGucX7IAzlZj36RP4lN3gZqFRSO/OiTOUYpE6Uv1hYRxeMzAYo5sBdCiiypjV9z
+H0Ew5NuWRf2/0nFWoywB9ktHcfD8lRFI3o8zUFXmE2JSUPQtKhW3tBkPPjYBlgzD
+RD8cq8dVK9P7i3tUENP+MNHJToNLFBqfA9De6bKnhCWHhZkfB0VeeSm4Ja9HkCg/
+DB+PAKMfbLCH5T8gCpEWxNlvj09r9mn37fNjtJPO/goAcNZNO2AURmb/ZQ4ggdry
+xb6lm832qplMUMWx//Ore0faEodlEc5d2kEtmcjj79gAypcLmm74q7CPt7xmniyd
+XvNT33S2tkh4dSirpCVwq0xyqOP3ZqTsTjudTveTBaTZNhTbCjDbaV7ga47TcH9/
++OZ3fQKjt2LAC6162wgEFZf10nUgaAXvSlI74gru93vEwWd8Pd3sWfGwuAFX3oKI
+JuwL2kxEuoZQmeRiVJu6KQb+Im7d5CIoWViDmfxcSDJfdtSePTqmDURIx87fw14Z
+XBWJP4PiK5PRmG/L0cGiDckmDKm/MuD13Z2I/NMl81GNY/q3WY2O7BmddPpAG5dr
+sc5hOqA9+jX08XbxKnfBPYllK5skYMkFH5tN
+-----END CERTIFICATE-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/intermediate.cert.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFaDCCA1CgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEQMA4GA1UEAwwHcm9vdCBj
+YTAeFw0xNzExMDIwMDM3NTlaFw0yNzEwMzEwMDM3NTlaMEcxCzAJBgNVBAYTAnVz
+MQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxGDAWBgNVBAMMD2ludGVy
+bWVkaWF0ZSBjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKuM2iFz
+CCKmbs4uLj/8wjtxf+jFmkwzN4Pps4hqJ3NJqWB326LhzWyqM4WiTAJWE02wSJbS
+16RPfbjkVC77OI4+PUdwqxU9vNAP/95w0h6hBSFtkysuT5VVUt5jiY7wnUKgqTCi
+MYhYOl+HEP32O4cnxAazkUKKvtyrd4/PvejJ9zz+iYexRnaaGfOFR3co7jQ5QKar
+oK4UgJC3mVDZQEMBV0oljkpgVQMAVb4XQU7e+o25JOUkOoK6LdK/b/95khR0jTyD
+OBLqd4mCEzxNi+jZ0jLTLPk0c+DnGmRfoNUxnFb40R8QnEIEKwf+JKyl6p89oqOl
+pvIZFLZlUWIS4qL+993l1SCqPkWJOAdTg+s/Zh6DeAOhrUn9/wk0aQwZrK7wQQLJ
+4GGhxC/FfuUGsLqZszAVkP8jDEWnzhN2rw3V+C7v6lj4qHhUwqGHuYzCx2Hxl+B8
+UyBmZb9gXKVUtAvaZjaL2PDj1ZAxT0KVMlw1ZVrZu45OsHNQuBx/4uIAt6Rga8yt
+av1lsq+hFqwI4bU/oZC/oPMacOsB4qEkAA1101WjMc5bg6JOPWobwIqmUXQR1WJE
+j30e99HCpk1Cc2+9sUCzNu8KvU5kUY2K90zwqProvj5IfMuDetAVXsEjgW+ZqSho
+UMIpJ2M/hzAFl8Z5IRlG+YNfZNXl0FqJ5LzLAgMBAAGjZjBkMB0GA1UdDgQWBBTl
+sy3e3gScEjW80ajbA+zEAwh4PzAfBgNVHSMEGDAWgBTgTvUeshGOijhPmkgpcA9g
+ULVlyjASBgNVHRMBAf8ECDAGAQH/AgEAMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG
+9w0BAQsFAAOCAgEAOS7DtliOUPcVosRfyx9dHcSUc3O+uY8uKjSHhdIrxDJm4lwP
+Q6lKg5j8CdMVb+sDQmyBkqQIA/6E13corP6R283jO6W8D4A8kjOiQWpXfjW6OcP3
+4rrDEWhCdeLFSNJIYOFkr2qWJpI/k0VpyDnmY0YluS5WbNjg6zTzGelzhFbV7/S1
+cteNAZD0vHD8NmbLVDJjjIY3E/iwzoUzBncLYbDwqyVS1g6utWdSy8LEJxzzqqWJ
+pBKlNYILAdh8efBgvotafaxsn2nfjmVmekPn3KcQZuE4Kzv1EQ2PrHpGeJKwh6up
+YBL2tav5cAki8bWoGPr2oGmWUf9L2tB57SdWdaY60ifzmQaeGiWPZBSmAz7PRSrz
+sR9SMIkBfYVRxXgWwlvr8JYnd2h/Ef5K9fI32nGfje+7/0kPEjNyjehri7sV4Sjt
+zzkDiFO+JklrRuLBPMFYOokq6Pcko32FKlE82pe8QkMDS8Sk//9PqCTK9ceB7y6E
+NYLNBW/X9SAw/TR5kdRinHHgHyEug7N4+DCU3lU1wl72ZjoiGE7V6c2AssFC2VcE
+E+WYxJT1ROJ1/5+U6BKdaIpTwMtRIFRomOEI66iOwOSEwqLIztkqxwpQ7THraWKm
+2W5e54u/efapIDcQFnP3E8r7TD0PdIeU6mD28o0+WiK3uL/OZpvyKaHPeFU=
+-----END CERTIFICATE-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/localhost-1.cert.pem

@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFFzCCAv+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwRzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEYMBYGA1UEAwwPaW50ZXJt
+ZWRpYXRlIGNhMB4XDTE3MTEwMjAwMzc1OVoXDTI3MTAzMTAwMzc1OVowTTELMAkG
+A1UEBhMCdXMxDjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEKMAgGA1UE
+CwwBMTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAycY/7H1xk3/XHZRopULV7YsOzPIrMG25zoACbpDZxjS0I+2r1c7V
+wnvE8TszAkloLi+Skku5CYC7IvVEEEuKuIuV+8M48FJEwlCPge8LPiy18C+npCEd
+fgDzCV/O9DfJj6UaiCUayVE7UujXoke7AlKQEJcqvnD/CoTv2Y8jV1A6mPf6CTEI
+Sl1BMeFSmeFyvZll+xJ8Up1KfQZxKhtpP1s/rp6ZNlqSs1LM5+vcDHHZ6COTbq7t
+2vvcmGDTqeCLsqicBg1kJyMPRtqa0bNPj2bcVtcK0Ndfn6eL2hi+EoBy2nIXi6aG
+PpXf85b9bCLd5pZI80nHzFlhdvV+SxqrfwIDAQABo4IBBTCCAQEwCQYDVR0TBAIw
+ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu
+ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUoYjECaDz/ZELru/r
+jfTB1ShlVrAwaAYDVR0jBGEwX4AU5bMt3t4EnBI1vNGo2wPsxAMIeD+hQ6RBMD8x
+CzAJBgNVBAYTAnVzMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxEDAO
+BgNVBAMMB3Jvb3QgY2GCAhAAMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggr
+BgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEAiiR2knMMym4O+3fD1KlYSnc2UR3v
+0FlRVAsr8wvTlVjJhx7DbRusBNJHWX66mUgK9x5OLnhyvyqlFVhR9AwlnxgfLWz9
+nnACeXzcjQZnKWFQWu8bJSC6Ene6rd1g2acK6SOjxavVbj7JVFnmlHF/naZUzvMl
+mJivYta4k7ob8UcX0I5TlJpzglU3UHyyJd5d9zhbF8wqbBq63zR2ovWci4pYCg+F
+jYcTGYVZJti3SHO+9/EqTC9x2KDNs3o0+rreJ3GuoonkInKZMQQZJQ6qILvkxlhT
+jyU5xlcaJ+0tSaiFK3eF0nXIpFYdZbIHYPCdLjh9AZ2dkFcAgSa/L8+tsVt60k8D
+HTO0Hz6dW5D2ckeebZvz5LACMN89gVzrc/rVkeg7QmpSbjkTSLC2KJS53hJzWcEI
+3KB73B9iY+ZYytcYBTYLizsAxd5g7j9z8UXrmVQ4mWbh2+xKiG+9aVOzCZ09AYi6
+WVK2aRcMQshgkkqPOloN9OeQNCE8Exf7N/zHsBhygorJXoD/PFgnV1VZm8xkOdiJ
+zTb3bpGdmL5+bzzS6wP8Q7pGZGYdlnB7JNO8oMYPPtzX8OOx92BTkPnqJnnRWTpR
+SjMEEdQe8K7iXxejQkjaAq5BlwaAOjCjPTqYomECcYjC0WaXsmrPcnZwSqpnHZZ2
+OiINYJub5cvBLNo=
+-----END CERTIFICATE-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/private/client.key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAyxZFTLqvGd9SpFAykyRyLQgHcR5hgD55mz+9fl1OfnMoAc7y
+TdPVLksDLmeFUlxcvCtLHysJklIBX62c6LzbsVcfLg/DPJlQxFnkhJCRKen4fp7x
+9h62qqJkDFVXsiEFza9L1lsN4OwqU8i4RRgZ/xggM/s/wVBtynioeW9QADNmKZ1n
+6HVKkYwdOynbFSggYfFrL3HL54bC9roZUETin0G5wZ9QU+srgivT0a/KC3ourBYH
+XAI40iHuuOBf3syDVJ6xId/r3UO3qkiQ5q7pwglg+8Nx7Q3CFtGZY3ewxSSSDo6B
+OyweGYMsBaxMO3EyTqecyfXn3n4XPqwmDalWYQIDAQABAoIBAFhIOR3OtVlw3BLz
+jdiq6jsrF1kUFNxTzDcxsSUiWIHde1G17Vzpre0uzJY6iBkyb1mZFFHbOpDxtwkp
+hmEh3/qqXbJ/RaatGxAP56e81G28+LnKTHJqDYwFhapa2wFjG4u7HSN0d4cEAq5j
+Pb9DZ+GdUjpmiON3HBL8+ne3bLZ42uI+DSVe8d3irbqg2rqsiANf0gdimMW4nuI4
+rVxf8HrY43PdQn/Vby+7qLRE3tmIlpbTqJGRtWRjdeBBI91APCrRljjXrKqT6Zpa
+E6Daz3YIQvXkIT0q+WkeN1VmQbtRnk7kRsPNp15kSwpHfmv6o/vkO9OUb1n71P2F
+wnB0WDECgYEA8iltnKxXnjqwZ/vzIWzcd94j+mdZg/K2/JCOqjwMvpSGCvx2zUmq
+Y2nxO2K85AVeOm/Yt87SMODB6AQ9CsrVGEUAzzacvCJDb8oUhaOL5gypnyvZiGCy
+snzXfgB+v/xuGekIjs2y7E8h3GG40j0aNQnUY1Fuc6iaeJG4BtjkuQUCgYEA1rE4
+DrTSsUh3hLYQusIHZR8Lecrrd4QUZSMKLkWjobiSTw3m4mglx1s2G4eZ3WuzOyFq
+Dp3/b3yfT8prdPBGA6shHNFf+1TO1q1/pIt15dc3sFwxMkuunai8N4QZJRqZLbYq
+FkNFkZ20hFHcH/NHDsAsRL/0tJdEmJ2ruP+Qdq0CgYBsdPGKwgVb8J0hdU4nIkJ7
+zRoABFmrJwGdjIDY7Zwnnw2JzhjHSL7vV3ubRVWkKmNReNZvPEoXahJuf7d3JfDa
+tczvAV6hRBc/8hnO4Li/h9xQVatP0T83gYJiBIbAJaaKJDyY+Lex7p8TvRCx2Hvs
+VUKyWL5HPrQwW9M3/dwyoQKBgQCNQoPA4Wcz8Jt7PZQaXaoh9eBGHab6t3P366s6
+MOXudZQG4f3FgINC/ZfHW1x43PFL+btfrMOyJkxoYqZ7hdB7f3DFFlpR80Y46GVw
+7bYAKbBhoPdZwYQ+BhT5bjhhOnQJKK/egBrZKevpmDb+6sIZSYaXIbovzMv8otmn
+WrhB7QKBgAdl+KYBQULCUBp8qCQH5sAQoWErpyuD2FNN6LGknpPqn4DdujvwEP0Z
+OSvbauLkI0Qc9/MezKPTeYXlFqdbpItwyySJsUkiI3HhVYlBgDkZ7xb6uHIH5E6I
+bKgIW5JEf5I7Eu1iurORkXxCCGMkiQmEs4X5kSXXRYgXfNgAD0FX
+-----END RSA PRIVATE KEY-----

src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/private/localhost-1.key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAycY/7H1xk3/XHZRopULV7YsOzPIrMG25zoACbpDZxjS0I+2r
+1c7VwnvE8TszAkloLi+Skku5CYC7IvVEEEuKuIuV+8M48FJEwlCPge8LPiy18C+n
+pCEdfgDzCV/O9DfJj6UaiCUayVE7UujXoke7AlKQEJcqvnD/CoTv2Y8jV1A6mPf6
+CTEISl1BMeFSmeFyvZll+xJ8Up1KfQZxKhtpP1s/rp6ZNlqSs1LM5+vcDHHZ6COT
+bq7t2vvcmGDTqeCLsqicBg1kJyMPRtqa0bNPj2bcVtcK0Ndfn6eL2hi+EoBy2nIX
+i6aGPpXf85b9bCLd5pZI80nHzFlhdvV+SxqrfwIDAQABAoIBAQC022161aoTEtjH
+m7n8v56vUCCRFVQfEYsljFohrtZ0sdLyDVwjxkSWEYiizXRYTWIDXALd/N+7o9aZ
+bAx5Kq0J45wpUYBc8PDO15T6W0DRlxPxWVXDaSddRQ6TTXxcLREPH2dbtx5+asBo
+/Woi/Haki0q0hDr8/p2sWSH/+SwtWpOezGVlrWrkMeIhlBwHZfdHVoZvSx65Uv7x
+WU07vsjrbXNDwf+2fmklAQrzhedCeh8loGyjtN3cfrTjrE1zqpEsHnlZcJxe6sRB
+1nOqpoUnpZXklDDIYC8EmeubmDJ0jnXOQCDDep3MzVcnZGyF5E/+szaa1NL70Ayj
+rbKk1Y3ZAoGBAPy/1ym7Cjl4OGHN2fdkR6iL68ebJozpr+eTSxDNLuBSi5IJxJyG
+1+B4+v1u0RwZ3DjrSQsO5DCbZ+DHU6O/DAJK2CxUED+M+G2kRyffailRQmNzjpRG
+75dIhSkSRYH8vdvEOnGpeQBZwBcCRH/2YUMlZeSfx9fHJhk1nyUxJeHjAoGBAMxe
+k+cBb0zYok+Ww1xTwOdq0PwKj0oDsEg8hOdWc8pH0SlOAB4BI5kmfd1JDMHfRc49
+7tpNqjsPrnlb9xd8l0281Lj2NoVSE5KX1JtsOsKecQsvHH5zRk4eJ3h/mNixpjfe
+79Zc/O40T4rWpQRqhat+WHveJC0/ON4AH4uT0BK1AoGBAPcTioCu6YXYsjVaCJPB
+IhPwBGOylfL2lxDoel9IVWTRDMOMbPkfEHXNjn6lECJKXW//Af6fZg7mPJwN/wN5
+xYGQLNbYrrGRW2HDUBP4YU1WtHGIC3+EAL+BEztdMzmpGuh1YTSvmSvwkMltXA1D
+iz0amArw72lOsz29n3+6FfBFAoGAIpRqMC8k9vq80/yth6TAQifnvo3G2v4uyLo8
+vqv5IaPvNy70hB8rN9G0gEnI99Dgjdoa3SNBB4dKvUwbTgUN0OB/meBHL13I5Af+
+uGGiu6V1eS/6gUbeAX/Gq/PjF99PQareKAZJ4cBGKTbSayHfBjp1nFflBSbqZ13b
++JEFJvUCgYBOs2J2XXamPbI7gu7B2TE9j/62v0SJyoHq2LHMmYUDRuPdPk3eKCt3
+283w+E8XUIFbctaxsbo8msNjjvV22D/Nci3d87aPe8bn1SVto3GnTuwnOpRq3E+3
+wAarqrhiZbGZSCcAkEOk7FlxAwYnCM6paqMxDEMCJ4qChMM42E9ZyQ==
+-----END RSA PRIVATE KEY-----

src/python/grpcio_tests/tests/unit/resources.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Constants and functions for data used in interoperability testing."""
+"""Constants and functions for data used in testing."""
 
 import os
 
@@ -34,3 +34,81 @@ def private_key():
 def certificate_chain():
     return pkg_resources.resource_string(__name__,
                                          _CERTIFICATE_CHAIN_RESOURCE_PATH)
+
+
+def cert_hier_1_root_ca_cert():
+    return pkg_resources.resource_string(
+        __name__, 'credentials/certificate_hierarchy_1/certs/ca.cert.pem')
+
+
+def cert_hier_1_intermediate_ca_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/certs/intermediate.cert.pem'
+    )
+
+
+def cert_hier_1_client_1_key():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/private/client.key.pem'
+    )
+
+
+def cert_hier_1_client_1_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/certs/client.cert.pem')
+
+
+def cert_hier_1_server_1_key():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/private/localhost-1.key.pem'
+    )
+
+
+def cert_hier_1_server_1_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/certs/localhost-1.cert.pem'
+    )
+
+
+def cert_hier_2_root_ca_cert():
+    return pkg_resources.resource_string(
+        __name__, 'credentials/certificate_hierarchy_2/certs/ca.cert.pem')
+
+
+def cert_hier_2_intermediate_ca_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/certs/intermediate.cert.pem'
+    )
+
+
+def cert_hier_2_client_1_key():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/private/client.key.pem'
+    )
+
+
+def cert_hier_2_client_1_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/certs/client.cert.pem')
+
+
+def cert_hier_2_server_1_key():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/private/localhost-1.key.pem'
+    )
+
+
+def cert_hier_2_server_1_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/certs/localhost-1.cert.pem'
+    )

templates/Makefile.template

@@ -215,7 +215,7 @@
   ifeq ($(SYSTEM),Darwin)
   CXXFLAGS += -stdlib=libc++
   endif
-  % for arg in ['CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'LDFLAGS', 'DEFINES']:
+  % for arg in ['CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'COREFLAGS', 'LDFLAGS', 'DEFINES']:
   %  if defaults.get('global', []).get(arg, None) is not None:
   ${arg} += ${defaults.get('global').get(arg)}
   %  endif
@@ -1268,6 +1268,16 @@
   	$(Q) mkdir -p `dirname $@`
   	$(Q) $(HOST_CXX) $(HOST_CXXFLAGS) $(HOST_CPPFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $<
 
+  $(OBJDIR)/$(CONFIG)/src/core/%.o : src/core/%.cc
+  	$(E) "[CXX]     Compiling $<"
+  	$(Q) mkdir -p `dirname $@`
+  	$(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(COREFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $<
+
+  $(OBJDIR)/$(CONFIG)/test/core/%.o : test/core/%.cc
+  	$(E) "[CXX]     Compiling $<"
+  	$(Q) mkdir -p `dirname $@`
+  	$(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(COREFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $<
+
   $(OBJDIR)/$(CONFIG)/%.o : %.cc
   	$(E) "[CXX]     Compiling $<"
   	$(Q) mkdir -p `dirname $@`

test/core/end2end/fixtures/http_proxy_fixture.cc

@@ -88,9 +88,11 @@ typedef struct proxy_connection {
 
   grpc_slice_buffer client_read_buffer;
   grpc_slice_buffer client_deferred_write_buffer;
+  bool client_is_writing;
   grpc_slice_buffer client_write_buffer;
   grpc_slice_buffer server_read_buffer;
   grpc_slice_buffer server_deferred_write_buffer;
+  bool server_is_writing;
   grpc_slice_buffer server_write_buffer;
 
   grpc_http_parser http_parser;
@@ -148,6 +150,7 @@ static void proxy_connection_failed(grpc_exec_ctx* exec_ctx,
 static void on_client_write_done(grpc_exec_ctx* exec_ctx, void* arg,
                                  grpc_error* error) {
   proxy_connection* conn = (proxy_connection*)arg;
+  conn->client_is_writing = false;
   if (error != GRPC_ERROR_NONE) {
     proxy_connection_failed(exec_ctx, conn, true /* is_client */,
                             "HTTP proxy client write", error);
@@ -160,6 +163,7 @@ static void on_client_write_done(grpc_exec_ctx* exec_ctx, void* arg,
   if (conn->client_deferred_write_buffer.length > 0) {
     grpc_slice_buffer_move_into(&conn->client_deferred_write_buffer,
                                 &conn->client_write_buffer);
+    conn->client_is_writing = true;
     grpc_endpoint_write(exec_ctx, conn->client_endpoint,
                         &conn->client_write_buffer,
                         &conn->on_client_write_done);
@@ -173,6 +177,7 @@ static void on_client_write_done(grpc_exec_ctx* exec_ctx, void* arg,
 static void on_server_write_done(grpc_exec_ctx* exec_ctx, void* arg,
                                  grpc_error* error) {
   proxy_connection* conn = (proxy_connection*)arg;
+  conn->server_is_writing = false;
   if (error != GRPC_ERROR_NONE) {
     proxy_connection_failed(exec_ctx, conn, false /* is_client */,
                             "HTTP proxy server write", error);
@@ -185,6 +190,7 @@ static void on_server_write_done(grpc_exec_ctx* exec_ctx, void* arg,
   if (conn->server_deferred_write_buffer.length > 0) {
     grpc_slice_buffer_move_into(&conn->server_deferred_write_buffer,
                                 &conn->server_write_buffer);
+    conn->server_is_writing = true;
     grpc_endpoint_write(exec_ctx, conn->server_endpoint,
                         &conn->server_write_buffer,
                         &conn->on_server_write_done);
@@ -210,13 +216,14 @@ static void on_client_read_done(grpc_exec_ctx* exec_ctx, void* arg,
   // the current write is finished.
   //
   // Otherwise, move the read data into the write buffer and write it.
-  if (conn->server_write_buffer.length > 0) {
+  if (conn->server_is_writing) {
     grpc_slice_buffer_move_into(&conn->client_read_buffer,
                                 &conn->server_deferred_write_buffer);
   } else {
     grpc_slice_buffer_move_into(&conn->client_read_buffer,
                                 &conn->server_write_buffer);
     proxy_connection_ref(conn, "client_read");
+    conn->server_is_writing = true;
     grpc_endpoint_write(exec_ctx, conn->server_endpoint,
                         &conn->server_write_buffer,
                         &conn->on_server_write_done);
@@ -242,13 +249,14 @@ static void on_server_read_done(grpc_exec_ctx* exec_ctx, void* arg,
   // the current write is finished.
   //
   // Otherwise, move the read data into the write buffer and write it.
-  if (conn->client_write_buffer.length > 0) {
+  if (conn->client_is_writing) {
     grpc_slice_buffer_move_into(&conn->server_read_buffer,
                                 &conn->client_deferred_write_buffer);
   } else {
     grpc_slice_buffer_move_into(&conn->server_read_buffer,
                                 &conn->client_write_buffer);
     proxy_connection_ref(conn, "server_read");
+    conn->client_is_writing = true;
     grpc_endpoint_write(exec_ctx, conn->client_endpoint,
                         &conn->client_write_buffer,
                         &conn->on_client_write_done);
@@ -262,6 +270,7 @@ static void on_server_read_done(grpc_exec_ctx* exec_ctx, void* arg,
 static void on_write_response_done(grpc_exec_ctx* exec_ctx, void* arg,
                                    grpc_error* error) {
   proxy_connection* conn = (proxy_connection*)arg;
+  conn->client_is_writing = false;
   if (error != GRPC_ERROR_NONE) {
     proxy_connection_failed(exec_ctx, conn, true /* is_client */,
                             "HTTP proxy write response", error);
@@ -302,6 +311,7 @@ static void on_server_connect_done(grpc_exec_ctx* exec_ctx, void* arg,
   grpc_slice slice =
       grpc_slice_from_copied_string("HTTP/1.0 200 connected\r\n\r\n");
   grpc_slice_buffer_add(&conn->client_write_buffer, slice);
+  conn->client_is_writing = true;
   grpc_endpoint_write(exec_ctx, conn->client_endpoint,
                       &conn->client_write_buffer,
                       &conn->on_write_response_done);
@@ -450,9 +460,11 @@ static void on_accept(grpc_exec_ctx* exec_ctx, void* arg,
                     grpc_combiner_scheduler(conn->proxy->combiner));
   grpc_slice_buffer_init(&conn->client_read_buffer);
   grpc_slice_buffer_init(&conn->client_deferred_write_buffer);
+  conn->client_is_writing = false;
   grpc_slice_buffer_init(&conn->client_write_buffer);
   grpc_slice_buffer_init(&conn->server_read_buffer);
   grpc_slice_buffer_init(&conn->server_deferred_write_buffer);
+  conn->server_is_writing = false;
   grpc_slice_buffer_init(&conn->server_write_buffer);
   grpc_http_parser_init(&conn->http_parser, GRPC_HTTP_REQUEST,
                         &conn->http_request);

test/core/support/BUILD

@@ -138,6 +138,16 @@ grpc_cc_test(
     ],
 )
 
+grpc_cc_test(
+    name = "manual_constructor_test",
+    srcs = ["manual_constructor_test.cc"],
+    language = "C++",
+    deps = [
+        "//:gpr",
+        "//test/core/util:gpr_test_util",
+    ],
+)
+
 grpc_cc_test(
     name = "spinlock_test",
     srcs = ["spinlock_test.cc"],

test/core/support/manual_constructor_test.cc

@@ -0,0 +1,99 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/* Test of gpr synchronization support. */
+
+#include "src/core/lib/support/manual_constructor.h"
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/thd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <cstring>
+#include "src/core/lib/support/abstract.h"
+#include "test/core/util/test_config.h"
+
+class A {
+ public:
+  A() {}
+  virtual ~A() {}
+  virtual const char* foo() { return "A_foo"; }
+  virtual const char* bar() { return "A_bar"; }
+  GRPC_ABSTRACT_BASE_CLASS
+};
+
+class B : public A {
+ public:
+  B() {}
+  ~B() {}
+  const char* foo() override { return "B_foo"; }
+  char get_junk() { return junk[0]; }
+
+ private:
+  char junk[1000];
+};
+
+class C : public B {
+ public:
+  C() {}
+  ~C() {}
+  virtual const char* bar() { return "C_bar"; }
+  char get_more_junk() { return more_junk[0]; }
+
+ private:
+  char more_junk[1000];
+};
+
+class D : public A {
+ public:
+  virtual const char* bar() { return "D_bar"; }
+};
+
+static void basic_test() {
+  grpc_core::PolymorphicManualConstructor<A, B> poly;
+  poly.Init<B>();
+  GPR_ASSERT(!strcmp(poly->foo(), "B_foo"));
+  GPR_ASSERT(!strcmp(poly->bar(), "A_bar"));
+}
+
+static void complex_test() {
+  grpc_core::PolymorphicManualConstructor<A, B, C, D> polyB;
+  polyB.Init<B>();
+  GPR_ASSERT(!strcmp(polyB->foo(), "B_foo"));
+  GPR_ASSERT(!strcmp(polyB->bar(), "A_bar"));
+
+  grpc_core::PolymorphicManualConstructor<A, B, C, D> polyC;
+  polyC.Init<C>();
+  GPR_ASSERT(!strcmp(polyC->foo(), "B_foo"));
+  GPR_ASSERT(!strcmp(polyC->bar(), "C_bar"));
+
+  grpc_core::PolymorphicManualConstructor<A, B, C, D> polyD;
+  polyD.Init<D>();
+  GPR_ASSERT(!strcmp(polyD->foo(), "A_foo"));
+  GPR_ASSERT(!strcmp(polyD->bar(), "D_bar"));
+}
+
+/* ------------------------------------------------- */
+
+int main(int argc, char* argv[]) {
+  grpc_test_init(argc, argv);
+  basic_test();
+  complex_test();
+  return 0;
+}

test/cpp/microbenchmarks/bm_error.cc

@@ -251,7 +251,7 @@ static void BM_ErrorGetStatus(benchmark::State& state) {
     grpc_status_code status;
     grpc_slice slice;
     grpc_error_get_status(&exec_ctx, fixture.error(), fixture.deadline(),
-                          &status, &slice, nullptr);
+                          &status, &slice, nullptr, nullptr);
   }
   grpc_exec_ctx_finish(&exec_ctx);
   track_counters.Finish(state);
@@ -265,7 +265,7 @@ static void BM_ErrorGetStatusCode(benchmark::State& state) {
   while (state.KeepRunning()) {
     grpc_status_code status;
     grpc_error_get_status(&exec_ctx, fixture.error(), fixture.deadline(),
-                          &status, nullptr, nullptr);
+                          &status, nullptr, nullptr, nullptr);
   }
   grpc_exec_ctx_finish(&exec_ctx);
   track_counters.Finish(state);
@@ -279,7 +279,7 @@ static void BM_ErrorHttpError(benchmark::State& state) {
   while (state.KeepRunning()) {
     grpc_http2_error_code error;
     grpc_error_get_status(&exec_ctx, fixture.error(), fixture.deadline(),
-                          nullptr, nullptr, &error);
+                          nullptr, nullptr, &error, nullptr);
   }
   grpc_exec_ctx_finish(&exec_ctx);
   track_counters.Finish(state);

third_party/cares/cares.BUILD

@@ -8,7 +8,7 @@ config_setting(
 config_setting(
     name = "android",
     values = {
-      "crosstool_top": "//external:android/crosstool",
+        "crosstool_top": "//external:android/crosstool",
     },
 )
 
@@ -18,120 +18,128 @@ config_setting(
     name = "ios_x86_64",
     values = {"cpu": "ios_x86_64"},
 )
+
 config_setting(
     name = "ios_armv7",
     values = {"cpu": "ios_armv7"},
 )
+
 config_setting(
     name = "ios_armv7s",
     values = {"cpu": "ios_armv7s"},
 )
+
 config_setting(
     name = "ios_arm64",
     values = {"cpu": "ios_arm64"},
 )
 
+genrule(
+    name = "ares_build",
+    srcs = ["@cares_local_files//:ares_build_h"],
+    outs = ["ares_build.h"],
+    cmd = "cat $(location @cares_local_files//:ares_build_h) > $@",
+)
+
+# cc_library(
+#     name = "ares_build_h",
+#     hdrs = ["ares_build.h"],
+#     data = [":ares_build"],
+#     includes = ["."],
+# )
+
+genrule(
+    name = "ares_config",
+    srcs = ["@cares_local_files//:ares_config_h"],
+    outs = ["ares_config.h"],
+    cmd = "cat $(location @cares_local_files//:ares_config_h) > $@",
+)
+
+# cc_library(
+#     name = "ares_config_h",
+#     hdrs = ["ares_config.h"],
+#     data = [":ares_config"],
+#     includes = ["."],
+# )
+
 cc_library(
     name = "ares",
     srcs = [
-        "cares/ares__close_sockets.c",
+        "ares__close_sockets.c",
-        "cares/ares__get_hostent.c",
+        "ares__get_hostent.c",
-        "cares/ares__read_line.c",
+        "ares__read_line.c",
-        "cares/ares__timeval.c",
+        "ares__timeval.c",
-        "cares/ares_cancel.c",
+        "ares_cancel.c",
-        "cares/ares_create_query.c",
+        "ares_create_query.c",
-        "cares/ares_data.c",
+        "ares_data.c",
-        "cares/ares_destroy.c",
+        "ares_destroy.c",
-        "cares/ares_expand_name.c",
+        "ares_expand_name.c",
-        "cares/ares_expand_string.c",
+        "ares_expand_string.c",
-        "cares/ares_fds.c",
+        "ares_fds.c",
-        "cares/ares_free_hostent.c",
+        "ares_free_hostent.c",
-        "cares/ares_free_string.c",
+        "ares_free_string.c",
-        "cares/ares_getenv.c",
+        "ares_getenv.c",
-        "cares/ares_gethostbyaddr.c",
+        "ares_gethostbyaddr.c",
-        "cares/ares_gethostbyname.c",
+        "ares_gethostbyname.c",
-        "cares/ares_getnameinfo.c",
+        "ares_getnameinfo.c",
-        "cares/ares_getopt.c",
+        "ares_getopt.c",
-        "cares/ares_getsock.c",
+        "ares_getsock.c",
-        "cares/ares_init.c",
+        "ares_init.c",
-        "cares/ares_library_init.c",
+        "ares_library_init.c",
-        "cares/ares_llist.c",
+        "ares_llist.c",
-        "cares/ares_mkquery.c",
+        "ares_mkquery.c",
-        "cares/ares_nowarn.c",
+        "ares_nowarn.c",
-        "cares/ares_options.c",
+        "ares_options.c",
-        "cares/ares_parse_a_reply.c",
+        "ares_parse_a_reply.c",
-        "cares/ares_parse_aaaa_reply.c",
+        "ares_parse_aaaa_reply.c",
-        "cares/ares_parse_mx_reply.c",
+        "ares_parse_mx_reply.c",
-        "cares/ares_parse_naptr_reply.c",
+        "ares_parse_naptr_reply.c",
-        "cares/ares_parse_ns_reply.c",
+        "ares_parse_ns_reply.c",
-        "cares/ares_parse_ptr_reply.c",
+        "ares_parse_ptr_reply.c",
-        "cares/ares_parse_soa_reply.c",
+        "ares_parse_soa_reply.c",
-        "cares/ares_parse_srv_reply.c",
+        "ares_parse_srv_reply.c",
-        "cares/ares_parse_txt_reply.c",
+        "ares_parse_txt_reply.c",
-        "cares/ares_platform.c",
+        "ares_platform.c",
-        "cares/ares_process.c",
+        "ares_process.c",
-        "cares/ares_query.c",
+        "ares_query.c",
-        "cares/ares_search.c",
+        "ares_search.c",
-        "cares/ares_send.c",
+        "ares_send.c",
-        "cares/ares_strcasecmp.c",
+        "ares_strcasecmp.c",
-        "cares/ares_strdup.c",
+        "ares_strdup.c",
-        "cares/ares_strerror.c",
+        "ares_strerror.c",
-        "cares/ares_timeout.c",
+        "ares_timeout.c",
-        "cares/ares_version.c",
+        "ares_version.c",
-        "cares/ares_writev.c",
+        "ares_writev.c",
-        "cares/bitncmp.c",
+        "bitncmp.c",
-        "cares/inet_net_pton.c",
+        "inet_net_pton.c",
-        "cares/inet_ntop.c",
+        "inet_ntop.c",
-        "cares/windows_port.c",
+        "windows_port.c",
     ],
     hdrs = [
+        "ares.h",
         "ares_build.h",
-        "cares/ares.h",
+        "ares_config.h",
-        "cares/ares_data.h",
+        "ares_data.h",
-        "cares/ares_dns.h",
+        "ares_dns.h",
-        "cares/ares_getenv.h",
+        "ares_getenv.h",
-        "cares/ares_getopt.h",
+        "ares_getopt.h",
-        "cares/ares_inet_net_pton.h",
+        "ares_inet_net_pton.h",
-        "cares/ares_iphlpapi.h",
+        "ares_iphlpapi.h",
-        "cares/ares_ipv6.h",
+        "ares_ipv6.h",
-        "cares/ares_library_init.h",
+        "ares_library_init.h",
-        "cares/ares_llist.h",
+        "ares_llist.h",
-        "cares/ares_nowarn.h",
+        "ares_nowarn.h",
-        "cares/ares_platform.h",
+        "ares_platform.h",
-        "cares/ares_private.h",
+        "ares_private.h",
-        "cares/ares_rules.h",
+        "ares_rules.h",
-        "cares/ares_setup.h",
+        "ares_setup.h",
-        "cares/ares_strcasecmp.h",
+        "ares_strcasecmp.h",
-        "cares/ares_strdup.h",
+        "ares_strdup.h",
-        "cares/ares_version.h",
+        "ares_version.h",
-        "cares/bitncmp.h",
+        "bitncmp.h",
-        "cares/config-win32.h",
+        "config-win32.h",
-        "cares/nameser.h",
+        "nameser.h",
-        "cares/setup_once.h",
+        "setup_once.h",
-    ] + select({
-        ":ios_x86_64": ["config_darwin/ares_config.h"],
-        ":ios_armv7": ["config_darwin/ares_config.h"],
-        ":ios_armv7s": ["config_darwin/ares_config.h"],
-        ":ios_arm64": ["config_darwin/ares_config.h"],
-        ":darwin": ["config_darwin/ares_config.h"],
-        ":android": ["config_android/ares_config.h"],
-        "//conditions:default": ["config_linux/ares_config.h"],
-    }),
-    includes = [
-        ".",
-        "cares"
-    ] + select({
-        ":ios_x86_64": ["config_darwin"],
-        ":ios_armv7": ["config_darwin"],
-        ":ios_armv7s": ["config_darwin"],
-        ":ios_arm64": ["config_darwin"],
-        ":darwin": ["config_darwin"],
-        ":android": ["config_android"],
-        "//conditions:default": ["config_linux"],
-    }),
-    linkstatic = 1,
-    visibility = [
-        "//visibility:public",
     ],
     copts = [
         "-D_GNU_SOURCE",
@@ -139,4 +147,13 @@ cc_library(
         "-DNOMINMAX",
         "-DHAVE_CONFIG_H",
     ],
+    data = [
+        ":ares_build",
+        ":ares_config",
+    ],
+    includes = ["."],
+    linkstatic = 1,
+    visibility = [
+        "//visibility:public",
+    ],
 )

third_party/cares/cares_local_files.BUILD

@@ -0,0 +1,57 @@
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "darwin",
+    values = {"cpu": "darwin"},
+)
+
+# Android is not officially supported through C++.
+# This just helps with the build for now.
+config_setting(
+    name = "android",
+    values = {
+        "crosstool_top": "//external:android/crosstool",
+    },
+)
+
+# iOS is not officially supported through C++.
+# This just helps with the build for now.
+config_setting(
+    name = "ios_x86_64",
+    values = {"cpu": "ios_x86_64"},
+)
+
+config_setting(
+    name = "ios_armv7",
+    values = {"cpu": "ios_armv7"},
+)
+
+config_setting(
+    name = "ios_armv7s",
+    values = {"cpu": "ios_armv7s"},
+)
+
+config_setting(
+    name = "ios_arm64",
+    values = {"cpu": "ios_arm64"},
+)
+
+filegroup(
+    name = "ares_build_h",
+    srcs = ["ares_build.h"],
+)
+
+filegroup(
+    name = "ares_config_h",
+    srcs = select({
+        ":ios_x86_64": ["config_darwin/ares_config.h"],
+        ":ios_armv7": ["config_darwin/ares_config.h"],
+        ":ios_armv7s": ["config_darwin/ares_config.h"],
+        ":ios_arm64": ["config_darwin/ares_config.h"],
+        ":darwin": ["config_darwin/ares_config.h"],
+        ":android": ["config_android/ares_config.h"],
+        "//conditions:default": ["config_linux/ares_config.h"],
+    }),
+)

tools/distrib/python/check_grpcio_tools.py

@@ -14,17 +14,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import cStringIO
+import make_grpcio_tools as _make
-
-import make_grpcio_tools as make
 
 OUT_OF_DATE_MESSAGE = """file {} is out of date
 
 Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?"""
 
-check_protoc_lib_deps_content = make.get_deps()
+submodule_commit_hash = _make.protobuf_submodule_commit_hash()
+
+with open(_make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as _protoc_lib_deps_file:
+  content = _protoc_lib_deps_file.read().splitlines()
+
+testString = (_make.COMMIT_HASH_PREFIX +
+              submodule_commit_hash +
+              _make.COMMIT_HASH_SUFFIX)
 
-with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file:
+if testString not in content:
-  if protoc_lib_deps_file.read() != check_protoc_lib_deps_content:
+  print(OUT_OF_DATE_MESSAGE.format(_make.GRPC_PYTHON_PROTOC_LIB_DEPS))
-    print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_LIB_DEPS))
+  raise SystemExit(1)
-    raise SystemExit(1)

tools/distrib/python/grpcio_tools/protoc_lib_deps.py

@@ -1,5 +1,5 @@
 
-# Copyright 2016 gRPC authors.
+# Copyright 2017 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,3 +19,5 @@ PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'go
 
 CC_INCLUDE='third_party/protobuf/src'
 PROTO_INCLUDE='third_party/protobuf/src'
+
+PROTOBUF_SUBMODULE_VERSION="80a37e0782d2d702d52234b62dd4b9ec74fd2c95"

tools/distrib/python/make_grpcio_tools.py

@@ -28,7 +28,7 @@ import traceback
 import uuid
 
 DEPS_FILE_CONTENT="""
-# Copyright 2016 gRPC authors.
+# Copyright 2017 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,8 +48,13 @@ PROTO_FILES={proto_files}
 
 CC_INCLUDE={cc_include}
 PROTO_INCLUDE={proto_include}
+
+{commit_hash}
 """
 
+COMMIT_HASH_PREFIX = 'PROTOBUF_SUBMODULE_VERSION="'
+COMMIT_HASH_SUFFIX = '"'
+
 # Bazel query result prefix for expected source files in protobuf.
 PROTOBUF_CC_PREFIX = '//:src/'
 PROTOBUF_PROTO_PREFIX = '//:src/'
@@ -63,6 +68,7 @@ GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib',
 
 GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', 'src')
 GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)
+GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join(GRPC_ROOT, 'third_party', 'protobuf')
 GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src', 'compiler')
 GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party', 'protobuf',
                                     'src')
@@ -78,6 +84,14 @@ BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', 'bazel_deps.s
 BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
 BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos'
 
+def protobuf_submodule_commit_hash():
+  """Gets the commit hash for the HEAD of the protobuf submodule currently
+     checked out."""
+  cwd = os.getcwd()
+  os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT)
+  output = subprocess.check_output(['git', 'rev-parse', 'HEAD'])
+  os.chdir(cwd)
+  return output.splitlines()[0].strip()
 
 def bazel_query(query):
   output = subprocess.check_output([BAZEL_DEPS, query])
@@ -94,11 +108,13 @@ def get_deps():
   proto_files = [
       name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output
       if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX)]
+  commit_hash = protobuf_submodule_commit_hash()
   deps_file_content = DEPS_FILE_CONTENT.format(
       cc_files=cc_files,
       proto_files=proto_files,
       cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
-      proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
+      proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
+      commit_hash=COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX)
   return deps_file_content
 
 def long_path(path):

tools/doxygen/Doxyfile.c++.internal

@@ -1026,6 +1026,7 @@ src/core/lib/slice/percent_encoding.h \
 src/core/lib/slice/slice_hash_table.h \
 src/core/lib/slice/slice_internal.h \
 src/core/lib/slice/slice_string_helpers.h \
+src/core/lib/support/abstract.h \
 src/core/lib/support/arena.h \
 src/core/lib/support/atomic.h \
 src/core/lib/support/atomic_with_atm.h \

tools/doxygen/Doxyfile.core.internal

@@ -1267,6 +1267,7 @@ src/core/lib/slice/slice_intern.cc \
 src/core/lib/slice/slice_internal.h \
 src/core/lib/slice/slice_string_helpers.cc \
 src/core/lib/slice/slice_string_helpers.h \
+src/core/lib/support/abstract.h \
 src/core/lib/support/alloc.cc \
 src/core/lib/support/arena.cc \
 src/core/lib/support/arena.h \

tools/run_tests/generated/sources_and_headers.json

@@ -748,6 +748,21 @@
     "third_party": false, 
     "type": "target"
   }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c", 
+    "name": "gpr_manual_constructor_test", 
+    "src": [
+      "test/core/support/manual_constructor_test.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
   {
     "deps": [
       "gpr", 
@@ -7814,6 +7829,7 @@
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
       "src/core/lib/profiling/timers.h", 
+      "src/core/lib/support/abstract.h", 
       "src/core/lib/support/arena.h", 
       "src/core/lib/support/atomic.h", 
       "src/core/lib/support/atomic_with_atm.h", 
@@ -7862,6 +7878,7 @@
       "include/grpc/support/tls_pthread.h", 
       "include/grpc/support/useful.h", 
       "src/core/lib/profiling/timers.h", 
+      "src/core/lib/support/abstract.h", 
       "src/core/lib/support/arena.h", 
       "src/core/lib/support/atomic.h", 
       "src/core/lib/support/atomic_with_atm.h", 

tools/run_tests/generated/tests.json

@@ -911,6 +911,30 @@
     ], 
     "uses_polling": false
   }, 
+  {
+    "args": [], 
+    "benchmark": false, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 3, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": false, 
+    "language": "c", 
+    "name": "gpr_manual_constructor_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "uses_polling": false
+  }, 
   {
     "args": [], 
     "benchmark": false, 

tools/run_tests/sanity/check_bazel_workspace.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+# Copyright 2016 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import ast
+import os
+import re
+import subprocess
+import sys
+
+os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..'))
+
+git_hash_pattern = re.compile('[0-9a-f]{40}')
+
+# Parse git hashes from submodules
+git_submodules = subprocess.check_output('git submodule', shell=True).strip().split('\n')
+git_submodule_hashes = {re.search(git_hash_pattern, s).group() for s in git_submodules}
+
+# Parse git hashes from Bazel WORKSPACE {new_}http_archive rules
+with open('WORKSPACE', 'r') as f:
+  workspace_rules = [expr.value for expr in ast.parse(f.read()).body]
+
+http_archive_rules = [rule for rule in workspace_rules if rule.func.id.endswith('http_archive')]
+archive_urls = [kw.value.s for rule in http_archive_rules for kw in rule.keywords if kw.arg == 'url']
+workspace_git_hashes = {re.search(git_hash_pattern, url).group() for url in archive_urls}
+
+# Validate the equivalence of the git submodules and Bazel git dependencies. The
+# condition we impose is that there is a git submodule for every dependency in
+# the workspace, but not necessarily conversely. E.g. Bloaty is a dependency
+# not used by any of the targets built by Bazel.
+if len(workspace_git_hashes - git_submodule_hashes) > 0:
+    print("Found discrepancies between git submodules and Bazel WORKSPACE dependencies")
+    sys.exit(1)
+
+sys.exit(0)

tools/run_tests/sanity/sanity_tests.yaml

@@ -1,4 +1,5 @@
 # a set of tests that are run in parallel for sanity tests
+- script: tools/run_tests/sanity/check_bazel_workspace.py
 - script: tools/run_tests/sanity/check_cache_mk.sh
 - script: tools/run_tests/sanity/check_owners.sh
 - script: tools/run_tests/sanity/check_sources_and_headers.py