Merge remote-tracking branch 'upstream/master' into plugin_credentials_api_fix

BUILD

@@ -577,6 +577,7 @@ grpc_cc_library(
         "src/core/lib/http/format_request.c",
         "src/core/lib/http/httpcli.c",
         "src/core/lib/http/parser.c",
+        "src/core/lib/iomgr/call_combiner.c",
         "src/core/lib/iomgr/closure.c",
         "src/core/lib/iomgr/combiner.c",
         "src/core/lib/iomgr/endpoint.c",
@@ -585,8 +586,6 @@ grpc_cc_library(
         "src/core/lib/iomgr/endpoint_pair_windows.c",
         "src/core/lib/iomgr/error.c",
         "src/core/lib/iomgr/ev_epoll1_linux.c",
-        "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c",
-        "src/core/lib/iomgr/ev_epoll_thread_pool_linux.c",
         "src/core/lib/iomgr/ev_epollex_linux.c",
         "src/core/lib/iomgr/ev_epollsig_linux.c",
         "src/core/lib/iomgr/ev_poll_posix.c",
@@ -709,6 +708,7 @@ grpc_cc_library(
         "src/core/lib/http/format_request.h",
         "src/core/lib/http/httpcli.h",
         "src/core/lib/http/parser.h",
+        "src/core/lib/iomgr/call_combiner.h",
         "src/core/lib/iomgr/closure.h",
         "src/core/lib/iomgr/combiner.h",
         "src/core/lib/iomgr/endpoint.h",
@@ -716,8 +716,6 @@ grpc_cc_library(
         "src/core/lib/iomgr/error.h",
         "src/core/lib/iomgr/error_internal.h",
         "src/core/lib/iomgr/ev_epoll1_linux.h",
-        "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h",
-        "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h",
         "src/core/lib/iomgr/ev_epollex_linux.h",
         "src/core/lib/iomgr/ev_epollsig_linux.h",
         "src/core/lib/iomgr/ev_poll_posix.h",
@@ -1601,4 +1599,19 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "grpc++_core_stats",
+    srcs = [
+        "src/cpp/util/core_stats.cc",
+    ],
+    hdrs = [
+        "src/cpp/util/core_stats.h",
+    ],
+    language = "c++",
+    deps = [
+        ":grpc++",
+        "//src/proto/grpc/core:stats_proto",
+    ],
+)
+
 grpc_generate_one_off_targets()

CMakeLists.txt

@@ -421,6 +421,9 @@ add_dependencies(buildtests_c ev_epollsig_linux_test)
 endif()
 add_dependencies(buildtests_c fake_resolver_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+add_dependencies(buildtests_c fake_transport_security_test)
+endif()
+if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_c fd_conservation_posix_test)
 endif()
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
@@ -458,6 +461,7 @@ add_dependencies(buildtests_c grpc_auth_context_test)
 add_dependencies(buildtests_c grpc_b64_test)
 add_dependencies(buildtests_c grpc_byte_buffer_reader_test)
 add_dependencies(buildtests_c grpc_channel_args_test)
+add_dependencies(buildtests_c grpc_channel_stack_builder_test)
 add_dependencies(buildtests_c grpc_channel_stack_test)
 add_dependencies(buildtests_c grpc_completion_queue_test)
 add_dependencies(buildtests_c grpc_completion_queue_threading_test)
@@ -530,6 +534,9 @@ add_dependencies(buildtests_c sockaddr_utils_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_c socket_utils_test)
 endif()
+if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+add_dependencies(buildtests_c ssl_transport_security_test)
+endif()
 add_dependencies(buildtests_c status_conversion_test)
 add_dependencies(buildtests_c stream_compression_test)
 add_dependencies(buildtests_c stream_owned_slice_test)
@@ -756,6 +763,7 @@ endif()
 add_dependencies(buildtests_cxx server_crash_test_client)
 add_dependencies(buildtests_cxx server_request_call_test)
 add_dependencies(buildtests_cxx shutdown_test)
+add_dependencies(buildtests_cxx stats_test)
 add_dependencies(buildtests_cxx status_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx streaming_throughput_test)
@@ -967,6 +975,7 @@ add_library(grpc
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
+  src/core/lib/iomgr/call_combiner.c
   src/core/lib/iomgr/closure.c
   src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
@@ -975,8 +984,6 @@ add_library(grpc
   src/core/lib/iomgr/endpoint_pair_windows.c
   src/core/lib/iomgr/error.c
   src/core/lib/iomgr/ev_epoll1_linux.c
-  src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
-  src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
   src/core/lib/iomgr/ev_epollex_linux.c
   src/core/lib/iomgr/ev_epollsig_linux.c
   src/core/lib/iomgr/ev_poll_posix.c
@@ -1318,6 +1325,7 @@ add_library(grpc_cronet
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
+  src/core/lib/iomgr/call_combiner.c
   src/core/lib/iomgr/closure.c
   src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
@@ -1326,8 +1334,6 @@ add_library(grpc_cronet
   src/core/lib/iomgr/endpoint_pair_windows.c
   src/core/lib/iomgr/error.c
   src/core/lib/iomgr/ev_epoll1_linux.c
-  src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
-  src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
   src/core/lib/iomgr/ev_epollex_linux.c
   src/core/lib/iomgr/ev_epollsig_linux.c
   src/core/lib/iomgr/ev_poll_posix.c
@@ -1637,6 +1643,7 @@ add_library(grpc_test_util
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
+  src/core/lib/iomgr/call_combiner.c
   src/core/lib/iomgr/closure.c
   src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
@@ -1645,8 +1652,6 @@ add_library(grpc_test_util
   src/core/lib/iomgr/endpoint_pair_windows.c
   src/core/lib/iomgr/error.c
   src/core/lib/iomgr/ev_epoll1_linux.c
-  src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
-  src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
   src/core/lib/iomgr/ev_epollex_linux.c
   src/core/lib/iomgr/ev_epollsig_linux.c
   src/core/lib/iomgr/ev_poll_posix.c
@@ -1900,6 +1905,7 @@ add_library(grpc_test_util_unsecure
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
+  src/core/lib/iomgr/call_combiner.c
   src/core/lib/iomgr/closure.c
   src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
@@ -1908,8 +1914,6 @@ add_library(grpc_test_util_unsecure
   src/core/lib/iomgr/endpoint_pair_windows.c
   src/core/lib/iomgr/error.c
   src/core/lib/iomgr/ev_epoll1_linux.c
-  src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
-  src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
   src/core/lib/iomgr/ev_epollex_linux.c
   src/core/lib/iomgr/ev_epollsig_linux.c
   src/core/lib/iomgr/ev_poll_posix.c
@@ -2149,6 +2153,7 @@ add_library(grpc_unsecure
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
+  src/core/lib/iomgr/call_combiner.c
   src/core/lib/iomgr/closure.c
   src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
@@ -2157,8 +2162,6 @@ add_library(grpc_unsecure
   src/core/lib/iomgr/endpoint_pair_windows.c
   src/core/lib/iomgr/error.c
   src/core/lib/iomgr/ev_epoll1_linux.c
-  src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
-  src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
   src/core/lib/iomgr/ev_epollex_linux.c
   src/core/lib/iomgr/ev_epollsig_linux.c
   src/core/lib/iomgr/ev_poll_posix.c
@@ -2770,6 +2773,68 @@ if (gRPC_INSTALL)
   )
 endif()
 
+if (gRPC_BUILD_TESTS)
+
+add_library(grpc++_core_stats
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/core/stats.pb.cc
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/core/stats.grpc.pb.cc
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/core/stats.pb.h
+  ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/core/stats.grpc.pb.h
+  src/cpp/util/core_stats.cc
+)
+
+if(WIN32 AND MSVC)
+  set_target_properties(grpc++_core_stats PROPERTIES COMPILE_PDB_NAME "grpc++_core_stats"
+    COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}"
+  )
+  if (gRPC_INSTALL)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpc++_core_stats.pdb
+      DESTINATION ${gRPC_INSTALL_LIBDIR} OPTIONAL
+    )
+  endif()
+endif()
+
+protobuf_generate_grpc_cpp(
+  src/proto/grpc/core/stats.proto
+)
+
+target_include_directories(grpc++_core_stats
+  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${BORINGSSL_ROOT_DIR}/include
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${ZLIB_INCLUDE_DIR}
+  PRIVATE ${BENCHMARK}/include
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_BUILD_INCLUDE_DIR}
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CARES_PLATFORM_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(grpc++_core_stats
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++
+)
+
+foreach(_hdr
+  src/cpp/util/core_stats.h
+)
+  string(REPLACE "include/" "" _path ${_hdr})
+  get_filename_component(_path ${_path} PATH)
+  install(FILES ${_hdr}
+    DESTINATION "${gRPC_INSTALL_INCLUDEDIR}/${_path}"
+  )
+endforeach()
+
+endif (gRPC_BUILD_TESTS)
 
 add_library(grpc++_cronet
   src/cpp/client/cronet_credentials.cc
@@ -2850,6 +2915,7 @@ add_library(grpc++_cronet
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
+  src/core/lib/iomgr/call_combiner.c
   src/core/lib/iomgr/closure.c
   src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
@@ -2858,8 +2924,6 @@ add_library(grpc++_cronet
   src/core/lib/iomgr/endpoint_pair_windows.c
   src/core/lib/iomgr/error.c
   src/core/lib/iomgr/ev_epoll1_linux.c
-  src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
-  src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
   src/core/lib/iomgr/ev_epollex_linux.c
   src/core/lib/iomgr/ev_epollsig_linux.c
   src/core/lib/iomgr/ev_poll_posix.c
@@ -4596,6 +4660,7 @@ target_link_libraries(qps
   ${_gRPC_ALLTARGETS_LIBRARIES}
   grpc_test_util
   grpc++_test_util
+  grpc++_core_stats
   grpc++
   grpc
 )
@@ -5966,6 +6031,39 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 
+add_executable(fake_transport_security_test
+  test/core/tsi/fake_transport_security_test.c
+  test/core/tsi/transport_security_test_lib.c
+)
+
+
+target_include_directories(fake_transport_security_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${BORINGSSL_ROOT_DIR}/include
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_BUILD_INCLUDE_DIR}
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CARES_PLATFORM_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+)
+
+target_link_libraries(fake_transport_security_test
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  gpr_test_util
+  gpr
+  grpc
+)
+
+endif()
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+
 add_executable(fd_conservation_posix_test
   test/core/iomgr/fd_conservation_posix_test.c
 )
@@ -6913,6 +7011,37 @@ target_link_libraries(grpc_channel_args_test
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
+add_executable(grpc_channel_stack_builder_test
+  test/core/channel/channel_stack_builder_test.c
+)
+
+
+target_include_directories(grpc_channel_stack_builder_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${BORINGSSL_ROOT_DIR}/include
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_BUILD_INCLUDE_DIR}
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CARES_PLATFORM_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+)
+
+target_link_libraries(grpc_channel_stack_builder_test
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc_test_util
+  grpc
+  gpr_test_util
+  gpr
+)
+
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
 add_executable(grpc_channel_stack_test
   test/core/channel/channel_stack_test.c
 )
@@ -8708,6 +8837,39 @@ target_link_libraries(socket_utils_test
   gpr
 )
 
+endif()
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+
+add_executable(ssl_transport_security_test
+  test/core/tsi/ssl_transport_security_test.c
+  test/core/tsi/transport_security_test_lib.c
+)
+
+
+target_include_directories(ssl_transport_security_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${BORINGSSL_ROOT_DIR}/include
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_BUILD_INCLUDE_DIR}
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CARES_PLATFORM_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+)
+
+target_link_libraries(ssl_transport_security_test
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  gpr_test_util
+  gpr
+  grpc
+)
+
 endif()
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
@@ -10398,6 +10560,7 @@ target_include_directories(codegen_test_full
 target_link_libraries(codegen_test_full
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_core_stats
   grpc++
   grpc
   gpr
@@ -10473,6 +10636,7 @@ target_include_directories(codegen_test_minimal
 target_link_libraries(codegen_test_minimal
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_core_stats
   grpc
   gpr
   ${_gRPC_GFLAGS_LIBRARIES}
@@ -12044,6 +12208,7 @@ target_link_libraries(qps_json_driver
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   qps
+  grpc++_core_stats
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -12089,6 +12254,7 @@ target_link_libraries(qps_openloop_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   qps
+  grpc++_core_stats
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -12134,6 +12300,7 @@ target_link_libraries(qps_worker
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   qps
+  grpc++_core_stats
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -12351,6 +12518,7 @@ target_link_libraries(secure_sync_unary_ping_pong_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
   qps
+  grpc++_core_stats
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -12688,6 +12856,47 @@ target_link_libraries(shutdown_test
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
+add_executable(stats_test
+  test/core/debug/stats_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(stats_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${BORINGSSL_ROOT_DIR}/include
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CARES_BUILD_INCLUDE_DIR}
+  PRIVATE ${CARES_INCLUDE_DIR}
+  PRIVATE ${CARES_PLATFORM_INCLUDE_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(stats_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_util
+  grpc_test_util
+  grpc
+  gpr_test_util
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
 add_executable(status_test
   test/cpp/util/status_test.cc
   third_party/googletest/googletest/src/gtest-all.cc

Makefile

@@ -410,7 +410,7 @@ E = @echo
 Q = @
 endif
 
-CORE_VERSION = 4.0.0-dev
+CORE_VERSION = 5.0.0-dev
 CPP_VERSION = 1.7.0-dev
 CSHARP_VERSION = 1.7.0-dev
 
@@ -460,7 +460,7 @@ SHARED_EXT_CORE = dll
 SHARED_EXT_CPP = dll
 SHARED_EXT_CSHARP = dll
 SHARED_PREFIX =
-SHARED_VERSION_CORE = -4
+SHARED_VERSION_CORE = -5
 SHARED_VERSION_CPP = -1
 SHARED_VERSION_CSHARP = -1
 else ifeq ($(SYSTEM),Darwin)
@@ -976,6 +976,7 @@ endpoint_pair_test: $(BINDIR)/$(CONFIG)/endpoint_pair_test
 error_test: $(BINDIR)/$(CONFIG)/error_test
 ev_epollsig_linux_test: $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test
 fake_resolver_test: $(BINDIR)/$(CONFIG)/fake_resolver_test
+fake_transport_security_test: $(BINDIR)/$(CONFIG)/fake_transport_security_test
 fd_conservation_posix_test: $(BINDIR)/$(CONFIG)/fd_conservation_posix_test
 fd_posix_test: $(BINDIR)/$(CONFIG)/fd_posix_test
 fling_client: $(BINDIR)/$(CONFIG)/fling_client
@@ -1007,6 +1008,7 @@ grpc_auth_context_test: $(BINDIR)/$(CONFIG)/grpc_auth_context_test
 grpc_b64_test: $(BINDIR)/$(CONFIG)/grpc_b64_test
 grpc_byte_buffer_reader_test: $(BINDIR)/$(CONFIG)/grpc_byte_buffer_reader_test
 grpc_channel_args_test: $(BINDIR)/$(CONFIG)/grpc_channel_args_test
+grpc_channel_stack_builder_test: $(BINDIR)/$(CONFIG)/grpc_channel_stack_builder_test
 grpc_channel_stack_test: $(BINDIR)/$(CONFIG)/grpc_channel_stack_test
 grpc_completion_queue_test: $(BINDIR)/$(CONFIG)/grpc_completion_queue_test
 grpc_completion_queue_threading_test: $(BINDIR)/$(CONFIG)/grpc_completion_queue_threading_test
@@ -1075,6 +1077,7 @@ sockaddr_resolver_test: $(BINDIR)/$(CONFIG)/sockaddr_resolver_test
 sockaddr_utils_test: $(BINDIR)/$(CONFIG)/sockaddr_utils_test
 socket_utils_test: $(BINDIR)/$(CONFIG)/socket_utils_test
 ssl_server_fuzzer: $(BINDIR)/$(CONFIG)/ssl_server_fuzzer
+ssl_transport_security_test: $(BINDIR)/$(CONFIG)/ssl_transport_security_test
 status_conversion_test: $(BINDIR)/$(CONFIG)/status_conversion_test
 stream_compression_test: $(BINDIR)/$(CONFIG)/stream_compression_test
 stream_owned_slice_test: $(BINDIR)/$(CONFIG)/stream_owned_slice_test
@@ -1170,6 +1173,7 @@ server_crash_test: $(BINDIR)/$(CONFIG)/server_crash_test
 server_crash_test_client: $(BINDIR)/$(CONFIG)/server_crash_test_client
 server_request_call_test: $(BINDIR)/$(CONFIG)/server_request_call_test
 shutdown_test: $(BINDIR)/$(CONFIG)/shutdown_test
+stats_test: $(BINDIR)/$(CONFIG)/stats_test
 status_test: $(BINDIR)/$(CONFIG)/status_test
 streaming_throughput_test: $(BINDIR)/$(CONFIG)/streaming_throughput_test
 stress_test: $(BINDIR)/$(CONFIG)/stress_test
@@ -1328,9 +1332,9 @@ pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc
 pc_cxx_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc
 
 ifeq ($(EMBED_OPENSSL),true)
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_constant_time_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_spake25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_p256-x86_64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_sign_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_verify_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_lhash_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_gcm_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_obj_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pool_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_refcount_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_constant_time_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_spake25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_p256-x86_64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_sign_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_verify_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_lhash_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_gcm_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_obj_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pool_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_refcount_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 else
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 endif
 
 
@@ -1366,6 +1370,7 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/error_test \
   $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test \
   $(BINDIR)/$(CONFIG)/fake_resolver_test \
+  $(BINDIR)/$(CONFIG)/fake_transport_security_test \
   $(BINDIR)/$(CONFIG)/fd_conservation_posix_test \
   $(BINDIR)/$(CONFIG)/fd_posix_test \
   $(BINDIR)/$(CONFIG)/fling_client \
@@ -1394,6 +1399,7 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/grpc_b64_test \
   $(BINDIR)/$(CONFIG)/grpc_byte_buffer_reader_test \
   $(BINDIR)/$(CONFIG)/grpc_channel_args_test \
+  $(BINDIR)/$(CONFIG)/grpc_channel_stack_builder_test \
   $(BINDIR)/$(CONFIG)/grpc_channel_stack_test \
   $(BINDIR)/$(CONFIG)/grpc_completion_queue_test \
   $(BINDIR)/$(CONFIG)/grpc_completion_queue_threading_test \
@@ -1448,6 +1454,7 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/sockaddr_resolver_test \
   $(BINDIR)/$(CONFIG)/sockaddr_utils_test \
   $(BINDIR)/$(CONFIG)/socket_utils_test \
+  $(BINDIR)/$(CONFIG)/ssl_transport_security_test \
   $(BINDIR)/$(CONFIG)/status_conversion_test \
   $(BINDIR)/$(CONFIG)/stream_compression_test \
   $(BINDIR)/$(CONFIG)/stream_owned_slice_test \
@@ -1600,6 +1607,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/server_crash_test_client \
   $(BINDIR)/$(CONFIG)/server_request_call_test \
   $(BINDIR)/$(CONFIG)/shutdown_test \
+  $(BINDIR)/$(CONFIG)/stats_test \
   $(BINDIR)/$(CONFIG)/status_test \
   $(BINDIR)/$(CONFIG)/streaming_throughput_test \
   $(BINDIR)/$(CONFIG)/stress_test \
@@ -1715,6 +1723,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/server_crash_test_client \
   $(BINDIR)/$(CONFIG)/server_request_call_test \
   $(BINDIR)/$(CONFIG)/shutdown_test \
+  $(BINDIR)/$(CONFIG)/stats_test \
   $(BINDIR)/$(CONFIG)/status_test \
   $(BINDIR)/$(CONFIG)/streaming_throughput_test \
   $(BINDIR)/$(CONFIG)/stress_test \
@@ -1788,6 +1797,8 @@ test_c: buildtests_c
 	$(Q) $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test || ( echo test ev_epollsig_linux_test failed ; exit 1 )
 	$(E) "[RUN]     Testing fake_resolver_test"
 	$(Q) $(BINDIR)/$(CONFIG)/fake_resolver_test || ( echo test fake_resolver_test failed ; exit 1 )
+	$(E) "[RUN]     Testing fake_transport_security_test"
+	$(Q) $(BINDIR)/$(CONFIG)/fake_transport_security_test || ( echo test fake_transport_security_test failed ; exit 1 )
 	$(E) "[RUN]     Testing fd_conservation_posix_test"
 	$(Q) $(BINDIR)/$(CONFIG)/fd_conservation_posix_test || ( echo test fd_conservation_posix_test failed ; exit 1 )
 	$(E) "[RUN]     Testing fd_posix_test"
@@ -1840,6 +1851,8 @@ test_c: buildtests_c
 	$(Q) $(BINDIR)/$(CONFIG)/grpc_byte_buffer_reader_test || ( echo test grpc_byte_buffer_reader_test failed ; exit 1 )
 	$(E) "[RUN]     Testing grpc_channel_args_test"
 	$(Q) $(BINDIR)/$(CONFIG)/grpc_channel_args_test || ( echo test grpc_channel_args_test failed ; exit 1 )
+	$(E) "[RUN]     Testing grpc_channel_stack_builder_test"
+	$(Q) $(BINDIR)/$(CONFIG)/grpc_channel_stack_builder_test || ( echo test grpc_channel_stack_builder_test failed ; exit 1 )
 	$(E) "[RUN]     Testing grpc_channel_stack_test"
 	$(Q) $(BINDIR)/$(CONFIG)/grpc_channel_stack_test || ( echo test grpc_channel_stack_test failed ; exit 1 )
 	$(E) "[RUN]     Testing grpc_completion_queue_test"
@@ -1936,6 +1949,8 @@ test_c: buildtests_c
 	$(Q) $(BINDIR)/$(CONFIG)/sockaddr_utils_test || ( echo test sockaddr_utils_test failed ; exit 1 )
 	$(E) "[RUN]     Testing socket_utils_test"
 	$(Q) $(BINDIR)/$(CONFIG)/socket_utils_test || ( echo test socket_utils_test failed ; exit 1 )
+	$(E) "[RUN]     Testing ssl_transport_security_test"
+	$(Q) $(BINDIR)/$(CONFIG)/ssl_transport_security_test || ( echo test ssl_transport_security_test failed ; exit 1 )
 	$(E) "[RUN]     Testing status_conversion_test"
 	$(Q) $(BINDIR)/$(CONFIG)/status_conversion_test || ( echo test status_conversion_test failed ; exit 1 )
 	$(E) "[RUN]     Testing stream_compression_test"
@@ -2114,6 +2129,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/server_request_call_test || ( echo test server_request_call_test failed ; exit 1 )
 	$(E) "[RUN]     Testing shutdown_test"
 	$(Q) $(BINDIR)/$(CONFIG)/shutdown_test || ( echo test shutdown_test failed ; exit 1 )
+	$(E) "[RUN]     Testing stats_test"
+	$(Q) $(BINDIR)/$(CONFIG)/stats_test || ( echo test stats_test failed ; exit 1 )
 	$(E) "[RUN]     Testing status_test"
 	$(Q) $(BINDIR)/$(CONFIG)/status_test || ( echo test status_test failed ; exit 1 )
 	$(E) "[RUN]     Testing streaming_throughput_test"
@@ -2240,6 +2257,22 @@ $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc:
 	$(Q) mkdir -p $(@D)
 	$(Q) echo "$(GRPCXX_UNSECURE_PC_FILE)" | tr , '\n' >$@
 
+ifeq ($(NO_PROTOC),true)
+$(GENDIR)/src/proto/grpc/core/stats.pb.cc: protoc_dep_error
+$(GENDIR)/src/proto/grpc/core/stats.grpc.pb.cc: protoc_dep_error
+else
+
+$(GENDIR)/src/proto/grpc/core/stats.pb.cc: src/proto/grpc/core/stats.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
+	$(E) "[PROTOC]  Generating protobuf CC file from $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --cpp_out=$(GENDIR) $<
+
+$(GENDIR)/src/proto/grpc/core/stats.grpc.pb.cc: src/proto/grpc/core/stats.proto $(GENDIR)/src/proto/grpc/core/stats.pb.cc $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
+	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin$(EXECUTABLE_SUFFIX) $<
+endif
+
 ifeq ($(NO_PROTOC),true)
 $(GENDIR)/src/proto/grpc/health/v1/health.pb.cc: protoc_dep_error
 $(GENDIR)/src/proto/grpc/health/v1/health.grpc.pb.cc: protoc_dep_error
@@ -2471,12 +2504,12 @@ $(GENDIR)/src/proto/grpc/testing/stats.pb.cc: protoc_dep_error
 $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc: protoc_dep_error
 else
 
-$(GENDIR)/src/proto/grpc/testing/stats.pb.cc: src/proto/grpc/testing/stats.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
+$(GENDIR)/src/proto/grpc/testing/stats.pb.cc: src/proto/grpc/testing/stats.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/core/stats.pb.cc
 	$(E) "[PROTOC]  Generating protobuf CC file from $<"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --cpp_out=$(GENDIR) $<
 
-$(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc: src/proto/grpc/testing/stats.proto $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
+$(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc: src/proto/grpc/testing/stats.proto $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/core/stats.pb.cc $(GENDIR)/src/proto/grpc/core/stats.grpc.pb.cc
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin$(EXECUTABLE_SUFFIX) $<
@@ -2595,7 +2628,7 @@ install-shared_c: shared_c strip-shared_c install-pkg-config_c
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgpr.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgpr.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgpr.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgpr.so
 endif
 	$(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)"
@@ -2604,7 +2637,7 @@ endif
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgrpc.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc.so
 endif
 	$(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)"
@@ -2613,7 +2646,7 @@ endif
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgrpc_cronet.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_cronet.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_cronet.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_cronet.so
 endif
 	$(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)"
@@ -2622,7 +2655,7 @@ endif
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgrpc_unsecure.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_unsecure.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_unsecure.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_unsecure.so
 endif
 ifneq ($(SYSTEM),MINGW32)
@@ -2639,7 +2672,7 @@ install-shared_cxx: shared_cxx strip-shared_cxx install-shared_c install-pkg-con
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++.so
 endif
 	$(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
@@ -2648,7 +2681,7 @@ endif
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_cronet$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++_cronet.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_cronet.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_cronet.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_cronet.so
 endif
 	$(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
@@ -2657,7 +2690,7 @@ endif
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_error_details$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++_error_details.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_error_details.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_error_details.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_error_details.so
 endif
 	$(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
@@ -2666,7 +2699,7 @@ endif
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_reflection$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++_reflection.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_reflection.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_reflection.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_reflection.so
 endif
 	$(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)"
@@ -2675,7 +2708,7 @@ endif
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++_unsecure.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_unsecure.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_unsecure.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_unsecure.so
 endif
 ifneq ($(SYSTEM),MINGW32)
@@ -2692,7 +2725,7 @@ install-shared_csharp: shared_csharp strip-shared_csharp
 ifeq ($(SYSTEM),MINGW32)
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CSHARP)-dll.a $(prefix)/lib/libgrpc_csharp_ext.a
 else ifneq ($(SYSTEM),Darwin)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CSHARP).$(SHARED_EXT_CSHARP) $(prefix)/lib/libgrpc_csharp_ext.so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CSHARP).$(SHARED_EXT_CSHARP) $(prefix)/lib/libgrpc_csharp_ext.so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CSHARP).$(SHARED_EXT_CSHARP) $(prefix)/lib/libgrpc_csharp_ext.so
 endif
 ifneq ($(SYSTEM),MINGW32)
@@ -2859,8 +2892,8 @@ $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGPR_OB
 ifeq ($(SYSTEM),Darwin)
 	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
 else
-	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.4 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
+	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.5 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
-	$(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.4
+	$(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.5
 	$(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so
 endif
 endif
@@ -2912,6 +2945,7 @@ LIBGRPC_SRC = \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
+    src/core/lib/iomgr/call_combiner.c \
     src/core/lib/iomgr/closure.c \
     src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
@@ -2920,8 +2954,6 @@ LIBGRPC_SRC = \
     src/core/lib/iomgr/endpoint_pair_windows.c \
     src/core/lib/iomgr/error.c \
     src/core/lib/iomgr/ev_epoll1_linux.c \
-    src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \
-    src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \
     src/core/lib/iomgr/ev_epollex_linux.c \
     src/core/lib/iomgr/ev_epollsig_linux.c \
     src/core/lib/iomgr/ev_poll_posix.c \
@@ -3229,8 +3261,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_
 ifeq ($(SYSTEM),Darwin)
 	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
 else
-	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.4 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
+	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.5 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so
 endif
 endif
@@ -3261,6 +3293,7 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
+    src/core/lib/iomgr/call_combiner.c \
     src/core/lib/iomgr/closure.c \
     src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
@@ -3269,8 +3302,6 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/iomgr/endpoint_pair_windows.c \
     src/core/lib/iomgr/error.c \
     src/core/lib/iomgr/ev_epoll1_linux.c \
-    src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \
-    src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \
     src/core/lib/iomgr/ev_epollex_linux.c \
     src/core/lib/iomgr/ev_epollsig_linux.c \
     src/core/lib/iomgr/ev_poll_posix.c \
@@ -3526,8 +3557,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(L
 ifeq ($(SYSTEM),Darwin)
 	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CRONET_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
 else
-	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_cronet.so.4 -o $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CRONET_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
+	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_cronet.so.5 -o $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CRONET_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).so
 endif
 endif
@@ -3577,6 +3608,7 @@ LIBGRPC_TEST_UTIL_SRC = \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
+    src/core/lib/iomgr/call_combiner.c \
     src/core/lib/iomgr/closure.c \
     src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
@@ -3585,8 +3617,6 @@ LIBGRPC_TEST_UTIL_SRC = \
     src/core/lib/iomgr/endpoint_pair_windows.c \
     src/core/lib/iomgr/error.c \
     src/core/lib/iomgr/ev_epoll1_linux.c \
-    src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \
-    src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \
     src/core/lib/iomgr/ev_epollex_linux.c \
     src/core/lib/iomgr/ev_epollsig_linux.c \
     src/core/lib/iomgr/ev_poll_posix.c \
@@ -3829,6 +3859,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
+    src/core/lib/iomgr/call_combiner.c \
     src/core/lib/iomgr/closure.c \
     src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
@@ -3837,8 +3868,6 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
     src/core/lib/iomgr/endpoint_pair_windows.c \
     src/core/lib/iomgr/error.c \
     src/core/lib/iomgr/ev_epoll1_linux.c \
-    src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \
-    src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \
     src/core/lib/iomgr/ev_epollex_linux.c \
     src/core/lib/iomgr/ev_epollsig_linux.c \
     src/core/lib/iomgr/ev_poll_posix.c \
@@ -4054,6 +4083,7 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
+    src/core/lib/iomgr/call_combiner.c \
     src/core/lib/iomgr/closure.c \
     src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
@@ -4062,8 +4092,6 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/lib/iomgr/endpoint_pair_windows.c \
     src/core/lib/iomgr/error.c \
     src/core/lib/iomgr/ev_epoll1_linux.c \
-    src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \
-    src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \
     src/core/lib/iomgr/ev_epollex_linux.c \
     src/core/lib/iomgr/ev_epollsig_linux.c \
     src/core/lib/iomgr/ev_poll_posix.c \
@@ -4327,8 +4355,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $
 ifeq ($(SYSTEM),Darwin)
 	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
 else
-	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.4 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
+	$(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.5 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(LDLIBS)
-	$(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.4
+	$(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.5
 	$(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so
 endif
 endif
@@ -4659,6 +4687,58 @@ endif
 endif
 
 
+LIBGRPC++_CORE_STATS_SRC = \
+    $(GENDIR)/src/proto/grpc/core/stats.pb.cc $(GENDIR)/src/proto/grpc/core/stats.grpc.pb.cc \
+    src/cpp/util/core_stats.cc \
+
+PUBLIC_HEADERS_CXX += \
+    src/cpp/util/core_stats.h \
+
+LIBGRPC++_CORE_STATS_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_CORE_STATS_SRC))))
+
+
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure libraries if you don't have OpenSSL.
+
+$(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a: openssl_dep_error
+
+
+else
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay.
+
+$(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a: protobuf_dep_error
+
+
+else
+
+$(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_CORE_STATS_OBJS) 
+	$(E) "[AR]      Creating $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBGRPC++_CORE_STATS_OBJS) 
+ifeq ($(SYSTEM),Darwin)
+	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a
+endif
+
+
+
+
+endif
+
+endif
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(LIBGRPC++_CORE_STATS_OBJS:.o=.dep)
+endif
+endif
+$(OBJDIR)/$(CONFIG)/src/cpp/util/core_stats.o: $(GENDIR)/src/proto/grpc/core/stats.pb.cc $(GENDIR)/src/proto/grpc/core/stats.grpc.pb.cc
+
+
 LIBGRPC++_CRONET_SRC = \
     src/cpp/client/cronet_credentials.cc \
     src/cpp/client/insecure_credentials.cc \
@@ -4738,6 +4818,7 @@ LIBGRPC++_CRONET_SRC = \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
+    src/core/lib/iomgr/call_combiner.c \
     src/core/lib/iomgr/closure.c \
     src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
@@ -4746,8 +4827,6 @@ LIBGRPC++_CRONET_SRC = \
     src/core/lib/iomgr/endpoint_pair_windows.c \
     src/core/lib/iomgr/error.c \
     src/core/lib/iomgr/ev_epoll1_linux.c \
-    src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \
-    src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \
     src/core/lib/iomgr/ev_epollex_linux.c \
     src/core/lib/iomgr/ev_epollsig_linux.c \
     src/core/lib/iomgr/ev_poll_posix.c \
@@ -9602,6 +9681,41 @@ endif
 endif
 
 
+FAKE_TRANSPORT_SECURITY_TEST_SRC = \
+    test/core/tsi/fake_transport_security_test.c \
+    test/core/tsi/transport_security_test_lib.c \
+
+FAKE_TRANSPORT_SECURITY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(FAKE_TRANSPORT_SECURITY_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/fake_transport_security_test: openssl_dep_error
+
+else
+
+
+
+$(BINDIR)/$(CONFIG)/fake_transport_security_test: $(FAKE_TRANSPORT_SECURITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LD) $(LDFLAGS) $(FAKE_TRANSPORT_SECURITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/fake_transport_security_test
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/tsi/fake_transport_security_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a
+
+$(OBJDIR)/$(CONFIG)/test/core/tsi/transport_security_test_lib.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a
+
+deps_fake_transport_security_test: $(FAKE_TRANSPORT_SECURITY_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(FAKE_TRANSPORT_SECURITY_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 FD_CONSERVATION_POSIX_TEST_SRC = \
     test/core/iomgr/fd_conservation_posix_test.c \
 
@@ -10594,6 +10708,38 @@ endif
 endif
 
 
+GRPC_CHANNEL_STACK_BUILDER_TEST_SRC = \
+    test/core/channel/channel_stack_builder_test.c \
+
+GRPC_CHANNEL_STACK_BUILDER_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_CHANNEL_STACK_BUILDER_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/grpc_channel_stack_builder_test: openssl_dep_error
+
+else
+
+
+
+$(BINDIR)/$(CONFIG)/grpc_channel_stack_builder_test: $(GRPC_CHANNEL_STACK_BUILDER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LD) $(LDFLAGS) $(GRPC_CHANNEL_STACK_BUILDER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/grpc_channel_stack_builder_test
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/channel/channel_stack_builder_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_grpc_channel_stack_builder_test: $(GRPC_CHANNEL_STACK_BUILDER_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(GRPC_CHANNEL_STACK_BUILDER_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 GRPC_CHANNEL_STACK_TEST_SRC = \
     test/core/channel/channel_stack_test.c \
 
@@ -12770,6 +12916,41 @@ endif
 endif
 
 
+SSL_TRANSPORT_SECURITY_TEST_SRC = \
+    test/core/tsi/ssl_transport_security_test.c \
+    test/core/tsi/transport_security_test_lib.c \
+
+SSL_TRANSPORT_SECURITY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SSL_TRANSPORT_SECURITY_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/ssl_transport_security_test: openssl_dep_error
+
+else
+
+
+
+$(BINDIR)/$(CONFIG)/ssl_transport_security_test: $(SSL_TRANSPORT_SECURITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LD) $(LDFLAGS) $(SSL_TRANSPORT_SECURITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/ssl_transport_security_test
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/tsi/ssl_transport_security_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a
+
+$(OBJDIR)/$(CONFIG)/test/core/tsi/transport_security_test_lib.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a
+
+deps_ssl_transport_security_test: $(SSL_TRANSPORT_SECURITY_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(SSL_TRANSPORT_SECURITY_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 STATUS_CONVERSION_TEST_SRC = \
     test/core/transport/status_conversion_test.c \
 
@@ -14441,26 +14622,26 @@ $(BINDIR)/$(CONFIG)/codegen_test_full: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/codegen_test_full: $(PROTOBUF_DEP) $(CODEGEN_TEST_FULL_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(BINDIR)/$(CONFIG)/codegen_test_full: $(PROTOBUF_DEP) $(CODEGEN_TEST_FULL_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(CODEGEN_TEST_FULL_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/codegen_test_full
+	$(Q) $(LDXX) $(LDFLAGS) $(CODEGEN_TEST_FULL_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/codegen_test_full
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/control.o:  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/control.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/messages.o:  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/messages.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/payloads.o:  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/payloads.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/services.o:  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/services.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/stats.o:  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/stats.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/test/cpp/codegen/codegen_test_full.o:  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/cpp/codegen/codegen_test_full.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_codegen_test_full: $(CODEGEN_TEST_FULL_OBJS:.o=.dep)
 
@@ -14501,28 +14682,28 @@ $(BINDIR)/$(CONFIG)/codegen_test_minimal: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/codegen_test_minimal: $(PROTOBUF_DEP) $(CODEGEN_TEST_MINIMAL_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(BINDIR)/$(CONFIG)/codegen_test_minimal: $(PROTOBUF_DEP) $(CODEGEN_TEST_MINIMAL_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(CODEGEN_TEST_MINIMAL_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/codegen_test_minimal
+	$(Q) $(LDXX) $(LDFLAGS) $(CODEGEN_TEST_MINIMAL_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/codegen_test_minimal
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/control.o:  $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/control.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/messages.o:  $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/messages.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/payloads.o:  $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/payloads.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/services.o:  $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/services.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/stats.o:  $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/stats.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/test/cpp/codegen/codegen_test_minimal.o:  $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/cpp/codegen/codegen_test_minimal.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/src/cpp/codegen/codegen_init.o:  $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/cpp/codegen/codegen_init.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_codegen_test_minimal: $(CODEGEN_TEST_MINIMAL_OBJS:.o=.dep)
 
@@ -16018,16 +16199,16 @@ $(BINDIR)/$(CONFIG)/qps_json_driver: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/qps_json_driver: $(PROTOBUF_DEP) $(QPS_JSON_DRIVER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/qps_json_driver: $(PROTOBUF_DEP) $(QPS_JSON_DRIVER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(QPS_JSON_DRIVER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/qps_json_driver
+	$(Q) $(LDXX) $(LDFLAGS) $(QPS_JSON_DRIVER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/qps_json_driver
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_json_driver.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_json_driver.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_qps_json_driver: $(QPS_JSON_DRIVER_OBJS:.o=.dep)
 
@@ -16061,16 +16242,16 @@ $(BINDIR)/$(CONFIG)/qps_openloop_test: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/qps_openloop_test: $(PROTOBUF_DEP) $(QPS_OPENLOOP_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/qps_openloop_test: $(PROTOBUF_DEP) $(QPS_OPENLOOP_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(QPS_OPENLOOP_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/qps_openloop_test
+	$(Q) $(LDXX) $(LDFLAGS) $(QPS_OPENLOOP_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/qps_openloop_test
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_openloop_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_openloop_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_qps_openloop_test: $(QPS_OPENLOOP_TEST_OBJS:.o=.dep)
 
@@ -16104,16 +16285,16 @@ $(BINDIR)/$(CONFIG)/qps_worker: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/qps_worker: $(PROTOBUF_DEP) $(QPS_WORKER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/qps_worker: $(PROTOBUF_DEP) $(QPS_WORKER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(QPS_WORKER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/qps_worker
+	$(Q) $(LDXX) $(LDFLAGS) $(QPS_WORKER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/qps_worker
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/worker.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/qps/worker.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_qps_worker: $(QPS_WORKER_OBJS:.o=.dep)
 
@@ -16296,16 +16477,16 @@ $(BINDIR)/$(CONFIG)/secure_sync_unary_ping_pong_test: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/secure_sync_unary_ping_pong_test: $(PROTOBUF_DEP) $(SECURE_SYNC_UNARY_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/secure_sync_unary_ping_pong_test: $(PROTOBUF_DEP) $(SECURE_SYNC_UNARY_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(SECURE_SYNC_UNARY_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/secure_sync_unary_ping_pong_test
+	$(Q) $(LDXX) $(LDFLAGS) $(SECURE_SYNC_UNARY_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/secure_sync_unary_ping_pong_test
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/secure_sync_unary_ping_pong_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/qps/secure_sync_unary_ping_pong_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_secure_sync_unary_ping_pong_test: $(SECURE_SYNC_UNARY_PING_PONG_TEST_OBJS:.o=.dep)
 
@@ -16631,6 +16812,49 @@ endif
 endif
 
 
+STATS_TEST_SRC = \
+    test/core/debug/stats_test.cc \
+
+STATS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(STATS_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/stats_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
+
+$(BINDIR)/$(CONFIG)/stats_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/stats_test: $(PROTOBUF_DEP) $(STATS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(STATS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/stats_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/debug/stats_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_stats_test: $(STATS_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(STATS_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 STATUS_TEST_SRC = \
     test/cpp/util/status_test.cc \
 
@@ -19766,6 +19990,7 @@ src/cpp/common/secure_create_auth_context.cc: $(OPENSSL_DEP)
 src/cpp/ext/proto_server_reflection.cc: $(OPENSSL_DEP)
 src/cpp/ext/proto_server_reflection_plugin.cc: $(OPENSSL_DEP)
 src/cpp/server/secure_server_credentials.cc: $(OPENSSL_DEP)
+src/cpp/util/core_stats.cc: $(OPENSSL_DEP)
 src/cpp/util/error_details.cc: $(OPENSSL_DEP)
 src/csharp/ext/grpc_csharp_ext.c: $(OPENSSL_DEP)
 test/core/bad_client/bad_client.c: $(OPENSSL_DEP)

WORKSPACE

@@ -38,6 +38,11 @@ bind(
     actual = "@submodule_gtest//:gtest",
 )
 
+bind(
+    name = "gmock",
+    actual = "@submodule_gtest//:gmock",
+)
+
 bind(
     name = "benchmark",
     actual = "@submodule_benchmark//:benchmark",

binding.gyp

@@ -672,6 +672,7 @@
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',
+        'src/core/lib/iomgr/call_combiner.c',
         'src/core/lib/iomgr/closure.c',
         'src/core/lib/iomgr/combiner.c',
         'src/core/lib/iomgr/endpoint.c',
@@ -680,8 +681,6 @@
         'src/core/lib/iomgr/endpoint_pair_windows.c',
         'src/core/lib/iomgr/error.c',
         'src/core/lib/iomgr/ev_epoll1_linux.c',
-        'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c',
-        'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c',
         'src/core/lib/iomgr/ev_epollex_linux.c',
         'src/core/lib/iomgr/ev_epollsig_linux.c',
         'src/core/lib/iomgr/ev_poll_posix.c',

build.yaml

@@ -12,7 +12,7 @@ settings:
   '#08': Use "-preN" suffixes to identify pre-release versions
   '#09': Per-language overrides are possible with (eg) ruby_version tag here
   '#10': See the expand_version.py for all the quirks here
-  core_version: 4.0.0-dev
+  core_version: 5.0.0-dev
   g_stands_for: gambit
   version: 1.7.0-dev
 filegroups:
@@ -200,6 +200,7 @@ filegroups:
   - src/core/lib/http/format_request.c
   - src/core/lib/http/httpcli.c
   - src/core/lib/http/parser.c
+  - src/core/lib/iomgr/call_combiner.c
   - src/core/lib/iomgr/closure.c
   - src/core/lib/iomgr/combiner.c
   - src/core/lib/iomgr/endpoint.c
@@ -208,8 +209,6 @@ filegroups:
   - src/core/lib/iomgr/endpoint_pair_windows.c
   - src/core/lib/iomgr/error.c
   - src/core/lib/iomgr/ev_epoll1_linux.c
-  - src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
-  - src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
   - src/core/lib/iomgr/ev_epollex_linux.c
   - src/core/lib/iomgr/ev_epollsig_linux.c
   - src/core/lib/iomgr/ev_poll_posix.c
@@ -352,6 +351,7 @@ filegroups:
   - src/core/lib/http/format_request.h
   - src/core/lib/http/httpcli.h
   - src/core/lib/http/parser.h
+  - src/core/lib/iomgr/call_combiner.h
   - src/core/lib/iomgr/closure.h
   - src/core/lib/iomgr/combiner.h
   - src/core/lib/iomgr/endpoint.h
@@ -359,8 +359,6 @@ filegroups:
   - src/core/lib/iomgr/error.h
   - src/core/lib/iomgr/error_internal.h
   - src/core/lib/iomgr/ev_epoll1_linux.h
-  - src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h
-  - src/core/lib/iomgr/ev_epoll_thread_pool_linux.h
   - src/core/lib/iomgr/ev_epollex_linux.h
   - src/core/lib/iomgr/ev_epollsig_linux.h
   - src/core/lib/iomgr/ev_poll_posix.h
@@ -750,6 +748,7 @@ filegroups:
   - test/core/util/trickle_endpoint.c
   deps:
   - gpr_test_util
+  - gpr
   uses:
   - grpc_base
   - grpc_client_channel
@@ -924,6 +923,14 @@ filegroups:
   - third_party/nanopb/pb_common.h
   - third_party/nanopb/pb_decode.h
   - third_party/nanopb/pb_encode.h
+- name: transport_security_test_lib
+  build: test
+  headers:
+  - test/core/tsi/transport_security_test_lib.h
+  src:
+  - test/core/tsi/transport_security_test_lib.c
+  deps:
+  - grpc
 - name: tsi
   headers:
   - src/core/tsi/fake_transport_security.h
@@ -1323,6 +1330,16 @@ libs:
   - grpc++_codegen_base_src
   secure: check
   vs_project_guid: '{C187A093-A0FE-489D-A40A-6E33DE0F9FEB}'
+- name: grpc++_core_stats
+  build: private
+  language: c++
+  public_headers:
+  - src/cpp/util/core_stats.h
+  src:
+  - src/proto/grpc/core/stats.proto
+  - src/cpp/util/core_stats.cc
+  deps:
+  - grpc++
 - name: grpc++_cronet
   build: all
   language: c++
@@ -1665,6 +1682,7 @@ libs:
   deps:
   - grpc_test_util
   - grpc++_test_util
+  - grpc++_core_stats
   - grpc++
   - grpc
 - name: grpc_csharp_ext
@@ -2034,6 +2052,21 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
+- name: fake_transport_security_test
+  build: test
+  language: c
+  src:
+  - test/core/tsi/fake_transport_security_test.c
+  deps:
+  - gpr_test_util
+  - gpr
+  - grpc
+  filegroups:
+  - transport_security_test_lib
+  platforms:
+  - linux
+  - posix
+  - mac
 - name: fd_conservation_posix_test
   build: test
   language: c
@@ -2337,6 +2370,16 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
+- name: grpc_channel_stack_builder_test
+  build: test
+  language: c
+  src:
+  - test/core/channel/channel_stack_builder_test.c
+  deps:
+  - grpc_test_util
+  - grpc
+  - gpr_test_util
+  - gpr
 - name: grpc_channel_stack_test
   build: test
   language: c
@@ -3105,6 +3148,21 @@ targets:
   corpus_dirs:
   - test/core/security/corpus/ssl_server_corpus
   maxlen: 2048
+- name: ssl_transport_security_test
+  build: test
+  language: c
+  src:
+  - test/core/tsi/ssl_transport_security_test.c
+  deps:
+  - gpr_test_util
+  - gpr
+  - grpc
+  filegroups:
+  - transport_security_test_lib
+  platforms:
+  - linux
+  - posix
+  - mac
 - name: status_conversion_test
   build: test
   language: c
@@ -3563,6 +3621,8 @@ targets:
 - name: bm_fullstack_streaming_ping_pong
   build: test
   language: c++
+  headers:
+  - test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h
   src:
   - test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
   deps:
@@ -3588,6 +3648,8 @@ targets:
 - name: bm_fullstack_streaming_pump
   build: test
   language: c++
+  headers:
+  - test/cpp/microbenchmarks/fullstack_streaming_pump.h
   src:
   - test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc
   deps:
@@ -3639,6 +3701,8 @@ targets:
 - name: bm_fullstack_unary_ping_pong
   build: test
   language: c++
+  headers:
+  - test/cpp/microbenchmarks/fullstack_unary_ping_pong.h
   src:
   - test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc
   deps:
@@ -3796,6 +3860,7 @@ targets:
   - src/proto/grpc/testing/stats.proto
   - test/cpp/codegen/codegen_test_full.cc
   deps:
+  - grpc++_core_stats
   - grpc++
   - grpc
   - gpr
@@ -3813,6 +3878,7 @@ targets:
   - src/proto/grpc/testing/stats.proto
   - test/cpp/codegen/codegen_test_minimal.cc
   deps:
+  - grpc++_core_stats
   - grpc
   - gpr
   filegroups:
@@ -4303,6 +4369,7 @@ targets:
   - test/cpp/qps/qps_json_driver.cc
   deps:
   - qps
+  - grpc++_core_stats
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -4318,6 +4385,7 @@ targets:
   - test/cpp/qps/qps_openloop_test.cc
   deps:
   - qps
+  - grpc++_core_stats
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -4340,6 +4408,7 @@ targets:
   - test/cpp/qps/worker.cc
   deps:
   - qps
+  - grpc++_core_stats
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -4403,6 +4472,7 @@ targets:
   - test/cpp/qps/secure_sync_unary_ping_pong_test.cc
   deps:
   - qps
+  - grpc++_core_stats
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -4515,6 +4585,18 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
+- name: stats_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/core/debug/stats_test.cc
+  deps:
+  - grpc++_test_util
+  - grpc_test_util
+  - grpc
+  - gpr_test_util
+  - gpr
 - name: status_test
   build: test
   language: c++

build_config.rb

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 module GrpcBuildConfig
-  CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-4.dll'
+  CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-5.dll'
 end

config.m4

@@ -101,6 +101,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
+    src/core/lib/iomgr/call_combiner.c \
     src/core/lib/iomgr/closure.c \
     src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
@@ -109,8 +110,6 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/iomgr/endpoint_pair_windows.c \
     src/core/lib/iomgr/error.c \
     src/core/lib/iomgr/ev_epoll1_linux.c \
-    src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \
-    src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \
     src/core/lib/iomgr/ev_epollex_linux.c \
     src/core/lib/iomgr/ev_epollsig_linux.c \
     src/core/lib/iomgr/ev_poll_posix.c \

config.w32

@@ -78,6 +78,7 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\http\\format_request.c " +
     "src\\core\\lib\\http\\httpcli.c " +
     "src\\core\\lib\\http\\parser.c " +
+    "src\\core\\lib\\iomgr\\call_combiner.c " +
     "src\\core\\lib\\iomgr\\closure.c " +
     "src\\core\\lib\\iomgr\\combiner.c " +
     "src\\core\\lib\\iomgr\\endpoint.c " +
@@ -86,8 +87,6 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\iomgr\\endpoint_pair_windows.c " +
     "src\\core\\lib\\iomgr\\error.c " +
     "src\\core\\lib\\iomgr\\ev_epoll1_linux.c " +
-    "src\\core\\lib\\iomgr\\ev_epoll_limited_pollers_linux.c " +
-    "src\\core\\lib\\iomgr\\ev_epoll_thread_pool_linux.c " +
     "src\\core\\lib\\iomgr\\ev_epollex_linux.c " +
     "src\\core\\lib\\iomgr\\ev_epollsig_linux.c " +
     "src\\core\\lib\\iomgr\\ev_poll_posix.c " +

doc/environment_variables.md

@@ -39,6 +39,7 @@ some configuration as environment variables that can be set.
   gRPC C core is processing requests via debug logs. Available tracers include:
   - api - traces api calls to the C core
   - bdp_estimator - traces behavior of bdp estimation logic
+  - call_combiner - traces call combiner state
   - call_error - traces the possible errors contributing to final call status
   - channel - traces operations on the C core channel stack
   - client_channel - traces client channel activity, including resolver
@@ -113,3 +114,11 @@ some configuration as environment variables that can be set.
   - native (default)- a DNS resolver based around getaddrinfo(), creates a new thread to
     perform name resolution
   - ares - a DNS resolver based around the c-ares library
+
+* GRPC_DISABLE_CHANNEL_CONNECTIVITY_WATCHER
+  The channel connectivity watcher uses one extra thread to check the channel
+  state every 500 ms on the client side. It can help reconnect disconnected
+  client channels (mostly due to idleness), so that the next RPC on this channel
+  won't fail. Set to 1 to turn off this watcher and save a thread. Please note
+  this is a temporary work-around, it will be removed in the future once we have
+  support for automatically reestablishing failed connections.

gRPC-Core.podspec

@@ -334,6 +334,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/http/format_request.h',
                       'src/core/lib/http/httpcli.h',
                       'src/core/lib/http/parser.h',
+                      'src/core/lib/iomgr/call_combiner.h',
                       'src/core/lib/iomgr/closure.h',
                       'src/core/lib/iomgr/combiner.h',
                       'src/core/lib/iomgr/endpoint.h',
@@ -341,8 +342,6 @@ Pod::Spec.new do |s|
                       'src/core/lib/iomgr/error.h',
                       'src/core/lib/iomgr/error_internal.h',
                       'src/core/lib/iomgr/ev_epoll1_linux.h',
-                      'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h',
-                      'src/core/lib/iomgr/ev_epoll_thread_pool_linux.h',
                       'src/core/lib/iomgr/ev_epollex_linux.h',
                       'src/core/lib/iomgr/ev_epollsig_linux.h',
                       'src/core/lib/iomgr/ev_poll_posix.h',
@@ -484,6 +483,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/http/format_request.c',
                       'src/core/lib/http/httpcli.c',
                       'src/core/lib/http/parser.c',
+                      'src/core/lib/iomgr/call_combiner.c',
                       'src/core/lib/iomgr/closure.c',
                       'src/core/lib/iomgr/combiner.c',
                       'src/core/lib/iomgr/endpoint.c',
@@ -492,8 +492,6 @@ Pod::Spec.new do |s|
                       'src/core/lib/iomgr/endpoint_pair_windows.c',
                       'src/core/lib/iomgr/error.c',
                       'src/core/lib/iomgr/ev_epoll1_linux.c',
-                      'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c',
-                      'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c',
                       'src/core/lib/iomgr/ev_epollex_linux.c',
                       'src/core/lib/iomgr/ev_epollsig_linux.c',
                       'src/core/lib/iomgr/ev_poll_posix.c',
@@ -831,6 +829,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/http/format_request.h',
                               'src/core/lib/http/httpcli.h',
                               'src/core/lib/http/parser.h',
+                              'src/core/lib/iomgr/call_combiner.h',
                               'src/core/lib/iomgr/closure.h',
                               'src/core/lib/iomgr/combiner.h',
                               'src/core/lib/iomgr/endpoint.h',
@@ -838,8 +837,6 @@ Pod::Spec.new do |s|
                               'src/core/lib/iomgr/error.h',
                               'src/core/lib/iomgr/error_internal.h',
                               'src/core/lib/iomgr/ev_epoll1_linux.h',
-                              'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h',
-                              'src/core/lib/iomgr/ev_epoll_thread_pool_linux.h',
                               'src/core/lib/iomgr/ev_epollex_linux.h',
                               'src/core/lib/iomgr/ev_epollsig_linux.h',
                               'src/core/lib/iomgr/ev_poll_posix.h',
@@ -994,6 +991,7 @@ Pod::Spec.new do |s|
                       'test/core/end2end/end2end_tests.{c,h}',
                       'test/core/end2end/end2end_test_utils.c',
                       'test/core/end2end/tests/*.{c,h}',
+                      'test/core/end2end/fixtures/*.h',
                       'test/core/end2end/data/*.{c,h}',
                       'test/core/util/debugger_macros.{c,h}',
                       'test/core/util/test_config.{c,h}',

grpc.def

@@ -65,11 +65,13 @@ EXPORTS
     grpc_completion_queue_shutdown
     grpc_completion_queue_destroy
     grpc_alarm_create
+    grpc_alarm_set
     grpc_alarm_cancel
     grpc_alarm_destroy
     grpc_channel_check_connectivity_state
     grpc_channel_num_external_connectivity_watchers
     grpc_channel_watch_connectivity_state
+    grpc_channel_support_connectivity_watcher
     grpc_channel_create_call
     grpc_channel_ping
     grpc_channel_register_call

grpc.gemspec

@@ -33,12 +33,12 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'bundler',            '~> 1.9'
   s.add_development_dependency 'facter',             '~> 2.4'
   s.add_development_dependency 'logging',            '~> 2.0'
-  s.add_development_dependency 'simplecov',          '~> 0.9'
+  s.add_development_dependency 'simplecov',          '~> 0.14.1'
-  s.add_development_dependency 'rake',               '~> 10.4'
+  s.add_development_dependency 'rake',               '~> 12.0'
   s.add_development_dependency 'rake-compiler',      '~> 1.0'
   s.add_development_dependency 'rake-compiler-dock', '~> 0.5.1'
-  s.add_development_dependency 'rspec',              '~> 3.2'
+  s.add_development_dependency 'rspec',              '~> 3.6'
-  s.add_development_dependency 'rubocop',            '~> 0.30.0'
+  s.add_development_dependency 'rubocop',            '~> 0.49.1'
   s.add_development_dependency 'signet',             '~> 0.7.0'
 
   s.extensions = %w(src/ruby/ext/grpc/extconf.rb)
@@ -266,6 +266,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/http/format_request.h )
   s.files += %w( src/core/lib/http/httpcli.h )
   s.files += %w( src/core/lib/http/parser.h )
+  s.files += %w( src/core/lib/iomgr/call_combiner.h )
   s.files += %w( src/core/lib/iomgr/closure.h )
   s.files += %w( src/core/lib/iomgr/combiner.h )
   s.files += %w( src/core/lib/iomgr/endpoint.h )
@@ -273,8 +274,6 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/iomgr/error.h )
   s.files += %w( src/core/lib/iomgr/error_internal.h )
   s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.h )
-  s.files += %w( src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h )
-  s.files += %w( src/core/lib/iomgr/ev_epoll_thread_pool_linux.h )
   s.files += %w( src/core/lib/iomgr/ev_epollex_linux.h )
   s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.h )
   s.files += %w( src/core/lib/iomgr/ev_poll_posix.h )
@@ -420,6 +419,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/http/format_request.c )
   s.files += %w( src/core/lib/http/httpcli.c )
   s.files += %w( src/core/lib/http/parser.c )
+  s.files += %w( src/core/lib/iomgr/call_combiner.c )
   s.files += %w( src/core/lib/iomgr/closure.c )
   s.files += %w( src/core/lib/iomgr/combiner.c )
   s.files += %w( src/core/lib/iomgr/endpoint.c )
@@ -428,8 +428,6 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c )
   s.files += %w( src/core/lib/iomgr/error.c )
   s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.c )
-  s.files += %w( src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c )
-  s.files += %w( src/core/lib/iomgr/ev_epoll_thread_pool_linux.c )
   s.files += %w( src/core/lib/iomgr/ev_epollex_linux.c )
   s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.c )
   s.files += %w( src/core/lib/iomgr/ev_poll_posix.c )

grpc.gyp

@@ -238,6 +238,7 @@
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',
+        'src/core/lib/iomgr/call_combiner.c',
         'src/core/lib/iomgr/closure.c',
         'src/core/lib/iomgr/combiner.c',
         'src/core/lib/iomgr/endpoint.c',
@@ -246,8 +247,6 @@
         'src/core/lib/iomgr/endpoint_pair_windows.c',
         'src/core/lib/iomgr/error.c',
         'src/core/lib/iomgr/ev_epoll1_linux.c',
-        'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c',
-        'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c',
         'src/core/lib/iomgr/ev_epollex_linux.c',
         'src/core/lib/iomgr/ev_epollsig_linux.c',
         'src/core/lib/iomgr/ev_poll_posix.c',
@@ -538,6 +537,7 @@
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',
+        'src/core/lib/iomgr/call_combiner.c',
         'src/core/lib/iomgr/closure.c',
         'src/core/lib/iomgr/combiner.c',
         'src/core/lib/iomgr/endpoint.c',
@@ -546,8 +546,6 @@
         'src/core/lib/iomgr/endpoint_pair_windows.c',
         'src/core/lib/iomgr/error.c',
         'src/core/lib/iomgr/ev_epoll1_linux.c',
-        'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c',
-        'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c',
         'src/core/lib/iomgr/ev_epollex_linux.c',
         'src/core/lib/iomgr/ev_epollsig_linux.c',
         'src/core/lib/iomgr/ev_poll_posix.c',
@@ -743,6 +741,7 @@
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',
+        'src/core/lib/iomgr/call_combiner.c',
         'src/core/lib/iomgr/closure.c',
         'src/core/lib/iomgr/combiner.c',
         'src/core/lib/iomgr/endpoint.c',
@@ -751,8 +750,6 @@
         'src/core/lib/iomgr/endpoint_pair_windows.c',
         'src/core/lib/iomgr/error.c',
         'src/core/lib/iomgr/ev_epoll1_linux.c',
-        'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c',
-        'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c',
         'src/core/lib/iomgr/ev_epollex_linux.c',
         'src/core/lib/iomgr/ev_epollsig_linux.c',
         'src/core/lib/iomgr/ev_poll_posix.c',
@@ -933,6 +930,7 @@
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',
+        'src/core/lib/iomgr/call_combiner.c',
         'src/core/lib/iomgr/closure.c',
         'src/core/lib/iomgr/combiner.c',
         'src/core/lib/iomgr/endpoint.c',
@@ -941,8 +939,6 @@
         'src/core/lib/iomgr/endpoint_pair_windows.c',
         'src/core/lib/iomgr/error.c',
         'src/core/lib/iomgr/ev_epoll1_linux.c',
-        'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c',
-        'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c',
         'src/core/lib/iomgr/ev_epollex_linux.c',
         'src/core/lib/iomgr/ev_epollsig_linux.c',
         'src/core/lib/iomgr/ev_poll_posix.c',
@@ -1226,6 +1222,17 @@
         'src/cpp/codegen/codegen_init.cc',
       ],
     },
+    {
+      'target_name': 'grpc++_core_stats',
+      'type': 'static_library',
+      'dependencies': [
+        'grpc++',
+      ],
+      'sources': [
+        'src/proto/grpc/core/stats.proto',
+        'src/cpp/util/core_stats.cc',
+      ],
+    },
     {
       'target_name': 'grpc++_error_details',
       'type': 'static_library',
@@ -1508,6 +1515,7 @@
       'dependencies': [
         'grpc_test_util',
         'grpc++_test_util',
+        'grpc++_core_stats',
         'grpc++',
         'grpc',
       ],

include/grpc++/alarm.h

@@ -37,20 +37,33 @@ class CompletionQueue;
 /// A thin wrapper around \a grpc_alarm (see / \a / src/core/surface/alarm.h).
 class Alarm : private GrpcLibraryCodegen {
  public:
-  /// Create a completion queue alarm instance associated to \a cq.
+  /// Create an unset completion queue alarm
-  ///
+  Alarm() : tag_(nullptr), alarm_(grpc_alarm_create(nullptr)) {}
-  /// Once the alarm expires (at \a deadline) or it's cancelled (see \a Cancel),
+
-  /// an event with tag \a tag will be added to \a cq. If the alarm expired, the
+  /// DEPRECATED: Create and set a completion queue alarm instance associated to
-  /// event's success bit will be true, false otherwise (ie, upon cancellation).
+  /// \a cq.
+  /// This form is deprecated because it is inherently racy.
   /// \internal We rely on the presence of \a cq for grpc initialization. If \a
   /// cq were ever to be removed, a reference to a static
   /// internal::GrpcLibraryInitializer instance would need to be introduced
   /// here. \endinternal.
   template <typename T>
   Alarm(CompletionQueue* cq, const T& deadline, void* tag)
-      : tag_(tag),
+      : tag_(tag), alarm_(grpc_alarm_create(nullptr)) {
-        alarm_(grpc_alarm_create(cq->cq(), TimePoint<T>(deadline).raw_time(),
+    grpc_alarm_set(alarm_, cq->cq(), TimePoint<T>(deadline).raw_time(),
-                                 static_cast<void*>(&tag_))) {}
+                   static_cast<void*>(&tag_), nullptr);
+  }
+
+  /// Trigger an alarm instance on completion queue \a cq at the specified time.
+  /// Once the alarm expires (at \a deadline) or it's cancelled (see \a Cancel),
+  /// an event with tag \a tag will be added to \a cq. If the alarm expired, the
+  /// event's success bit will be true, false otherwise (ie, upon cancellation).
+  template <typename T>
+  void Set(CompletionQueue* cq, const T& deadline, void* tag) {
+    tag_.Set(tag);
+    grpc_alarm_set(alarm_, cq->cq(), TimePoint<T>(deadline).raw_time(),
+                   static_cast<void*>(&tag_), nullptr);
+  }
 
   /// Alarms aren't copyable.
   Alarm(const Alarm&) = delete;
@@ -69,17 +82,20 @@ class Alarm : private GrpcLibraryCodegen {
 
   /// Destroy the given completion queue alarm, cancelling it in the process.
   ~Alarm() {
-    if (alarm_ != nullptr) grpc_alarm_destroy(alarm_);
+    if (alarm_ != nullptr) grpc_alarm_destroy(alarm_, nullptr);
   }
 
   /// Cancel a completion queue alarm. Calling this function over an alarm that
   /// has already fired has no effect.
-  void Cancel() { grpc_alarm_cancel(alarm_); }
+  void Cancel() {
+    if (alarm_ != nullptr) grpc_alarm_cancel(alarm_, nullptr);
+  }
 
  private:
   class AlarmEntry : public CompletionQueueTag {
    public:
     AlarmEntry(void* tag) : tag_(tag) {}
+    void Set(void* tag) { tag_ = tag; }
     bool FinalizeResult(void** tag, bool* status) override {
       *tag = tag_;
       return true;

include/grpc++/impl/codegen/sync_stream.h

@@ -244,7 +244,7 @@ class ClientWriterInterface : public ClientStreamingInterface,
                               public WriterInterface<W> {
  public:
   /// Half close writing from the client. (signal that the stream of messages
-  /// coming from the clinet is complete).
+  /// coming from the client is complete).
   /// Blocks until currently-pending writes are completed.
   /// Thread safe with respect to \a ReaderInterface::Read operations only
   ///
@@ -375,7 +375,7 @@ class ClientReaderWriterInterface : public ClientStreamingInterface,
   virtual void WaitForInitialMetadata() = 0;
 
   /// Half close writing from the client. (signal that the stream of messages
-  /// coming from the clinet is complete).
+  /// coming from the client is complete).
   /// Blocks until currently-pending writes are completed.
   /// Thread-safe with respect to \a ReaderInterface::Read
   ///

include/grpc++/server_builder.h

@@ -151,7 +151,8 @@ class ServerBuilder {
   /// Add a completion queue for handling asynchronous services.
   ///
   /// Caller is required to shutdown the server prior to shutting down the
-  /// returned completion queue. A typical usage scenario:
+  /// returned completion queue. Caller is also required to drain the
+  /// completion queue after shutting it down. A typical usage scenario:
   ///
   /// // While building the server:
   /// ServerBuilder builder;
@@ -162,6 +163,10 @@ class ServerBuilder {
   /// // While shutting down the server;
   /// server_->Shutdown();
   /// cq_->Shutdown();  // Always *after* the associated server's Shutdown()!
+  /// // Drain the cq_ that was created
+  /// void* ignored_tag;
+  /// bool ignored_ok;
+  /// while (cq_->Next(&ignored_tag, &ignored_ok)) { }
   ///
   /// \param is_frequently_polled This is an optional parameter to inform gRPC
   /// library about whether this completion queue would be frequently polled

include/grpc/grpc.h

@@ -143,21 +143,24 @@ GRPCAPI void grpc_completion_queue_shutdown(grpc_completion_queue *cq);
     drained and no threads are executing grpc_completion_queue_next */
 GRPCAPI void grpc_completion_queue_destroy(grpc_completion_queue *cq);
 
-/** Create a completion queue alarm instance associated to \a cq.
+/** Create a completion queue alarm instance */
+GRPCAPI grpc_alarm *grpc_alarm_create(void *reserved);
+
+/** Set a completion queue alarm instance associated to \a cq.
  *
  * Once the alarm expires (at \a deadline) or it's cancelled (see \a
  * grpc_alarm_cancel), an event with tag \a tag will be added to \a cq. If the
  * alarm expired, the event's success bit will be true, false otherwise (ie,
  * upon cancellation). */
-GRPCAPI grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq,
+GRPCAPI void grpc_alarm_set(grpc_alarm *alarm, grpc_completion_queue *cq,
-                                      gpr_timespec deadline, void *tag);
+                            gpr_timespec deadline, void *tag, void *reserved);
 
 /** Cancel a completion queue alarm. Calling this function over an alarm that
  * has already fired has no effect. */
-GRPCAPI void grpc_alarm_cancel(grpc_alarm *alarm);
+GRPCAPI void grpc_alarm_cancel(grpc_alarm *alarm, void *reserved);
 
 /** Destroy the given completion queue alarm, cancelling it in the process. */
-GRPCAPI void grpc_alarm_destroy(grpc_alarm *alarm);
+GRPCAPI void grpc_alarm_destroy(grpc_alarm *alarm, void *reserved);
 
 /** Check the connectivity state of a channel. */
 GRPCAPI grpc_connectivity_state grpc_channel_check_connectivity_state(
@@ -178,6 +181,9 @@ GRPCAPI void grpc_channel_watch_connectivity_state(
     grpc_channel *channel, grpc_connectivity_state last_observed_state,
     gpr_timespec deadline, grpc_completion_queue *cq, void *tag);
 
+/** Check whether a grpc channel supports connectivity watcher */
+GRPCAPI int grpc_channel_support_connectivity_watcher(grpc_channel *channel);
+
 /** Create a call given a grpc_channel, in order to call 'method'. All
     completions are sent to 'completion_queue'. 'method' and 'host' need only
     live through the invocation of this function.

include/grpc/impl/codegen/atm.h

@@ -46,6 +46,7 @@
 
    // Atomically return *p, with acquire semantics.
    gpr_atm gpr_atm_acq_load(gpr_atm *p);
+   gpr_atm gpr_atm_no_barrier_load(gpr_atm *p);
 
    // Atomically set *p = value, with release semantics.
    void gpr_atm_rel_store(gpr_atm *p, gpr_atm value);

include/grpc/impl/codegen/slice.h

@@ -62,7 +62,12 @@ typedef struct grpc_slice_refcount {
   struct grpc_slice_refcount *sub_refcount;
 } grpc_slice_refcount;
 
-#define GRPC_SLICE_INLINED_SIZE (sizeof(size_t) + sizeof(uint8_t *) - 1)
+/* Inlined half of grpc_slice is allowed to expand the size of the overall type
+   by this many bytes */
+#define GRPC_SLICE_INLINE_EXTRA_SIZE sizeof(void *)
+
+#define GRPC_SLICE_INLINED_SIZE \
+  (sizeof(size_t) + sizeof(uint8_t *) - 1 + GRPC_SLICE_INLINE_EXTRA_SIZE)
 
 /** A grpc_slice s, if initialized, represents the byte range
    s.bytes[0..s.length-1].

package.xml

@@ -276,6 +276,7 @@
     <file baseinstalldir="/" name="src/core/lib/http/format_request.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/httpcli.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/parser.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/iomgr/call_combiner.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/closure.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/combiner.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint.h" role="src" />
@@ -283,8 +284,6 @@
     <file baseinstalldir="/" name="src/core/lib/iomgr/error.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/error_internal.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epoll1_linux.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epoll_thread_pool_linux.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epollex_linux.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epollsig_linux.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/ev_poll_posix.h" role="src" />
@@ -430,6 +429,7 @@
     <file baseinstalldir="/" name="src/core/lib/http/format_request.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/httpcli.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/parser.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/iomgr/call_combiner.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/closure.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/combiner.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint.c" role="src" />
@@ -438,8 +438,6 @@
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint_pair_windows.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/error.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epoll1_linux.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epoll_thread_pool_linux.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epollex_linux.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epollsig_linux.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/ev_poll_posix.c" role="src" />

src/core/ext/census/context.c

@@ -141,7 +141,7 @@ static char *decode_tag(struct raw_tag *tag, char *header, int offset) {
 // Make a copy (in 'to') of an existing tag_set.
 static void tag_set_copy(struct tag_set *to, const struct tag_set *from) {
   memcpy(to, from, sizeof(struct tag_set));
-  to->kvm = gpr_malloc(to->kvm_size);
+  to->kvm = (char *)gpr_malloc(to->kvm_size);
   memcpy(to->kvm, from->kvm, from->kvm_used);
 }
 
@@ -184,7 +184,7 @@ static bool tag_set_add_tag(struct tag_set *tags, const census_tag *tag,
   if (tags->kvm_used + tag_size > tags->kvm_size) {
     // allocate new memory if needed
     tags->kvm_size += 2 * CENSUS_MAX_TAG_KV_LEN + TAG_HEADER_SIZE;
-    char *new_kvm = gpr_malloc(tags->kvm_size);
+    char *new_kvm = (char *)gpr_malloc(tags->kvm_size);
     if (tags->kvm_used > 0) memcpy(new_kvm, tags->kvm, tags->kvm_used);
     gpr_free(tags->kvm);
     tags->kvm = new_kvm;
@@ -274,7 +274,8 @@ static void tag_set_flatten(struct tag_set *tags) {
 census_context *census_context_create(const census_context *base,
                                       const census_tag *tags, int ntags,
                                       census_context_status const **status) {
-  census_context *context = gpr_malloc(sizeof(census_context));
+  census_context *context =
+      (census_context *)gpr_malloc(sizeof(census_context));
   // If we are given a base, copy it into our new tag set. Otherwise set it
   // to zero/NULL everything.
   if (base == NULL) {
@@ -459,7 +460,7 @@ static void tag_set_decode(struct tag_set *tags, const char *buffer,
   }
   tags->kvm_used = size - header_size;
   tags->kvm_size = tags->kvm_used + CENSUS_MAX_TAG_KV_LEN;
-  tags->kvm = gpr_malloc(tags->kvm_size);
+  tags->kvm = (char *)gpr_malloc(tags->kvm_size);
   if (tag_header_size != TAG_HEADER_SIZE) {
     // something new in the tag information. I don't understand it, so
     // don't copy it over.
@@ -481,7 +482,8 @@ static void tag_set_decode(struct tag_set *tags, const char *buffer,
 }
 
 census_context *census_context_decode(const char *buffer, size_t size) {
-  census_context *context = gpr_malloc(sizeof(census_context));
+  census_context *context =
+      (census_context *)gpr_malloc(sizeof(census_context));
   memset(&context->tags[LOCAL_TAGS], 0, sizeof(struct tag_set));
   if (buffer == NULL) {
     memset(&context->tags[PROPAGATED_TAGS], 0, sizeof(struct tag_set));

src/core/ext/census/grpc_filter.c

@@ -60,8 +60,8 @@ static void extract_and_annotate_method_tag(grpc_metadata_batch *md,
 
 static void client_mutate_op(grpc_call_element *elem,
                              grpc_transport_stream_op_batch *op) {
-  call_data *calld = elem->call_data;
+  call_data *calld = (call_data *)elem->call_data;
-  channel_data *chand = elem->channel_data;
+  channel_data *chand = (channel_data *)elem->channel_data;
   if (op->send_initial_metadata) {
     extract_and_annotate_method_tag(
         op->payload->send_initial_metadata.send_initial_metadata, calld, chand);
@@ -78,9 +78,9 @@ static void client_start_transport_op(grpc_exec_ctx *exec_ctx,
 static void server_on_done_recv(grpc_exec_ctx *exec_ctx, void *ptr,
                                 grpc_error *error) {
   GPR_TIMER_BEGIN("census-server:server_on_done_recv", 0);
-  grpc_call_element *elem = ptr;
+  grpc_call_element *elem = (grpc_call_element *)ptr;
-  call_data *calld = elem->call_data;
+  call_data *calld = (call_data *)elem->call_data;
-  channel_data *chand = elem->channel_data;
+  channel_data *chand = (channel_data *)elem->channel_data;
   if (error == GRPC_ERROR_NONE) {
     extract_and_annotate_method_tag(calld->recv_initial_metadata, calld, chand);
   }
@@ -90,7 +90,7 @@ static void server_on_done_recv(grpc_exec_ctx *exec_ctx, void *ptr,
 
 static void server_mutate_op(grpc_call_element *elem,
                              grpc_transport_stream_op_batch *op) {
-  call_data *calld = elem->call_data;
+  call_data *calld = (call_data *)elem->call_data;
   if (op->recv_initial_metadata) {
     /* substitute our callback for the op callback */
     calld->recv_initial_metadata =
@@ -117,7 +117,7 @@ static void server_start_transport_op(grpc_exec_ctx *exec_ctx,
 static grpc_error *client_init_call_elem(grpc_exec_ctx *exec_ctx,
                                          grpc_call_element *elem,
                                          const grpc_call_element_args *args) {
-  call_data *d = elem->call_data;
+  call_data *d = (call_data *)elem->call_data;
   GPR_ASSERT(d != NULL);
   memset(d, 0, sizeof(*d));
   d->start_ts = args->start_time;
@@ -128,7 +128,7 @@ static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_call_element *elem,
                                      const grpc_call_final_info *final_info,
                                      grpc_closure *ignored) {
-  call_data *d = elem->call_data;
+  call_data *d = (call_data *)elem->call_data;
   GPR_ASSERT(d != NULL);
   /* TODO(hongyu): record rpc client stats and census_rpc_end_op here */
 }
@@ -136,7 +136,7 @@ static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx,
 static grpc_error *server_init_call_elem(grpc_exec_ctx *exec_ctx,
                                          grpc_call_element *elem,
                                          const grpc_call_element_args *args) {
-  call_data *d = elem->call_data;
+  call_data *d = (call_data *)elem->call_data;
   GPR_ASSERT(d != NULL);
   memset(d, 0, sizeof(*d));
   d->start_ts = args->start_time;
@@ -150,7 +150,7 @@ static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_call_element *elem,
                                      const grpc_call_final_info *final_info,
                                      grpc_closure *ignored) {
-  call_data *d = elem->call_data;
+  call_data *d = (call_data *)elem->call_data;
   GPR_ASSERT(d != NULL);
   /* TODO(hongyu): record rpc server stats and census_tracing_end_op here */
 }
@@ -158,14 +158,14 @@ static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx,
 static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_element *elem,
                                      grpc_channel_element_args *args) {
-  channel_data *chand = elem->channel_data;
+  channel_data *chand = (channel_data *)elem->channel_data;
   GPR_ASSERT(chand != NULL);
   return GRPC_ERROR_NONE;
 }
 
 static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem) {
-  channel_data *chand = elem->channel_data;
+  channel_data *chand = (channel_data *)elem->channel_data;
   GPR_ASSERT(chand != NULL);
 }
 
@@ -179,7 +179,6 @@ const grpc_channel_filter grpc_client_census_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "census-client"};
 
@@ -193,6 +192,5 @@ const grpc_channel_filter grpc_server_census_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "census-server"};

src/core/ext/census/mlog.c

@@ -467,7 +467,8 @@ void census_log_initialize(size_t size_in_mb, int discard_old_records) {
   g_log.blocks = (cl_block*)gpr_malloc_aligned(
       g_log.num_blocks * sizeof(cl_block), GPR_CACHELINE_SIZE_LOG);
   memset(g_log.blocks, 0, g_log.num_blocks * sizeof(cl_block));
-  g_log.buffer = gpr_malloc(g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE);
+  g_log.buffer =
+      (char*)gpr_malloc(g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE);
   memset(g_log.buffer, 0, g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE);
   cl_block_list_initialize(&g_log.free_block_list);
   cl_block_list_initialize(&g_log.dirty_block_list);

src/core/ext/census/resource.c

@@ -87,7 +87,7 @@ static bool validate_string(pb_istream_t *stream, const pb_field_t *field,
         gpr_log(GPR_INFO, "Zero-length Resource name.");
         return false;
       }
-      vresource->name = gpr_malloc(stream->bytes_left + 1);
+      vresource->name = (char *)gpr_malloc(stream->bytes_left + 1);
       vresource->name[stream->bytes_left] = '\0';
       if (!pb_read(stream, (uint8_t *)vresource->name, stream->bytes_left)) {
         return false;
@@ -106,7 +106,7 @@ static bool validate_string(pb_istream_t *stream, const pb_field_t *field,
       if (stream->bytes_left == 0) {
         return true;
       }
-      vresource->description = gpr_malloc(stream->bytes_left + 1);
+      vresource->description = (char *)gpr_malloc(stream->bytes_left + 1);
       vresource->description[stream->bytes_left] = '\0';
       if (!pb_read(stream, (uint8_t *)vresource->description,
                    stream->bytes_left)) {
@@ -134,7 +134,8 @@ static bool validate_units_helper(pb_istream_t *stream, int *count,
     // Have to allocate a new array of values. Normal case is 0 or 1, so
     // this should normally not be an issue.
     google_census_Resource_BasicUnit *new_bup =
-        gpr_malloc((size_t)*count * sizeof(google_census_Resource_BasicUnit));
+        (google_census_Resource_BasicUnit *)gpr_malloc(
+            (size_t)*count * sizeof(google_census_Resource_BasicUnit));
     if (*count != 1) {
       memcpy(new_bup, *bup,
              (size_t)(*count - 1) * sizeof(google_census_Resource_BasicUnit));
@@ -207,7 +208,8 @@ size_t allocate_resource(void) {
   // Expand resources if needed.
   if (n_resources == n_defined_resources) {
     size_t new_n_resources = n_resources ? n_resources * 2 : 2;
-    resource **new_resources = gpr_malloc(new_n_resources * sizeof(resource *));
+    resource **new_resources =
+        (resource **)gpr_malloc(new_n_resources * sizeof(resource *));
     if (n_resources != 0) {
       memcpy(new_resources, resources, n_resources * sizeof(resource *));
     }
@@ -226,7 +228,7 @@ size_t allocate_resource(void) {
     }
   }
   GPR_ASSERT(id < n_resources && resources[id] == NULL);
-  resources[id] = gpr_malloc(sizeof(resource));
+  resources[id] = (resource *)gpr_malloc(sizeof(resource));
   memset(resources[id], 0, sizeof(resource));
   n_defined_resources++;
   next_id = (id + 1) % n_resources;
@@ -276,22 +278,24 @@ int32_t define_resource(const resource *base) {
   gpr_mu_lock(&resource_lock);
   size_t id = allocate_resource();
   size_t len = strlen(base->name) + 1;
-  resources[id]->name = gpr_malloc(len);
+  resources[id]->name = (char *)gpr_malloc(len);
   memcpy(resources[id]->name, base->name, len);
   if (base->description) {
     len = strlen(base->description) + 1;
-    resources[id]->description = gpr_malloc(len);
+    resources[id]->description = (char *)gpr_malloc(len);
     memcpy(resources[id]->description, base->description, len);
   }
   resources[id]->prefix = base->prefix;
   resources[id]->n_numerators = base->n_numerators;
   len = (size_t)base->n_numerators * sizeof(*base->numerators);
-  resources[id]->numerators = gpr_malloc(len);
+  resources[id]->numerators =
+      (google_census_Resource_BasicUnit *)gpr_malloc(len);
   memcpy(resources[id]->numerators, base->numerators, len);
   resources[id]->n_denominators = base->n_denominators;
   if (base->n_denominators != 0) {
     len = (size_t)base->n_denominators * sizeof(*base->denominators);
-    resources[id]->denominators = gpr_malloc(len);
+    resources[id]->denominators =
+        (google_census_Resource_BasicUnit *)gpr_malloc(len);
     memcpy(resources[id]->denominators, base->denominators, len);
   }
   gpr_mu_unlock(&resource_lock);

src/core/ext/filters/client_channel/channel_connectivity.c

@@ -191,6 +191,12 @@ static void watcher_timer_init(grpc_exec_ctx *exec_ctx, void *arg,
   gpr_free(wa);
 }
 
+int grpc_channel_support_connectivity_watcher(grpc_channel *channel) {
+  grpc_channel_element *client_channel_elem =
+      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel));
+  return client_channel_elem->filter != &grpc_client_channel_filter ? 0 : 1;
+}
+
 void grpc_channel_watch_connectivity_state(
     grpc_channel *channel, grpc_connectivity_state last_observed_state,
     gpr_timespec deadline, grpc_completion_queue *cq, void *tag) {

src/core/ext/filters/client_channel/client_channel.c

@@ -796,7 +796,8 @@ static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 //   send_message
 //   recv_trailing_metadata
 //   send_trailing_metadata
-#define MAX_WAITING_BATCHES 6
+// We also add room for a single cancel_stream batch.
+#define MAX_WAITING_BATCHES 7
 
 /** Call data.  Holds a pointer to grpc_subchannel_call and the
     associated machinery to create such a pointer.
@@ -807,24 +808,27 @@ typedef struct client_channel_call_data {
   // State for handling deadlines.
   // The code in deadline_filter.c requires this to be the first field.
   // TODO(roth): This is slightly sub-optimal in that grpc_deadline_state
-  // and this struct both independently store a pointer to the call
+  // and this struct both independently store pointers to the call stack
-  // stack and each has its own mutex.  If/when we have time, find a way
+  // and call combiner.  If/when we have time, find a way to avoid this
-  // to avoid this without breaking the grpc_deadline_state abstraction.
+  // without breaking the grpc_deadline_state abstraction.
   grpc_deadline_state deadline_state;
 
   grpc_slice path;  // Request path.
   gpr_timespec call_start_time;
   gpr_timespec deadline;
+  gpr_arena *arena;
+  grpc_call_stack *owning_call;
+  grpc_call_combiner *call_combiner;
+
   grpc_server_retry_throttle_data *retry_throttle_data;
   method_parameters *method_params;
 
-  /** either 0 for no call, a pointer to a grpc_subchannel_call (if the lowest
+  grpc_subchannel_call *subchannel_call;
-      bit is 0), or a pointer to an error (if the lowest bit is 1) */
+  grpc_error *error;
-  gpr_atm subchannel_call_or_error;
-  gpr_arena *arena;
 
   grpc_lb_policy *lb_policy;  // Holds ref while LB pick is pending.
   grpc_closure lb_pick_closure;
+  grpc_closure lb_pick_cancel_closure;
 
   grpc_connected_subchannel *connected_subchannel;
   grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
@@ -832,10 +836,9 @@ typedef struct client_channel_call_data {
 
   grpc_transport_stream_op_batch *waiting_for_pick_batches[MAX_WAITING_BATCHES];
   size_t waiting_for_pick_batches_count;
+  grpc_closure handle_pending_batch_in_call_combiner[MAX_WAITING_BATCHES];
 
-  grpc_transport_stream_op_batch_payload *initial_metadata_payload;
+  grpc_transport_stream_op_batch *initial_metadata_batch;
-
-  grpc_call_stack *owning_call;
 
   grpc_linked_mdelem lb_token_mdelem;
 
@@ -843,53 +846,40 @@ typedef struct client_channel_call_data {
   grpc_closure *original_on_complete;
 } call_data;
 
-typedef struct {
-  grpc_subchannel_call *subchannel_call;
-  grpc_error *error;
-} call_or_error;
-
-static call_or_error get_call_or_error(call_data *p) {
-  gpr_atm c = gpr_atm_acq_load(&p->subchannel_call_or_error);
-  if (c == 0)
-    return (call_or_error){NULL, NULL};
-  else if (c & 1)
-    return (call_or_error){NULL, (grpc_error *)((c) & ~(gpr_atm)1)};
-  else
-    return (call_or_error){(grpc_subchannel_call *)c, NULL};
-}
-
-static bool set_call_or_error(call_data *p, call_or_error coe) {
-  // this should always be under a lock
-  call_or_error existing = get_call_or_error(p);
-  if (existing.error != GRPC_ERROR_NONE) {
-    GRPC_ERROR_UNREF(coe.error);
-    return false;
-  }
-  GPR_ASSERT(existing.subchannel_call == NULL);
-  if (coe.error != GRPC_ERROR_NONE) {
-    GPR_ASSERT(coe.subchannel_call == NULL);
-    gpr_atm_rel_store(&p->subchannel_call_or_error, 1 | (gpr_atm)coe.error);
-  } else {
-    GPR_ASSERT(coe.subchannel_call != NULL);
-    gpr_atm_rel_store(&p->subchannel_call_or_error,
-                      (gpr_atm)coe.subchannel_call);
-  }
-  return true;
-}
-
 grpc_subchannel_call *grpc_client_channel_get_subchannel_call(
-    grpc_call_element *call_elem) {
+    grpc_call_element *elem) {
-  return get_call_or_error(call_elem->call_data).subchannel_call;
+  call_data *calld = elem->call_data;
+  return calld->subchannel_call;
 }
 
-static void waiting_for_pick_batches_add_locked(
+// This is called via the call combiner, so access to calld is synchronized.
+static void waiting_for_pick_batches_add(
     call_data *calld, grpc_transport_stream_op_batch *batch) {
+  if (batch->send_initial_metadata) {
+    GPR_ASSERT(calld->initial_metadata_batch == NULL);
+    calld->initial_metadata_batch = batch;
+  } else {
     GPR_ASSERT(calld->waiting_for_pick_batches_count < MAX_WAITING_BATCHES);
     calld->waiting_for_pick_batches[calld->waiting_for_pick_batches_count++] =
         batch;
   }
+}
 
-static void waiting_for_pick_batches_fail_locked(grpc_exec_ctx *exec_ctx,
+// This is called via the call combiner, so access to calld is synchronized.
+static void fail_pending_batch_in_call_combiner(grpc_exec_ctx *exec_ctx,
+                                                void *arg, grpc_error *error) {
+  call_data *calld = arg;
+  if (calld->waiting_for_pick_batches_count > 0) {
+    --calld->waiting_for_pick_batches_count;
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx,
+        calld->waiting_for_pick_batches[calld->waiting_for_pick_batches_count],
+        GRPC_ERROR_REF(error), calld->call_combiner);
+  }
+}
+
+// This is called via the call combiner, so access to calld is synchronized.
+static void waiting_for_pick_batches_fail(grpc_exec_ctx *exec_ctx,
                                           grpc_call_element *elem,
                                           grpc_error *error) {
   call_data *calld = elem->call_data;
@@ -900,34 +890,60 @@ static void waiting_for_pick_batches_fail_locked(grpc_exec_ctx *exec_ctx,
             grpc_error_string(error));
   }
   for (size_t i = 0; i < calld->waiting_for_pick_batches_count; ++i) {
+    GRPC_CLOSURE_INIT(&calld->handle_pending_batch_in_call_combiner[i],
+                      fail_pending_batch_in_call_combiner, calld,
+                      grpc_schedule_on_exec_ctx);
+    GRPC_CALL_COMBINER_START(exec_ctx, calld->call_combiner,
+                             &calld->handle_pending_batch_in_call_combiner[i],
+                             GRPC_ERROR_REF(error),
+                             "waiting_for_pick_batches_fail");
+  }
+  if (calld->initial_metadata_batch != NULL) {
     grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, calld->waiting_for_pick_batches[i], GRPC_ERROR_REF(error));
+        exec_ctx, calld->initial_metadata_batch, GRPC_ERROR_REF(error),
+        calld->call_combiner);
+  } else {
+    GRPC_CALL_COMBINER_STOP(exec_ctx, calld->call_combiner,
+                            "waiting_for_pick_batches_fail");
   }
-  calld->waiting_for_pick_batches_count = 0;
   GRPC_ERROR_UNREF(error);
 }
 
-static void waiting_for_pick_batches_resume_locked(grpc_exec_ctx *exec_ctx,
+// This is called via the call combiner, so access to calld is synchronized.
+static void run_pending_batch_in_call_combiner(grpc_exec_ctx *exec_ctx,
+                                               void *arg, grpc_error *ignored) {
+  call_data *calld = arg;
+  if (calld->waiting_for_pick_batches_count > 0) {
+    --calld->waiting_for_pick_batches_count;
+    grpc_subchannel_call_process_op(
+        exec_ctx, calld->subchannel_call,
+        calld->waiting_for_pick_batches[calld->waiting_for_pick_batches_count]);
+  }
+}
+
+// This is called via the call combiner, so access to calld is synchronized.
+static void waiting_for_pick_batches_resume(grpc_exec_ctx *exec_ctx,
                                             grpc_call_element *elem) {
+  channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
-  if (calld->waiting_for_pick_batches_count == 0) return;
-  call_or_error coe = get_call_or_error(calld);
-  if (coe.error != GRPC_ERROR_NONE) {
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem,
-                                         GRPC_ERROR_REF(coe.error));
-    return;
-  }
   if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: sending %" PRIdPTR
                        " pending batches to subchannel_call=%p",
-            elem->channel_data, calld, calld->waiting_for_pick_batches_count,
+            chand, calld, calld->waiting_for_pick_batches_count,
-            coe.subchannel_call);
+            calld->subchannel_call);
   }
   for (size_t i = 0; i < calld->waiting_for_pick_batches_count; ++i) {
-    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call,
+    GRPC_CLOSURE_INIT(&calld->handle_pending_batch_in_call_combiner[i],
-                                    calld->waiting_for_pick_batches[i]);
+                      run_pending_batch_in_call_combiner, calld,
+                      grpc_schedule_on_exec_ctx);
+    GRPC_CALL_COMBINER_START(exec_ctx, calld->call_combiner,
+                             &calld->handle_pending_batch_in_call_combiner[i],
+                             GRPC_ERROR_NONE,
+                             "waiting_for_pick_batches_resume");
   }
-  calld->waiting_for_pick_batches_count = 0;
+  GPR_ASSERT(calld->initial_metadata_batch != NULL);
+  grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call,
+                                  calld->initial_metadata_batch);
 }
 
 // Applies service config to the call.  Must be invoked once we know
@@ -968,29 +984,28 @@ static void apply_service_config_to_call_locked(grpc_exec_ctx *exec_ctx,
 static void create_subchannel_call_locked(grpc_exec_ctx *exec_ctx,
                                           grpc_call_element *elem,
                                           grpc_error *error) {
+  channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
-  grpc_subchannel_call *subchannel_call = NULL;
   const grpc_connected_subchannel_call_args call_args = {
       .pollent = calld->pollent,
       .path = calld->path,
       .start_time = calld->call_start_time,
       .deadline = calld->deadline,
       .arena = calld->arena,
-      .context = calld->subchannel_call_context};
+      .context = calld->subchannel_call_context,
+      .call_combiner = calld->call_combiner};
   grpc_error *new_error = grpc_connected_subchannel_create_call(
-      exec_ctx, calld->connected_subchannel, &call_args, &subchannel_call);
+      exec_ctx, calld->connected_subchannel, &call_args,
+      &calld->subchannel_call);
   if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
     gpr_log(GPR_DEBUG, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
-            elem->channel_data, calld, subchannel_call,
+            chand, calld, calld->subchannel_call, grpc_error_string(new_error));
-            grpc_error_string(new_error));
   }
-  GPR_ASSERT(set_call_or_error(
-      calld, (call_or_error){.subchannel_call = subchannel_call}));
   if (new_error != GRPC_ERROR_NONE) {
     new_error = grpc_error_add_child(new_error, error);
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem, new_error);
+    waiting_for_pick_batches_fail(exec_ctx, elem, new_error);
   } else {
-    waiting_for_pick_batches_resume_locked(exec_ctx, elem);
+    waiting_for_pick_batches_resume(exec_ctx, elem);
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -1002,11 +1017,10 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx,
   channel_data *chand = elem->channel_data;
   grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent,
                                            chand->interested_parties);
-  call_or_error coe = get_call_or_error(calld);
   if (calld->connected_subchannel == NULL) {
     // Failed to create subchannel.
-    grpc_error *failure =
+    GRPC_ERROR_UNREF(calld->error);
-        error == GRPC_ERROR_NONE
+    calld->error = error == GRPC_ERROR_NONE
                        ? GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                              "Call dropped by load balancing policy")
                        : GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
@@ -1014,48 +1028,16 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx,
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG,
               "chand=%p calld=%p: failed to create subchannel: error=%s", chand,
-              calld, grpc_error_string(failure));
+              calld, grpc_error_string(calld->error));
-    }
-    set_call_or_error(calld, (call_or_error){.error = GRPC_ERROR_REF(failure)});
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem, failure);
-  } else if (coe.error != GRPC_ERROR_NONE) {
-    /* already cancelled before subchannel became ready */
-    grpc_error *child_errors[] = {error, coe.error};
-    grpc_error *cancellation_error =
-        GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-            "Cancelled before creating subchannel", child_errors,
-            GPR_ARRAY_SIZE(child_errors));
-    /* if due to deadline, attach the deadline exceeded status to the error */
-    if (gpr_time_cmp(calld->deadline, gpr_now(GPR_CLOCK_MONOTONIC)) < 0) {
-      cancellation_error =
-          grpc_error_set_int(cancellation_error, GRPC_ERROR_INT_GRPC_STATUS,
-                             GRPC_STATUS_DEADLINE_EXCEEDED);
-    }
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG,
-              "chand=%p calld=%p: cancelled before subchannel became ready: %s",
-              chand, calld, grpc_error_string(cancellation_error));
     }
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem, cancellation_error);
+    waiting_for_pick_batches_fail(exec_ctx, elem, GRPC_ERROR_REF(calld->error));
   } else {
     /* Create call on subchannel. */
     create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
   }
-  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
   GRPC_ERROR_UNREF(error);
 }
 
-static char *cc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  call_data *calld = elem->call_data;
-  grpc_subchannel_call *subchannel_call =
-      get_call_or_error(calld).subchannel_call;
-  if (subchannel_call == NULL) {
-    return NULL;
-  } else {
-    return grpc_subchannel_call_get_peer(exec_ctx, subchannel_call);
-  }
-}
-
 /** Return true if subchannel is available immediately (in which case
     subchannel_ready_locked() should not be called), or false otherwise (in
     which case subchannel_ready_locked() should be called when the subchannel
@@ -1065,39 +1047,78 @@ static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
 
 typedef struct {
   grpc_call_element *elem;
-  bool cancelled;
+  bool finished;
   grpc_closure closure;
+  grpc_closure cancel_closure;
 } pick_after_resolver_result_args;
 
+// Note: This runs under the client_channel combiner, but will NOT be
+// holding the call combiner.
+static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx,
+                                                     void *arg,
+                                                     grpc_error *error) {
+  pick_after_resolver_result_args *args = arg;
+  if (args->finished) {
+    gpr_free(args);
+    return;
+  }
+  args->finished = true;
+  grpc_call_element *elem = args->elem;
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+  // If we don't yet have a resolver result, then a closure for
+  // pick_after_resolver_result_done_locked() will have been added to
+  // chand->waiting_for_resolver_result_closures, and it may not be invoked
+  // until after this call has been destroyed.  We mark the operation as
+  // finished, so that when pick_after_resolver_result_done_locked()
+  // is called, it will be a no-op.  We also immediately invoke
+  // subchannel_ready_locked() to propagate the error back to the caller.
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG,
+            "chand=%p calld=%p: cancelling pick waiting for resolver result",
+            chand, calld);
+  }
+  // Note: Although we are not in the call combiner here, we are
+  // basically stealing the call combiner from the pending pick, so
+  // it's safe to call subchannel_ready_locked() here -- we are
+  // essentially calling it here instead of calling it in
+  // pick_after_resolver_result_done_locked().
+  subchannel_ready_locked(exec_ctx, elem,
+                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+                              "Pick cancelled", &error, 1));
+}
+
 static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx,
                                                    void *arg,
                                                    grpc_error *error) {
   pick_after_resolver_result_args *args = arg;
-  if (args->cancelled) {
+  if (args->finished) {
     /* cancelled, do nothing */
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG, "call cancelled before resolver result");
     }
-  } else {
+    gpr_free(args);
-    channel_data *chand = args->elem->channel_data;
+    return;
-    call_data *calld = args->elem->call_data;
+  }
+  args->finished = true;
+  grpc_call_element *elem = args->elem;
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
   if (error != GRPC_ERROR_NONE) {
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver failed to return data",
               chand, calld);
     }
-      subchannel_ready_locked(exec_ctx, args->elem, GRPC_ERROR_REF(error));
+    subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
   } else {
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver returned, doing pick",
               chand, calld);
     }
-      if (pick_subchannel_locked(exec_ctx, args->elem)) {
+    if (pick_subchannel_locked(exec_ctx, elem)) {
-        subchannel_ready_locked(exec_ctx, args->elem, GRPC_ERROR_NONE);
+      subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_NONE);
-      }
     }
   }
-  gpr_free(args);
 }
 
 static void pick_after_resolver_result_start_locked(grpc_exec_ctx *exec_ctx,
@@ -1116,41 +1137,34 @@ static void pick_after_resolver_result_start_locked(grpc_exec_ctx *exec_ctx,
                     args, grpc_combiner_scheduler(chand->combiner));
   grpc_closure_list_append(&chand->waiting_for_resolver_result_closures,
                            &args->closure, GRPC_ERROR_NONE);
+  grpc_call_combiner_set_notify_on_cancel(
+      exec_ctx, calld->call_combiner,
+      GRPC_CLOSURE_INIT(&args->cancel_closure,
+                        pick_after_resolver_result_cancel_locked, args,
+                        grpc_combiner_scheduler(chand->combiner)));
 }
 
-static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx,
+// Note: This runs under the client_channel combiner, but will NOT be
-                                                     grpc_call_element *elem,
+// holding the call combiner.
+static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                         grpc_error *error) {
+  grpc_call_element *elem = arg;
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
-  // If we don't yet have a resolver result, then a closure for
+  if (error != GRPC_ERROR_NONE && calld->lb_policy != NULL) {
-  // pick_after_resolver_result_done_locked() will have been added to
-  // chand->waiting_for_resolver_result_closures, and it may not be invoked
-  // until after this call has been destroyed.  We mark the operation as
-  // cancelled, so that when pick_after_resolver_result_done_locked()
-  // is called, it will be a no-op.  We also immediately invoke
-  // subchannel_ready_locked() to propagate the error back to the caller.
-  for (grpc_closure *closure = chand->waiting_for_resolver_result_closures.head;
-       closure != NULL; closure = closure->next_data.next) {
-    pick_after_resolver_result_args *args = closure->cb_arg;
-    if (!args->cancelled && args->elem == elem) {
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-        gpr_log(GPR_DEBUG,
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
-                "chand=%p calld=%p: "
+              chand, calld, calld->lb_policy);
-                "cancelling pick waiting for resolver result",
-                chand, calld);
-      }
-      args->cancelled = true;
-      subchannel_ready_locked(exec_ctx, elem,
-                              GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                                  "Pick cancelled", &error, 1));
     }
+    grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy,
+                                      &calld->connected_subchannel,
+                                      GRPC_ERROR_REF(error));
   }
-  GRPC_ERROR_UNREF(error);
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_callback_cancel");
 }
 
 // Callback invoked by grpc_lb_policy_pick_locked() for async picks.
-// Unrefs the LB policy after invoking subchannel_ready_locked().
+// Unrefs the LB policy and invokes subchannel_ready_locked().
 static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                       grpc_error *error) {
   grpc_call_element *elem = arg;
@@ -1194,24 +1208,17 @@ static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx,
     }
     GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
     calld->lb_policy = NULL;
+  } else {
+    GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel");
+    grpc_call_combiner_set_notify_on_cancel(
+        exec_ctx, calld->call_combiner,
+        GRPC_CLOSURE_INIT(&calld->lb_pick_cancel_closure,
+                          pick_callback_cancel_locked, elem,
+                          grpc_combiner_scheduler(chand->combiner)));
   }
   return pick_done;
 }
 
-static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_element *elem,
-                                        grpc_error *error) {
-  channel_data *chand = elem->channel_data;
-  call_data *calld = elem->call_data;
-  GPR_ASSERT(calld->lb_policy != NULL);
-  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-    gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
-            chand, calld, calld->lb_policy);
-  }
-  grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy,
-                                    &calld->connected_subchannel, error);
-}
-
 static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
                                    grpc_call_element *elem) {
   GPR_TIMER_BEGIN("pick_subchannel", 0);
@@ -1224,7 +1231,7 @@ static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
     // Otherwise, if the service config specified a value for this
     // method, use that.
     uint32_t initial_metadata_flags =
-        calld->initial_metadata_payload->send_initial_metadata
+        calld->initial_metadata_batch->payload->send_initial_metadata
             .send_initial_metadata_flags;
     const bool wait_for_ready_set_from_api =
         initial_metadata_flags &
@@ -1241,7 +1248,7 @@ static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
       }
     }
     const grpc_lb_policy_pick_args inputs = {
-        calld->initial_metadata_payload->send_initial_metadata
+        calld->initial_metadata_batch->payload->send_initial_metadata
             .send_initial_metadata,
         initial_metadata_flags, &calld->lb_token_mdelem};
     pick_done = pick_callback_start_locked(exec_ctx, elem, &inputs);
@@ -1258,91 +1265,33 @@ static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
   return pick_done;
 }
 
-static void start_transport_stream_op_batch_locked(grpc_exec_ctx *exec_ctx,
+static void start_pick_locked(grpc_exec_ctx *exec_ctx, void *arg,
-                                                   void *arg,
                               grpc_error *error_ignored) {
-  GPR_TIMER_BEGIN("start_transport_stream_op_batch_locked", 0);
+  GPR_TIMER_BEGIN("start_pick_locked", 0);
-  grpc_transport_stream_op_batch *batch = arg;
+  grpc_call_element *elem = (grpc_call_element *)arg;
-  grpc_call_element *elem = batch->handler_private.extra_arg;
+  call_data *calld = (call_data *)elem->call_data;
-  call_data *calld = elem->call_data;
+  channel_data *chand = (channel_data *)elem->channel_data;
-  channel_data *chand = elem->channel_data;
-  /* need to recheck that another thread hasn't set the call */
-  call_or_error coe = get_call_or_error(calld);
-  if (coe.error != GRPC_ERROR_NONE) {
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG, "chand=%p calld=%p: failing batch with error: %s",
-              chand, calld, grpc_error_string(coe.error));
-    }
-    grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, batch, GRPC_ERROR_REF(coe.error));
-    goto done;
-  }
-  if (coe.subchannel_call != NULL) {
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG,
-              "chand=%p calld=%p: sending batch to subchannel_call=%p", chand,
-              calld, coe.subchannel_call);
-    }
-    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call, batch);
-    goto done;
-  }
-  // Add to waiting-for-pick list.  If we succeed in getting a
-  // subchannel call below, we'll handle this batch (along with any
-  // other waiting batches) in waiting_for_pick_batches_resume_locked().
-  waiting_for_pick_batches_add_locked(calld, batch);
-  // If this is a cancellation, cancel the pending pick (if any) and
-  // fail any pending batches.
-  if (batch->cancel_stream) {
-    grpc_error *error = batch->payload->cancel_stream.cancel_error;
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG, "chand=%p calld=%p: recording cancel_error=%s", chand,
-              calld, grpc_error_string(error));
-    }
-    /* Stash a copy of cancel_error in our call data, so that we can use
-       it for subsequent operations.  This ensures that if the call is
-       cancelled before any batches are passed down (e.g., if the deadline
-       is in the past when the call starts), we can return the right
-       error to the caller when the first batch does get passed down. */
-    set_call_or_error(calld, (call_or_error){.error = GRPC_ERROR_REF(error)});
-    if (calld->lb_policy != NULL) {
-      pick_callback_cancel_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
-    } else {
-      pick_after_resolver_result_cancel_locked(exec_ctx, elem,
-                                               GRPC_ERROR_REF(error));
-    }
-    waiting_for_pick_batches_fail_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
-    goto done;
-  }
-  /* if we don't have a subchannel, try to get one */
-  if (batch->send_initial_metadata) {
   GPR_ASSERT(calld->connected_subchannel == NULL);
-    calld->initial_metadata_payload = batch->payload;
-    GRPC_CALL_STACK_REF(calld->owning_call, "pick_subchannel");
-    /* If a subchannel is not available immediately, the polling entity from
-       call_data should be provided to channel_data's interested_parties, so
-       that IO of the lb_policy and resolver could be done under it. */
   if (pick_subchannel_locked(exec_ctx, elem)) {
     // Pick was returned synchronously.
-      GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
     if (calld->connected_subchannel == NULL) {
-        grpc_error *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+      GRPC_ERROR_UNREF(calld->error);
+      calld->error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
           "Call dropped by load balancing policy");
-        set_call_or_error(calld,
+      waiting_for_pick_batches_fail(exec_ctx, elem,
-                          (call_or_error){.error = GRPC_ERROR_REF(error)});
+                                    GRPC_ERROR_REF(calld->error));
-        waiting_for_pick_batches_fail_locked(exec_ctx, elem, error);
     } else {
       // Create subchannel call.
       create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_NONE);
     }
   } else {
+    // Pick will be done asynchronously.  Add the call's polling entity to
+    // the channel's interested_parties, so that I/O for the resolver
+    // and LB policy can be done under it.
     grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent,
                                            chand->interested_parties);
   }
-  }
+  GPR_TIMER_END("start_pick_locked", 0);
-done:
-  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call,
-                        "start_transport_stream_op_batch");
-  GPR_TIMER_END("start_transport_stream_op_batch_locked", 0);
 }
 
 static void on_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
@@ -1365,27 +1314,49 @@ static void on_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
                    GRPC_ERROR_REF(error));
 }
 
-/* The logic here is fairly complicated, due to (a) the fact that we
-   need to handle the case where we receive the send op before the
-   initial metadata op, and (b) the need for efficiency, especially in
-   the streaming case.
-
-   We use double-checked locking to initially see if initialization has been
-   performed. If it has not, we acquire the combiner and perform initialization.
-   If it has, we proceed on the fast path. */
 static void cc_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  if (GRPC_TRACER_ON(grpc_client_channel_trace) ||
-      GRPC_TRACER_ON(grpc_trace_channel)) {
-    grpc_call_log_op(GPR_INFO, elem, batch);
-  }
   if (chand->deadline_checking_enabled) {
     grpc_deadline_state_client_start_transport_stream_op_batch(exec_ctx, elem,
                                                                batch);
   }
+  GPR_TIMER_BEGIN("cc_start_transport_stream_op_batch", 0);
+  // If we've previously been cancelled, immediately fail any new batches.
+  if (calld->error != GRPC_ERROR_NONE) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: failing batch with error: %s",
+              chand, calld, grpc_error_string(calld->error));
+    }
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, batch, GRPC_ERROR_REF(calld->error), calld->call_combiner);
+    goto done;
+  }
+  if (batch->cancel_stream) {
+    // Stash a copy of cancel_error in our call data, so that we can use
+    // it for subsequent operations.  This ensures that if the call is
+    // cancelled before any batches are passed down (e.g., if the deadline
+    // is in the past when the call starts), we can return the right
+    // error to the caller when the first batch does get passed down.
+    GRPC_ERROR_UNREF(calld->error);
+    calld->error = GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error);
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: recording cancel_error=%s", chand,
+              calld, grpc_error_string(calld->error));
+    }
+    // If we have a subchannel call, send the cancellation batch down.
+    // Otherwise, fail all pending batches.
+    if (calld->subchannel_call != NULL) {
+      grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call, batch);
+    } else {
+      waiting_for_pick_batches_add(calld, batch);
+      waiting_for_pick_batches_fail(exec_ctx, elem,
+                                    GRPC_ERROR_REF(calld->error));
+    }
+    goto done;
+  }
   // Intercept on_complete for recv_trailing_metadata so that we can
   // check retry throttle status.
   if (batch->recv_trailing_metadata) {
@@ -1395,38 +1366,43 @@ static void cc_start_transport_stream_op_batch(
                       grpc_schedule_on_exec_ctx);
     batch->on_complete = &calld->on_complete;
   }
-  /* try to (atomically) get the call */
+  // Check if we've already gotten a subchannel call.
-  call_or_error coe = get_call_or_error(calld);
+  // Note that once we have completed the pick, we do not need to enter
-  GPR_TIMER_BEGIN("cc_start_transport_stream_op_batch", 0);
+  // the channel combiner, which is more efficient (especially for
-  if (coe.error != GRPC_ERROR_NONE) {
+  // streaming calls).
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+  if (calld->subchannel_call != NULL) {
-      gpr_log(GPR_DEBUG, "chand=%p calld=%p: failing batch with error: %s",
-              chand, calld, grpc_error_string(coe.error));
-    }
-    grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, batch, GRPC_ERROR_REF(coe.error));
-    goto done;
-  }
-  if (coe.subchannel_call != NULL) {
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG,
               "chand=%p calld=%p: sending batch to subchannel_call=%p", chand,
-              calld, coe.subchannel_call);
+              calld, calld->subchannel_call);
     }
-    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call, batch);
+    grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call, batch);
     goto done;
   }
-  /* we failed; lock and figure out what to do */
+  // We do not yet have a subchannel call.
+  // Add the batch to the waiting-for-pick list.
+  waiting_for_pick_batches_add(calld, batch);
+  // For batches containing a send_initial_metadata op, enter the channel
+  // combiner to start a pick.
+  if (batch->send_initial_metadata) {
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering combiner", chand, calld);
     }
-  GRPC_CALL_STACK_REF(calld->owning_call, "start_transport_stream_op_batch");
-  batch->handler_private.extra_arg = elem;
     GRPC_CLOSURE_SCHED(
-      exec_ctx, GRPC_CLOSURE_INIT(&batch->handler_private.closure,
+        exec_ctx,
-                                  start_transport_stream_op_batch_locked, batch,
+        GRPC_CLOSURE_INIT(&batch->handler_private.closure, start_pick_locked,
-                                  grpc_combiner_scheduler(chand->combiner)),
+                          elem, grpc_combiner_scheduler(chand->combiner)),
         GRPC_ERROR_NONE);
+  } else {
+    // For all other batches, release the call combiner.
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG,
+              "chand=%p calld=%p: saved batch, yeilding call combiner", chand,
+              calld);
+    }
+    GRPC_CALL_COMBINER_STOP(exec_ctx, calld->call_combiner,
+                            "batch does not include send_initial_metadata");
+  }
 done:
   GPR_TIMER_END("cc_start_transport_stream_op_batch", 0);
 }
@@ -1441,10 +1417,12 @@ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx,
   calld->path = grpc_slice_ref_internal(args->path);
   calld->call_start_time = args->start_time;
   calld->deadline = gpr_convert_clock_type(args->deadline, GPR_CLOCK_MONOTONIC);
-  calld->owning_call = args->call_stack;
   calld->arena = args->arena;
+  calld->owning_call = args->call_stack;
+  calld->call_combiner = args->call_combiner;
   if (chand->deadline_checking_enabled) {
-    grpc_deadline_state_init(exec_ctx, elem, args->call_stack, calld->deadline);
+    grpc_deadline_state_init(exec_ctx, elem, args->call_stack,
+                             args->call_combiner, calld->deadline);
   }
   return GRPC_ERROR_NONE;
 }
@@ -1463,13 +1441,12 @@ static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx,
   if (calld->method_params != NULL) {
     method_parameters_unref(calld->method_params);
   }
-  call_or_error coe = get_call_or_error(calld);
+  GRPC_ERROR_UNREF(calld->error);
-  GRPC_ERROR_UNREF(coe.error);
+  if (calld->subchannel_call != NULL) {
-  if (coe.subchannel_call != NULL) {
+    grpc_subchannel_call_set_cleanup_closure(calld->subchannel_call,
-    grpc_subchannel_call_set_cleanup_closure(coe.subchannel_call,
                                              then_schedule_closure);
     then_schedule_closure = NULL;
-    GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, coe.subchannel_call,
+    GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, calld->subchannel_call,
                                "client_channel_destroy_call");
   }
   GPR_ASSERT(calld->lb_policy == NULL);
@@ -1508,7 +1485,6 @@ const grpc_channel_filter grpc_client_channel_filter = {
     sizeof(channel_data),
     cc_init_channel_elem,
     cc_destroy_channel_elem,
-    cc_get_peer,
     cc_get_channel_info,
     "client-channel",
 };

src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c

@@ -132,6 +132,5 @@ const grpc_channel_filter grpc_client_load_reporting_filter = {
     0,  // sizeof(channel_data)
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "client_load_reporting"};

src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c

@@ -32,6 +32,7 @@
 #include "src/core/ext/filters/client_channel/parse_address.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/unix_sockets_posix.h"
@@ -125,7 +126,6 @@ static const grpc_resolver_vtable fake_resolver_vtable = {
 
 struct grpc_fake_resolver_response_generator {
   fake_resolver* resolver;  // Set by the fake_resolver constructor to itself.
-  grpc_channel_args* next_response;
   gpr_refcount refcount;
 };
 
@@ -151,19 +151,26 @@ void grpc_fake_resolver_response_generator_unref(
   }
 }
 
-static void set_response_cb(grpc_exec_ctx* exec_ctx, void* arg,
+typedef struct set_response_closure_arg {
+  grpc_closure set_response_closure;
+  grpc_fake_resolver_response_generator* generator;
+  grpc_channel_args* next_response;
+} set_response_closure_arg;
+
+static void set_response_closure_fn(grpc_exec_ctx* exec_ctx, void* arg,
                                     grpc_error* error) {
-  grpc_fake_resolver_response_generator* generator =
+  set_response_closure_arg* closure_arg = arg;
-      (grpc_fake_resolver_response_generator*)arg;
+  grpc_fake_resolver_response_generator* generator = closure_arg->generator;
   fake_resolver* r = generator->resolver;
   if (r->next_results != NULL) {
     grpc_channel_args_destroy(exec_ctx, r->next_results);
   }
-  r->next_results = generator->next_response;
+  r->next_results = closure_arg->next_response;
   if (r->results_upon_error != NULL) {
     grpc_channel_args_destroy(exec_ctx, r->results_upon_error);
   }
-  r->results_upon_error = grpc_channel_args_copy(generator->next_response);
+  r->results_upon_error = grpc_channel_args_copy(closure_arg->next_response);
+  gpr_free(closure_arg);
   fake_resolver_maybe_finish_next_locked(exec_ctx, r);
 }
 
@@ -171,9 +178,12 @@ void grpc_fake_resolver_response_generator_set_response(
     grpc_exec_ctx* exec_ctx, grpc_fake_resolver_response_generator* generator,
     grpc_channel_args* next_response) {
   GPR_ASSERT(generator->resolver != NULL);
-  generator->next_response = grpc_channel_args_copy(next_response);
+  set_response_closure_arg* closure_arg = gpr_zalloc(sizeof(*closure_arg));
-  GRPC_CLOSURE_SCHED(
+  closure_arg->generator = generator;
-      exec_ctx, GRPC_CLOSURE_CREATE(set_response_cb, generator,
+  closure_arg->next_response = grpc_channel_args_copy(next_response);
+  GRPC_CLOSURE_SCHED(exec_ctx,
+                     GRPC_CLOSURE_INIT(&closure_arg->set_response_closure,
+                                       set_response_closure_fn, closure_arg,
                                        grpc_combiner_scheduler(
                                            generator->resolver->base.combiner)),
                      GRPC_ERROR_NONE);

src/core/ext/filters/client_channel/subchannel.c

@@ -724,20 +724,14 @@ void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx,
   GRPC_CALL_STACK_UNREF(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON);
 }
 
-char *grpc_subchannel_call_get_peer(grpc_exec_ctx *exec_ctx,
-                                    grpc_subchannel_call *call) {
-  grpc_call_stack *call_stack = SUBCHANNEL_CALL_TO_CALL_STACK(call);
-  grpc_call_element *top_elem = grpc_call_stack_element(call_stack, 0);
-  return top_elem->filter->get_peer(exec_ctx, top_elem);
-}
-
 void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx,
                                      grpc_subchannel_call *call,
-                                     grpc_transport_stream_op_batch *op) {
+                                     grpc_transport_stream_op_batch *batch) {
   GPR_TIMER_BEGIN("grpc_subchannel_call_process_op", 0);
   grpc_call_stack *call_stack = SUBCHANNEL_CALL_TO_CALL_STACK(call);
   grpc_call_element *top_elem = grpc_call_stack_element(call_stack, 0);
-  top_elem->filter->start_transport_stream_op_batch(exec_ctx, top_elem, op);
+  GRPC_CALL_LOG_OP(GPR_INFO, top_elem, batch);
+  top_elem->filter->start_transport_stream_op_batch(exec_ctx, top_elem, batch);
   GPR_TIMER_END("grpc_subchannel_call_process_op", 0);
 }
 
@@ -760,13 +754,15 @@ grpc_error *grpc_connected_subchannel_create_call(
       args->arena, sizeof(grpc_subchannel_call) + chanstk->call_stack_size);
   grpc_call_stack *callstk = SUBCHANNEL_CALL_TO_CALL_STACK(*call);
   (*call)->connection = GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call");
-  const grpc_call_element_args call_args = {.call_stack = callstk,
+  const grpc_call_element_args call_args = {
+      .call_stack = callstk,
       .server_transport_data = NULL,
       .context = args->context,
       .path = args->path,
       .start_time = args->start_time,
       .deadline = args->deadline,
-                                            .arena = args->arena};
+      .arena = args->arena,
+      .call_combiner = args->call_combiner};
   grpc_error *error = grpc_call_stack_init(
       exec_ctx, chanstk, 1, subchannel_call_destroy, *call, &call_args);
   if (error != GRPC_ERROR_NONE) {

src/core/ext/filters/client_channel/subchannel.h

@@ -106,6 +106,7 @@ typedef struct {
   gpr_timespec deadline;
   gpr_arena *arena;
   grpc_call_context_element *context;
+  grpc_call_combiner *call_combiner;
 } grpc_connected_subchannel_call_args;
 
 grpc_error *grpc_connected_subchannel_create_call(
@@ -150,10 +151,6 @@ void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx,
                                      grpc_subchannel_call *subchannel_call,
                                      grpc_transport_stream_op_batch *op);
 
-/** continue querying for peer */
-char *grpc_subchannel_call_get_peer(grpc_exec_ctx *exec_ctx,
-                                    grpc_subchannel_call *subchannel_call);
-
 /** Must be called once per call. Sets the 'then_schedule_closure' argument for
     call stack destruction. */
 void grpc_subchannel_call_set_cleanup_closure(

src/core/ext/filters/deadline/deadline_filter.c

@@ -34,22 +34,56 @@
 // grpc_deadline_state
 //
 
+// The on_complete callback used when sending a cancel_error batch down the
+// filter stack.  Yields the call combiner when the batch returns.
+static void yield_call_combiner(grpc_exec_ctx* exec_ctx, void* arg,
+                                grpc_error* ignored) {
+  grpc_deadline_state* deadline_state = arg;
+  GRPC_CALL_COMBINER_STOP(exec_ctx, deadline_state->call_combiner,
+                          "got on_complete from cancel_stream batch");
+  GRPC_CALL_STACK_UNREF(exec_ctx, deadline_state->call_stack, "deadline_timer");
+}
+
+// This is called via the call combiner, so access to deadline_state is
+// synchronized.
+static void send_cancel_op_in_call_combiner(grpc_exec_ctx* exec_ctx, void* arg,
+                                            grpc_error* error) {
+  grpc_call_element* elem = arg;
+  grpc_deadline_state* deadline_state = elem->call_data;
+  grpc_transport_stream_op_batch* batch = grpc_make_transport_stream_op(
+      GRPC_CLOSURE_INIT(&deadline_state->timer_callback, yield_call_combiner,
+                        deadline_state, grpc_schedule_on_exec_ctx));
+  batch->cancel_stream = true;
+  batch->payload->cancel_stream.cancel_error = GRPC_ERROR_REF(error);
+  elem->filter->start_transport_stream_op_batch(exec_ctx, elem, batch);
+}
+
 // Timer callback.
 static void timer_callback(grpc_exec_ctx* exec_ctx, void* arg,
                            grpc_error* error) {
   grpc_call_element* elem = (grpc_call_element*)arg;
   grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   if (error != GRPC_ERROR_CANCELLED) {
-    grpc_call_element_signal_error(
+    error = grpc_error_set_int(
-        exec_ctx, elem,
-        grpc_error_set_int(
         GRPC_ERROR_CREATE_FROM_STATIC_STRING("Deadline Exceeded"),
-            GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_DEADLINE_EXCEEDED));
+        GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_DEADLINE_EXCEEDED);
+    grpc_call_combiner_cancel(exec_ctx, deadline_state->call_combiner,
+                              GRPC_ERROR_REF(error));
+    GRPC_CLOSURE_INIT(&deadline_state->timer_callback,
+                      send_cancel_op_in_call_combiner, elem,
+                      grpc_schedule_on_exec_ctx);
+    GRPC_CALL_COMBINER_START(exec_ctx, deadline_state->call_combiner,
+                             &deadline_state->timer_callback, error,
+                             "deadline exceeded -- sending cancel_stream op");
+  } else {
+    GRPC_CALL_STACK_UNREF(exec_ctx, deadline_state->call_stack,
+                          "deadline_timer");
   }
-  GRPC_CALL_STACK_UNREF(exec_ctx, deadline_state->call_stack, "deadline_timer");
 }
 
 // Starts the deadline timer.
+// This is called via the call combiner, so access to deadline_state is
+// synchronized.
 static void start_timer_if_needed(grpc_exec_ctx* exec_ctx,
                                   grpc_call_element* elem,
                                   gpr_timespec deadline) {
@@ -58,51 +92,39 @@ static void start_timer_if_needed(grpc_exec_ctx* exec_ctx,
     return;
   }
   grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
-  grpc_deadline_timer_state cur_state;
   grpc_closure* closure = NULL;
-retry:
+  switch (deadline_state->timer_state) {
-  cur_state =
-      (grpc_deadline_timer_state)gpr_atm_acq_load(&deadline_state->timer_state);
-  switch (cur_state) {
     case GRPC_DEADLINE_STATE_PENDING:
       // Note: We do not start the timer if there is already a timer
       return;
     case GRPC_DEADLINE_STATE_FINISHED:
-      if (gpr_atm_rel_cas(&deadline_state->timer_state,
+      deadline_state->timer_state = GRPC_DEADLINE_STATE_PENDING;
-                          GRPC_DEADLINE_STATE_FINISHED,
-                          GRPC_DEADLINE_STATE_PENDING)) {
       // If we've already created and destroyed a timer, we always create a
       // new closure: we have no other guarantee that the inlined closure is
       // not in use (it may hold a pending call to timer_callback)
-        closure = GRPC_CLOSURE_CREATE(timer_callback, elem,
+      closure =
-                                      grpc_schedule_on_exec_ctx);
+          GRPC_CLOSURE_CREATE(timer_callback, elem, grpc_schedule_on_exec_ctx);
-      } else {
-        goto retry;
-      }
       break;
     case GRPC_DEADLINE_STATE_INITIAL:
-      if (gpr_atm_rel_cas(&deadline_state->timer_state,
+      deadline_state->timer_state = GRPC_DEADLINE_STATE_PENDING;
-                          GRPC_DEADLINE_STATE_INITIAL,
-                          GRPC_DEADLINE_STATE_PENDING)) {
       closure =
           GRPC_CLOSURE_INIT(&deadline_state->timer_callback, timer_callback,
                             elem, grpc_schedule_on_exec_ctx);
-      } else {
-        goto retry;
-      }
       break;
   }
-  GPR_ASSERT(closure);
+  GPR_ASSERT(closure != NULL);
   GRPC_CALL_STACK_REF(deadline_state->call_stack, "deadline_timer");
   grpc_timer_init(exec_ctx, &deadline_state->timer, deadline, closure,
                   gpr_now(GPR_CLOCK_MONOTONIC));
 }
 
 // Cancels the deadline timer.
+// This is called via the call combiner, so access to deadline_state is
+// synchronized.
 static void cancel_timer_if_needed(grpc_exec_ctx* exec_ctx,
                                    grpc_deadline_state* deadline_state) {
-  if (gpr_atm_rel_cas(&deadline_state->timer_state, GRPC_DEADLINE_STATE_PENDING,
+  if (deadline_state->timer_state == GRPC_DEADLINE_STATE_PENDING) {
-                      GRPC_DEADLINE_STATE_FINISHED)) {
+    deadline_state->timer_state = GRPC_DEADLINE_STATE_FINISHED;
     grpc_timer_cancel(exec_ctx, &deadline_state->timer);
   } else {
     // timer was either in STATE_INITAL (nothing to cancel)
@@ -131,6 +153,7 @@ static void inject_on_complete_cb(grpc_deadline_state* deadline_state,
 // Callback and associated state for starting the timer after call stack
 // initialization has been completed.
 struct start_timer_after_init_state {
+  bool in_call_combiner;
   grpc_call_element* elem;
   gpr_timespec deadline;
   grpc_closure closure;
@@ -138,15 +161,29 @@ struct start_timer_after_init_state {
 static void start_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg,
                                    grpc_error* error) {
   struct start_timer_after_init_state* state = arg;
+  grpc_deadline_state* deadline_state = state->elem->call_data;
+  if (!state->in_call_combiner) {
+    // We are initially called without holding the call combiner, so we
+    // need to bounce ourselves into it.
+    state->in_call_combiner = true;
+    GRPC_CALL_COMBINER_START(exec_ctx, deadline_state->call_combiner,
+                             &state->closure, GRPC_ERROR_REF(error),
+                             "scheduling deadline timer");
+    return;
+  }
   start_timer_if_needed(exec_ctx, state->elem, state->deadline);
   gpr_free(state);
+  GRPC_CALL_COMBINER_STOP(exec_ctx, deadline_state->call_combiner,
+                          "done scheduling deadline timer");
 }
 
 void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                               grpc_call_stack* call_stack,
+                              grpc_call_combiner* call_combiner,
                               gpr_timespec deadline) {
   grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   deadline_state->call_stack = call_stack;
+  deadline_state->call_combiner = call_combiner;
   // Deadline will always be infinite on servers, so the timer will only be
   // set on clients with a finite deadline.
   deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
@@ -158,7 +195,7 @@ void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
     // call stack initialization is finished.  To avoid that problem, we
     // create a closure to start the timer, and we schedule that closure
     // to be run after call stack initialization is done.
-    struct start_timer_after_init_state* state = gpr_malloc(sizeof(*state));
+    struct start_timer_after_init_state* state = gpr_zalloc(sizeof(*state));
     state->elem = elem;
     state->deadline = deadline;
     GRPC_CLOSURE_INIT(&state->closure, start_timer_after_init, state,
@@ -232,7 +269,8 @@ typedef struct server_call_data {
 static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
                                   grpc_call_element* elem,
                                   const grpc_call_element_args* args) {
-  grpc_deadline_state_init(exec_ctx, elem, args->call_stack, args->deadline);
+  grpc_deadline_state_init(exec_ctx, elem, args->call_stack,
+                           args->call_combiner, args->deadline);
   return GRPC_ERROR_NONE;
 }
 
@@ -310,7 +348,6 @@ const grpc_channel_filter grpc_client_deadline_filter = {
     0,  // sizeof(channel_data)
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "deadline",
 };
@@ -325,7 +362,6 @@ const grpc_channel_filter grpc_server_deadline_filter = {
     0,  // sizeof(channel_data)
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "deadline",
 };

src/core/ext/filters/deadline/deadline_filter.h

@@ -31,7 +31,8 @@ typedef enum grpc_deadline_timer_state {
 typedef struct grpc_deadline_state {
   // We take a reference to the call stack for the timer callback.
   grpc_call_stack* call_stack;
-  gpr_atm timer_state;
+  grpc_call_combiner* call_combiner;
+  grpc_deadline_timer_state timer_state;
   grpc_timer timer;
   grpc_closure timer_callback;
   // Closure to invoke when the call is complete.
@@ -50,6 +51,7 @@ typedef struct grpc_deadline_state {
 // assumes elem->call_data is zero'd
 void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                               grpc_call_stack* call_stack,
+                              grpc_call_combiner* call_combiner,
                               gpr_timespec deadline);
 void grpc_deadline_state_destroy(grpc_exec_ctx* exec_ctx,
                                  grpc_call_element* elem);
@@ -61,6 +63,8 @@ void grpc_deadline_state_destroy(grpc_exec_ctx* exec_ctx,
 // to ensure that the timer callback is not invoked while it is in the
 // process of being reset, which means that attempting to increase the
 // deadline may result in the timer being called twice.
+//
+// Note: Must be called while holding the call combiner.
 void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                                gpr_timespec new_deadline);
 
@@ -70,6 +74,8 @@ void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
 //
 // Note: It is the caller's responsibility to chain to the next filter if
 // necessary after this function returns.
+//
+// Note: Must be called while holding the call combiner.
 void grpc_deadline_state_client_start_transport_stream_op_batch(
     grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
     grpc_transport_stream_op_batch* op);

src/core/ext/filters/http/client/http_client_filter.c

@@ -36,6 +36,7 @@
 static const size_t kMaxPayloadSizeForGet = 2048;
 
 typedef struct call_data {
+  grpc_call_combiner *call_combiner;
   // State for handling send_initial_metadata ops.
   grpc_linked_mdelem method;
   grpc_linked_mdelem scheme;
@@ -215,13 +216,13 @@ static void on_send_message_next_done(grpc_exec_ctx *exec_ctx, void *arg,
   call_data *calld = (call_data *)elem->call_data;
   if (error != GRPC_ERROR_NONE) {
     grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, calld->send_message_batch, error);
+        exec_ctx, calld->send_message_batch, error, calld->call_combiner);
     return;
   }
   error = pull_slice_from_send_message(exec_ctx, calld);
   if (error != GRPC_ERROR_NONE) {
     grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, calld->send_message_batch, error);
+        exec_ctx, calld->send_message_batch, error, calld->call_combiner);
     return;
   }
   // There may or may not be more to read, but we don't care.  If we got
@@ -302,7 +303,6 @@ static void hc_start_transport_stream_op_batch(
   call_data *calld = elem->call_data;
   channel_data *channeld = elem->channel_data;
   GPR_TIMER_BEGIN("hc_start_transport_stream_op_batch", 0);
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
 
   if (batch->recv_initial_metadata) {
     /* substitute our callback for the higher callback */
@@ -414,7 +414,7 @@ static void hc_start_transport_stream_op_batch(
 done:
   if (error != GRPC_ERROR_NONE) {
     grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, calld->send_message_batch, error);
+        exec_ctx, calld->send_message_batch, error, calld->call_combiner);
   } else if (!batch_will_be_handled_asynchronously) {
     grpc_call_next_op(exec_ctx, elem, batch);
   }
@@ -426,6 +426,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
   call_data *calld = (call_data *)elem->call_data;
+  calld->call_combiner = args->call_combiner;
   GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready,
                     recv_initial_metadata_ready, elem,
                     grpc_schedule_on_exec_ctx);
@@ -565,6 +566,5 @@ const grpc_channel_filter grpc_http_client_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "http-client"};

src/core/ext/filters/http/message_compress/message_compress_filter.c

@@ -35,35 +35,29 @@
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/transport/static_metadata.h"
 
-#define INITIAL_METADATA_UNSEEN 0
+typedef enum {
-#define HAS_COMPRESSION_ALGORITHM 2
+  // Initial metadata not yet seen.
-#define NO_COMPRESSION_ALGORITHM 4
+  INITIAL_METADATA_UNSEEN = 0,
-
+  // Initial metadata seen; compression algorithm set.
-#define CANCELLED_BIT ((gpr_atm)1)
+  HAS_COMPRESSION_ALGORITHM,
+  // Initial metadata seen; no compression algorithm set.
+  NO_COMPRESSION_ALGORITHM,
+} initial_metadata_state;
 
 typedef struct call_data {
-  grpc_slice_buffer slices; /**< Buffers up input slices to be compressed */
+  grpc_call_combiner *call_combiner;
   grpc_linked_mdelem compression_algorithm_storage;
   grpc_linked_mdelem stream_compression_algorithm_storage;
   grpc_linked_mdelem accept_encoding_storage;
   grpc_linked_mdelem accept_stream_encoding_storage;
-  uint32_t remaining_slice_bytes;
   /** Compression algorithm we'll try to use. It may be given by incoming
    * metadata, or by the channel's default compression settings. */
   grpc_compression_algorithm compression_algorithm;
-
+  initial_metadata_state send_initial_metadata_state;
-  /* Atomic recording the state of initial metadata; allowed values:
+  grpc_error *cancel_error;
-     INITIAL_METADATA_UNSEEN - initial metadata op not seen
+  grpc_closure start_send_message_batch_in_call_combiner;
-     HAS_COMPRESSION_ALGORITHM - initial metadata seen; compression algorithm
-                                 set
-     NO_COMPRESSION_ALGORITHM - initial metadata seen; no compression algorithm
-                                set
-     pointer - a stalled op containing a send_message that's waiting on initial
-               metadata
-     pointer | CANCELLED_BIT - request was cancelled with error pointed to */
-  gpr_atm send_initial_metadata_state;
-
   grpc_transport_stream_op_batch *send_message_batch;
+  grpc_slice_buffer slices; /**< Buffers up input slices to be compressed */
   grpc_slice_buffer_stream replacement_stream;
   grpc_closure *original_send_message_on_complete;
   grpc_closure send_message_on_complete;
@@ -92,13 +86,13 @@ static bool skip_compression(grpc_call_element *elem, uint32_t flags,
   channel_data *channeld = elem->channel_data;
 
   if (flags & (GRPC_WRITE_NO_COMPRESS | GRPC_WRITE_INTERNAL_COMPRESS)) {
-    return 1;
+    return true;
   }
   if (has_compression_algorithm) {
     if (calld->compression_algorithm == GRPC_COMPRESS_NONE) {
-      return 1;
+      return true;
     }
-    return 0; /* we have an actual call-specific algorithm */
+    return false; /* we have an actual call-specific algorithm */
   }
   /* no per-call compression override */
   return channeld->default_compression_algorithm == GRPC_COMPRESS_NONE;
@@ -226,6 +220,18 @@ static void send_message_on_complete(grpc_exec_ctx *exec_ctx, void *arg,
                    GRPC_ERROR_REF(error));
 }
 
+static void send_message_batch_continue(grpc_exec_ctx *exec_ctx,
+                                        grpc_call_element *elem) {
+  call_data *calld = (call_data *)elem->call_data;
+  // Note: The call to grpc_call_next_op() results in yielding the
+  // call combiner, so we need to clear calld->send_message_batch
+  // before we do that.
+  grpc_transport_stream_op_batch *send_message_batch =
+      calld->send_message_batch;
+  calld->send_message_batch = NULL;
+  grpc_call_next_op(exec_ctx, elem, send_message_batch);
+}
+
 static void finish_send_message(grpc_exec_ctx *exec_ctx,
                                 grpc_call_element *elem) {
   call_data *calld = (call_data *)elem->call_data;
@@ -234,8 +240,8 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx,
   grpc_slice_buffer_init(&tmp);
   uint32_t send_flags =
       calld->send_message_batch->payload->send_message.send_message->flags;
-  const bool did_compress = grpc_msg_compress(
+  bool did_compress = grpc_msg_compress(exec_ctx, calld->compression_algorithm,
-      exec_ctx, calld->compression_algorithm, &calld->slices, &tmp);
+                                        &calld->slices, &tmp);
   if (did_compress) {
     if (GRPC_TRACER_ON(grpc_compression_trace)) {
       char *algo_name;
@@ -273,7 +279,19 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx,
   calld->original_send_message_on_complete =
       calld->send_message_batch->on_complete;
   calld->send_message_batch->on_complete = &calld->send_message_on_complete;
-  grpc_call_next_op(exec_ctx, elem, calld->send_message_batch);
+  send_message_batch_continue(exec_ctx, elem);
+}
+
+static void fail_send_message_batch_in_call_combiner(grpc_exec_ctx *exec_ctx,
+                                                     void *arg,
+                                                     grpc_error *error) {
+  call_data *calld = arg;
+  if (calld->send_message_batch != NULL) {
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, calld->send_message_batch, GRPC_ERROR_REF(error),
+        calld->call_combiner);
+    calld->send_message_batch = NULL;
+  }
 }
 
 // Pulls a slice from the send_message byte stream and adds it to calld->slices.
@@ -293,21 +311,25 @@ static grpc_error *pull_slice_from_send_message(grpc_exec_ctx *exec_ctx,
 // If all data has been read, invokes finish_send_message().  Otherwise,
 // an async call to grpc_byte_stream_next() has been started, which will
 // eventually result in calling on_send_message_next_done().
-static grpc_error *continue_reading_send_message(grpc_exec_ctx *exec_ctx,
+static void continue_reading_send_message(grpc_exec_ctx *exec_ctx,
                                           grpc_call_element *elem) {
   call_data *calld = (call_data *)elem->call_data;
   while (grpc_byte_stream_next(
       exec_ctx, calld->send_message_batch->payload->send_message.send_message,
       ~(size_t)0, &calld->on_send_message_next_done)) {
     grpc_error *error = pull_slice_from_send_message(exec_ctx, calld);
-    if (error != GRPC_ERROR_NONE) return error;
+    if (error != GRPC_ERROR_NONE) {
+      // Closure callback; does not take ownership of error.
+      fail_send_message_batch_in_call_combiner(exec_ctx, calld, error);
+      GRPC_ERROR_UNREF(error);
+      return;
+    }
     if (calld->slices.length ==
         calld->send_message_batch->payload->send_message.send_message->length) {
       finish_send_message(exec_ctx, elem);
       break;
     }
   }
-  return GRPC_ERROR_NONE;
 }
 
 // Async callback for grpc_byte_stream_next().
@@ -315,46 +337,37 @@ static void on_send_message_next_done(grpc_exec_ctx *exec_ctx, void *arg,
                                       grpc_error *error) {
   grpc_call_element *elem = (grpc_call_element *)arg;
   call_data *calld = (call_data *)elem->call_data;
-  if (error != GRPC_ERROR_NONE) goto fail;
+  if (error != GRPC_ERROR_NONE) {
+    // Closure callback; does not take ownership of error.
+    fail_send_message_batch_in_call_combiner(exec_ctx, calld, error);
+    return;
+  }
   error = pull_slice_from_send_message(exec_ctx, calld);
-  if (error != GRPC_ERROR_NONE) goto fail;
+  if (error != GRPC_ERROR_NONE) {
+    // Closure callback; does not take ownership of error.
+    fail_send_message_batch_in_call_combiner(exec_ctx, calld, error);
+    GRPC_ERROR_UNREF(error);
+    return;
+  }
   if (calld->slices.length ==
       calld->send_message_batch->payload->send_message.send_message->length) {
     finish_send_message(exec_ctx, elem);
   } else {
-    // This will either finish reading all of the data and invoke
+    continue_reading_send_message(exec_ctx, elem);
-    // finish_send_message(), or else it will make an async call to
-    // grpc_byte_stream_next(), which will eventually result in calling
-    // this function again.
-    error = continue_reading_send_message(exec_ctx, elem);
-    if (error != GRPC_ERROR_NONE) goto fail;
   }
-  return;
-fail:
-  grpc_transport_stream_op_batch_finish_with_failure(
-      exec_ctx, calld->send_message_batch, error);
 }
 
-static void start_send_message_batch(grpc_exec_ctx *exec_ctx,
+static void start_send_message_batch(grpc_exec_ctx *exec_ctx, void *arg,
-                                     grpc_call_element *elem,
+                                     grpc_error *unused) {
-                                     grpc_transport_stream_op_batch *batch,
+  grpc_call_element *elem = (grpc_call_element *)arg;
-                                     bool has_compression_algorithm) {
   call_data *calld = (call_data *)elem->call_data;
-  if (!skip_compression(elem, batch->payload->send_message.send_message->flags,
+  if (skip_compression(
-                        has_compression_algorithm)) {
+          elem,
-    calld->send_message_batch = batch;
+          calld->send_message_batch->payload->send_message.send_message->flags,
-    // This will either finish reading all of the data and invoke
+          calld->send_initial_metadata_state == HAS_COMPRESSION_ALGORITHM)) {
-    // finish_send_message(), or else it will make an async call to
+    send_message_batch_continue(exec_ctx, elem);
-    // grpc_byte_stream_next(), which will eventually result in calling
-    // on_send_message_next_done().
-    grpc_error *error = continue_reading_send_message(exec_ctx, elem);
-    if (error != GRPC_ERROR_NONE) {
-      grpc_transport_stream_op_batch_finish_with_failure(
-          exec_ctx, calld->send_message_batch, error);
-    }
   } else {
-    /* pass control down the stack */
+    continue_reading_send_message(exec_ctx, elem);
-    grpc_call_next_op(exec_ctx, elem, batch);
   }
 }
 
@@ -362,95 +375,80 @@ static void compress_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
-
   GPR_TIMER_BEGIN("compress_start_transport_stream_op_batch", 0);
-
+  // Handle cancel_stream.
   if (batch->cancel_stream) {
-    // TODO(roth): As part of the upcoming call combiner work, change
+    GRPC_ERROR_UNREF(calld->cancel_error);
-    // this to call grpc_byte_stream_shutdown() on the incoming byte
+    calld->cancel_error =
-    // stream, to cancel any in-flight calls to grpc_byte_stream_next().
         GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error);
-    gpr_atm cur = gpr_atm_full_xchg(
+    if (calld->send_message_batch != NULL) {
-        &calld->send_initial_metadata_state,
+      if (calld->send_initial_metadata_state == INITIAL_METADATA_UNSEEN) {
-        CANCELLED_BIT | (gpr_atm)batch->payload->cancel_stream.cancel_error);
+        GRPC_CALL_COMBINER_START(
-    switch (cur) {
+            exec_ctx, calld->call_combiner,
-      case HAS_COMPRESSION_ALGORITHM:
+            GRPC_CLOSURE_CREATE(fail_send_message_batch_in_call_combiner, calld,
-      case NO_COMPRESSION_ALGORITHM:
+                                grpc_schedule_on_exec_ctx),
-      case INITIAL_METADATA_UNSEEN:
+            GRPC_ERROR_REF(calld->cancel_error), "failing send_message op");
-        break;
-      default:
-        if ((cur & CANCELLED_BIT) == 0) {
-          grpc_transport_stream_op_batch_finish_with_failure(
-              exec_ctx, (grpc_transport_stream_op_batch *)cur,
-              GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error));
       } else {
-          GRPC_ERROR_UNREF((grpc_error *)(cur & ~CANCELLED_BIT));
+        grpc_byte_stream_shutdown(
+            exec_ctx,
+            calld->send_message_batch->payload->send_message.send_message,
+            GRPC_ERROR_REF(calld->cancel_error));
       }
-        break;
     }
+  } else if (calld->cancel_error != GRPC_ERROR_NONE) {
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, batch, GRPC_ERROR_REF(calld->cancel_error),
+        calld->call_combiner);
+    goto done;
   }
-
+  // Handle send_initial_metadata.
   if (batch->send_initial_metadata) {
+    GPR_ASSERT(calld->send_initial_metadata_state == INITIAL_METADATA_UNSEEN);
     bool has_compression_algorithm;
     grpc_error *error = process_send_initial_metadata(
         exec_ctx, elem,
         batch->payload->send_initial_metadata.send_initial_metadata,
         &has_compression_algorithm);
     if (error != GRPC_ERROR_NONE) {
-      grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch,
+      grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error,
-                                                         error);
+                                                         calld->call_combiner);
-      return;
+      goto done;
     }
-    gpr_atm cur;
+    calld->send_initial_metadata_state = has_compression_algorithm
-  retry_send_im:
-    cur = gpr_atm_acq_load(&calld->send_initial_metadata_state);
-    GPR_ASSERT(cur != HAS_COMPRESSION_ALGORITHM &&
-               cur != NO_COMPRESSION_ALGORITHM);
-    if ((cur & CANCELLED_BIT) == 0) {
-      if (!gpr_atm_rel_cas(&calld->send_initial_metadata_state, cur,
-                           has_compression_algorithm
                                              ? HAS_COMPRESSION_ALGORITHM
-                               : NO_COMPRESSION_ALGORITHM)) {
+                                             : NO_COMPRESSION_ALGORITHM;
-        goto retry_send_im;
+    // If we had previously received a batch containing a send_message op,
-      }
+    // handle it now.  Note that we need to re-enter the call combiner
-      if (cur != INITIAL_METADATA_UNSEEN) {
+    // for this, since we can't send two batches down while holding the
-        start_send_message_batch(exec_ctx, elem,
+    // call combiner, since the connected_channel filter (at the bottom of
-                                 (grpc_transport_stream_op_batch *)cur,
+    // the call stack) will release the call combiner for each batch it sees.
-                                 has_compression_algorithm);
+    if (calld->send_message_batch != NULL) {
-      }
+      GRPC_CALL_COMBINER_START(
-    }
+          exec_ctx, calld->call_combiner,
-  }
+          &calld->start_send_message_batch_in_call_combiner, GRPC_ERROR_NONE,
+          "starting send_message after send_initial_metadata");
+    }
+  }
+  // Handle send_message.
   if (batch->send_message) {
-    gpr_atm cur;
+    GPR_ASSERT(calld->send_message_batch == NULL);
-  retry_send:
+    calld->send_message_batch = batch;
-    cur = gpr_atm_acq_load(&calld->send_initial_metadata_state);
+    // If we have not yet seen send_initial_metadata, then we have to
-    switch (cur) {
+    // wait.  We save the batch in calld and then drop the call
-      case INITIAL_METADATA_UNSEEN:
+    // combiner, which we'll have to pick up again later when we get
-        if (!gpr_atm_rel_cas(&calld->send_initial_metadata_state, cur,
+    // send_initial_metadata.
-                             (gpr_atm)batch)) {
+    if (calld->send_initial_metadata_state == INITIAL_METADATA_UNSEEN) {
-          goto retry_send;
+      GRPC_CALL_COMBINER_STOP(
-        }
+          exec_ctx, calld->call_combiner,
-        break;
+          "send_message batch pending send_initial_metadata");
-      case HAS_COMPRESSION_ALGORITHM:
+      goto done;
-      case NO_COMPRESSION_ALGORITHM:
+    }
-        start_send_message_batch(exec_ctx, elem, batch,
+    start_send_message_batch(exec_ctx, elem, GRPC_ERROR_NONE);
-                                 cur == HAS_COMPRESSION_ALGORITHM);
-        break;
-      default:
-        if (cur & CANCELLED_BIT) {
-          grpc_transport_stream_op_batch_finish_with_failure(
-              exec_ctx, batch,
-              GRPC_ERROR_REF((grpc_error *)(cur & ~CANCELLED_BIT)));
-        } else {
-          /* >1 send_message concurrently */
-          GPR_UNREACHABLE_CODE(break);
-        }
-    }
   } else {
-    /* pass control down the stack */
+    // Pass control down the stack.
     grpc_call_next_op(exec_ctx, elem, batch);
   }
-
+done:
   GPR_TIMER_END("compress_start_transport_stream_op_batch", 0);
 }
 
@@ -458,16 +456,16 @@ static void compress_start_transport_stream_op_batch(
 static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
-  /* grab pointers to our data from the call element */
+  call_data *calld = (call_data *)elem->call_data;
-  call_data *calld = elem->call_data;
+  calld->call_combiner = args->call_combiner;
-
+  calld->cancel_error = GRPC_ERROR_NONE;
-  /* initialize members */
   grpc_slice_buffer_init(&calld->slices);
+  GRPC_CLOSURE_INIT(&calld->start_send_message_batch_in_call_combiner,
+                    start_send_message_batch, elem, grpc_schedule_on_exec_ctx);
   GRPC_CLOSURE_INIT(&calld->on_send_message_next_done,
                     on_send_message_next_done, elem, grpc_schedule_on_exec_ctx);
   GRPC_CLOSURE_INIT(&calld->send_message_on_complete, send_message_on_complete,
                     elem, grpc_schedule_on_exec_ctx);
-
   return GRPC_ERROR_NONE;
 }
 
@@ -475,14 +473,9 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
                               grpc_closure *ignored) {
-  /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   grpc_slice_buffer_destroy_internal(exec_ctx, &calld->slices);
-  gpr_atm imstate =
+  GRPC_ERROR_UNREF(calld->cancel_error);
-      gpr_atm_no_barrier_load(&calld->send_initial_metadata_state);
-  if (imstate & CANCELLED_BIT) {
-    GRPC_ERROR_UNREF((grpc_error *)(imstate & ~CANCELLED_BIT));
-  }
 }
 
 /* Constructor for channel_data */
@@ -550,6 +543,5 @@ const grpc_channel_filter grpc_message_compress_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
-    "compress"};
+    "message_compress"};

src/core/ext/filters/http/server/http_server_filter.c

@@ -32,6 +32,8 @@
 #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
 typedef struct call_data {
+  grpc_call_combiner *call_combiner;
+
   grpc_linked_mdelem status;
   grpc_linked_mdelem content_type;
 
@@ -281,7 +283,11 @@ static void hs_on_complete(grpc_exec_ctx *exec_ctx, void *user_data,
     *calld->pp_recv_message = calld->payload_bin_delivered
                                   ? NULL
                                   : (grpc_byte_stream *)&calld->read_stream;
-    GRPC_CLOSURE_RUN(exec_ctx, calld->recv_message_ready, GRPC_ERROR_REF(err));
+    // Re-enter call combiner for recv_message_ready, since the surface
+    // code will release the call combiner for each callback it receives.
+    GRPC_CALL_COMBINER_START(exec_ctx, calld->call_combiner,
+                             calld->recv_message_ready, GRPC_ERROR_REF(err),
+                             "resuming recv_message_ready from on_complete");
     calld->recv_message_ready = NULL;
     calld->payload_bin_delivered = true;
   }
@@ -293,14 +299,19 @@ static void hs_recv_message_ready(grpc_exec_ctx *exec_ctx, void *user_data,
   grpc_call_element *elem = user_data;
   call_data *calld = elem->call_data;
   if (calld->seen_path_with_query) {
-    /* do nothing. This is probably a GET request, and payload will be returned
+    // Do nothing. This is probably a GET request, and payload will be
-    in hs_on_complete callback. */
+    // returned in hs_on_complete callback.
+    // Note that we release the call combiner here, so that other
+    // callbacks can run.
+    GRPC_CALL_COMBINER_STOP(exec_ctx, calld->call_combiner,
+                            "pausing recv_message_ready until on_complete");
   } else {
     GRPC_CLOSURE_RUN(exec_ctx, calld->recv_message_ready, GRPC_ERROR_REF(err));
   }
 }
 
-static void hs_mutate_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+static grpc_error *hs_mutate_op(grpc_exec_ctx *exec_ctx,
+                                grpc_call_element *elem,
                                 grpc_transport_stream_op_batch *op) {
   /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
@@ -323,10 +334,7 @@ static void hs_mutate_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
               server_filter_outgoing_metadata(
                   exec_ctx, elem,
                   op->payload->send_initial_metadata.send_initial_metadata));
-    if (error != GRPC_ERROR_NONE) {
+    if (error != GRPC_ERROR_NONE) return error;
-      grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error);
-      return;
-    }
   }
 
   if (op->recv_initial_metadata) {
@@ -359,21 +367,25 @@ static void hs_mutate_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_error *error = server_filter_outgoing_metadata(
         exec_ctx, elem,
         op->payload->send_trailing_metadata.send_trailing_metadata);
-    if (error != GRPC_ERROR_NONE) {
+    if (error != GRPC_ERROR_NONE) return error;
-      grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error);
-      return;
-    }
   }
+
+  return GRPC_ERROR_NONE;
 }
 
-static void hs_start_transport_op(grpc_exec_ctx *exec_ctx,
+static void hs_start_transport_stream_op_batch(
-                                  grpc_call_element *elem,
+    grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *op) {
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
+  call_data *calld = elem->call_data;
-  GPR_TIMER_BEGIN("hs_start_transport_op", 0);
+  GPR_TIMER_BEGIN("hs_start_transport_stream_op_batch", 0);
-  hs_mutate_op(exec_ctx, elem, op);
+  grpc_error *error = hs_mutate_op(exec_ctx, elem, op);
+  if (error != GRPC_ERROR_NONE) {
+    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error,
+                                                       calld->call_combiner);
+  } else {
     grpc_call_next_op(exec_ctx, elem, op);
-  GPR_TIMER_END("hs_start_transport_op", 0);
+  }
+  GPR_TIMER_END("hs_start_transport_stream_op_batch", 0);
 }
 
 /* Constructor for call_data */
@@ -383,6 +395,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
   /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   /* initialize members */
+  calld->call_combiner = args->call_combiner;
   GRPC_CLOSURE_INIT(&calld->hs_on_recv, hs_on_recv, elem,
                     grpc_schedule_on_exec_ctx);
   GRPC_CLOSURE_INIT(&calld->hs_on_complete, hs_on_complete, elem,
@@ -414,7 +427,7 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem) {}
 
 const grpc_channel_filter grpc_http_server_filter = {
-    hs_start_transport_op,
+    hs_start_transport_stream_op_batch,
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
@@ -423,6 +436,5 @@ const grpc_channel_filter grpc_http_server_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "http-server"};

src/core/ext/filters/load_reporting/load_reporting.c

@@ -42,9 +42,15 @@ static bool maybe_add_load_reporting_filter(grpc_exec_ctx *exec_ctx,
                                             void *arg) {
   const grpc_channel_args *args =
       grpc_channel_stack_builder_get_channel_arguments(builder);
-  if (is_load_reporting_enabled(args)) {
+  const grpc_channel_filter *filter = arg;
-    return grpc_channel_stack_builder_prepend_filter(
+  grpc_channel_stack_builder_iterator *it =
-        builder, (const grpc_channel_filter *)arg, NULL, NULL);
+      grpc_channel_stack_builder_iterator_find(builder, filter->name);
+  const bool already_has_load_reporting_filter =
+      !grpc_channel_stack_builder_iterator_is_end(it);
+  grpc_channel_stack_builder_iterator_destroy(it);
+  if (is_load_reporting_enabled(args) && !already_has_load_reporting_filter) {
+    return grpc_channel_stack_builder_prepend_filter(builder, filter, NULL,
+                                                     NULL);
   }
   return true;
 }

src/core/ext/filters/load_reporting/load_reporting_filter.c

@@ -223,6 +223,5 @@ const grpc_channel_filter grpc_load_reporting_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "load_reporting"};

src/core/ext/filters/max_age/max_age_filter.c

@@ -273,7 +273,7 @@ static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
 static void destroy_call_elem(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                               const grpc_call_final_info* final_info,
                               grpc_closure* ignored) {
-  channel_data* chand = elem->channel_data;
+  channel_data* chand = (channel_data*)elem->channel_data;
   decrease_call_count(exec_ctx, chand);
 }
 
@@ -391,7 +391,6 @@ const grpc_channel_filter grpc_max_age_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "max_age"};
 

src/core/ext/filters/message_size/message_size_filter.c

@@ -68,6 +68,7 @@ static void* message_size_limits_create_from_json(const grpc_json* json) {
 }
 
 typedef struct call_data {
+  grpc_call_combiner* call_combiner;
   message_size_limits limits;
   // Receive closures are chained: we inject this closure as the
   // recv_message_ready up-call on transport_stream_op, and remember to
@@ -131,7 +132,8 @@ static void start_transport_stream_op_batch(
         exec_ctx, op,
         grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(message_string),
                            GRPC_ERROR_INT_GRPC_STATUS,
-                           GRPC_STATUS_RESOURCE_EXHAUSTED));
+                           GRPC_STATUS_RESOURCE_EXHAUSTED),
+        calld->call_combiner);
     gpr_free(message_string);
     return;
   }
@@ -152,6 +154,7 @@ static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
                                   const grpc_call_element_args* args) {
   channel_data* chand = (channel_data*)elem->channel_data;
   call_data* calld = (call_data*)elem->call_data;
+  calld->call_combiner = args->call_combiner;
   calld->next_recv_message_ready = NULL;
   GRPC_CLOSURE_INIT(&calld->recv_message_ready, recv_message_ready, elem,
                     grpc_schedule_on_exec_ctx);
@@ -259,7 +262,6 @@ const grpc_channel_filter grpc_message_size_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "message_size"};
 

src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c

@@ -177,7 +177,6 @@ const grpc_channel_filter grpc_workaround_cronet_compression_filter = {
     0,
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "workaround_cronet_compression"};
 

src/core/ext/transport/chttp2/transport/chttp2_transport.c

@@ -34,6 +34,7 @@
 #include "src/core/ext/transport/chttp2/transport/varint.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/compression/stream_compression.h"
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/timer.h"
@@ -1258,6 +1259,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   grpc_transport_stream_op_batch_payload *op_payload = op->payload;
   grpc_chttp2_transport *t = s->t;
 
+  GRPC_STATS_INC_HTTP2_OP_BATCHES(exec_ctx);
+
   if (GRPC_TRACER_ON(grpc_http_trace)) {
     char *str = grpc_transport_stream_op_batch_string(op);
     gpr_log(GPR_DEBUG, "perform_stream_op_locked: %s; on_complete = %p", str,
@@ -1291,11 +1294,13 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->cancel_stream) {
+    GRPC_STATS_INC_HTTP2_OP_CANCEL(exec_ctx);
     grpc_chttp2_cancel_stream(exec_ctx, t, s,
                               op_payload->cancel_stream.cancel_error);
   }
 
   if (op->send_initial_metadata) {
+    GRPC_STATS_INC_HTTP2_OP_SEND_INITIAL_METADATA(exec_ctx);
     GPR_ASSERT(s->send_initial_metadata_finished == NULL);
     on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
 
@@ -1370,15 +1375,29 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
             "send_initial_metadata_finished");
       }
     }
+    if (op_payload->send_initial_metadata.peer_string != NULL) {
+      gpr_atm_rel_store(op_payload->send_initial_metadata.peer_string,
+                        (gpr_atm)gpr_strdup(t->peer_string));
+    }
   }
 
   if (op->send_message) {
+    GRPC_STATS_INC_HTTP2_OP_SEND_MESSAGE(exec_ctx);
+    GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE(
+        exec_ctx, op->payload->send_message.send_message->length);
     on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->fetching_send_message_finished = add_closure_barrier(op->on_complete);
     if (s->write_closed) {
+      // Return an error unless the client has already received trailing
+      // metadata from the server, since an application using a
+      // streaming call might send another message before getting a
+      // recv_message failure, breaking out of its loop, and then
+      // starting recv_trailing_metadata.
       grpc_chttp2_complete_closure_step(
           exec_ctx, t, s, &s->fetching_send_message_finished,
-          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+          t->is_client && s->received_trailing_metadata
+              ? GRPC_ERROR_NONE
+              : GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                     "Attempt to send message after stream was closed",
                     &s->write_closed_error, 1),
           "fetching_send_message_finished");
@@ -1410,6 +1429,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->send_trailing_metadata) {
+    GRPC_STATS_INC_HTTP2_OP_SEND_TRAILING_METADATA(exec_ctx);
     GPR_ASSERT(s->send_trailing_metadata_finished == NULL);
     on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->send_trailing_metadata_finished = add_closure_barrier(on_complete);
@@ -1459,6 +1479,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->recv_initial_metadata) {
+    GRPC_STATS_INC_HTTP2_OP_RECV_INITIAL_METADATA(exec_ctx);
     GPR_ASSERT(s->recv_initial_metadata_ready == NULL);
     s->recv_initial_metadata_ready =
         op_payload->recv_initial_metadata.recv_initial_metadata_ready;
@@ -1466,10 +1487,15 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
         op_payload->recv_initial_metadata.recv_initial_metadata;
     s->trailing_metadata_available =
         op_payload->recv_initial_metadata.trailing_metadata_available;
+    if (op_payload->recv_initial_metadata.peer_string != NULL) {
+      gpr_atm_rel_store(op_payload->recv_initial_metadata.peer_string,
+                        (gpr_atm)gpr_strdup(t->peer_string));
+    }
     grpc_chttp2_maybe_complete_recv_initial_metadata(exec_ctx, t, s);
   }
 
   if (op->recv_message) {
+    GRPC_STATS_INC_HTTP2_OP_RECV_MESSAGE(exec_ctx);
     size_t already_received;
     GPR_ASSERT(s->recv_message_ready == NULL);
     GPR_ASSERT(!s->pending_byte_stream);
@@ -1491,6 +1517,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->recv_trailing_metadata) {
+    GRPC_STATS_INC_HTTP2_OP_RECV_TRAILING_METADATA(exec_ctx);
     GPR_ASSERT(s->recv_trailing_metadata_finished == NULL);
     s->recv_trailing_metadata_finished = add_closure_barrier(on_complete);
     s->recv_trailing_metadata =
@@ -1824,8 +1851,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx,
         }
       }
     }
-    if (s->read_closed && s->frame_storage.length == 0 &&
+    if (s->read_closed && s->frame_storage.length == 0 && !pending_data &&
-        (!pending_data || s->seen_error) &&
         s->recv_trailing_metadata_finished != NULL) {
       grpc_chttp2_incoming_metadata_buffer_publish(
           exec_ctx, &s->metadata_buffer[1], s->recv_trailing_metadata);
@@ -2931,14 +2957,6 @@ static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg,
   GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "destructive_reclaimer");
 }
 
-/*******************************************************************************
- * INTEGRATION GLUE
- */
-
-static char *chttp2_get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *t) {
-  return gpr_strdup(((grpc_chttp2_transport *)t)->peer_string);
-}
-
 /*******************************************************************************
  * MONITORING
  */
@@ -2956,7 +2974,6 @@ static const grpc_transport_vtable vtable = {sizeof(grpc_chttp2_stream),
                                              perform_transport_op,
                                              destroy_stream,
                                              destroy_transport,
-                                             chttp2_get_peer,
                                              chttp2_get_endpoint};
 
 grpc_transport *grpc_create_chttp2_transport(

src/core/ext/transport/chttp2/transport/internal.h

@@ -509,6 +509,8 @@ struct grpc_chttp2_stream {
   /** Are we buffering writes on this stream? If yes, we won't become writable
       until there's enough queued up in the flow_controlled_buffer */
   bool write_buffering;
+  /** Has trailing metadata been received. */
+  bool received_trailing_metadata;
 
   /** the error that resulted in this stream being read-closed */
   grpc_error *read_closed_error;

src/core/ext/transport/chttp2/transport/parsing.c

@@ -623,6 +623,7 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx,
           *s->trailing_metadata_available = true;
         }
         t->hpack_parser.on_header = on_trailing_header;
+        s->received_trailing_metadata = true;
       } else {
         GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "parsing initial_metadata"));
         t->hpack_parser.on_header = on_initial_header;
@@ -631,6 +632,7 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx,
     case 1:
       GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "parsing trailing_metadata"));
       t->hpack_parser.on_header = on_trailing_header;
+      s->received_trailing_metadata = true;
       break;
     case 2:
       gpr_log(GPR_ERROR, "too many header frames received");

src/core/ext/transport/chttp2/transport/writing.c

@@ -22,6 +22,7 @@
 
 #include <grpc/support/log.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/transport/http2_errors.h"
@@ -116,6 +117,7 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx,
                          &pq->lists[GRPC_CHTTP2_PCL_INFLIGHT]);
   grpc_slice_buffer_add(&t->outbuf,
                         grpc_chttp2_ping_create(false, pq->inflight_id));
+  GRPC_STATS_INC_HTTP2_PINGS_SENT(exec_ctx);
   t->ping_state.last_ping_sent_time = now;
   t->ping_state.pings_before_data_required -=
       (t->ping_state.pings_before_data_required != 0);
@@ -154,23 +156,16 @@ static uint32_t target_write_size(grpc_chttp2_transport *t) {
 }
 
 // Returns true if initial_metadata contains only default headers.
-//
-// TODO(roth): The fact that we hard-code these particular headers here
-// is fairly ugly.  Need some better way to know which headers are
-// default, maybe via a bit in the static metadata table?
 static bool is_default_initial_metadata(grpc_metadata_batch *initial_metadata) {
-  int num_default_fields =
+  return initial_metadata->list.default_count == initial_metadata->list.count;
-      (initial_metadata->idx.named.status != NULL) +
-      (initial_metadata->idx.named.content_type != NULL) +
-      (initial_metadata->idx.named.grpc_encoding != NULL) +
-      (initial_metadata->idx.named.grpc_accept_encoding != NULL);
-  return (size_t)num_default_fields == initial_metadata->list.count;
 }
 
 grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) {
   grpc_chttp2_stream *s;
 
+  GRPC_STATS_INC_HTTP2_WRITES_BEGUN(exec_ctx);
+
   GPR_TIMER_BEGIN("grpc_chttp2_begin_write", 0);
 
   if (t->dirtied_local_settings && !t->sent_local_settings) {

src/core/ext/transport/cronet/transport/cronet_transport.c

@@ -187,9 +187,34 @@ struct stream_obj {
 
   /* Mutex to protect storage */
   gpr_mu mu;
+
+  /* Refcount object of the stream */
+  grpc_stream_refcount *refcount;
 };
 typedef struct stream_obj stream_obj;
 
+#ifndef NDEBUG
+#define GRPC_CRONET_STREAM_REF(stream, reason) \
+  grpc_cronet_stream_ref((stream), (reason))
+#define GRPC_CRONET_STREAM_UNREF(exec_ctx, stream, reason) \
+  grpc_cronet_stream_unref((exec_ctx), (stream), (reason))
+void grpc_cronet_stream_ref(stream_obj *s, const char *reason) {
+  grpc_stream_ref(s->refcount, reason);
+}
+void grpc_cronet_stream_unref(grpc_exec_ctx *exec_ctx, stream_obj *s,
+                              const char *reason) {
+  grpc_stream_unref(exec_ctx, s->refcount, reason);
+}
+#else
+#define GRPC_CRONET_STREAM_REF(stream, reason) grpc_cronet_stream_ref((stream))
+#define GRPC_CRONET_STREAM_UNREF(exec_ctx, stream, reason) \
+  grpc_cronet_stream_unref((exec_ctx), (stream))
+void grpc_cronet_stream_ref(stream_obj *s) { grpc_stream_ref(s->refcount); }
+void grpc_cronet_stream_unref(grpc_exec_ctx *exec_ctx, stream_obj *s) {
+  grpc_stream_unref(exec_ctx, s->refcount);
+}
+#endif
+
 static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
                                           struct op_and_state *oas);
 
@@ -346,13 +371,12 @@ static void remove_from_storage(struct stream_obj *s,
   This can get executed from the Cronet network thread via cronet callback
   or on the application supplied thread via the perform_stream_op function.
 */
-static void execute_from_storage(stream_obj *s) {
+static void execute_from_storage(grpc_exec_ctx *exec_ctx, stream_obj *s) {
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   gpr_mu_lock(&s->mu);
   for (struct op_and_state *curr = s->storage.head; curr != NULL;) {
     CRONET_LOG(GPR_DEBUG, "calling op at %p. done = %d", curr, curr->done);
     GPR_ASSERT(curr->done == 0);
-    enum e_op_result result = execute_stream_op(&exec_ctx, curr);
+    enum e_op_result result = execute_stream_op(exec_ctx, curr);
     CRONET_LOG(GPR_DEBUG, "execute_stream_op[%p] returns %s", curr,
                op_result_string(result));
     /* if this op is done, then remove it and free memory */
@@ -369,7 +393,6 @@ static void execute_from_storage(stream_obj *s) {
     }
   }
   gpr_mu_unlock(&s->mu);
-  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 /*
@@ -377,6 +400,8 @@ static void execute_from_storage(stream_obj *s) {
 */
 static void on_failed(bidirectional_stream *stream, int net_error) {
   CRONET_LOG(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
   stream_obj *s = (stream_obj *)stream->annotation;
   gpr_mu_lock(&s->mu);
   bidirectional_stream_destroy(s->cbs);
@@ -392,7 +417,9 @@ static void on_failed(bidirectional_stream *stream, int net_error) {
   }
   null_and_maybe_free_read_buffer(s);
   gpr_mu_unlock(&s->mu);
-  execute_from_storage(s);
+  execute_from_storage(&exec_ctx, s);
+  GRPC_CRONET_STREAM_UNREF(&exec_ctx, s, "cronet transport");
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 /*
@@ -400,6 +427,8 @@ static void on_failed(bidirectional_stream *stream, int net_error) {
 */
 static void on_canceled(bidirectional_stream *stream) {
   CRONET_LOG(GPR_DEBUG, "on_canceled(%p)", stream);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
   stream_obj *s = (stream_obj *)stream->annotation;
   gpr_mu_lock(&s->mu);
   bidirectional_stream_destroy(s->cbs);
@@ -415,7 +444,9 @@ static void on_canceled(bidirectional_stream *stream) {
   }
   null_and_maybe_free_read_buffer(s);
   gpr_mu_unlock(&s->mu);
-  execute_from_storage(s);
+  execute_from_storage(&exec_ctx, s);
+  GRPC_CRONET_STREAM_UNREF(&exec_ctx, s, "cronet transport");
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 /*
@@ -423,6 +454,8 @@ static void on_canceled(bidirectional_stream *stream) {
 */
 static void on_succeeded(bidirectional_stream *stream) {
   CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
   stream_obj *s = (stream_obj *)stream->annotation;
   gpr_mu_lock(&s->mu);
   bidirectional_stream_destroy(s->cbs);
@@ -430,7 +463,9 @@ static void on_succeeded(bidirectional_stream *stream) {
   s->cbs = NULL;
   null_and_maybe_free_read_buffer(s);
   gpr_mu_unlock(&s->mu);
-  execute_from_storage(s);
+  execute_from_storage(&exec_ctx, s);
+  GRPC_CRONET_STREAM_UNREF(&exec_ctx, s, "cronet transport");
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 /*
@@ -438,6 +473,7 @@ static void on_succeeded(bidirectional_stream *stream) {
 */
 static void on_stream_ready(bidirectional_stream *stream) {
   CRONET_LOG(GPR_DEBUG, "W: on_stream_ready(%p)", stream);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   stream_obj *s = (stream_obj *)stream->annotation;
   grpc_cronet_transport *t = (grpc_cronet_transport *)s->curr_ct;
   gpr_mu_lock(&s->mu);
@@ -457,7 +493,8 @@ static void on_stream_ready(bidirectional_stream *stream) {
     }
   }
   gpr_mu_unlock(&s->mu);
-  execute_from_storage(s);
+  execute_from_storage(&exec_ctx, s);
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 /*
@@ -513,14 +550,15 @@ static void on_response_headers_received(
     s->state.pending_read_from_cronet = true;
   }
   gpr_mu_unlock(&s->mu);
+  execute_from_storage(&exec_ctx, s);
   grpc_exec_ctx_finish(&exec_ctx);
-  execute_from_storage(s);
 }
 
 /*
   Cronet callback
 */
 static void on_write_completed(bidirectional_stream *stream, const char *data) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   stream_obj *s = (stream_obj *)stream->annotation;
   CRONET_LOG(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data);
   gpr_mu_lock(&s->mu);
@@ -530,7 +568,8 @@ static void on_write_completed(bidirectional_stream *stream, const char *data) {
   }
   s->state.state_callback_received[OP_SEND_MESSAGE] = true;
   gpr_mu_unlock(&s->mu);
-  execute_from_storage(s);
+  execute_from_storage(&exec_ctx, s);
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 /*
@@ -538,6 +577,7 @@ static void on_write_completed(bidirectional_stream *stream, const char *data) {
 */
 static void on_read_completed(bidirectional_stream *stream, char *data,
                               int count) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   stream_obj *s = (stream_obj *)stream->annotation;
   CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data,
              count);
@@ -563,14 +603,15 @@ static void on_read_completed(bidirectional_stream *stream, char *data,
       gpr_mu_unlock(&s->mu);
     } else {
       gpr_mu_unlock(&s->mu);
-      execute_from_storage(s);
+      execute_from_storage(&exec_ctx, s);
     }
   } else {
     null_and_maybe_free_read_buffer(s);
     s->state.rs.read_stream_closed = true;
     gpr_mu_unlock(&s->mu);
-    execute_from_storage(s);
+    execute_from_storage(&exec_ctx, s);
   }
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 /*
@@ -625,12 +666,11 @@ static void on_response_trailers_received(
     s->state.state_op_done[OP_SEND_TRAILING_METADATA] = true;
 
     gpr_mu_unlock(&s->mu);
-    grpc_exec_ctx_finish(&exec_ctx);
   } else {
     gpr_mu_unlock(&s->mu);
-    grpc_exec_ctx_finish(&exec_ctx);
+    execute_from_storage(&exec_ctx, s);
-    execute_from_storage(s);
   }
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 /*
@@ -1313,6 +1353,9 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
                        grpc_stream *gs, grpc_stream_refcount *refcount,
                        const void *server_data, gpr_arena *arena) {
   stream_obj *s = (stream_obj *)gs;
+
+  s->refcount = refcount;
+  GRPC_CRONET_STREAM_REF(s, "cronet transport");
   memset(&s->storage, 0, sizeof(s->storage));
   s->storage.head = NULL;
   memset(&s->state, 0, sizeof(s->state));
@@ -1370,7 +1413,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   }
   stream_obj *s = (stream_obj *)gs;
   add_to_storage(s, op);
-  execute_from_storage(s);
+  execute_from_storage(exec_ctx, s);
 }
 
 static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
@@ -1386,10 +1429,6 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
 
 static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {}
 
-static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {
-  return NULL;
-}
-
 static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx,
                                    grpc_transport *gt) {
   return NULL;
@@ -1408,7 +1447,6 @@ static const grpc_transport_vtable grpc_cronet_vtable = {
     perform_op,
     destroy_stream,
     destroy_transport,
-    get_peer,
     get_endpoint};
 
 grpc_transport *grpc_create_cronet_transport(void *engine, const char *target,

src/core/ext/transport/inproc/inproc_transport.c

@@ -1251,20 +1251,14 @@ static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   // Nothing to do here
 }
 
-static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *t) {
-  return gpr_strdup("inproc");
-}
-
 static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) {
   return NULL;
 }
 
 static const grpc_transport_vtable inproc_vtable = {
-    sizeof(inproc_stream), "inproc",
+    sizeof(inproc_stream), "inproc",        init_stream,
-    init_stream,           set_pollset,
+    set_pollset,           set_pollset_set, perform_stream_op,
-    set_pollset_set,       perform_stream_op,
+    perform_transport_op,  destroy_stream,  destroy_transport,
-    perform_transport_op,  destroy_stream,
-    destroy_transport,     get_peer,
     get_endpoint};
 
 /*******************************************************************************

src/core/lib/channel/channel_stack.c

@@ -233,15 +233,10 @@ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack,
 void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                        grpc_transport_stream_op_batch *op) {
   grpc_call_element *next_elem = elem + 1;
+  GRPC_CALL_LOG_OP(GPR_INFO, next_elem, op);
   next_elem->filter->start_transport_stream_op_batch(exec_ctx, next_elem, op);
 }
 
-char *grpc_call_next_get_peer(grpc_exec_ctx *exec_ctx,
-                              grpc_call_element *elem) {
-  grpc_call_element *next_elem = elem + 1;
-  return next_elem->filter->get_peer(exec_ctx, next_elem);
-}
-
 void grpc_channel_next_get_info(grpc_exec_ctx *exec_ctx,
                                 grpc_channel_element *elem,
                                 const grpc_channel_info *channel_info) {
@@ -265,12 +260,3 @@ grpc_call_stack *grpc_call_stack_from_top_element(grpc_call_element *elem) {
   return (grpc_call_stack *)((char *)(elem)-ROUND_UP_TO_ALIGNMENT_SIZE(
       sizeof(grpc_call_stack)));
 }
-
-void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx,
-                                    grpc_call_element *elem,
-                                    grpc_error *error) {
-  grpc_transport_stream_op_batch *op = grpc_make_transport_stream_op(NULL);
-  op->cancel_stream = true;
-  op->payload->cancel_stream.cancel_error = error;
-  elem->filter->start_transport_stream_op_batch(exec_ctx, elem, op);
-}

src/core/lib/channel/channel_stack.h

@@ -40,6 +40,7 @@
 #include <grpc/support/time.h>
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/support/arena.h"
 #include "src/core/lib/transport/transport.h"
@@ -71,6 +72,7 @@ typedef struct {
   gpr_timespec start_time;
   gpr_timespec deadline;
   gpr_arena *arena;
+  grpc_call_combiner *call_combiner;
 } grpc_call_element_args;
 
 typedef struct {
@@ -150,9 +152,6 @@ typedef struct {
   void (*destroy_channel_elem)(grpc_exec_ctx *exec_ctx,
                                grpc_channel_element *elem);
 
-  /* Implement grpc_call_get_peer() */
-  char *(*get_peer)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem);
-
   /* Implement grpc_channel_get_info() */
   void (*get_channel_info)(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
                            const grpc_channel_info *channel_info);
@@ -271,8 +270,6 @@ void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
    stack */
 void grpc_channel_next_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
                           grpc_transport_op *op);
-/* Pass through a request to get_peer to the next child element */
-char *grpc_call_next_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem);
 /* Pass through a request to get_channel_info() to the next child element */
 void grpc_channel_next_get_info(grpc_exec_ctx *exec_ctx,
                                 grpc_channel_element *elem,
@@ -288,10 +285,6 @@ void grpc_call_log_op(char *file, int line, gpr_log_severity severity,
                       grpc_call_element *elem,
                       grpc_transport_stream_op_batch *op);
 
-void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx,
-                                    grpc_call_element *cur_elem,
-                                    grpc_error *error);
-
 extern grpc_tracer_flag grpc_trace_channel;
 
 #define GRPC_CALL_LOG_OP(sev, elem, op) \

src/core/lib/channel/channel_stack_builder.c

@@ -124,6 +124,20 @@ bool grpc_channel_stack_builder_move_prev(
   return true;
 }
 
+grpc_channel_stack_builder_iterator *grpc_channel_stack_builder_iterator_find(
+    grpc_channel_stack_builder *builder, const char *filter_name) {
+  GPR_ASSERT(filter_name != NULL);
+  grpc_channel_stack_builder_iterator *it =
+      grpc_channel_stack_builder_create_iterator_at_first(builder);
+  while (grpc_channel_stack_builder_move_next(it)) {
+    if (grpc_channel_stack_builder_iterator_is_end(it)) break;
+    const char *filter_name_at_it =
+        grpc_channel_stack_builder_iterator_filter_name(it);
+    if (strcmp(filter_name, filter_name_at_it) == 0) break;
+  }
+  return it;
+}
+
 bool grpc_channel_stack_builder_move_prev(
     grpc_channel_stack_builder_iterator *iterator);
 
@@ -169,6 +183,21 @@ bool grpc_channel_stack_builder_append_filter(
   return ok;
 }
 
+bool grpc_channel_stack_builder_remove_filter(
+    grpc_channel_stack_builder *builder, const char *filter_name) {
+  grpc_channel_stack_builder_iterator *it =
+      grpc_channel_stack_builder_iterator_find(builder, filter_name);
+  if (grpc_channel_stack_builder_iterator_is_end(it)) {
+    grpc_channel_stack_builder_iterator_destroy(it);
+    return false;
+  }
+  it->node->prev->next = it->node->next;
+  it->node->next->prev = it->node->prev;
+  gpr_free(it->node);
+  grpc_channel_stack_builder_iterator_destroy(it);
+  return true;
+}
+
 bool grpc_channel_stack_builder_prepend_filter(
     grpc_channel_stack_builder *builder, const grpc_channel_filter *filter,
     grpc_post_filter_create_init_func post_init_func, void *user_data) {

src/core/lib/channel/channel_stack_builder.h

@@ -95,6 +95,11 @@ bool grpc_channel_stack_builder_move_next(
 bool grpc_channel_stack_builder_move_prev(
     grpc_channel_stack_builder_iterator *iterator);
 
+/// Return an iterator at \a filter_name, or at the end of the list if not
+/// found.
+grpc_channel_stack_builder_iterator *grpc_channel_stack_builder_iterator_find(
+    grpc_channel_stack_builder *builder, const char *filter_name);
+
 typedef void (*grpc_post_filter_create_init_func)(
     grpc_channel_stack *channel_stack, grpc_channel_element *elem, void *arg);
 
@@ -132,6 +137,11 @@ bool grpc_channel_stack_builder_append_filter(
     grpc_post_filter_create_init_func post_init_func,
     void *user_data) GRPC_MUST_USE_RESULT;
 
+/// Remove any filter whose name is \a filter_name from \a builder. Returns true
+/// if \a filter_name was not found.
+bool grpc_channel_stack_builder_remove_filter(
+    grpc_channel_stack_builder *builder, const char *filter_name);
+
 /// Terminate iteration and destroy \a iterator
 void grpc_channel_stack_builder_iterator_destroy(
     grpc_channel_stack_builder_iterator *iterator);

src/core/lib/channel/connected_channel.c

@@ -36,7 +36,57 @@ typedef struct connected_channel_channel_data {
   grpc_transport *transport;
 } channel_data;
 
-typedef struct connected_channel_call_data { void *unused; } call_data;
+typedef struct {
+  grpc_closure closure;
+  grpc_closure *original_closure;
+  grpc_call_combiner *call_combiner;
+  const char *reason;
+} callback_state;
+
+typedef struct connected_channel_call_data {
+  grpc_call_combiner *call_combiner;
+  // Closures used for returning results on the call combiner.
+  callback_state on_complete[6];  // Max number of pending batches.
+  callback_state recv_initial_metadata_ready;
+  callback_state recv_message_ready;
+} call_data;
+
+static void run_in_call_combiner(grpc_exec_ctx *exec_ctx, void *arg,
+                                 grpc_error *error) {
+  callback_state *state = (callback_state *)arg;
+  GRPC_CALL_COMBINER_START(exec_ctx, state->call_combiner,
+                           state->original_closure, GRPC_ERROR_REF(error),
+                           state->reason);
+}
+
+static void run_cancel_in_call_combiner(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  run_in_call_combiner(exec_ctx, arg, error);
+  gpr_free(arg);
+}
+
+static void intercept_callback(call_data *calld, callback_state *state,
+                               bool free_when_done, const char *reason,
+                               grpc_closure **original_closure) {
+  state->original_closure = *original_closure;
+  state->call_combiner = calld->call_combiner;
+  state->reason = reason;
+  *original_closure = GRPC_CLOSURE_INIT(
+      &state->closure,
+      free_when_done ? run_cancel_in_call_combiner : run_in_call_combiner,
+      state, grpc_schedule_on_exec_ctx);
+}
+
+static callback_state *get_state_for_batch(
+    call_data *calld, grpc_transport_stream_op_batch *batch) {
+  if (batch->send_initial_metadata) return &calld->on_complete[0];
+  if (batch->send_message) return &calld->on_complete[1];
+  if (batch->send_trailing_metadata) return &calld->on_complete[2];
+  if (batch->recv_initial_metadata) return &calld->on_complete[3];
+  if (batch->recv_message) return &calld->on_complete[4];
+  if (batch->recv_trailing_metadata) return &calld->on_complete[5];
+  GPR_UNREACHABLE_CODE(return NULL);
+}
 
 /* We perform a small hack to locate transport data alongside the connected
    channel data in call allocations, to allow everything to be pulled in minimal
@@ -49,13 +99,38 @@ typedef struct connected_channel_call_data { void *unused; } call_data;
    into transport stream operations */
 static void con_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-    grpc_transport_stream_op_batch *op) {
+    grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
+  if (batch->recv_initial_metadata) {
-
+    callback_state *state = &calld->recv_initial_metadata_ready;
+    intercept_callback(
+        calld, state, false, "recv_initial_metadata_ready",
+        &batch->payload->recv_initial_metadata.recv_initial_metadata_ready);
+  }
+  if (batch->recv_message) {
+    callback_state *state = &calld->recv_message_ready;
+    intercept_callback(calld, state, false, "recv_message_ready",
+                       &batch->payload->recv_message.recv_message_ready);
+  }
+  if (batch->cancel_stream) {
+    // There can be more than one cancellation batch in flight at any
+    // given time, so we can't just pick out a fixed index into
+    // calld->on_complete like we can for the other ops.  However,
+    // cancellation isn't in the fast path, so we just allocate a new
+    // closure for each one.
+    callback_state *state = (callback_state *)gpr_malloc(sizeof(*state));
+    intercept_callback(calld, state, true, "on_complete (cancel_stream)",
+                       &batch->on_complete);
+  } else {
+    callback_state *state = get_state_for_batch(calld, batch);
+    intercept_callback(calld, state, false, "on_complete", &batch->on_complete);
+  }
   grpc_transport_perform_stream_op(exec_ctx, chand->transport,
-                                   TRANSPORT_STREAM_FROM_CALL_DATA(calld), op);
+                                   TRANSPORT_STREAM_FROM_CALL_DATA(calld),
+                                   batch);
+  GRPC_CALL_COMBINER_STOP(exec_ctx, calld->call_combiner,
+                          "passed batch to transport");
 }
 
 static void con_start_transport_op(grpc_exec_ctx *exec_ctx,
@@ -71,6 +146,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   const grpc_call_element_args *args) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
+  calld->call_combiner = args->call_combiner;
   int r = grpc_transport_init_stream(
       exec_ctx, chand->transport, TRANSPORT_STREAM_FROM_CALL_DATA(calld),
       &args->call_stack->refcount, args->server_transport_data, args->arena);
@@ -118,11 +194,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
   }
 }
 
-static char *con_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  channel_data *chand = elem->channel_data;
-  return grpc_transport_get_peer(exec_ctx, chand->transport);
-}
-
 /* No-op. */
 static void con_get_channel_info(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem,
@@ -138,7 +209,6 @@ const grpc_channel_filter grpc_connected_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    con_get_peer,
     con_get_channel_info,
     "connected",
 };

src/core/lib/debug/stats.c

@@ -45,8 +45,96 @@ void grpc_stats_collect(grpc_stats_data *output) {
       output->counters[i] += gpr_atm_no_barrier_load(
           &grpc_stats_per_cpu_storage[core].counters[i]);
     }
+    for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_BUCKETS; i++) {
+      output->histograms[i] += gpr_atm_no_barrier_load(
+          &grpc_stats_per_cpu_storage[core].histograms[i]);
     }
   }
+}
+
+void grpc_stats_diff(const grpc_stats_data *b, const grpc_stats_data *a,
+                     grpc_stats_data *c) {
+  for (size_t i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
+    c->counters[i] = b->counters[i] - a->counters[i];
+  }
+  for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_BUCKETS; i++) {
+    c->histograms[i] = b->histograms[i] - a->histograms[i];
+  }
+}
+
+int grpc_stats_histo_find_bucket_slow(grpc_exec_ctx *exec_ctx, int value,
+                                      const int *table, int table_size) {
+  GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx);
+  const int *const start = table;
+  while (table_size > 0) {
+    int step = table_size / 2;
+    const int *it = table + step;
+    if (value >= *it) {
+      table = it + 1;
+      table_size -= step + 1;
+    } else {
+      table_size = step;
+    }
+  }
+  return (int)(table - start) - 1;
+}
+
+size_t grpc_stats_histo_count(const grpc_stats_data *stats,
+                              grpc_stats_histograms histogram) {
+  size_t sum = 0;
+  for (int i = 0; i < grpc_stats_histo_buckets[histogram]; i++) {
+    sum += (size_t)stats->histograms[grpc_stats_histo_start[histogram] + i];
+  }
+  return sum;
+}
+
+static double threshold_for_count_below(const gpr_atm *bucket_counts,
+                                        const int *bucket_boundaries,
+                                        int num_buckets, double count_below) {
+  double count_so_far;
+  double lower_bound;
+  double upper_bound;
+  int lower_idx;
+  int upper_idx;
+
+  /* find the lowest bucket that gets us above count_below */
+  count_so_far = 0.0;
+  for (lower_idx = 0; lower_idx < num_buckets; lower_idx++) {
+    count_so_far += (double)bucket_counts[lower_idx];
+    if (count_so_far >= count_below) {
+      break;
+    }
+  }
+  if (count_so_far == count_below) {
+    /* this bucket hits the threshold exactly... we should be midway through
+       any run of zero values following the bucket */
+    for (upper_idx = lower_idx + 1; upper_idx < num_buckets; upper_idx++) {
+      if (bucket_counts[upper_idx]) {
+        break;
+      }
+    }
+    return (bucket_boundaries[lower_idx] + bucket_boundaries[upper_idx]) / 2.0;
+  } else {
+    /* treat values as uniform throughout the bucket, and find where this value
+       should lie */
+    lower_bound = bucket_boundaries[lower_idx];
+    upper_bound = bucket_boundaries[lower_idx + 1];
+    return upper_bound -
+           (upper_bound - lower_bound) * (count_so_far - count_below) /
+               (double)bucket_counts[lower_idx];
+  }
+}
+
+double grpc_stats_histo_percentile(const grpc_stats_data *stats,
+                                   grpc_stats_histograms histogram,
+                                   double percentile) {
+  size_t count = grpc_stats_histo_count(stats, histogram);
+  if (count == 0) return 0.0;
+  return threshold_for_count_below(
+      stats->histograms + grpc_stats_histo_start[histogram],
+      grpc_stats_histo_bucket_boundaries[histogram],
+      grpc_stats_histo_buckets[histogram], (double)count * percentile / 100.0);
+}
 
 char *grpc_stats_data_as_json(const grpc_stats_data *data) {
   gpr_strvec v;
@@ -60,6 +148,25 @@ char *grpc_stats_data_as_json(const grpc_stats_data *data) {
     gpr_strvec_add(&v, tmp);
     is_first = false;
   }
+  for (size_t i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) {
+    gpr_asprintf(&tmp, "%s\"%s\": [", is_first ? "" : ", ",
+                 grpc_stats_histogram_name[i]);
+    gpr_strvec_add(&v, tmp);
+    for (int j = 0; j < grpc_stats_histo_buckets[i]; j++) {
+      gpr_asprintf(&tmp, "%s%" PRIdPTR, j == 0 ? "" : ",",
+                   data->histograms[grpc_stats_histo_start[i] + j]);
+      gpr_strvec_add(&v, tmp);
+    }
+    gpr_asprintf(&tmp, "], \"%s_bkt\": [", grpc_stats_histogram_name[i]);
+    gpr_strvec_add(&v, tmp);
+    for (int j = 0; j < grpc_stats_histo_buckets[i]; j++) {
+      gpr_asprintf(&tmp, "%s%d", j == 0 ? "" : ",",
+                   grpc_stats_histo_bucket_boundaries[i][j]);
+      gpr_strvec_add(&v, tmp);
+    }
+    gpr_strvec_add(&v, gpr_strdup("]"));
+    is_first = false;
+  }
   gpr_strvec_add(&v, gpr_strdup("}"));
   tmp = gpr_strvec_flatten(&v, NULL);
   gpr_strvec_destroy(&v);

src/core/lib/debug/stats.h

@@ -25,6 +25,7 @@
 
 typedef struct grpc_stats_data {
   gpr_atm counters[GRPC_STATS_COUNTER_COUNT];
+  gpr_atm histograms[GRPC_STATS_HISTOGRAM_BUCKETS];
 } grpc_stats_data;
 
 extern grpc_stats_data *grpc_stats_per_cpu_storage;
@@ -36,9 +37,25 @@ extern grpc_stats_data *grpc_stats_per_cpu_storage;
   (gpr_atm_no_barrier_fetch_add(              \
       &GRPC_THREAD_STATS_DATA((exec_ctx))->counters[(ctr)], 1))
 
+#define GRPC_STATS_INC_HISTOGRAM(exec_ctx, histogram, index) \
+  (gpr_atm_no_barrier_fetch_add(                             \
+      &GRPC_THREAD_STATS_DATA((exec_ctx))                    \
+           ->histograms[histogram##_FIRST_SLOT + (index)],   \
+      1))
+
 void grpc_stats_init(void);
 void grpc_stats_shutdown(void);
 void grpc_stats_collect(grpc_stats_data *output);
+// c = b-a
+void grpc_stats_diff(const grpc_stats_data *b, const grpc_stats_data *a,
+                     grpc_stats_data *c);
 char *grpc_stats_data_as_json(const grpc_stats_data *data);
+int grpc_stats_histo_find_bucket_slow(grpc_exec_ctx *exec_ctx, int value,
+                                      const int *table, int table_size);
+double grpc_stats_histo_percentile(const grpc_stats_data *data,
+                                   grpc_stats_histograms histogram,
+                                   double percentile);
+size_t grpc_stats_histo_count(const grpc_stats_data *data,
+                              grpc_stats_histograms histogram);
 
 #endif

src/core/lib/debug/stats_data.c

@@ -19,7 +19,227 @@
  */
 
 #include "src/core/lib/debug/stats_data.h"
+#include <grpc/support/useful.h>
+#include "src/core/lib/debug/stats.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
-    "client_calls_created", "server_calls_created", "syscall_write",
+    "client_calls_created",
-    "syscall_read",         "syscall_poll",         "syscall_wait",
+    "server_calls_created",
+    "syscall_poll",
+    "syscall_wait",
+    "histogram_slow_lookups",
+    "syscall_write",
+    "syscall_read",
+    "http2_op_batches",
+    "http2_op_cancel",
+    "http2_op_send_initial_metadata",
+    "http2_op_send_message",
+    "http2_op_send_trailing_metadata",
+    "http2_op_recv_initial_metadata",
+    "http2_op_recv_message",
+    "http2_op_recv_trailing_metadata",
+    "http2_pings_sent",
+    "http2_writes_begun",
+    "combiner_locks_initiated",
+    "combiner_locks_scheduled_items",
+    "combiner_locks_scheduled_final_items",
+    "combiner_locks_offloaded",
+    "executor_scheduled_items",
+    "executor_scheduled_to_self",
+    "executor_wakeup_initiated",
+    "executor_queue_drained",
 };
+const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = {
+    "tcp_write_size", "tcp_write_iov_size", "tcp_read_size",
+    "tcp_read_offer", "tcp_read_iov_size",  "http2_send_message_size",
+};
+const int grpc_stats_table_0[65] = {
+    0,       1,       2,       3,       4,       6,       8,        11,
+    15,      20,      26,      34,      44,      57,      73,       94,
+    121,     155,     199,     255,     327,     419,     537,      688,
+    881,     1128,    1444,    1848,    2365,    3026,    3872,     4954,
+    6338,    8108,    10373,   13270,   16976,   21717,   27782,    35541,
+    45467,   58165,   74409,   95189,   121772,  155778,  199281,   254933,
+    326126,  417200,  533707,  682750,  873414,  1117323, 1429345,  1828502,
+    2339127, 2992348, 3827987, 4896985, 6264509, 8013925, 10251880, 13114801,
+    16777216};
+const uint8_t grpc_stats_table_1[87] = {
+    0,  0,  1,  1,  2,  3,  3,  4,  4,  5,  6,  6,  7,  8,  8,  9,  10, 11,
+    11, 12, 13, 13, 14, 15, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22, 22, 23,
+    24, 25, 25, 26, 27, 27, 28, 29, 29, 30, 31, 31, 32, 33, 34, 34, 35, 36,
+    36, 37, 38, 39, 39, 40, 41, 41, 42, 43, 44, 44, 45, 45, 46, 47, 48, 48,
+    49, 50, 51, 51, 52, 53, 53, 54, 55, 56, 56, 57, 58, 58, 59};
+const int grpc_stats_table_2[65] = {
+    0,   1,   2,   3,   4,   5,   6,   7,   8,   9,   10,  11,  12,
+    14,  16,  18,  20,  22,  24,  27,  30,  33,  36,  39,  43,  47,
+    51,  56,  61,  66,  72,  78,  85,  92,  100, 109, 118, 128, 139,
+    151, 164, 178, 193, 209, 226, 244, 264, 285, 308, 333, 359, 387,
+    418, 451, 486, 524, 565, 609, 656, 707, 762, 821, 884, 952, 1024};
+const uint8_t grpc_stats_table_3[102] = {
+    0,  0,  0,  1,  1,  1,  1,  2,  2,  3,  3,  4,  4,  5,  5,  6,  6,
+    6,  7,  7,  7,  8,  8,  9,  9,  10, 11, 11, 12, 12, 13, 13, 14, 14,
+    14, 15, 15, 16, 16, 17, 17, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23,
+    23, 24, 24, 24, 25, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32,
+    32, 33, 33, 34, 35, 35, 36, 37, 37, 38, 38, 39, 39, 40, 40, 41, 41,
+    42, 42, 43, 44, 44, 45, 46, 46, 47, 48, 48, 49, 49, 50, 50, 51, 51};
+void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 16777216);
+  if (value < 5) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
+                             value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4683743612465315840ull) {
+    int bucket =
+        grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5;
+    _bkt.dbl = grpc_stats_table_0[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
+                             bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_0, 64));
+}
+void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 1024);
+  if (value < 13) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx),
+                             GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4637863191261478912ull) {
+    int bucket =
+        grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13;
+    _bkt.dbl = grpc_stats_table_2[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx),
+                             GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_2, 64));
+}
+void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 16777216);
+  if (value < 5) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
+                             value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4683743612465315840ull) {
+    int bucket =
+        grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5;
+    _bkt.dbl = grpc_stats_table_0[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
+                             bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_0, 64));
+}
+void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 16777216);
+  if (value < 5) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER,
+                             value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4683743612465315840ull) {
+    int bucket =
+        grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5;
+    _bkt.dbl = grpc_stats_table_0[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER,
+                             bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_0, 64));
+}
+void grpc_stats_inc_tcp_read_iov_size(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 1024);
+  if (value < 13) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE,
+                             value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4637863191261478912ull) {
+    int bucket =
+        grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13;
+    _bkt.dbl = grpc_stats_table_2[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE,
+                             bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_2, 64));
+}
+void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx,
+                                            int value) {
+  value = GPR_CLAMP(value, 0, 16777216);
+  if (value < 5) {
+    GRPC_STATS_INC_HISTOGRAM(
+        (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4683743612465315840ull) {
+    int bucket =
+        grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)] + 5;
+    _bkt.dbl = grpc_stats_table_0[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM(
+        (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx),
+                           GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_0, 64));
+}
+const int grpc_stats_histo_buckets[6] = {64, 64, 64, 64, 64, 64};
+const int grpc_stats_histo_start[6] = {0, 64, 128, 192, 256, 320};
+const int *const grpc_stats_histo_bucket_boundaries[6] = {
+    grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0,
+    grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0};
+void (*const grpc_stats_inc_histogram[6])(grpc_exec_ctx *exec_ctx, int x) = {
+    grpc_stats_inc_tcp_write_size,    grpc_stats_inc_tcp_write_iov_size,
+    grpc_stats_inc_tcp_read_size,     grpc_stats_inc_tcp_read_offer,
+    grpc_stats_inc_tcp_read_iov_size, grpc_stats_inc_http2_send_message_size};

src/core/lib/debug/stats_data.h

@@ -21,27 +21,146 @@
 #ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H
 #define GRPC_CORE_LIB_DEBUG_STATS_DATA_H
 
+#include <inttypes.h>
+#include "src/core/lib/iomgr/exec_ctx.h"
+
 typedef enum {
   GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED,
   GRPC_STATS_COUNTER_SERVER_CALLS_CREATED,
-  GRPC_STATS_COUNTER_SYSCALL_WRITE,
-  GRPC_STATS_COUNTER_SYSCALL_READ,
   GRPC_STATS_COUNTER_SYSCALL_POLL,
   GRPC_STATS_COUNTER_SYSCALL_WAIT,
+  GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS,
+  GRPC_STATS_COUNTER_SYSCALL_WRITE,
+  GRPC_STATS_COUNTER_SYSCALL_READ,
+  GRPC_STATS_COUNTER_HTTP2_OP_BATCHES,
+  GRPC_STATS_COUNTER_HTTP2_OP_CANCEL,
+  GRPC_STATS_COUNTER_HTTP2_OP_SEND_INITIAL_METADATA,
+  GRPC_STATS_COUNTER_HTTP2_OP_SEND_MESSAGE,
+  GRPC_STATS_COUNTER_HTTP2_OP_SEND_TRAILING_METADATA,
+  GRPC_STATS_COUNTER_HTTP2_OP_RECV_INITIAL_METADATA,
+  GRPC_STATS_COUNTER_HTTP2_OP_RECV_MESSAGE,
+  GRPC_STATS_COUNTER_HTTP2_OP_RECV_TRAILING_METADATA,
+  GRPC_STATS_COUNTER_HTTP2_PINGS_SENT,
+  GRPC_STATS_COUNTER_HTTP2_WRITES_BEGUN,
+  GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED,
+  GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_ITEMS,
+  GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS,
+  GRPC_STATS_COUNTER_COMBINER_LOCKS_OFFLOADED,
+  GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_ITEMS,
+  GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_TO_SELF,
+  GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED,
+  GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED,
   GRPC_STATS_COUNTER_COUNT
 } grpc_stats_counters;
+extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];
+typedef enum {
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE,
+  GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
+  GRPC_STATS_HISTOGRAM_TCP_READ_OFFER,
+  GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE,
+  GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE,
+  GRPC_STATS_HISTOGRAM_COUNT
+} grpc_stats_histograms;
+extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT];
+typedef enum {
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 0,
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 64,
+  GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 128,
+  GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 192,
+  GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_FIRST_SLOT = 256,
+  GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 320,
+  GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_BUCKETS = 64,
+  GRPC_STATS_HISTOGRAM_BUCKETS = 384
+} grpc_stats_histogram_constants;
 #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \
   GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED)
 #define GRPC_STATS_INC_SERVER_CALLS_CREATED(exec_ctx) \
   GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_CALLS_CREATED)
-#define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \
-  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WRITE)
-#define GRPC_STATS_INC_SYSCALL_READ(exec_ctx) \
-  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_READ)
 #define GRPC_STATS_INC_SYSCALL_POLL(exec_ctx) \
   GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL)
 #define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \
   GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WAIT)
-extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];
+#define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS)
+#define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WRITE)
+#define GRPC_STATS_INC_SYSCALL_READ(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_READ)
+#define GRPC_STATS_INC_HTTP2_OP_BATCHES(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_OP_BATCHES)
+#define GRPC_STATS_INC_HTTP2_OP_CANCEL(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_OP_CANCEL)
+#define GRPC_STATS_INC_HTTP2_OP_SEND_INITIAL_METADATA(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                            \
+                         GRPC_STATS_COUNTER_HTTP2_OP_SEND_INITIAL_METADATA)
+#define GRPC_STATS_INC_HTTP2_OP_SEND_MESSAGE(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_OP_SEND_MESSAGE)
+#define GRPC_STATS_INC_HTTP2_OP_SEND_TRAILING_METADATA(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                             \
+                         GRPC_STATS_COUNTER_HTTP2_OP_SEND_TRAILING_METADATA)
+#define GRPC_STATS_INC_HTTP2_OP_RECV_INITIAL_METADATA(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                            \
+                         GRPC_STATS_COUNTER_HTTP2_OP_RECV_INITIAL_METADATA)
+#define GRPC_STATS_INC_HTTP2_OP_RECV_MESSAGE(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_OP_RECV_MESSAGE)
+#define GRPC_STATS_INC_HTTP2_OP_RECV_TRAILING_METADATA(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                             \
+                         GRPC_STATS_COUNTER_HTTP2_OP_RECV_TRAILING_METADATA)
+#define GRPC_STATS_INC_HTTP2_PINGS_SENT(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_PINGS_SENT)
+#define GRPC_STATS_INC_HTTP2_WRITES_BEGUN(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_WRITES_BEGUN)
+#define GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                      \
+                         GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED)
+#define GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_ITEMS(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                            \
+                         GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_ITEMS)
+#define GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS(exec_ctx) \
+  GRPC_STATS_INC_COUNTER(                                             \
+      (exec_ctx), GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS)
+#define GRPC_STATS_INC_COMBINER_LOCKS_OFFLOADED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                      \
+                         GRPC_STATS_COUNTER_COMBINER_LOCKS_OFFLOADED)
+#define GRPC_STATS_INC_EXECUTOR_SCHEDULED_ITEMS(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                      \
+                         GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_ITEMS)
+#define GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                        \
+                         GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_TO_SELF)
+#define GRPC_STATS_INC_EXECUTOR_WAKEUP_INITIATED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                       \
+                         GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED)
+#define GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED)
+#define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \
+  grpc_stats_inc_tcp_write_size((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(exec_ctx, value) \
+  grpc_stats_inc_tcp_write_iov_size((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_TCP_READ_SIZE(exec_ctx, value) \
+  grpc_stats_inc_tcp_read_size((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_TCP_READ_OFFER(exec_ctx, value) \
+  grpc_stats_inc_tcp_read_offer((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_TCP_READ_IOV_SIZE(exec_ctx, value) \
+  grpc_stats_inc_tcp_read_iov_size((exec_ctx), (int)(value))
+void grpc_stats_inc_tcp_read_iov_size(grpc_exec_ctx *exec_ctx, int x);
+#define GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE(exec_ctx, value) \
+  grpc_stats_inc_http2_send_message_size((exec_ctx), (int)(value))
+void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, int x);
+extern const int grpc_stats_histo_buckets[6];
+extern const int grpc_stats_histo_start[6];
+extern const int *const grpc_stats_histo_bucket_boundaries[6];
+extern void (*const grpc_stats_inc_histogram[6])(grpc_exec_ctx *exec_ctx,
+                                                 int x);
 
 #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */

src/core/lib/debug/stats_data.yaml

@@ -1,9 +1,67 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Stats data declaration
 # use tools / codegen / core / gen_stats_data.py to turn this into stats_data.h
 
+# overall
 - counter: client_calls_created
 - counter: server_calls_created
-- counter: syscall_write
+# polling
-- counter: syscall_read
 - counter: syscall_poll
 - counter: syscall_wait
+# stats system
+- counter: histogram_slow_lookups
+# tcp
+- counter: syscall_write
+- counter: syscall_read
+- histogram: tcp_write_size
+  max: 16777216 # 16 meg max write tracked
+  buckets: 64
+- histogram: tcp_write_iov_size
+  max: 1024
+  buckets: 64
+- histogram: tcp_read_size
+  max: 16777216
+  buckets: 64
+- histogram: tcp_read_offer
+  max: 16777216
+  buckets: 64
+- histogram: tcp_read_iov_size
+  max: 1024
+  buckets: 64
+# chttp2
+- counter: http2_op_batches
+- counter: http2_op_cancel
+- counter: http2_op_send_initial_metadata
+- counter: http2_op_send_message
+- counter: http2_op_send_trailing_metadata
+- counter: http2_op_recv_initial_metadata
+- counter: http2_op_recv_message
+- counter: http2_op_recv_trailing_metadata
+- histogram: http2_send_message_size
+  max: 16777216
+  buckets: 64
+- counter: http2_pings_sent
+- counter: http2_writes_begun
+# combiner locks
+- counter: combiner_locks_initiated
+- counter: combiner_locks_scheduled_items
+- counter: combiner_locks_scheduled_final_items
+- counter: combiner_locks_offloaded
+# executor
+- counter: executor_scheduled_items
+- counter: executor_scheduled_to_self
+- counter: executor_wakeup_initiated
+- counter: executor_queue_drained

src/core/lib/iomgr/call_combiner.c

@@ -0,0 +1,202 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/iomgr/call_combiner.h"
+
+#include <grpc/support/log.h>
+
+grpc_tracer_flag grpc_call_combiner_trace =
+    GRPC_TRACER_INITIALIZER(false, "call_combiner");
+
+static grpc_error* decode_cancel_state_error(gpr_atm cancel_state) {
+  if (cancel_state & 1) {
+    return (grpc_error*)(cancel_state & ~(gpr_atm)1);
+  }
+  return GRPC_ERROR_NONE;
+}
+
+static gpr_atm encode_cancel_state_error(grpc_error* error) {
+  return (gpr_atm)1 | (gpr_atm)error;
+}
+
+void grpc_call_combiner_init(grpc_call_combiner* call_combiner) {
+  gpr_mpscq_init(&call_combiner->queue);
+}
+
+void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner) {
+  gpr_mpscq_destroy(&call_combiner->queue);
+  GRPC_ERROR_UNREF(decode_cancel_state_error(call_combiner->cancel_state));
+}
+
+#ifndef NDEBUG
+#define DEBUG_ARGS , const char *file, int line
+#define DEBUG_FMT_STR "%s:%d: "
+#define DEBUG_FMT_ARGS , file, line
+#else
+#define DEBUG_ARGS
+#define DEBUG_FMT_STR
+#define DEBUG_FMT_ARGS
+#endif
+
+void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx,
+                              grpc_call_combiner* call_combiner,
+                              grpc_closure* closure,
+                              grpc_error* error DEBUG_ARGS,
+                              const char* reason) {
+  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG,
+            "==> grpc_call_combiner_start() [%p] closure=%p [" DEBUG_FMT_STR
+            "%s] error=%s",
+            call_combiner, closure DEBUG_FMT_ARGS, reason,
+            grpc_error_string(error));
+  }
+  size_t prev_size =
+      (size_t)gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)1);
+  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
+            prev_size + 1);
+  }
+  if (prev_size == 0) {
+    if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+      gpr_log(GPR_DEBUG, "  EXECUTING IMMEDIATELY");
+    }
+    // Queue was empty, so execute this closure immediately.
+    GRPC_CLOSURE_SCHED(exec_ctx, closure, error);
+  } else {
+    if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+      gpr_log(GPR_INFO, "  QUEUING");
+    }
+    // Queue was not empty, so add closure to queue.
+    closure->error_data.error = error;
+    gpr_mpscq_push(&call_combiner->queue, (gpr_mpscq_node*)closure);
+  }
+}
+
+void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx,
+                             grpc_call_combiner* call_combiner DEBUG_ARGS,
+                             const char* reason) {
+  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG,
+            "==> grpc_call_combiner_stop() [%p] [" DEBUG_FMT_STR "%s]",
+            call_combiner DEBUG_FMT_ARGS, reason);
+  }
+  size_t prev_size =
+      (size_t)gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)-1);
+  if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
+            prev_size - 1);
+  }
+  GPR_ASSERT(prev_size >= 1);
+  if (prev_size > 1) {
+    while (true) {
+      if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+        gpr_log(GPR_DEBUG, "  checking queue");
+      }
+      bool empty;
+      grpc_closure* closure = (grpc_closure*)gpr_mpscq_pop_and_check_end(
+          &call_combiner->queue, &empty);
+      if (closure == NULL) {
+        // This can happen either due to a race condition within the mpscq
+        // code or because of a race with grpc_call_combiner_start().
+        if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+          gpr_log(GPR_DEBUG, "  queue returned no result; checking again");
+        }
+        continue;
+      }
+      if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+        gpr_log(GPR_DEBUG, "  EXECUTING FROM QUEUE: closure=%p error=%s",
+                closure, grpc_error_string(closure->error_data.error));
+      }
+      GRPC_CLOSURE_SCHED(exec_ctx, closure, closure->error_data.error);
+      break;
+    }
+  } else if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+    gpr_log(GPR_DEBUG, "  queue empty");
+  }
+}
+
+void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
+                                             grpc_call_combiner* call_combiner,
+                                             grpc_closure* closure) {
+  while (true) {
+    // Decode original state.
+    gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state);
+    grpc_error* original_error = decode_cancel_state_error(original_state);
+    // If error is set, invoke the cancellation closure immediately.
+    // Otherwise, store the new closure.
+    if (original_error != GRPC_ERROR_NONE) {
+      if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+        gpr_log(GPR_DEBUG,
+                "call_combiner=%p: scheduling notify_on_cancel callback=%p "
+                "for pre-existing cancellation",
+                call_combiner, closure);
+      }
+      GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_ERROR_REF(original_error));
+      break;
+    } else {
+      if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state,
+                           (gpr_atm)closure)) {
+        if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+          gpr_log(GPR_DEBUG, "call_combiner=%p: setting notify_on_cancel=%p",
+                  call_combiner, closure);
+        }
+        // If we replaced an earlier closure, invoke the original
+        // closure with GRPC_ERROR_NONE.  This allows callers to clean
+        // up any resources they may be holding for the callback.
+        if (original_state != 0) {
+          closure = (grpc_closure*)original_state;
+          if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+            gpr_log(GPR_DEBUG,
+                    "call_combiner=%p: scheduling old cancel callback=%p",
+                    call_combiner, closure);
+          }
+          GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_ERROR_NONE);
+        }
+        break;
+      }
+    }
+    // cas failed, try again.
+  }
+}
+
+void grpc_call_combiner_cancel(grpc_exec_ctx* exec_ctx,
+                               grpc_call_combiner* call_combiner,
+                               grpc_error* error) {
+  while (true) {
+    gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state);
+    grpc_error* original_error = decode_cancel_state_error(original_state);
+    if (original_error != GRPC_ERROR_NONE) {
+      GRPC_ERROR_UNREF(error);
+      break;
+    }
+    if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state,
+                         encode_cancel_state_error(error))) {
+      if (original_state != 0) {
+        grpc_closure* notify_on_cancel = (grpc_closure*)original_state;
+        if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+          gpr_log(GPR_DEBUG,
+                  "call_combiner=%p: scheduling notify_on_cancel callback=%p",
+                  call_combiner, notify_on_cancel);
+        }
+        GRPC_CLOSURE_SCHED(exec_ctx, notify_on_cancel, GRPC_ERROR_REF(error));
+      }
+      break;
+    }
+    // cas failed, try again.
+  }
+}

src/core/lib/iomgr/call_combiner.h

@@ -0,0 +1,121 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H
+#define GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H
+
+#include <stddef.h>
+
+#include <grpc/support/atm.h>
+
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/support/mpscq.h"
+
+// A simple, lock-free mechanism for serializing activity related to a
+// single call.  This is similar to a combiner but is more lightweight.
+//
+// It requires the callback (or, in the common case where the callback
+// actually kicks off a chain of callbacks, the last callback in that
+// chain) to explicitly indicate (by calling GRPC_CALL_COMBINER_STOP())
+// when it is done with the action that was kicked off by the original
+// callback.
+
+extern grpc_tracer_flag grpc_call_combiner_trace;
+
+typedef struct {
+  gpr_atm size;  // size_t, num closures in queue or currently executing
+  gpr_mpscq queue;
+  // Either 0 (if not cancelled and no cancellation closure set),
+  // a grpc_closure* (if the lowest bit is 0),
+  // or a grpc_error* (if the lowest bit is 1).
+  gpr_atm cancel_state;
+} grpc_call_combiner;
+
+// Assumes memory was initialized to zero.
+void grpc_call_combiner_init(grpc_call_combiner* call_combiner);
+
+void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner);
+
+#ifndef NDEBUG
+#define GRPC_CALL_COMBINER_START(exec_ctx, call_combiner, closure, error,   \
+                                 reason)                                    \
+  grpc_call_combiner_start((exec_ctx), (call_combiner), (closure), (error), \
+                           __FILE__, __LINE__, (reason))
+#define GRPC_CALL_COMBINER_STOP(exec_ctx, call_combiner, reason)           \
+  grpc_call_combiner_stop((exec_ctx), (call_combiner), __FILE__, __LINE__, \
+                          (reason))
+/// Starts processing \a closure on \a call_combiner.
+void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx,
+                              grpc_call_combiner* call_combiner,
+                              grpc_closure* closure, grpc_error* error,
+                              const char* file, int line, const char* reason);
+/// Yields the call combiner to the next closure in the queue, if any.
+void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx,
+                             grpc_call_combiner* call_combiner,
+                             const char* file, int line, const char* reason);
+#else
+#define GRPC_CALL_COMBINER_START(exec_ctx, call_combiner, closure, error,   \
+                                 reason)                                    \
+  grpc_call_combiner_start((exec_ctx), (call_combiner), (closure), (error), \
+                           (reason))
+#define GRPC_CALL_COMBINER_STOP(exec_ctx, call_combiner, reason) \
+  grpc_call_combiner_stop((exec_ctx), (call_combiner), (reason))
+/// Starts processing \a closure on \a call_combiner.
+void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx,
+                              grpc_call_combiner* call_combiner,
+                              grpc_closure* closure, grpc_error* error,
+                              const char* reason);
+/// Yields the call combiner to the next closure in the queue, if any.
+void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx,
+                             grpc_call_combiner* call_combiner,
+                             const char* reason);
+#endif
+
+/// Registers \a closure to be invoked by \a call_combiner when
+/// grpc_call_combiner_cancel() is called.
+///
+/// Once a closure is registered, it will always be scheduled exactly
+/// once; this allows the closure to hold references that will be freed
+/// regardless of whether or not the call was cancelled.  If a cancellation
+/// does occur, the closure will be scheduled with the cancellation error;
+/// otherwise, it will be scheduled with GRPC_ERROR_NONE.
+///
+/// The closure will be scheduled in the following cases:
+/// - If grpc_call_combiner_cancel() was called prior to registering the
+///   closure, it will be scheduled immediately with the cancelation error.
+/// - If grpc_call_combiner_cancel() is called after registering the
+///   closure, the closure will be scheduled with the cancellation error.
+/// - If grpc_call_combiner_set_notify_on_cancel() is called again to
+///   register a new cancellation closure, the previous cancellation
+///   closure will be scheduled with GRPC_ERROR_NONE.
+///
+/// If \a closure is NULL, then no closure will be invoked on
+/// cancellation; this effectively unregisters the previously set closure.
+/// However, most filters will not need to explicitly unregister their
+/// callbacks, as this is done automatically when the call is destroyed.
+void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
+                                             grpc_call_combiner* call_combiner,
+                                             grpc_closure* closure);
+
+/// Indicates that the call has been cancelled.
+void grpc_call_combiner_cancel(grpc_exec_ctx* exec_ctx,
+                               grpc_call_combiner* call_combiner,
+                               grpc_error* error);
+
+#endif /* GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H */

src/core/lib/iomgr/combiner.c

@@ -24,6 +24,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/profiling/timers.h"
 
@@ -153,6 +154,7 @@ static void push_first_on_exec_ctx(grpc_exec_ctx *exec_ctx,
 
 static void combiner_exec(grpc_exec_ctx *exec_ctx, grpc_closure *cl,
                           grpc_error *error) {
+  GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_ITEMS(exec_ctx);
   GPR_TIMER_BEGIN("combiner.execute", 0);
   grpc_combiner *lock = COMBINER_FROM_CLOSURE_SCHEDULER(cl, scheduler);
   gpr_atm last = gpr_atm_full_fetch_add(&lock->state, STATE_ELEM_COUNT_LOW_BIT);
@@ -160,6 +162,7 @@ static void combiner_exec(grpc_exec_ctx *exec_ctx, grpc_closure *cl,
                               "C:%p grpc_combiner_execute c=%p last=%" PRIdPTR,
                               lock, cl, last));
   if (last == 1) {
+    GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx);
     gpr_atm_no_barrier_store(&lock->initiating_exec_ctx_or_null,
                              (gpr_atm)exec_ctx);
     // first element on this list: add it to the list of combiner locks
@@ -195,6 +198,7 @@ static void offload(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
 }
 
 static void queue_offload(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) {
+  GRPC_STATS_INC_COMBINER_LOCKS_OFFLOADED(exec_ctx);
   move_next(exec_ctx);
   GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, "C:%p queue_offload", lock));
   GRPC_CLOSURE_SCHED(exec_ctx, &lock->offload, GRPC_ERROR_NONE);
@@ -325,6 +329,7 @@ static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure,
 
 static void combiner_finally_exec(grpc_exec_ctx *exec_ctx,
                                   grpc_closure *closure, grpc_error *error) {
+  GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS(exec_ctx);
   grpc_combiner *lock =
       COMBINER_FROM_CLOSURE_SCHEDULER(closure, finally_scheduler);
   GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG,

src/core/lib/iomgr/ev_epoll1_linux.c

@@ -698,14 +698,21 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
         gpr_mu_unlock(&pollset->mu);
         goto retry_lock_neighbourhood;
       }
+
+      /* In the brief time we released the pollset locks above, the worker MAY
+         have been kicked. In this case, the worker should get out of this
+         pollset ASAP and hence this should neither add the pollset to
+         neighbourhood nor mark the pollset as active.
+
+         On a side note, the only way a worker's kick state could have changed
+         at this point is if it were "kicked specifically". Since the worker has
+         not added itself to the pollset yet (by calling worker_insert()), it is
+         not visible in the "kick any" path yet */
+      if (worker->kick_state == UNKICKED) {
         pollset->seen_inactive = false;
         if (neighbourhood->active_root == NULL) {
           neighbourhood->active_root = pollset->next = pollset->prev = pollset;
-        /* TODO: sreek. Why would this worker state be other than UNKICKED
+          /* Make this the designated poller if there isn't one already */
-         * here ? (since the worker isn't added to the pollset yet, there is no
-         * way it can be "found" by other threads to get kicked). */
-
-        /* If there is no designated poller, make this the designated poller */
           if (worker->kick_state == UNKICKED &&
               gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) {
             SET_KICK_STATE(worker, DESIGNATED_POLLER);
@@ -716,6 +723,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
           pollset->next->prev = pollset->prev->next = pollset;
         }
       }
+    }
     if (is_reassigning) {
       GPR_ASSERT(pollset->reassigning_neighbourhood);
       pollset->reassigning_neighbourhood = false;
@@ -1001,6 +1009,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset,
     gpr_log(GPR_ERROR, "%s", tmp);
     gpr_free(tmp);
   }
+
   if (specific_worker == NULL) {
     if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) {
       grpc_pollset_worker *root_worker = pollset->root_worker;
@@ -1076,7 +1085,11 @@ static grpc_error *pollset_kick(grpc_pollset *pollset,
       }
       goto done;
     }
-  } else if (specific_worker->kick_state == KICKED) {
+
+    GPR_UNREACHABLE_CODE(goto done);
+  }
+
+  if (specific_worker->kick_state == KICKED) {
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
       gpr_log(GPR_ERROR, " .. specific worker already kicked");
     }

src/core/lib/iomgr/ev_epoll_thread_pool_linux.h

@@ -1,28 +0,0 @@
-/*
- *
- * Copyright 2017 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H
-#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H
-
-#include "src/core/lib/iomgr/ev_posix.h"
-#include "src/core/lib/iomgr/port.h"
-
-const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(
-    bool requested_explicitly);
-
-#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H */

src/core/lib/iomgr/ev_posix.c

@@ -31,8 +31,6 @@
 
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/ev_epoll1_linux.h"
-#include "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h"
-#include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h"
 #include "src/core/lib/iomgr/ev_epollex_linux.h"
 #include "src/core/lib/iomgr/ev_epollsig_linux.h"
 #include "src/core/lib/iomgr/ev_poll_posix.h"
@@ -66,8 +64,6 @@ typedef struct {
 static const event_engine_factory g_factories[] = {
     {"epoll1", grpc_init_epoll1_linux},
     {"epollsig", grpc_init_epollsig_linux},
-    {"epoll-threadpool", grpc_init_epoll_thread_pool_linux},
-    {"epoll-limited", grpc_init_epoll_limited_pollers_linux},
     {"poll", grpc_init_poll_posix},
     {"poll-cv", grpc_init_poll_cv_posix},
     {"epollex", grpc_init_epollex_linux},

src/core/lib/iomgr/executor.c

@@ -28,6 +28,7 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/support/spinlock.h"
 
@@ -145,6 +146,7 @@ static void executor_thread(void *arg) {
       gpr_mu_unlock(&ts->mu);
       break;
     }
+    GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx);
     grpc_closure_list exec = ts->elems;
     ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT;
     gpr_mu_unlock(&ts->mu);
@@ -158,6 +160,7 @@ static void executor_thread(void *arg) {
 static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
                           grpc_error *error) {
   size_t cur_thread_count = (size_t)gpr_atm_no_barrier_load(&g_cur_threads);
+  GRPC_STATS_INC_EXECUTOR_SCHEDULED_ITEMS(exec_ctx);
   if (cur_thread_count == 0) {
     grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
     return;
@@ -165,9 +168,12 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
   thread_state *ts = (thread_state *)gpr_tls_get(&g_this_thread_state);
   if (ts == NULL) {
     ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)];
+  } else {
+    GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx);
   }
   gpr_mu_lock(&ts->mu);
   if (grpc_closure_list_empty(ts->elems)) {
+    GRPC_STATS_INC_EXECUTOR_WAKEUP_INITIATED(exec_ctx);
     gpr_cv_signal(&ts->cv);
   }
   grpc_closure_list_append(&ts->elems, closure, error);

src/core/lib/iomgr/iomgr.c

@@ -164,13 +164,7 @@ void grpc_iomgr_unregister_object(grpc_iomgr_object *obj) {
 
 bool grpc_iomgr_abort_on_leaks(void) {
   char *env = gpr_getenv("GRPC_ABORT_ON_LEAKS");
-  if (env == NULL) return false;
+  bool should_we = gpr_is_true(env);
-  static const char *truthy[] = {"yes",  "Yes",  "YES", "true",
-                                 "True", "TRUE", "1"};
-  bool should_we = false;
-  for (size_t i = 0; i < GPR_ARRAY_SIZE(truthy); i++) {
-    if (0 == strcmp(env, truthy[i])) should_we = true;
-  }
   gpr_free(env);
   return should_we;
 }

src/core/lib/iomgr/tcp_posix.c

@@ -67,7 +67,6 @@ typedef struct {
   grpc_fd *em_fd;
   int fd;
   bool finished_edge;
-  msg_iovlen_type iov_size; /* Number of slices to allocate per read attempt */
   double target_length;
   double bytes_read_this_round;
   gpr_refcount refcount;
@@ -240,7 +239,6 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   size_t i;
 
   GPR_ASSERT(!tcp->finished_edge);
-  GPR_ASSERT(tcp->iov_size <= MAX_READ_IOVEC);
   GPR_ASSERT(tcp->incoming_buffer->count <= MAX_READ_IOVEC);
   GPR_TIMER_BEGIN("tcp_continue_read", 0);
 
@@ -252,11 +250,14 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   msg.msg_name = NULL;
   msg.msg_namelen = 0;
   msg.msg_iov = iov;
-  msg.msg_iovlen = tcp->iov_size;
+  msg.msg_iovlen = (msg_iovlen_type)tcp->incoming_buffer->count;
   msg.msg_control = NULL;
   msg.msg_controllen = 0;
   msg.msg_flags = 0;
 
+  GRPC_STATS_INC_TCP_READ_OFFER(exec_ctx, tcp->incoming_buffer->length);
+  GRPC_STATS_INC_TCP_READ_IOV_SIZE(exec_ctx, tcp->incoming_buffer->count);
+
   GPR_TIMER_BEGIN("recvmsg", 0);
   do {
     GRPC_STATS_INC_SYSCALL_READ(exec_ctx);
@@ -287,6 +288,7 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
             GRPC_ERROR_CREATE_FROM_STATIC_STRING("Socket closed"), tcp));
     TCP_UNREF(exec_ctx, tcp, "read");
   } else {
+    GRPC_STATS_INC_TCP_READ_SIZE(exec_ctx, read_bytes);
     add_to_estimate(tcp, (size_t)read_bytes);
     GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length);
     if ((size_t)read_bytes < tcp->incoming_buffer->length) {
@@ -403,6 +405,9 @@ static bool tcp_flush(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp,
     msg.msg_controllen = 0;
     msg.msg_flags = 0;
 
+    GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, sending_length);
+    GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(exec_ctx, iov_size);
+
     GPR_TIMER_BEGIN("sendmsg", 1);
     do {
       /* TODO(klempner): Cork if this is a partial write */
@@ -621,7 +626,6 @@ grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_fd *em_fd,
   tcp->min_read_chunk_size = tcp_min_read_chunk_size;
   tcp->max_read_chunk_size = tcp_max_read_chunk_size;
   tcp->bytes_read_this_round = 0;
-  tcp->iov_size = 1;
   tcp->finished_edge = true;
   /* paired with unref in grpc_tcp_destroy */
   gpr_ref_init(&tcp->refcount, 1);

src/core/lib/iomgr/timer.h

@@ -44,6 +44,10 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
                      gpr_timespec deadline, grpc_closure *closure,
                      gpr_timespec now);
 
+/* Initialize *timer without setting it. This can later be passed through
+   the regular init or cancel */
+void grpc_timer_init_unset(grpc_timer *timer);
+
 /* Note that there is no timer destroy function. This is because the
    timer is a one-time occurrence with a guarantee that the callback will
    be called exactly once, either at expiration or cancellation. Thus, all

src/core/lib/iomgr/timer_generic.c

@@ -234,6 +234,8 @@ static void note_deadline_change(timer_shard *shard) {
   }
 }
 
+void grpc_timer_init_unset(grpc_timer *timer) { timer->pending = false; }
+
 void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
                      gpr_timespec deadline, grpc_closure *closure,
                      gpr_timespec now) {

src/core/lib/iomgr/timer_uv.c

@@ -77,6 +77,8 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
   uv_unref((uv_handle_t *)uv_timer);
 }
 
+void grpc_timer_init_unset(grpc_timer *timer) { timer->pending = 0; }
+
 void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) {
   GRPC_UV_ASSERT_SAME_THREAD();
   if (timer->pending) {

src/core/lib/security/transport/client_auth_filter.c

@@ -39,6 +39,8 @@
 
 /* We can have a per-call credentials. */
 typedef struct {
+  grpc_call_stack *owning_call;
+  grpc_call_combiner *call_combiner;
   grpc_call_credentials *creds;
   bool have_host;
   bool have_method;
@@ -49,17 +51,12 @@ typedef struct {
      pollset_set so that work can progress when this call wants work to progress
   */
   grpc_polling_entity *pollent;
-  gpr_atm security_context_set;
-  gpr_mu security_context_mu;
   grpc_credentials_mdelem_array md_array;
   grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT];
   grpc_auth_metadata_context auth_md_context;
-  grpc_closure closure;
+  grpc_closure async_result_closure;
-  // Either 0 (no cancellation and no async operation in flight),
+  grpc_closure check_call_host_cancel_closure;
-  // a grpc_closure* (if the lowest bit is 0),
+  grpc_closure get_request_metadata_cancel_closure;
-  // or a grpc_error* (if the lowest bit is 1).
-  gpr_atm cancellation_state;
-  grpc_closure cancel_closure;
 } call_data;
 
 /* We can have a per-channel credentials. */
@@ -68,43 +65,6 @@ typedef struct {
   grpc_auth_context *auth_context;
 } channel_data;
 
-static void decode_cancel_state(gpr_atm cancel_state, grpc_closure **func,
-                                grpc_error **error) {
-  // If the lowest bit is 1, the value is a grpc_error*.
-  // Otherwise, if non-zdero, the value is a grpc_closure*.
-  if (cancel_state & 1) {
-    *error = (grpc_error *)(cancel_state & ~(gpr_atm)1);
-  } else if (cancel_state != 0) {
-    *func = (grpc_closure *)cancel_state;
-  }
-}
-
-static gpr_atm encode_cancel_state_error(grpc_error *error) {
-  // Set the lowest bit to 1 to indicate that it's an error.
-  return (gpr_atm)1 | (gpr_atm)error;
-}
-
-// Returns an error if the call has been cancelled.  Otherwise, sets the
-// cancellation function to be called upon cancellation.
-static grpc_error *set_cancel_func(grpc_call_element *elem,
-                                   grpc_iomgr_cb_func func) {
-  call_data *calld = (call_data *)elem->call_data;
-  // Decode original state.
-  gpr_atm original_state = gpr_atm_acq_load(&calld->cancellation_state);
-  grpc_error *original_error = GRPC_ERROR_NONE;
-  grpc_closure *original_func = NULL;
-  decode_cancel_state(original_state, &original_func, &original_error);
-  // If error is set, return it.
-  if (original_error != GRPC_ERROR_NONE) return GRPC_ERROR_REF(original_error);
-  // Otherwise, store func.
-  GRPC_CLOSURE_INIT(&calld->cancel_closure, func, elem,
-                    grpc_schedule_on_exec_ctx);
-  GPR_ASSERT(((gpr_atm)&calld->cancel_closure & (gpr_atm)1) == 0);
-  gpr_atm_rel_store(&calld->cancellation_state,
-                    (gpr_atm)&calld->cancel_closure);
-  return GRPC_ERROR_NONE;
-}
-
 static void reset_auth_metadata_context(
     grpc_auth_metadata_context *auth_md_context) {
   if (auth_md_context->service_url != NULL) {
@@ -153,7 +113,8 @@ static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *arg,
   } else {
     error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS,
                                GRPC_STATUS_UNAUTHENTICATED);
-    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error);
+    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error,
+                                                       calld->call_combiner);
   }
 }
 
@@ -191,9 +152,13 @@ static void cancel_get_request_metadata(grpc_exec_ctx *exec_ctx, void *arg,
                                         grpc_error *error) {
   grpc_call_element *elem = (grpc_call_element *)arg;
   call_data *calld = (call_data *)elem->call_data;
+  if (error != GRPC_ERROR_NONE) {
     grpc_call_credentials_cancel_get_request_metadata(
         exec_ctx, calld->creds, &calld->md_array, GRPC_ERROR_REF(error));
   }
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call,
+                        "cancel_get_request_metadata");
+}
 
 static void send_security_metadata(grpc_exec_ctx *exec_ctx,
                                    grpc_call_element *elem,
@@ -223,7 +188,8 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
           grpc_error_set_int(
               GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                   "Incompatible credentials set on channel and call."),
-              GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED));
+              GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED),
+          calld->call_combiner);
       return;
     }
   } else {
@@ -234,22 +200,25 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
   build_auth_metadata_context(&chand->security_connector->base,
                               chand->auth_context, calld);
 
-  grpc_error *cancel_error = set_cancel_func(elem, cancel_get_request_metadata);
-  if (cancel_error != GRPC_ERROR_NONE) {
-    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch,
-                                                       cancel_error);
-    return;
-  }
   GPR_ASSERT(calld->pollent != NULL);
-  GRPC_CLOSURE_INIT(&calld->closure, on_credentials_metadata, batch,
+
-                    grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&calld->async_result_closure, on_credentials_metadata,
+                    batch, grpc_schedule_on_exec_ctx);
   grpc_error *error = GRPC_ERROR_NONE;
   if (grpc_call_credentials_get_request_metadata(
           exec_ctx, calld->creds, calld->pollent, calld->auth_md_context,
-          &calld->md_array, &calld->closure, &error)) {
+          &calld->md_array, &calld->async_result_closure, &error)) {
     // Synchronous return; invoke on_credentials_metadata() directly.
     on_credentials_metadata(exec_ctx, batch, error);
     GRPC_ERROR_UNREF(error);
+  } else {
+    // Async return; register cancellation closure with call combiner.
+    GRPC_CALL_STACK_REF(calld->owning_call, "cancel_get_request_metadata");
+    grpc_call_combiner_set_notify_on_cancel(
+        exec_ctx, calld->call_combiner,
+        GRPC_CLOSURE_INIT(&calld->get_request_metadata_cancel_closure,
+                          cancel_get_request_metadata, elem,
+                          grpc_schedule_on_exec_ctx));
   }
 }
 
@@ -258,7 +227,6 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *arg,
   grpc_transport_stream_op_batch *batch = (grpc_transport_stream_op_batch *)arg;
   grpc_call_element *elem = batch->handler_private.extra_arg;
   call_data *calld = elem->call_data;
-
   if (error == GRPC_ERROR_NONE) {
     send_security_metadata(exec_ctx, elem, batch);
   } else {
@@ -271,7 +239,8 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *arg,
         exec_ctx, batch,
         grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg),
                            GRPC_ERROR_INT_GRPC_STATUS,
-                           GRPC_STATUS_UNAUTHENTICATED));
+                           GRPC_STATUS_UNAUTHENTICATED),
+        calld->call_combiner);
     gpr_free(error_msg);
   }
 }
@@ -281,10 +250,13 @@ static void cancel_check_call_host(grpc_exec_ctx *exec_ctx, void *arg,
   grpc_call_element *elem = (grpc_call_element *)arg;
   call_data *calld = (call_data *)elem->call_data;
   channel_data *chand = (channel_data *)elem->channel_data;
+  if (error != GRPC_ERROR_NONE) {
     grpc_channel_security_connector_cancel_check_call_host(
-      exec_ctx, chand->security_connector, &calld->closure,
+        exec_ctx, chand->security_connector, &calld->async_result_closure,
         GRPC_ERROR_REF(error));
   }
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "cancel_check_call_host");
+}
 
 static void auth_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
@@ -295,36 +267,7 @@ static void auth_start_transport_stream_op_batch(
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
 
-  if (batch->cancel_stream) {
+  if (!batch->cancel_stream) {
-    while (true) {
-      // Decode the original cancellation state.
-      gpr_atm original_state = gpr_atm_acq_load(&calld->cancellation_state);
-      grpc_error *cancel_error = GRPC_ERROR_NONE;
-      grpc_closure *func = NULL;
-      decode_cancel_state(original_state, &func, &cancel_error);
-      // If we had already set a cancellation error, there's nothing
-      // more to do.
-      if (cancel_error != GRPC_ERROR_NONE) break;
-      // If there's a cancel func, call it.
-      // Note that even if the cancel func has been changed by some
-      // other thread between when we decoded it and now, it will just
-      // be a no-op.
-      cancel_error = GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error);
-      if (func != NULL) {
-        GRPC_CLOSURE_SCHED(exec_ctx, func, GRPC_ERROR_REF(cancel_error));
-      }
-      // Encode the new error into cancellation state.
-      if (gpr_atm_full_cas(&calld->cancellation_state, original_state,
-                           encode_cancel_state_error(cancel_error))) {
-        break;  // Success.
-      }
-      // The cas failed, so try again.
-    }
-  } else {
-    /* double checked lock over security context to ensure it's set once */
-    if (gpr_atm_acq_load(&calld->security_context_set) == 0) {
-      gpr_mu_lock(&calld->security_context_mu);
-      if (gpr_atm_acq_load(&calld->security_context_set) == 0) {
     GPR_ASSERT(batch->payload->context != NULL);
     if (batch->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) {
       batch->payload->context[GRPC_CONTEXT_SECURITY].value =
@@ -337,10 +280,6 @@ static void auth_start_transport_stream_op_batch(
     GRPC_AUTH_CONTEXT_UNREF(sec_ctx->auth_context, "client auth filter");
     sec_ctx->auth_context =
         GRPC_AUTH_CONTEXT_REF(chand->auth_context, "client_auth_filter");
-        gpr_atm_rel_store(&calld->security_context_set, 1);
-      }
-      gpr_mu_unlock(&calld->security_context_mu);
-    }
   }
 
   if (batch->send_initial_metadata) {
@@ -365,26 +304,27 @@ static void auth_start_transport_stream_op_batch(
       }
     }
     if (calld->have_host) {
-      grpc_error *cancel_error = set_cancel_func(elem, cancel_check_call_host);
-      if (cancel_error != GRPC_ERROR_NONE) {
-        grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch,
-                                                           cancel_error);
-      } else {
-        char *call_host = grpc_slice_to_c_string(calld->host);
       batch->handler_private.extra_arg = elem;
+      GRPC_CLOSURE_INIT(&calld->async_result_closure, on_host_checked, batch,
+                        grpc_schedule_on_exec_ctx);
+      char *call_host = grpc_slice_to_c_string(calld->host);
       grpc_error *error = GRPC_ERROR_NONE;
       if (grpc_channel_security_connector_check_call_host(
               exec_ctx, chand->security_connector, call_host,
-                chand->auth_context,
+              chand->auth_context, &calld->async_result_closure, &error)) {
-                GRPC_CLOSURE_INIT(&calld->closure, on_host_checked, batch,
-                                  grpc_schedule_on_exec_ctx),
-                &error)) {
         // Synchronous return; invoke on_host_checked() directly.
         on_host_checked(exec_ctx, batch, error);
         GRPC_ERROR_UNREF(error);
+      } else {
+        // Async return; register cancellation closure with call combiner.
+        GRPC_CALL_STACK_REF(calld->owning_call, "cancel_check_call_host");
+        grpc_call_combiner_set_notify_on_cancel(
+            exec_ctx, calld->call_combiner,
+            GRPC_CLOSURE_INIT(&calld->check_call_host_cancel_closure,
+                              cancel_check_call_host, elem,
+                              grpc_schedule_on_exec_ctx));
       }
       gpr_free(call_host);
-      }
       GPR_TIMER_END("auth_start_transport_stream_op_batch", 0);
       return; /* early exit */
     }
@@ -400,8 +340,8 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
   call_data *calld = elem->call_data;
-  memset(calld, 0, sizeof(*calld));
+  calld->owning_call = args->call_stack;
-  gpr_mu_init(&calld->security_context_mu);
+  calld->call_combiner = args->call_combiner;
   return GRPC_ERROR_NONE;
 }
 
@@ -426,12 +366,6 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_slice_unref_internal(exec_ctx, calld->method);
   }
   reset_auth_metadata_context(&calld->auth_md_context);
-  gpr_mu_destroy(&calld->security_context_mu);
-  gpr_atm cancel_state = gpr_atm_acq_load(&calld->cancellation_state);
-  grpc_error *cancel_error = GRPC_ERROR_NONE;
-  grpc_closure *cancel_func = NULL;
-  decode_cancel_state(cancel_state, &cancel_func, &cancel_error);
-  GRPC_ERROR_UNREF(cancel_error);
 }
 
 /* Constructor for channel_data */
@@ -490,6 +424,5 @@ const grpc_channel_filter grpc_client_auth_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "client-auth"};

src/core/lib/security/transport/secure_endpoint.c

@@ -34,7 +34,7 @@
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
 #include "src/core/lib/support/string.h"
-#include "src/core/tsi/transport_security_interface.h"
+#include "src/core/tsi/transport_security_grpc.h"
 
 #define STAGING_BUFFER_SIZE 8192
 
@@ -42,6 +42,7 @@ typedef struct {
   grpc_endpoint base;
   grpc_endpoint *wrapped_ep;
   struct tsi_frame_protector *protector;
+  struct tsi_zero_copy_grpc_protector *zero_copy_protector;
   gpr_mu protector_mu;
   /* saved upper level callbacks and user_data. */
   grpc_closure *read_cb;
@@ -67,6 +68,7 @@ static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) {
   secure_endpoint *ep = secure_ep;
   grpc_endpoint_destroy(exec_ctx, ep->wrapped_ep);
   tsi_frame_protector_destroy(ep->protector);
+  tsi_zero_copy_grpc_protector_destroy(exec_ctx, ep->zero_copy_protector);
   grpc_slice_buffer_destroy_internal(exec_ctx, &ep->leftover_bytes);
   grpc_slice_unref_internal(exec_ctx, ep->read_staging_buffer);
   grpc_slice_unref_internal(exec_ctx, ep->write_staging_buffer);
@@ -159,6 +161,12 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
     return;
   }
 
+  if (ep->zero_copy_protector != NULL) {
+    // Use zero-copy grpc protector to unprotect.
+    result = tsi_zero_copy_grpc_protector_unprotect(
+        exec_ctx, ep->zero_copy_protector, &ep->source_buffer, ep->read_buffer);
+  } else {
+    // Use frame protector to unprotect.
     /* TODO(yangg) check error, maybe bail out early */
     for (i = 0; i < ep->source_buffer.count; i++) {
       grpc_slice encrypted = ep->source_buffer.slices[i];
@@ -169,8 +177,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
         size_t unprotected_buffer_size_written = (size_t)(end - cur);
         size_t processed_message_size = message_size;
         gpr_mu_lock(&ep->protector_mu);
-      result = tsi_frame_protector_unprotect(ep->protector, message_bytes,
+        result = tsi_frame_protector_unprotect(
-                                             &processed_message_size, cur,
+            ep->protector, message_bytes, &processed_message_size, cur,
             &unprotected_buffer_size_written);
         gpr_mu_unlock(&ep->protector_mu);
         if (result != TSI_OK) {
@@ -184,10 +192,10 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
 
         if (cur == end) {
           flush_read_staging_buffer(ep, &cur, &end);
-        /* Force to enter the loop again to extract buffered bytes in protector.
+          /* Force to enter the loop again to extract buffered bytes in
-           The bytes could be buffered because of running out of staging_buffer.
+             protector. The bytes could be buffered because of running out of
-           If this happens at the end of all slices, doing another unprotect
+             staging_buffer. If this happens at the end of all slices, doing
-           avoids leaving data in the protector. */
+             another unprotect avoids leaving data in the protector. */
           keep_looping = 1;
         } else if (unprotected_buffer_size_written > 0) {
           keep_looping = 1;
@@ -205,6 +213,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data,
               &ep->read_staging_buffer,
               (size_t)(cur - GRPC_SLICE_START_PTR(ep->read_staging_buffer))));
     }
+  }
 
   /* TODO(yangg) experiment with moving this block after read_cb to see if it
      helps latency */
@@ -270,6 +279,12 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
     }
   }
 
+  if (ep->zero_copy_protector != NULL) {
+    // Use zero-copy grpc protector to protect.
+    result = tsi_zero_copy_grpc_protector_protect(
+        exec_ctx, ep->zero_copy_protector, slices, &ep->output_buffer);
+  } else {
+    // Use frame protector to protect.
     for (i = 0; i < slices->count; i++) {
       grpc_slice plain = slices->slices[i];
       uint8_t *message_bytes = GRPC_SLICE_START_PTR(plain);
@@ -302,8 +317,8 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
       do {
         size_t protected_buffer_size_to_send = (size_t)(end - cur);
         gpr_mu_lock(&ep->protector_mu);
-      result = tsi_frame_protector_protect_flush(ep->protector, cur,
+        result = tsi_frame_protector_protect_flush(
-                                                 &protected_buffer_size_to_send,
+            ep->protector, cur, &protected_buffer_size_to_send,
             &still_pending_size);
         gpr_mu_unlock(&ep->protector_mu);
         if (result != TSI_OK) break;
@@ -317,7 +332,9 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
             &ep->output_buffer,
             grpc_slice_split_head(
                 &ep->write_staging_buffer,
-              (size_t)(cur - GRPC_SLICE_START_PTR(ep->write_staging_buffer))));
+                (size_t)(cur -
+                         GRPC_SLICE_START_PTR(ep->write_staging_buffer))));
+      }
     }
   }
 
@@ -389,13 +406,16 @@ static const grpc_endpoint_vtable vtable = {endpoint_read,
                                             endpoint_get_fd};
 
 grpc_endpoint *grpc_secure_endpoint_create(
-    struct tsi_frame_protector *protector, grpc_endpoint *transport,
+    struct tsi_frame_protector *protector,
-    grpc_slice *leftover_slices, size_t leftover_nslices) {
+    struct tsi_zero_copy_grpc_protector *zero_copy_protector,
+    grpc_endpoint *transport, grpc_slice *leftover_slices,
+    size_t leftover_nslices) {
   size_t i;
   secure_endpoint *ep = (secure_endpoint *)gpr_malloc(sizeof(secure_endpoint));
   ep->base.vtable = &vtable;
   ep->wrapped_ep = transport;
   ep->protector = protector;
+  ep->zero_copy_protector = zero_copy_protector;
   grpc_slice_buffer_init(&ep->leftover_bytes);
   for (i = 0; i < leftover_nslices; i++) {
     grpc_slice_buffer_add(&ep->leftover_bytes,

src/core/lib/security/transport/secure_endpoint.h

@@ -23,12 +23,17 @@
 #include "src/core/lib/iomgr/endpoint.h"
 
 struct tsi_frame_protector;
+struct tsi_zero_copy_grpc_protector;
 
 extern grpc_tracer_flag grpc_trace_secure_endpoint;
 
-/* Takes ownership of protector and to_wrap, and refs leftover_slices. */
+/* Takes ownership of protector, zero_copy_protector, and to_wrap, and refs
+ * leftover_slices. If zero_copy_protector is not NULL, protector will never be
+ * used. */
 grpc_endpoint *grpc_secure_endpoint_create(
-    struct tsi_frame_protector *protector, grpc_endpoint *to_wrap,
+    struct tsi_frame_protector *protector,
-    grpc_slice *leftover_slices, size_t leftover_nslices);
+    struct tsi_zero_copy_grpc_protector *zero_copy_protector,
+    grpc_endpoint *to_wrap, grpc_slice *leftover_slices,
+    size_t leftover_nslices);
 
 #endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H */

src/core/lib/security/transport/security_handshaker.c

@@ -32,6 +32,7 @@
 #include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/security/transport/tsi_error.h"
 #include "src/core/lib/slice/slice_internal.h"
+#include "src/core/tsi/transport_security_grpc.h"
 
 #define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
 
@@ -135,17 +136,31 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *arg,
     security_handshake_failed_locked(exec_ctx, h, GRPC_ERROR_REF(error));
     goto done;
   }
-  // Create frame protector.
+  // Create zero-copy frame protector, if implemented.
-  tsi_frame_protector *protector;
+  tsi_zero_copy_grpc_protector *zero_copy_protector = NULL;
-  tsi_result result = tsi_handshaker_result_create_frame_protector(
+  tsi_result result = tsi_handshaker_result_create_zero_copy_grpc_protector(
-      h->handshaker_result, NULL, &protector);
+      h->handshaker_result, NULL, &zero_copy_protector);
-  if (result != TSI_OK) {
+  if (result != TSI_OK && result != TSI_UNIMPLEMENTED) {
     error = grpc_set_tsi_error_result(
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Frame protector creation failed"),
+        GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "Zero-copy frame protector creation failed"),
+        result);
+    security_handshake_failed_locked(exec_ctx, h, error);
+    goto done;
+  }
+  // Create frame protector if zero-copy frame protector is NULL.
+  tsi_frame_protector *protector = NULL;
+  if (zero_copy_protector == NULL) {
+    result = tsi_handshaker_result_create_frame_protector(h->handshaker_result,
+                                                          NULL, &protector);
+    if (result != TSI_OK) {
+      error = grpc_set_tsi_error_result(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                                            "Frame protector creation failed"),
                                         result);
       security_handshake_failed_locked(exec_ctx, h, error);
       goto done;
     }
+  }
   // Get unused bytes.
   const unsigned char *unused_bytes = NULL;
   size_t unused_bytes_size = 0;
@@ -155,12 +170,12 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *arg,
   if (unused_bytes_size > 0) {
     grpc_slice slice =
         grpc_slice_from_copied_buffer((char *)unused_bytes, unused_bytes_size);
-    h->args->endpoint =
+    h->args->endpoint = grpc_secure_endpoint_create(
-        grpc_secure_endpoint_create(protector, h->args->endpoint, &slice, 1);
+        protector, zero_copy_protector, h->args->endpoint, &slice, 1);
     grpc_slice_unref_internal(exec_ctx, slice);
   } else {
-    h->args->endpoint =
+    h->args->endpoint = grpc_secure_endpoint_create(
-        grpc_secure_endpoint_create(protector, h->args->endpoint, NULL, 0);
+        protector, zero_copy_protector, h->args->endpoint, NULL, 0);
   }
   tsi_handshaker_result_destroy(h->handshaker_result);
   h->handshaker_result = NULL;

src/core/lib/security/transport/server_auth_filter.c

@@ -26,7 +26,15 @@
 #include "src/core/lib/security/transport/auth_filters.h"
 #include "src/core/lib/slice/slice_internal.h"
 
+typedef enum {
+  STATE_INIT = 0,
+  STATE_DONE,
+  STATE_CANCELLED,
+} async_state;
+
 typedef struct call_data {
+  grpc_call_combiner *call_combiner;
+  grpc_call_stack *owning_call;
   grpc_transport_stream_op_batch *recv_initial_metadata_batch;
   grpc_closure *original_recv_initial_metadata_ready;
   grpc_closure recv_initial_metadata_ready;
@@ -34,6 +42,8 @@ typedef struct call_data {
   const grpc_metadata *consumed_md;
   size_t num_consumed_md;
   grpc_auth_context *auth_context;
+  grpc_closure cancel_closure;
+  gpr_atm state;  // async_state
 } call_data;
 
 typedef struct channel_data {
@@ -78,54 +88,94 @@ static grpc_filtered_mdelem remove_consumed_md(grpc_exec_ctx *exec_ctx,
   return GRPC_FILTERED_MDELEM(md);
 }
 
-/* called from application code */
+static void on_md_processing_done_inner(grpc_exec_ctx *exec_ctx,
-static void on_md_processing_done(
+                                        grpc_call_element *elem,
-    void *user_data, const grpc_metadata *consumed_md, size_t num_consumed_md,
+                                        const grpc_metadata *consumed_md,
-    const grpc_metadata *response_md, size_t num_response_md,
+                                        size_t num_consumed_md,
-    grpc_status_code status, const char *error_details) {
+                                        const grpc_metadata *response_md,
-  grpc_call_element *elem = user_data;
+                                        size_t num_response_md,
+                                        grpc_error *error) {
   call_data *calld = elem->call_data;
   grpc_transport_stream_op_batch *batch = calld->recv_initial_metadata_batch;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   /* TODO(jboeuf): Implement support for response_md. */
   if (response_md != NULL && num_response_md > 0) {
     gpr_log(GPR_INFO,
             "response_md in auth metadata processing not supported for now. "
             "Ignoring...");
   }
-  grpc_error *error = GRPC_ERROR_NONE;
+  if (error == GRPC_ERROR_NONE) {
-  if (status == GRPC_STATUS_OK) {
     calld->consumed_md = consumed_md;
     calld->num_consumed_md = num_consumed_md;
     error = grpc_metadata_batch_filter(
-        &exec_ctx, batch->payload->recv_initial_metadata.recv_initial_metadata,
+        exec_ctx, batch->payload->recv_initial_metadata.recv_initial_metadata,
         remove_consumed_md, elem, "Response metadata filtering error");
-  } else {
+  }
+  GRPC_CLOSURE_SCHED(exec_ctx, calld->original_recv_initial_metadata_ready,
+                     error);
+}
+
+// Called from application code.
+static void on_md_processing_done(
+    void *user_data, const grpc_metadata *consumed_md, size_t num_consumed_md,
+    const grpc_metadata *response_md, size_t num_response_md,
+    grpc_status_code status, const char *error_details) {
+  grpc_call_element *elem = user_data;
+  call_data *calld = elem->call_data;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  // If the call was not cancelled while we were in flight, process the result.
+  if (gpr_atm_full_cas(&calld->state, (gpr_atm)STATE_INIT,
+                       (gpr_atm)STATE_DONE)) {
+    grpc_error *error = GRPC_ERROR_NONE;
+    if (status != GRPC_STATUS_OK) {
       if (error_details == NULL) {
         error_details = "Authentication metadata processing failed.";
       }
-    error =
+      error = grpc_error_set_int(
-        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_details),
+          GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_details),
           GRPC_ERROR_INT_GRPC_STATUS, status);
     }
+    on_md_processing_done_inner(&exec_ctx, elem, consumed_md, num_consumed_md,
+                                response_md, num_response_md, error);
+  }
+  // Clean up.
   for (size_t i = 0; i < calld->md.count; i++) {
     grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].key);
     grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].value);
   }
   grpc_metadata_array_destroy(&calld->md);
-  GRPC_CLOSURE_SCHED(&exec_ctx, calld->original_recv_initial_metadata_ready,
+  GRPC_CALL_STACK_UNREF(&exec_ctx, calld->owning_call, "server_auth_metadata");
-                     error);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
+static void cancel_call(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = elem->call_data;
+  // If the result was not already processed, invoke the callback now.
+  if (error != GRPC_ERROR_NONE &&
+      gpr_atm_full_cas(&calld->state, (gpr_atm)STATE_INIT,
+                       (gpr_atm)STATE_CANCELLED)) {
+    on_md_processing_done_inner(exec_ctx, elem, NULL, 0, NULL, 0,
+                                GRPC_ERROR_REF(error));
+  }
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "cancel_call");
+}
+
 static void recv_initial_metadata_ready(grpc_exec_ctx *exec_ctx, void *arg,
                                         grpc_error *error) {
-  grpc_call_element *elem = arg;
+  grpc_call_element *elem = (grpc_call_element *)arg;
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
   grpc_transport_stream_op_batch *batch = calld->recv_initial_metadata_batch;
   if (error == GRPC_ERROR_NONE) {
     if (chand->creds != NULL && chand->creds->processor.process != NULL) {
+      // We're calling out to the application, so we need to make sure
+      // to drop the call combiner early if we get cancelled.
+      GRPC_CALL_STACK_REF(calld->owning_call, "cancel_call");
+      GRPC_CLOSURE_INIT(&calld->cancel_closure, cancel_call, elem,
+                        grpc_schedule_on_exec_ctx);
+      grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,
+                                              &calld->cancel_closure);
+      GRPC_CALL_STACK_REF(calld->owning_call, "server_auth_metadata");
       calld->md = metadata_batch_to_md_array(
           batch->payload->recv_initial_metadata.recv_initial_metadata);
       chand->creds->processor.process(
@@ -159,6 +209,8 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   const grpc_call_element_args *args) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
+  calld->call_combiner = args->call_combiner;
+  calld->owning_call = args->call_stack;
   GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready,
                     recv_initial_metadata_ready, elem,
                     grpc_schedule_on_exec_ctx);
@@ -218,6 +270,5 @@ const grpc_channel_filter grpc_server_auth_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "server-auth"};

src/core/lib/support/string.c

@@ -298,3 +298,16 @@ void *gpr_memrchr(const void *s, int c, size_t n) {
   }
   return NULL;
 }
+
+bool gpr_is_true(const char *s) {
+  if (s == NULL) {
+    return false;
+  }
+  static const char *truthy[] = {"yes", "true", "1"};
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(truthy); i++) {
+    if (0 == gpr_stricmp(s, truthy[i])) {
+      return true;
+    }
+  }
+  return false;
+}

src/core/lib/support/string.h

@@ -19,6 +19,7 @@
 #ifndef GRPC_CORE_LIB_SUPPORT_STRING_H
 #define GRPC_CORE_LIB_SUPPORT_STRING_H
 
+#include <stdbool.h>
 #include <stddef.h>
 
 #include <grpc/support/port_platform.h>
@@ -106,6 +107,8 @@ int gpr_stricmp(const char *a, const char *b);
 
 void *gpr_memrchr(const void *s, int c, size_t n);
 
+/** Return true if lower(s) equals "true", "yes" or "1", otherwise false. */
+bool gpr_is_true(const char *s);
 #ifdef __cplusplus
 }
 #endif

src/core/lib/surface/alarm.c

@@ -44,7 +44,9 @@ static void alarm_ref(grpc_alarm *alarm) { gpr_ref(&alarm->refs); }
 static void alarm_unref(grpc_alarm *alarm) {
   if (gpr_unref(&alarm->refs)) {
     grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    if (alarm->cq != NULL) {
       GRPC_CQ_INTERNAL_UNREF(&exec_ctx, alarm->cq, "alarm");
+    }
     grpc_exec_ctx_finish(&exec_ctx);
     gpr_free(alarm);
   }
@@ -93,12 +95,8 @@ static void alarm_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
                  (void *)alarm, &alarm->completion);
 }
 
-grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline,
+grpc_alarm *grpc_alarm_create(void *reserved) {
-                              void *tag) {
   grpc_alarm *alarm = gpr_malloc(sizeof(grpc_alarm));
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-
-  gpr_ref_init(&alarm->refs, 1);
 
 #ifndef NDEBUG
   if (GRPC_TRACER_ON(grpc_trace_alarm_refcount)) {
@@ -106,27 +104,36 @@ grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline,
   }
 #endif
 
+  gpr_ref_init(&alarm->refs, 1);
+  grpc_timer_init_unset(&alarm->alarm);
+  alarm->cq = NULL;
+  GRPC_CLOSURE_INIT(&alarm->on_alarm, alarm_cb, alarm,
+                    grpc_schedule_on_exec_ctx);
+  return alarm;
+}
+
+void grpc_alarm_set(grpc_alarm *alarm, grpc_completion_queue *cq,
+                    gpr_timespec deadline, void *tag, void *reserved) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
   GRPC_CQ_INTERNAL_REF(cq, "alarm");
   alarm->cq = cq;
   alarm->tag = tag;
 
   GPR_ASSERT(grpc_cq_begin_op(cq, tag));
-  GRPC_CLOSURE_INIT(&alarm->on_alarm, alarm_cb, alarm,
-                    grpc_schedule_on_exec_ctx);
   grpc_timer_init(&exec_ctx, &alarm->alarm,
                   gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC),
                   &alarm->on_alarm, gpr_now(GPR_CLOCK_MONOTONIC));
   grpc_exec_ctx_finish(&exec_ctx);
-  return alarm;
 }
 
-void grpc_alarm_cancel(grpc_alarm *alarm) {
+void grpc_alarm_cancel(grpc_alarm *alarm, void *reserved) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_timer_cancel(&exec_ctx, &alarm->alarm);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-void grpc_alarm_destroy(grpc_alarm *alarm) {
+void grpc_alarm_destroy(grpc_alarm *alarm, void *reserved) {
-  grpc_alarm_cancel(alarm);
+  grpc_alarm_cancel(alarm, reserved);
   GRPC_ALARM_UNREF(alarm, "alarm_destroy");
 }

src/core/lib/surface/call.c

@@ -122,6 +122,7 @@ typedef struct batch_control {
       bool is_closure;
     } notify_tag;
   } completion_data;
+  grpc_closure start_batch;
   grpc_closure finish_batch;
   gpr_refcount steps_to_complete;
 
@@ -151,6 +152,7 @@ typedef struct {
 struct grpc_call {
   gpr_refcount ext_ref;
   gpr_arena *arena;
+  grpc_call_combiner call_combiner;
   grpc_completion_queue *cq;
   grpc_polling_entity pollent;
   grpc_channel *channel;
@@ -184,6 +186,11 @@ struct grpc_call {
      Element 0 is initial metadata, element 1 is trailing metadata. */
   grpc_metadata_array *buffered_metadata[2];
 
+  grpc_metadata compression_md;
+
+  // A char* indicating the peer name.
+  gpr_atm peer_string;
+
   /* Packed received call statuses from various sources */
   gpr_atm status[STATUS_SOURCE_COUNT];
 
@@ -262,8 +269,9 @@ grpc_tracer_flag grpc_compression_trace =
 #define CALL_FROM_TOP_ELEM(top_elem) \
   CALL_FROM_CALL_STACK(grpc_call_stack_from_top_element(top_elem))
 
-static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
+static void execute_batch(grpc_exec_ctx *exec_ctx, grpc_call *call,
-                       grpc_transport_stream_op_batch *op);
+                          grpc_transport_stream_op_batch *op,
+                          grpc_closure *start_batch_closure);
 static void cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
                                status_source source, grpc_status_code status,
                                const char *description);
@@ -328,6 +336,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx,
                          sizeof(grpc_call) + channel_stack->call_stack_size);
   gpr_ref_init(&call->ext_ref, 1);
   call->arena = arena;
+  grpc_call_combiner_init(&call->call_combiner);
   *out_call = call;
   call->channel = args->channel;
   call->cq = args->cq;
@@ -436,7 +445,8 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx,
       .path = path,
       .start_time = call->start_time,
       .deadline = send_deadline,
-      .arena = call->arena};
+      .arena = call->arena,
+      .call_combiner = &call->call_combiner};
   add_init_error(&error, grpc_call_stack_init(exec_ctx, channel_stack, 1,
                                               destroy_call, call, &call_args));
   if (error != GRPC_ERROR_NONE) {
@@ -503,6 +513,8 @@ static void release_call(grpc_exec_ctx *exec_ctx, void *call,
                          grpc_error *error) {
   grpc_call *c = call;
   grpc_channel *channel = c->channel;
+  grpc_call_combiner_destroy(&c->call_combiner);
+  gpr_free((char *)c->peer_string);
   grpc_channel_update_call_size_estimate(channel, gpr_arena_destroy(c->arena));
   GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call");
 }
@@ -586,6 +598,12 @@ void grpc_call_unref(grpc_call *c) {
   if (cancel) {
     cancel_with_error(&exec_ctx, c, STATUS_FROM_API_OVERRIDE,
                       GRPC_ERROR_CANCELLED);
+  } else {
+    // Unset the call combiner cancellation closure.  This has the
+    // effect of scheduling the previously set cancellation closure, if
+    // any, so that it can release any internal references it may be
+    // holding to the call stack.
+    grpc_call_combiner_set_notify_on_cancel(&exec_ctx, &c->call_combiner, NULL);
   }
   GRPC_CALL_INTERNAL_UNREF(&exec_ctx, c, "destroy");
   grpc_exec_ctx_finish(&exec_ctx);
@@ -602,30 +620,37 @@ grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved) {
   return GRPC_CALL_OK;
 }
 
-static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
+// This is called via the call combiner to start sending a batch down
-                       grpc_transport_stream_op_batch *op) {
+// the filter stack.
-  grpc_call_element *elem;
+static void execute_batch_in_call_combiner(grpc_exec_ctx *exec_ctx, void *arg,
-
+                                           grpc_error *ignored) {
-  GPR_TIMER_BEGIN("execute_op", 0);
+  grpc_transport_stream_op_batch *batch = arg;
-  elem = CALL_ELEM_FROM_CALL(call, 0);
+  grpc_call *call = batch->handler_private.extra_arg;
-  elem->filter->start_transport_stream_op_batch(exec_ctx, elem, op);
+  GPR_TIMER_BEGIN("execute_batch", 0);
-  GPR_TIMER_END("execute_op", 0);
+  grpc_call_element *elem = CALL_ELEM_FROM_CALL(call, 0);
+  GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
+  elem->filter->start_transport_stream_op_batch(exec_ctx, elem, batch);
+  GPR_TIMER_END("execute_batch", 0);
+}
+
+// start_batch_closure points to a caller-allocated closure to be used
+// for entering the call combiner.
+static void execute_batch(grpc_exec_ctx *exec_ctx, grpc_call *call,
+                          grpc_transport_stream_op_batch *batch,
+                          grpc_closure *start_batch_closure) {
+  batch->handler_private.extra_arg = call;
+  GRPC_CLOSURE_INIT(start_batch_closure, execute_batch_in_call_combiner, batch,
+                    grpc_schedule_on_exec_ctx);
+  GRPC_CALL_COMBINER_START(exec_ctx, &call->call_combiner, start_batch_closure,
+                           GRPC_ERROR_NONE, "executing batch");
 }
 
 char *grpc_call_get_peer(grpc_call *call) {
-  grpc_call_element *elem = CALL_ELEM_FROM_CALL(call, 0);
+  char *peer_string = (char *)gpr_atm_acq_load(&call->peer_string);
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  if (peer_string != NULL) return gpr_strdup(peer_string);
-  char *result;
+  peer_string = grpc_channel_get_target(call->channel);
-  GRPC_API_TRACE("grpc_call_get_peer(%p)", 1, (call));
+  if (peer_string != NULL) return peer_string;
-  result = elem->filter->get_peer(&exec_ctx, elem);
+  return gpr_strdup("unknown");
-  if (result == NULL) {
-    result = grpc_channel_get_target(call->channel);
-  }
-  if (result == NULL) {
-    result = gpr_strdup("unknown");
-  }
-  grpc_exec_ctx_finish(&exec_ctx);
-  return result;
 }
 
 grpc_call *grpc_call_from_top_element(grpc_call_element *elem) {
@@ -652,20 +677,41 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c,
   return GRPC_CALL_OK;
 }
 
-static void done_termination(grpc_exec_ctx *exec_ctx, void *call,
+typedef struct {
+  grpc_call *call;
+  grpc_closure start_batch;
+  grpc_closure finish_batch;
+} cancel_state;
+
+// The on_complete callback used when sending a cancel_stream batch down
+// the filter stack.  Yields the call combiner when the batch is done.
+static void done_termination(grpc_exec_ctx *exec_ctx, void *arg,
                              grpc_error *error) {
-  GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "termination");
+  cancel_state *state = (cancel_state *)arg;
+  GRPC_CALL_COMBINER_STOP(exec_ctx, &state->call->call_combiner,
+                          "on_complete for cancel_stream op");
+  GRPC_CALL_INTERNAL_UNREF(exec_ctx, state->call, "termination");
+  gpr_free(state);
 }
 
 static void cancel_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c,
                               status_source source, grpc_error *error) {
   GRPC_CALL_INTERNAL_REF(c, "termination");
+  // Inform the call combiner of the cancellation, so that it can cancel
+  // any in-flight asynchronous actions that may be holding the call
+  // combiner.  This ensures that the cancel_stream batch can be sent
+  // down the filter stack in a timely manner.
+  grpc_call_combiner_cancel(exec_ctx, &c->call_combiner, GRPC_ERROR_REF(error));
   set_status_from_error(exec_ctx, c, source, GRPC_ERROR_REF(error));
-  grpc_transport_stream_op_batch *op = grpc_make_transport_stream_op(
+  cancel_state *state = (cancel_state *)gpr_malloc(sizeof(*state));
-      GRPC_CLOSURE_CREATE(done_termination, c, grpc_schedule_on_exec_ctx));
+  state->call = c;
+  GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state,
+                    grpc_schedule_on_exec_ctx);
+  grpc_transport_stream_op_batch *op =
+      grpc_make_transport_stream_op(&state->finish_batch);
   op->cancel_stream = true;
   op->payload->cancel_stream.cancel_error = error;
-  execute_op(exec_ctx, c, op);
+  execute_batch(exec_ctx, c, op, &state->start_batch);
 }
 
 static grpc_error *error_from_status(grpc_status_code status,
@@ -1431,6 +1477,18 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
   }
 }
 
+// The recv_message_ready callback used when sending a batch containing
+// a recv_message op down the filter stack.  Yields the call combiner
+// before processing the received message.
+static void receiving_stream_ready_in_call_combiner(grpc_exec_ctx *exec_ctx,
+                                                    void *bctlp,
+                                                    grpc_error *error) {
+  batch_control *bctl = bctlp;
+  grpc_call *call = bctl->call;
+  GRPC_CALL_COMBINER_STOP(exec_ctx, &call->call_combiner, "recv_message_ready");
+  receiving_stream_ready(exec_ctx, bctlp, error);
+}
+
 static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx,
                                        batch_control *bctl) {
   grpc_call *call = bctl->call;
@@ -1537,6 +1595,9 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
   batch_control *bctl = bctlp;
   grpc_call *call = bctl->call;
 
+  GRPC_CALL_COMBINER_STOP(exec_ctx, &call->call_combiner,
+                          "recv_initial_metadata_ready");
+
   add_batch_error(exec_ctx, bctl, GRPC_ERROR_REF(error), false);
   if (error == GRPC_ERROR_NONE) {
     grpc_metadata_batch *md =
@@ -1590,7 +1651,8 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
 static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp,
                          grpc_error *error) {
   batch_control *bctl = bctlp;
-
+  grpc_call *call = bctl->call;
+  GRPC_CALL_COMBINER_STOP(exec_ctx, &call->call_combiner, "on_complete");
   add_batch_error(exec_ctx, bctl, GRPC_ERROR_REF(error), false);
   finish_batch_step(exec_ctx, bctl);
 }
@@ -1610,9 +1672,6 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
   int num_completion_callbacks_needed = 1;
   grpc_call_error error = GRPC_CALL_OK;
 
-  // sent_initial_metadata guards against variable reuse.
-  grpc_metadata compression_md;
-
   GPR_TIMER_BEGIN("grpc_call_start_batch", 0);
   GRPC_CALL_LOG_BATCH(GPR_INFO, call, ops, nops, notify_tag);
 
@@ -1660,7 +1719,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
           goto done_with_error;
         }
         /* process compression level */
-        memset(&compression_md, 0, sizeof(compression_md));
+        memset(&call->compression_md, 0, sizeof(call->compression_md));
         size_t additional_metadata_count = 0;
         grpc_compression_level effective_compression_level =
             GRPC_COMPRESS_LEVEL_NONE;
@@ -1698,9 +1757,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
             const grpc_stream_compression_algorithm calgo =
                 stream_compression_algorithm_for_level_locked(
                     call, effective_stream_compression_level);
-            compression_md.key =
+            call->compression_md.key =
                 GRPC_MDSTR_GRPC_INTERNAL_STREAM_ENCODING_REQUEST;
-            compression_md.value =
+            call->compression_md.value =
                 grpc_stream_compression_algorithm_slice(calgo);
           } else {
             const grpc_compression_algorithm calgo =
@@ -1708,8 +1767,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
                     call, effective_compression_level);
             /* the following will be picked up by the compress filter and used
              * as the call's compression algorithm. */
-            compression_md.key = GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST;
+            call->compression_md.key =
-            compression_md.value = grpc_compression_algorithm_slice(calgo);
+                GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST;
+            call->compression_md.value =
+                grpc_compression_algorithm_slice(calgo);
             additional_metadata_count++;
           }
         }
@@ -1724,7 +1785,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         if (!prepare_application_metadata(
                 exec_ctx, call, (int)op->data.send_initial_metadata.count,
                 op->data.send_initial_metadata.metadata, 0, call->is_client,
-                &compression_md, (int)additional_metadata_count)) {
+                &call->compression_md, (int)additional_metadata_count)) {
           error = GRPC_CALL_ERROR_INVALID_METADATA;
           goto done_with_error;
         }
@@ -1736,6 +1797,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
             &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */];
         stream_op_payload->send_initial_metadata.send_initial_metadata_flags =
             op->flags;
+        if (call->is_client) {
+          stream_op_payload->send_initial_metadata.peer_string =
+              &call->peer_string;
+        }
         break;
       case GRPC_OP_SEND_MESSAGE:
         if (!are_write_flags_valid(op->flags)) {
@@ -1868,6 +1933,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
             &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
         stream_op_payload->recv_initial_metadata.recv_initial_metadata_ready =
             &call->receiving_initial_metadata_ready;
+        if (!call->is_client) {
+          stream_op_payload->recv_initial_metadata.peer_string =
+              &call->peer_string;
+        }
         num_completion_callbacks_needed++;
         break;
       case GRPC_OP_RECV_MESSAGE:
@@ -1884,8 +1953,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         stream_op->recv_message = true;
         call->receiving_buffer = op->data.recv_message.recv_message;
         stream_op_payload->recv_message.recv_message = &call->receiving_stream;
-        GRPC_CLOSURE_INIT(&call->receiving_stream_ready, receiving_stream_ready,
+        GRPC_CLOSURE_INIT(&call->receiving_stream_ready,
-                          bctl, grpc_schedule_on_exec_ctx);
+                          receiving_stream_ready_in_call_combiner, bctl,
+                          grpc_schedule_on_exec_ctx);
         stream_op_payload->recv_message.recv_message_ready =
             &call->receiving_stream_ready;
         num_completion_callbacks_needed++;
@@ -1955,7 +2025,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
   stream_op->on_complete = &bctl->finish_batch;
   gpr_atm_rel_store(&call->any_ops_sent_atm, 1);
 
-  execute_op(exec_ctx, call, stream_op);
+  execute_batch(exec_ctx, call, stream_op, &bctl->start_batch);
 
 done:
   GPR_TIMER_END("grpc_call_start_batch", 0);

src/core/lib/surface/call.h

@@ -19,6 +19,10 @@
 #ifndef GRPC_CORE_LIB_SURFACE_CALL_H
 #define GRPC_CORE_LIB_SURFACE_CALL_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/context.h"
 #include "src/core/lib/surface/api_trace.h"
@@ -26,10 +30,6 @@
 #include <grpc/grpc.h>
 #include <grpc/impl/codegen/compression_types.h>
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 typedef void (*grpc_ioreq_completion_func)(grpc_exec_ctx *exec_ctx,
                                            grpc_call *call, int success,
                                            void *user_data);
@@ -89,7 +89,7 @@ grpc_call_error grpc_call_start_batch_and_execute(grpc_exec_ctx *exec_ctx,
 /* Given the top call_element, get the call object. */
 grpc_call *grpc_call_from_top_element(grpc_call_element *surface_element);
 
-void grpc_call_log_batch(char *file, int line, gpr_log_severity severity,
+void grpc_call_log_batch(const char *file, int line, gpr_log_severity severity,
                          grpc_call *call, const grpc_op *ops, size_t nops,
                          void *tag);
 

src/core/lib/surface/call_log_batch.c

@@ -103,7 +103,7 @@ char *grpc_op_string(const grpc_op *op) {
   return out;
 }
 
-void grpc_call_log_batch(char *file, int line, gpr_log_severity severity,
+void grpc_call_log_batch(const char *file, int line, gpr_log_severity severity,
                          grpc_call *call, const grpc_op *ops, size_t nops,
                          void *tag) {
   char *tmp;

src/core/lib/surface/init.c

@@ -31,6 +31,7 @@
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
@@ -129,6 +130,7 @@ void grpc_init(void) {
     grpc_register_tracer(&grpc_trace_channel_stack_builder);
     grpc_register_tracer(&grpc_http1_trace);
     grpc_register_tracer(&grpc_cq_pluck_trace);  // default on
+    grpc_register_tracer(&grpc_call_combiner_trace);
     grpc_register_tracer(&grpc_combiner_trace);
     grpc_register_tracer(&grpc_server_channel_trace);
     grpc_register_tracer(&grpc_bdp_estimator_trace);

src/core/lib/surface/lame_client.cc

@@ -40,6 +40,7 @@ namespace grpc_core {
 namespace {
 
 struct CallData {
+  grpc_call_combiner *call_combiner;
   grpc_linked_mdelem status;
   grpc_linked_mdelem details;
   grpc_core::atomic<bool> filled_metadata;
@@ -52,14 +53,14 @@ struct ChannelData {
 
 static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                           grpc_metadata_batch *mdb) {
-  CallData *calld = static_cast<CallData *>(elem->call_data);
+  CallData *calld = reinterpret_cast<CallData *>(elem->call_data);
   bool expected = false;
   if (!calld->filled_metadata.compare_exchange_strong(
           expected, true, grpc_core::memory_order_relaxed,
           grpc_core::memory_order_relaxed)) {
     return;
   }
-  ChannelData *chand = static_cast<ChannelData *>(elem->channel_data);
+  ChannelData *chand = reinterpret_cast<ChannelData *>(elem->channel_data);
   char tmp[GPR_LTOA_MIN_BUFSIZE];
   gpr_ltoa(chand->error_code, tmp);
   calld->status.md = grpc_mdelem_from_slices(
@@ -79,6 +80,7 @@ static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 static void lame_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *op) {
+  CallData *calld = reinterpret_cast<CallData *>(elem->call_data);
   if (op->recv_initial_metadata) {
     fill_metadata(exec_ctx, elem,
                   op->payload->recv_initial_metadata.recv_initial_metadata);
@@ -87,12 +89,8 @@ static void lame_start_transport_stream_op_batch(
                   op->payload->recv_trailing_metadata.recv_trailing_metadata);
   }
   grpc_transport_stream_op_batch_finish_with_failure(
-      exec_ctx, op,
+      exec_ctx, op, GRPC_ERROR_CREATE_FROM_STATIC_STRING("lame client channel"),
-      GRPC_ERROR_CREATE_FROM_STATIC_STRING("lame client channel"));
+      calld->call_combiner);
-}
-
-static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
-  return NULL;
 }
 
 static void lame_get_channel_info(grpc_exec_ctx *exec_ctx,
@@ -122,6 +120,8 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx,
 static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
+  CallData *calld = reinterpret_cast<CallData *>(elem->call_data);
+  calld->call_combiner = args->call_combiner;
   return GRPC_ERROR_NONE;
 }
 
@@ -156,7 +156,6 @@ extern "C" const grpc_channel_filter grpc_lame_filter = {
     sizeof(grpc_core::ChannelData),
     grpc_core::init_channel_elem,
     grpc_core::destroy_channel_elem,
-    grpc_core::lame_get_peer,
     grpc_core::lame_get_channel_info,
     "lame-client",
 };
@@ -176,7 +175,7 @@ grpc_channel *grpc_lame_client_channel_create(const char *target,
       "error_message=%s)",
       3, (target, (int)error_code, error_message));
   GPR_ASSERT(elem->filter == &grpc_lame_filter);
-  auto chand = static_cast<grpc_core::ChannelData *>(elem->channel_data);
+  auto chand = reinterpret_cast<grpc_core::ChannelData *>(elem->channel_data);
   chand->error_code = error_code;
   chand->error_message = error_message;
   grpc_exec_ctx_finish(&exec_ctx);

src/core/lib/surface/server.c

@@ -789,7 +789,6 @@ static void server_mutate_op(grpc_call_element *elem,
 static void server_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     grpc_transport_stream_op_batch *op) {
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
   server_mutate_op(elem, op);
   grpc_call_next_op(exec_ctx, elem, op);
 }
@@ -962,7 +961,6 @@ const grpc_channel_filter grpc_server_top_filter = {
     sizeof(channel_data),
     init_channel_elem,
     destroy_channel_elem,
-    grpc_call_next_get_peer,
     grpc_channel_next_get_info,
     "server",
 };

src/core/lib/surface/version.c

@@ -21,6 +21,6 @@
 
 #include <grpc/grpc.h>
 
-const char *grpc_version_string(void) { return "4.0.0-dev"; }
+const char *grpc_version_string(void) { return "5.0.0-dev"; }
 
 const char *grpc_g_stands_for(void) { return "gambit"; }

src/core/lib/transport/metadata_batch.c

@@ -105,6 +105,7 @@ static grpc_error *maybe_link_callout(grpc_metadata_batch *batch,
     return GRPC_ERROR_NONE;
   }
   if (batch->idx.array[idx] == NULL) {
+    if (grpc_static_callout_is_default[idx]) ++batch->list.default_count;
     batch->idx.array[idx] = storage;
     return GRPC_ERROR_NONE;
   }
@@ -120,6 +121,7 @@ static void maybe_unlink_callout(grpc_metadata_batch *batch,
   if (idx == GRPC_BATCH_CALLOUTS_COUNT) {
     return;
   }
+  if (grpc_static_callout_is_default[idx]) --batch->list.default_count;
   GPR_ASSERT(batch->idx.array[idx] != NULL);
   batch->idx.array[idx] = NULL;
 }

src/core/lib/transport/metadata_batch.h

@@ -41,6 +41,7 @@ typedef struct grpc_linked_mdelem {
 
 typedef struct grpc_mdelem_list {
   size_t count;
+  size_t default_count;  // Number of default keys.
   grpc_linked_mdelem *head;
   grpc_linked_mdelem *tail;
 } grpc_mdelem_list;

src/core/lib/transport/static_metadata.c

@@ -823,6 +823,31 @@ grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = {
      {.refcount = &grpc_static_metadata_refcounts[97],
       .data.refcounted = {g_bytes + 1040, 13}}},
 };
+bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = {
+    true,  // :path
+    true,  // :method
+    true,  // :status
+    true,  // :authority
+    true,  // :scheme
+    true,  // te
+    true,  // grpc-message
+    true,  // grpc-status
+    true,  // grpc-payload-bin
+    true,  // grpc-encoding
+    true,  // grpc-accept-encoding
+    true,  // grpc-server-stats-bin
+    true,  // grpc-tags-bin
+    true,  // grpc-trace-bin
+    true,  // content-type
+    true,  // content-encoding
+    true,  // accept-encoding
+    true,  // grpc-internal-encoding-request
+    true,  // grpc-internal-stream-encoding-request
+    true,  // user-agent
+    true,  // host
+    true,  // lb-token
+};
+
 const uint8_t grpc_static_accept_encoding_metadata[8] = {0,  76, 77, 78,
                                                          79, 80, 81, 82};
 

src/core/lib/transport/static_metadata.h

@@ -571,6 +571,8 @@ typedef union {
              GRPC_BATCH_CALLOUTS_COUNT)                 \
        : GRPC_BATCH_CALLOUTS_COUNT)
 
+extern bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT];
+
 extern const uint8_t grpc_static_accept_encoding_metadata[8];
 #define GRPC_MDELEM_ACCEPT_ENCODING_FOR_ALGORITHMS(algs)                       \
   (GRPC_MAKE_MDELEM(                                                           \

src/core/lib/transport/transport.c

@@ -197,11 +197,6 @@ void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx,
                                     then_schedule_closure);
 }
 
-char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx,
-                              grpc_transport *transport) {
-  return transport->vtable->get_peer(exec_ctx, transport);
-}
-
 grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx,
                                            grpc_transport *transport) {
   return transport->vtable->get_endpoint(exec_ctx, transport);
@@ -214,24 +209,24 @@ grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx,
 // is a function that must always unref cancel_error
 // though it lives in lib, it handles transport stream ops sure
 // it's grpc_transport_stream_op_batch_finish_with_failure
-
 void grpc_transport_stream_op_batch_finish_with_failure(
     grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_batch *batch,
-    grpc_error *error) {
+    grpc_error *error, grpc_call_combiner *call_combiner) {
   if (batch->send_message) {
     grpc_byte_stream_destroy(exec_ctx,
                              batch->payload->send_message.send_message);
   }
   if (batch->recv_message) {
-    GRPC_CLOSURE_SCHED(exec_ctx,
+    GRPC_CALL_COMBINER_START(exec_ctx, call_combiner,
                              batch->payload->recv_message.recv_message_ready,
-                       GRPC_ERROR_REF(error));
+                             GRPC_ERROR_REF(error),
+                             "failing recv_message_ready");
   }
   if (batch->recv_initial_metadata) {
-    GRPC_CLOSURE_SCHED(
+    GRPC_CALL_COMBINER_START(
-        exec_ctx,
+        exec_ctx, call_combiner,
         batch->payload->recv_initial_metadata.recv_initial_metadata_ready,
-        GRPC_ERROR_REF(error));
+        GRPC_ERROR_REF(error), "failing recv_initial_metadata_ready");
   }
   GRPC_CLOSURE_SCHED(exec_ctx, batch->on_complete, error);
   if (batch->cancel_stream) {

src/core/lib/transport/transport.h

@@ -22,6 +22,7 @@
 #include <stddef.h>
 
 #include "src/core/lib/channel/context.h"
+#include "src/core/lib/iomgr/call_combiner.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/pollset.h"
@@ -152,6 +153,9 @@ struct grpc_transport_stream_op_batch_payload {
     /** Iff send_initial_metadata != NULL, flags associated with
         send_initial_metadata: a bitfield of GRPC_INITIAL_METADATA_xxx */
     uint32_t send_initial_metadata_flags;
+    // If non-NULL, will be set by the transport to the peer string
+    // (a char*, which the caller takes ownership of).
+    gpr_atm *peer_string;
   } send_initial_metadata;
 
   struct {
@@ -176,6 +180,9 @@ struct grpc_transport_stream_op_batch_payload {
     // immediately available.  This may be a signal that we received a
     // Trailers-Only response.
     bool *trailing_metadata_available;
+    // If non-NULL, will be set by the transport to the peer string
+    // (a char*, which the caller takes ownership of).
+    gpr_atm *peer_string;
   } recv_initial_metadata;
 
   struct {
@@ -293,7 +300,7 @@ void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx,
 
 void grpc_transport_stream_op_batch_finish_with_failure(
     grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_batch *op,
-    grpc_error *error);
+    grpc_error *error, grpc_call_combiner *call_combiner);
 
 char *grpc_transport_stream_op_batch_string(grpc_transport_stream_op_batch *op);
 char *grpc_transport_op_string(grpc_transport_op *op);
@@ -332,10 +339,6 @@ void grpc_transport_close(grpc_transport *transport);
 /* Destroy the transport */
 void grpc_transport_destroy(grpc_exec_ctx *exec_ctx, grpc_transport *transport);
 
-/* Get the transports peer */
-char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx,
-                              grpc_transport *transport);
-
 /* Get the endpoint used by \a transport */
 grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx,
                                            grpc_transport *transport);

src/core/lib/transport/transport_impl.h

@@ -59,9 +59,6 @@ typedef struct grpc_transport_vtable {
   /* implementation of grpc_transport_destroy */
   void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_transport *self);
 
-  /* implementation of grpc_transport_get_peer */
-  char *(*get_peer)(grpc_exec_ctx *exec_ctx, grpc_transport *self);
-
   /* implementation of grpc_transport_get_endpoint */
   grpc_endpoint *(*get_endpoint)(grpc_exec_ctx *exec_ctx, grpc_transport *self);
 } grpc_transport_vtable;

src/core/lib/transport/transport_op_string.c

@@ -112,6 +112,13 @@ char *grpc_transport_stream_op_batch_string(
     gpr_strvec_add(&b, tmp);
   }
 
+  if (op->collect_stats) {
+    gpr_strvec_add(&b, gpr_strdup(" "));
+    gpr_asprintf(&tmp, "COLLECT_STATS:%p",
+                 op->payload->collect_stats.collect_stats);
+    gpr_strvec_add(&b, tmp);
+  }
+
   out = gpr_strvec_flatten(&b, NULL);
   gpr_strvec_destroy(&b);
 

src/core/tsi/fake_transport_security.c

@@ -25,7 +25,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/useful.h>
-#include "src/core/tsi/transport_security.h"
+#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/tsi/transport_security_grpc.h"
 
 /* --- Constants. ---*/
 #define TSI_FAKE_FRAME_HEADER_SIZE 4
@@ -74,6 +75,14 @@ typedef struct {
   size_t max_frame_size;
 } tsi_fake_frame_protector;
 
+typedef struct {
+  tsi_zero_copy_grpc_protector base;
+  grpc_slice_buffer header_sb;
+  grpc_slice_buffer protected_sb;
+  size_t max_frame_size;
+  size_t parsed_frame_size;
+} tsi_fake_zero_copy_grpc_protector;
+
 /* --- Utils. ---*/
 
 static const char *tsi_fake_handshake_message_strings[] = {
@@ -113,6 +122,28 @@ static void store32_little_endian(uint32_t value, unsigned char *buf) {
   buf[0] = (unsigned char)((value)&0xFF);
 }
 
+static uint32_t read_frame_size(const grpc_slice_buffer *sb) {
+  GPR_ASSERT(sb != NULL && sb->length >= TSI_FAKE_FRAME_HEADER_SIZE);
+  uint8_t frame_size_buffer[TSI_FAKE_FRAME_HEADER_SIZE];
+  uint8_t *buf = frame_size_buffer;
+  /* Copies the first 4 bytes to a temporary buffer.  */
+  size_t remaining = TSI_FAKE_FRAME_HEADER_SIZE;
+  for (size_t i = 0; i < sb->count; i++) {
+    size_t slice_length = GRPC_SLICE_LENGTH(sb->slices[i]);
+    if (remaining <= slice_length) {
+      memcpy(buf, GRPC_SLICE_START_PTR(sb->slices[i]), remaining);
+      remaining = 0;
+      break;
+    } else {
+      memcpy(buf, GRPC_SLICE_START_PTR(sb->slices[i]), slice_length);
+      buf += slice_length;
+      remaining -= slice_length;
+    }
+  }
+  GPR_ASSERT(remaining == 0);
+  return load32_little_endian(frame_size_buffer);
+}
+
 static void tsi_fake_frame_reset(tsi_fake_frame *frame, int needs_draining) {
   frame->offset = 0;
   frame->needs_draining = needs_draining;
@@ -363,6 +394,84 @@ static const tsi_frame_protector_vtable frame_protector_vtable = {
     fake_protector_unprotect, fake_protector_destroy,
 };
 
+/* --- tsi_zero_copy_grpc_protector methods implementation. ---*/
+
+static tsi_result fake_zero_copy_grpc_protector_protect(
+    grpc_exec_ctx *exec_ctx, tsi_zero_copy_grpc_protector *self,
+    grpc_slice_buffer *unprotected_slices,
+    grpc_slice_buffer *protected_slices) {
+  if (self == NULL || unprotected_slices == NULL || protected_slices == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  tsi_fake_zero_copy_grpc_protector *impl =
+      (tsi_fake_zero_copy_grpc_protector *)self;
+  /* Protects each frame. */
+  while (unprotected_slices->length > 0) {
+    size_t frame_length =
+        GPR_MIN(impl->max_frame_size,
+                unprotected_slices->length + TSI_FAKE_FRAME_HEADER_SIZE);
+    grpc_slice slice = GRPC_SLICE_MALLOC(TSI_FAKE_FRAME_HEADER_SIZE);
+    store32_little_endian((uint32_t)frame_length, GRPC_SLICE_START_PTR(slice));
+    grpc_slice_buffer_add(protected_slices, slice);
+    size_t data_length = frame_length - TSI_FAKE_FRAME_HEADER_SIZE;
+    grpc_slice_buffer_move_first(unprotected_slices, data_length,
+                                 protected_slices);
+  }
+  return TSI_OK;
+}
+
+static tsi_result fake_zero_copy_grpc_protector_unprotect(
+    grpc_exec_ctx *exec_ctx, tsi_zero_copy_grpc_protector *self,
+    grpc_slice_buffer *protected_slices,
+    grpc_slice_buffer *unprotected_slices) {
+  if (self == NULL || unprotected_slices == NULL || protected_slices == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  tsi_fake_zero_copy_grpc_protector *impl =
+      (tsi_fake_zero_copy_grpc_protector *)self;
+  grpc_slice_buffer_move_into(protected_slices, &impl->protected_sb);
+  /* Unprotect each frame, if we get a full frame. */
+  while (impl->protected_sb.length >= TSI_FAKE_FRAME_HEADER_SIZE) {
+    if (impl->parsed_frame_size == 0) {
+      impl->parsed_frame_size = read_frame_size(&impl->protected_sb);
+      if (impl->parsed_frame_size <= 4) {
+        gpr_log(GPR_ERROR, "Invalid frame size.");
+        return TSI_DATA_CORRUPTED;
+      }
+    }
+    /* If we do not have a full frame, return with OK status. */
+    if (impl->protected_sb.length < impl->parsed_frame_size) break;
+    /* Strips header bytes. */
+    grpc_slice_buffer_move_first(&impl->protected_sb,
+                                 TSI_FAKE_FRAME_HEADER_SIZE, &impl->header_sb);
+    /* Moves data to unprotected slices. */
+    grpc_slice_buffer_move_first(
+        &impl->protected_sb,
+        impl->parsed_frame_size - TSI_FAKE_FRAME_HEADER_SIZE,
+        unprotected_slices);
+    impl->parsed_frame_size = 0;
+    grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &impl->header_sb);
+  }
+  return TSI_OK;
+}
+
+static void fake_zero_copy_grpc_protector_destroy(
+    grpc_exec_ctx *exec_ctx, tsi_zero_copy_grpc_protector *self) {
+  if (self == NULL) return;
+  tsi_fake_zero_copy_grpc_protector *impl =
+      (tsi_fake_zero_copy_grpc_protector *)self;
+  grpc_slice_buffer_destroy_internal(exec_ctx, &impl->header_sb);
+  grpc_slice_buffer_destroy_internal(exec_ctx, &impl->protected_sb);
+  gpr_free(impl);
+}
+
+static const tsi_zero_copy_grpc_protector_vtable
+    zero_copy_grpc_protector_vtable = {
+        fake_zero_copy_grpc_protector_protect,
+        fake_zero_copy_grpc_protector_unprotect,
+        fake_zero_copy_grpc_protector_destroy,
+};
+
 /* --- tsi_handshaker_result methods implementation. ---*/
 
 typedef struct {
@@ -383,6 +492,14 @@ static tsi_result fake_handshaker_result_extract_peer(
   return result;
 }
 
+static tsi_result fake_handshaker_result_create_zero_copy_grpc_protector(
+    const tsi_handshaker_result *self, size_t *max_output_protected_frame_size,
+    tsi_zero_copy_grpc_protector **protector) {
+  *protector =
+      tsi_create_fake_zero_copy_grpc_protector(max_output_protected_frame_size);
+  return TSI_OK;
+}
+
 static tsi_result fake_handshaker_result_create_frame_protector(
     const tsi_handshaker_result *self, size_t *max_output_protected_frame_size,
     tsi_frame_protector **protector) {
@@ -407,7 +524,7 @@ static void fake_handshaker_result_destroy(tsi_handshaker_result *self) {
 
 static const tsi_handshaker_result_vtable handshaker_result_vtable = {
     fake_handshaker_result_extract_peer,
-    NULL, /* create_zero_copy_grpc_protector */
+    fake_handshaker_result_create_zero_copy_grpc_protector,
     fake_handshaker_result_create_frame_protector,
     fake_handshaker_result_get_unused_bytes,
     fake_handshaker_result_destroy,
@@ -631,3 +748,16 @@ tsi_frame_protector *tsi_create_fake_frame_protector(
   impl->base.vtable = &frame_protector_vtable;
   return &impl->base;
 }
+
+tsi_zero_copy_grpc_protector *tsi_create_fake_zero_copy_grpc_protector(
+    size_t *max_protected_frame_size) {
+  tsi_fake_zero_copy_grpc_protector *impl = gpr_zalloc(sizeof(*impl));
+  grpc_slice_buffer_init(&impl->header_sb);
+  grpc_slice_buffer_init(&impl->protected_sb);
+  impl->max_frame_size = (max_protected_frame_size == NULL)
+                             ? TSI_FAKE_DEFAULT_FRAME_SIZE
+                             : *max_protected_frame_size;
+  impl->parsed_frame_size = 0;
+  impl->base.vtable = &zero_copy_grpc_protector_vtable;
+  return &impl->base;
+}

src/core/tsi/fake_transport_security.h

@@ -39,6 +39,11 @@ tsi_handshaker *tsi_create_fake_handshaker(int is_client);
 tsi_frame_protector *tsi_create_fake_frame_protector(
     size_t *max_protected_frame_size);
 
+/* Creates a zero-copy protector directly without going through the handshake
+ * phase. */
+tsi_zero_copy_grpc_protector *tsi_create_fake_zero_copy_grpc_protector(
+    size_t *max_protected_frame_size);
+
 #ifdef __cplusplus
 }
 #endif

src/core/tsi/test_creds/BUILD

@@ -18,4 +18,12 @@ exports_files([
         "ca.pem",
         "server1.key",
         "server1.pem",
+        "server0.key",
+        "server0.pem",
+        "client.key",
+        "client.pem",
+        "badserver.key",
+        "badserver.pem",
+        "badclient.key",
+        "badclient.pem",
 ])

src/core/tsi/transport_security_grpc.c

@@ -37,28 +37,33 @@ tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector(
    Calls specific implementation after state/input validation. */
 
 tsi_result tsi_zero_copy_grpc_protector_protect(
-    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *unprotected_slices,
+    grpc_exec_ctx *exec_ctx, tsi_zero_copy_grpc_protector *self,
+    grpc_slice_buffer *unprotected_slices,
     grpc_slice_buffer *protected_slices) {
-  if (self == NULL || self->vtable == NULL || unprotected_slices == NULL ||
+  if (exec_ctx == NULL || self == NULL || self->vtable == NULL ||
-      protected_slices == NULL) {
+      unprotected_slices == NULL || protected_slices == NULL) {
     return TSI_INVALID_ARGUMENT;
   }
   if (self->vtable->protect == NULL) return TSI_UNIMPLEMENTED;
-  return self->vtable->protect(self, unprotected_slices, protected_slices);
+  return self->vtable->protect(exec_ctx, self, unprotected_slices,
+                               protected_slices);
 }
 
 tsi_result tsi_zero_copy_grpc_protector_unprotect(
-    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *protected_slices,
+    grpc_exec_ctx *exec_ctx, tsi_zero_copy_grpc_protector *self,
+    grpc_slice_buffer *protected_slices,
     grpc_slice_buffer *unprotected_slices) {
-  if (self == NULL || self->vtable == NULL || protected_slices == NULL ||
+  if (exec_ctx == NULL || self == NULL || self->vtable == NULL ||
-      unprotected_slices == NULL) {
+      protected_slices == NULL || unprotected_slices == NULL) {
     return TSI_INVALID_ARGUMENT;
   }
   if (self->vtable->unprotect == NULL) return TSI_UNIMPLEMENTED;
-  return self->vtable->unprotect(self, protected_slices, unprotected_slices);
+  return self->vtable->unprotect(exec_ctx, self, protected_slices,
+                                 unprotected_slices);
 }
 
-void tsi_zero_copy_grpc_protector_destroy(tsi_zero_copy_grpc_protector *self) {
+void tsi_zero_copy_grpc_protector_destroy(grpc_exec_ctx *exec_ctx,
+                                          tsi_zero_copy_grpc_protector *self) {
   if (self == NULL) return;
-  self->vtable->destroy(self);
+  self->vtable->destroy(exec_ctx, self);
 }

src/core/tsi/transport_security_grpc.h

@@ -42,8 +42,8 @@ tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector(
    - This method returns TSI_OK in case of success or a specific error code in
      case of failure.  */
 tsi_result tsi_zero_copy_grpc_protector_protect(
-    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *unprotected_slices,
+    grpc_exec_ctx *exec_ctx, tsi_zero_copy_grpc_protector *self,
-    grpc_slice_buffer *protected_slices);
+    grpc_slice_buffer *unprotected_slices, grpc_slice_buffer *protected_slices);
 
 /* Outputs unprotected bytes.
    - protected_slices is the bytes of protected frames.
@@ -52,21 +52,24 @@ tsi_result tsi_zero_copy_grpc_protector_protect(
      there is not enough data to output in which case unprotected_slices has 0
      bytes.  */
 tsi_result tsi_zero_copy_grpc_protector_unprotect(
-    tsi_zero_copy_grpc_protector *self, grpc_slice_buffer *protected_slices,
+    grpc_exec_ctx *exec_ctx, tsi_zero_copy_grpc_protector *self,
-    grpc_slice_buffer *unprotected_slices);
+    grpc_slice_buffer *protected_slices, grpc_slice_buffer *unprotected_slices);
 
 /* Destroys the tsi_zero_copy_grpc_protector object.  */
-void tsi_zero_copy_grpc_protector_destroy(tsi_zero_copy_grpc_protector *self);
+void tsi_zero_copy_grpc_protector_destroy(grpc_exec_ctx *exec_ctx,
+                                          tsi_zero_copy_grpc_protector *self);
 
 /* Base for tsi_zero_copy_grpc_protector implementations.  */
 typedef struct {
-  tsi_result (*protect)(tsi_zero_copy_grpc_protector *self,
+  tsi_result (*protect)(grpc_exec_ctx *exec_ctx,
+                        tsi_zero_copy_grpc_protector *self,
                         grpc_slice_buffer *unprotected_slices,
                         grpc_slice_buffer *protected_slices);
-  tsi_result (*unprotect)(tsi_zero_copy_grpc_protector *self,
+  tsi_result (*unprotect)(grpc_exec_ctx *exec_ctx,
+                          tsi_zero_copy_grpc_protector *self,
                           grpc_slice_buffer *protected_slices,
                           grpc_slice_buffer *unprotected_slices);
-  void (*destroy)(tsi_zero_copy_grpc_protector *self);
+  void (*destroy)(grpc_exec_ctx *exec_ctx, tsi_zero_copy_grpc_protector *self);
 } tsi_zero_copy_grpc_protector_vtable;
 
 struct tsi_zero_copy_grpc_protector {

src/cpp/client/channel_cc.cc

@@ -18,7 +18,10 @@
 
 #include <grpc++/channel.h>
 
+#include <chrono>
+#include <condition_variable>
 #include <memory>
+#include <mutex>
 
 #include <grpc++/client_context.h>
 #include <grpc++/completion_queue.h>
@@ -35,17 +38,197 @@
 #include <grpc/slice.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/thd.h>
+#include <grpc/support/time.h>
+#include <grpc/support/useful.h>
 #include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/support/env.h"
+#include "src/core/lib/support/string.h"
 
 namespace grpc {
 
+namespace {
+int kConnectivityCheckIntervalMsec = 500;
+void WatchStateChange(void* arg);
+
+class TagSaver final : public CompletionQueueTag {
+ public:
+  explicit TagSaver(void* tag) : tag_(tag) {}
+  ~TagSaver() override {}
+  bool FinalizeResult(void** tag, bool* status) override {
+    *tag = tag_;
+    delete this;
+    return true;
+  }
+
+ private:
+  void* tag_;
+};
+
+// Constantly watches channel connectivity status to reconnect a transiently
+// disconnected channel. This is a temporary work-around before we have retry
+// support.
+class ChannelConnectivityWatcher : private GrpcLibraryCodegen {
+ public:
+  static void StartWatching(grpc_channel* channel) {
+    if (!IsDisabled()) {
+      std::unique_lock<std::mutex> lock(g_watcher_mu_);
+      if (g_watcher_ == nullptr) {
+        g_watcher_ = new ChannelConnectivityWatcher();
+      }
+      g_watcher_->StartWatchingLocked(channel);
+    }
+  }
+
+  static void StopWatching() {
+    if (!IsDisabled()) {
+      std::unique_lock<std::mutex> lock(g_watcher_mu_);
+      if (g_watcher_->StopWatchingLocked()) {
+        delete g_watcher_;
+        g_watcher_ = nullptr;
+      }
+    }
+  }
+
+ private:
+  ChannelConnectivityWatcher() : channel_count_(0), shutdown_(false) {
+    gpr_ref_init(&ref_, 0);
+    gpr_thd_options options = gpr_thd_options_default();
+    gpr_thd_options_set_joinable(&options);
+    gpr_thd_new(&thd_id_, &WatchStateChange, this, &options);
+  }
+
+  static bool IsDisabled() {
+    char* env = gpr_getenv("GRPC_DISABLE_CHANNEL_CONNECTIVITY_WATCHER");
+    bool disabled = gpr_is_true(env);
+    gpr_free(env);
+    return disabled;
+  }
+
+  void WatchStateChangeImpl() {
+    bool ok = false;
+    void* tag = NULL;
+    CompletionQueue::NextStatus status = CompletionQueue::GOT_EVENT;
+    while (true) {
+      {
+        std::unique_lock<std::mutex> lock(shutdown_mu_);
+        if (shutdown_) {
+          // Drain cq_ if the watcher is shutting down
+          status = cq_.AsyncNext(&tag, &ok, gpr_inf_future(GPR_CLOCK_REALTIME));
+        } else {
+          status = cq_.AsyncNext(&tag, &ok, gpr_inf_past(GPR_CLOCK_REALTIME));
+          // Make sure we've seen 2 TIMEOUTs before going to sleep
+          if (status == CompletionQueue::TIMEOUT) {
+            status = cq_.AsyncNext(&tag, &ok, gpr_inf_past(GPR_CLOCK_REALTIME));
+            if (status == CompletionQueue::TIMEOUT) {
+              shutdown_cv_.wait_for(lock, std::chrono::milliseconds(
+                                              kConnectivityCheckIntervalMsec));
+              continue;
+            }
+          }
+        }
+      }
+      ChannelState* channel_state = static_cast<ChannelState*>(tag);
+      channel_state->state =
+          grpc_channel_check_connectivity_state(channel_state->channel, false);
+      if (channel_state->state == GRPC_CHANNEL_SHUTDOWN) {
+        void* shutdown_tag = NULL;
+        channel_state->shutdown_cq.Next(&shutdown_tag, &ok);
+        delete channel_state;
+        if (gpr_unref(&ref_)) {
+          break;
+        }
+      } else {
+        TagSaver* tag_saver = new TagSaver(channel_state);
+        grpc_channel_watch_connectivity_state(
+            channel_state->channel, channel_state->state,
+            gpr_inf_future(GPR_CLOCK_REALTIME), cq_.cq(), tag_saver);
+      }
+    }
+  }
+
+  void StartWatchingLocked(grpc_channel* channel) {
+    if (thd_id_ != 0) {
+      gpr_ref(&ref_);
+      ++channel_count_;
+      ChannelState* channel_state = new ChannelState(channel);
+      // The first grpc_channel_watch_connectivity_state() is not used to
+      // monitor the channel state change, but to hold a reference of the
+      // c channel. So that WatchStateChangeImpl() can observe state ==
+      // GRPC_CHANNEL_SHUTDOWN before the channel gets destroyed.
+      grpc_channel_watch_connectivity_state(
+          channel_state->channel, channel_state->state,
+          gpr_inf_future(GPR_CLOCK_REALTIME), channel_state->shutdown_cq.cq(),
+          new TagSaver(nullptr));
+      grpc_channel_watch_connectivity_state(
+          channel_state->channel, channel_state->state,
+          gpr_inf_future(GPR_CLOCK_REALTIME), cq_.cq(),
+          new TagSaver(channel_state));
+    }
+  }
+
+  bool StopWatchingLocked() {
+    if (--channel_count_ == 0) {
+      {
+        std::unique_lock<std::mutex> lock(shutdown_mu_);
+        shutdown_ = true;
+        shutdown_cv_.notify_one();
+      }
+      gpr_thd_join(thd_id_);
+      return true;
+    }
+    return false;
+  }
+
+  friend void WatchStateChange(void* arg);
+  struct ChannelState {
+    explicit ChannelState(grpc_channel* channel)
+        : channel(channel), state(GRPC_CHANNEL_IDLE){};
+    grpc_channel* channel;
+    grpc_connectivity_state state;
+    CompletionQueue shutdown_cq;
+  };
+  gpr_thd_id thd_id_;
+  CompletionQueue cq_;
+  gpr_refcount ref_;
+  int channel_count_;
+
+  std::mutex shutdown_mu_;
+  std::condition_variable shutdown_cv_;  // protected by shutdown_mu_
+  bool shutdown_;                        // protected by shutdown_mu_
+
+  static std::mutex g_watcher_mu_;
+  static ChannelConnectivityWatcher* g_watcher_;  // protected by g_watcher_mu_
+};
+
+std::mutex ChannelConnectivityWatcher::g_watcher_mu_;
+ChannelConnectivityWatcher* ChannelConnectivityWatcher::g_watcher_ = nullptr;
+
+void WatchStateChange(void* arg) {
+  ChannelConnectivityWatcher* watcher =
+      static_cast<ChannelConnectivityWatcher*>(arg);
+  watcher->WatchStateChangeImpl();
+}
+}  // namespace
+
 static internal::GrpcLibraryInitializer g_gli_initializer;
 Channel::Channel(const grpc::string& host, grpc_channel* channel)
     : host_(host), c_channel_(channel) {
   g_gli_initializer.summon();
+  if (grpc_channel_support_connectivity_watcher(channel)) {
+    ChannelConnectivityWatcher::StartWatching(channel);
+  }
 }
 
-Channel::~Channel() { grpc_channel_destroy(c_channel_); }
+Channel::~Channel() {
+  const bool stop_watching =
+      grpc_channel_support_connectivity_watcher(c_channel_);
+  grpc_channel_destroy(c_channel_);
+  if (stop_watching) {
+    ChannelConnectivityWatcher::StopWatching();
+  }
+}
 
 namespace {
 
@@ -130,23 +313,6 @@ grpc_connectivity_state Channel::GetState(bool try_to_connect) {
   return grpc_channel_check_connectivity_state(c_channel_, try_to_connect);
 }
 
-namespace {
-class TagSaver final : public CompletionQueueTag {
- public:
-  explicit TagSaver(void* tag) : tag_(tag) {}
-  ~TagSaver() override {}
-  bool FinalizeResult(void** tag, bool* status) override {
-    *tag = tag_;
-    delete this;
-    return true;
-  }
-
- private:
-  void* tag_;
-};
-
-}  // namespace
-
 void Channel::NotifyOnStateChangeImpl(grpc_connectivity_state last_observed,
                                       gpr_timespec deadline,
                                       CompletionQueue* cq, void* tag) {