Merge branch 'exec' into execp

8 years ago · 79eba775ca
parent f66ed28853 213d2744b5
commit 79eba775ca
102 changed files with 3449 additions and 1200 deletions
--- a/.gitignore
+++ b/.gitignore
@ -16,7 +16,7 @@ htmlcov/
 dist/
 *.egg
 py27/
-py34/
+py3[0-9]*/

 # Node installation output
 node_modules
--- a/2
+++ b/2
@ -989,6 +989,7 @@ grpc_cc_library(
    name = "grpc_codegen",
    language = "c",
    public_hdrs = [
+        "include/grpc/impl/codegen/byte_buffer.h",
        "include/grpc/impl/codegen/byte_buffer_reader.h",
        "include/grpc/impl/codegen/compression_types.h",
        "include/grpc/impl/codegen/connectivity_state.h",
@ -1487,6 +1488,7 @@ grpc_cc_library(
    public_hdrs = [
        "include/grpc++/impl/codegen/async_stream.h",
        "include/grpc++/impl/codegen/async_unary_call.h",
+        "include/grpc++/impl/codegen/byte_buffer.h",
        "include/grpc++/impl/codegen/call.h",
        "include/grpc++/impl/codegen/call_hook.h",
        "include/grpc++/impl/codegen/channel_interface.h",
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -123,10 +123,8 @@ if("${gRPC_ZLIB_PROVIDER}" STREQUAL "module")
    set(gRPC_INSTALL FALSE)
  endif()
 elseif("${gRPC_ZLIB_PROVIDER}" STREQUAL "package")
-  find_package(ZLIB)
-  if(TARGET ZLIB::ZLIB)
-    set(_gRPC_ZLIB_LIBRARIES ZLIB::ZLIB)
-  endif()
+  find_package(ZLIB REQUIRED)
+  set(_gRPC_ZLIB_LIBRARIES ${ZLIB_LIBRARIES})
  set(_gRPC_FIND_ZLIB "if(NOT ZLIB_FOUND)\n  find_package(ZLIB)\nendif()")
 endif()

@ -145,7 +143,7 @@ if("${gRPC_CARES_PROVIDER}" STREQUAL "module")
    set(gRPC_INSTALL FALSE)
  endif()
 elseif("${gRPC_CARES_PROVIDER}" STREQUAL "package")
-  find_package(c-ares CONFIG)
+  find_package(c-ares REQUIRED CONFIG)
  if(TARGET c-ares::cares)
    set(_gRPC_CARES_LIBRARIES c-ares::cares)
  endif()
@ -189,7 +187,7 @@ if("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "module")
    set(gRPC_INSTALL FALSE)
  endif()
 elseif("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "package")
-  find_package(Protobuf ${gRPC_PROTOBUF_PACKAGE_TYPE})
+  find_package(Protobuf REQUIRED ${gRPC_PROTOBUF_PACKAGE_TYPE})
  if(Protobuf_FOUND OR PROTOBUF_FOUND)
    if(TARGET protobuf::${_gRPC_PROTOBUF_LIBRARY_NAME})
      set(_gRPC_PROTOBUF_LIBRARIES protobuf::${_gRPC_PROTOBUF_LIBRARY_NAME})
@ -234,11 +232,9 @@ if("${gRPC_SSL_PROVIDER}" STREQUAL "module")
    set(gRPC_INSTALL FALSE)
  endif()
 elseif("${gRPC_SSL_PROVIDER}" STREQUAL "package")
-  find_package(OpenSSL)
-  if(TARGET OpenSSL::SSL)
-    set(_gRPC_SSL_LIBRARIES OpenSSL::SSL)
-  endif()
-  set(_gRPC_FIND_SSL "if(NOT OpenSSL_FOUND)\n  find_package(OpenSSL)\nendif()")
+  find_package(OpenSSL REQUIRED)
+  set(_gRPC_SSL_LIBRARIES ${OPENSSL_LIBRARIES})
+  set(_gRPC_FIND_SSL "if(NOT OPENSSL_FOUND)\n  find_package(OpenSSL)\nendif()")
 endif()

 if("${gRPC_GFLAGS_PROVIDER}" STREQUAL "module")
@ -832,7 +828,7 @@ endif()


 target_include_directories(gpr
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -924,7 +920,7 @@ endif()


 target_include_directories(gpr_test_util
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -1219,7 +1215,7 @@ endif()


 target_include_directories(grpc
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -1241,6 +1237,7 @@ target_link_libraries(grpc
 )

 foreach(_hdr
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -1525,7 +1522,7 @@ endif()


 target_include_directories(grpc_cronet
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -1547,6 +1544,7 @@ target_link_libraries(grpc_cronet
 )

 foreach(_hdr
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -1803,7 +1801,7 @@ endif()


 target_include_directories(grpc_test_util
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -1823,6 +1821,7 @@ target_link_libraries(grpc_test_util
 )

 foreach(_hdr
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -2063,7 +2062,7 @@ endif()


 target_include_directories(grpc_test_util_unsecure
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -2083,6 +2082,7 @@ target_link_libraries(grpc_test_util_unsecure
 )

 foreach(_hdr
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -2357,7 +2357,7 @@ endif()


 target_include_directories(grpc_unsecure
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -2378,6 +2378,7 @@ target_link_libraries(grpc_unsecure
 )

 foreach(_hdr
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -2446,7 +2447,7 @@ endif()


 target_include_directories(reconnect_server
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -2488,7 +2489,7 @@ endif()


 target_include_directories(test_tcp_server
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -2569,7 +2570,7 @@ endif()


 target_include_directories(grpc++
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -2687,6 +2688,7 @@ foreach(_hdr
  include/grpc/slice_buffer.h
  include/grpc/status.h
  include/grpc/support/workaround_list.h
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -2697,6 +2699,7 @@ foreach(_hdr
  include/grpc/impl/codegen/status.h
  include/grpc++/impl/codegen/async_stream.h
  include/grpc++/impl/codegen/async_unary_call.h
+  include/grpc++/impl/codegen/byte_buffer.h
  include/grpc++/impl/codegen/call.h
  include/grpc++/impl/codegen/call_hook.h
  include/grpc++/impl/codegen/channel_interface.h
@ -2769,7 +2772,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(grpc++_core_stats
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -3060,7 +3063,7 @@ endif()


 target_include_directories(grpc++_cronet
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -3179,6 +3182,7 @@ foreach(_hdr
  include/grpc/slice_buffer.h
  include/grpc/status.h
  include/grpc/support/workaround_list.h
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -3189,6 +3193,7 @@ foreach(_hdr
  include/grpc/impl/codegen/status.h
  include/grpc++/impl/codegen/async_stream.h
  include/grpc++/impl/codegen/async_unary_call.h
+  include/grpc++/impl/codegen/byte_buffer.h
  include/grpc++/impl/codegen/call.h
  include/grpc++/impl/codegen/call_hook.h
  include/grpc++/impl/codegen/channel_interface.h
@ -3259,7 +3264,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(grpc++_error_details
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -3324,7 +3329,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(grpc++_proto_reflection_desc_db
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -3385,7 +3390,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(grpc++_reflection
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -3443,7 +3448,7 @@ endif()


 target_include_directories(grpc++_test_config
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -3521,7 +3526,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(grpc++_test_util
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -3549,6 +3554,7 @@ target_link_libraries(grpc++_test_util
 foreach(_hdr
  include/grpc++/impl/codegen/async_stream.h
  include/grpc++/impl/codegen/async_unary_call.h
+  include/grpc++/impl/codegen/byte_buffer.h
  include/grpc++/impl/codegen/call.h
  include/grpc++/impl/codegen/call_hook.h
  include/grpc++/impl/codegen/channel_interface.h
@ -3576,6 +3582,7 @@ foreach(_hdr
  include/grpc++/impl/codegen/stub_options.h
  include/grpc++/impl/codegen/sync_stream.h
  include/grpc++/impl/codegen/time.h
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -3659,7 +3666,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(grpc++_test_util_unsecure
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -3687,6 +3694,7 @@ target_link_libraries(grpc++_test_util_unsecure
 foreach(_hdr
  include/grpc++/impl/codegen/async_stream.h
  include/grpc++/impl/codegen/async_unary_call.h
+  include/grpc++/impl/codegen/byte_buffer.h
  include/grpc++/impl/codegen/call.h
  include/grpc++/impl/codegen/call_hook.h
  include/grpc++/impl/codegen/channel_interface.h
@ -3714,6 +3722,7 @@ foreach(_hdr
  include/grpc++/impl/codegen/stub_options.h
  include/grpc++/impl/codegen/sync_stream.h
  include/grpc++/impl/codegen/time.h
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -3799,7 +3808,7 @@ endif()


 target_include_directories(grpc++_unsecure
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -3916,6 +3925,7 @@ foreach(_hdr
  include/grpc/slice_buffer.h
  include/grpc/status.h
  include/grpc/support/workaround_list.h
+  include/grpc/impl/codegen/byte_buffer.h
  include/grpc/impl/codegen/byte_buffer_reader.h
  include/grpc/impl/codegen/compression_types.h
  include/grpc/impl/codegen/connectivity_state.h
@ -3926,6 +3936,7 @@ foreach(_hdr
  include/grpc/impl/codegen/status.h
  include/grpc++/impl/codegen/async_stream.h
  include/grpc++/impl/codegen/async_unary_call.h
+  include/grpc++/impl/codegen/byte_buffer.h
  include/grpc++/impl/codegen/call.h
  include/grpc++/impl/codegen/call_hook.h
  include/grpc++/impl/codegen/channel_interface.h
@ -3989,7 +4000,7 @@ endif()


 target_include_directories(grpc_benchmark
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4048,7 +4059,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(grpc_cli_libs
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4108,7 +4119,7 @@ endif()


 target_include_directories(grpc_plugin_support
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4186,7 +4197,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(http2_client_main
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4241,7 +4252,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(interop_client_helper
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4311,7 +4322,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(interop_client_main
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4362,7 +4373,7 @@ endif()


 target_include_directories(interop_server_helper
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4431,7 +4442,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(interop_server_lib
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4482,7 +4493,7 @@ endif()


 target_include_directories(interop_server_main
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4570,7 +4581,7 @@ protobuf_generate_grpc_cpp(
 )

 target_include_directories(qps
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4617,7 +4628,7 @@ endif()


 target_include_directories(grpc_csharp_ext
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4712,7 +4723,7 @@ endif()


 target_include_directories(ares
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4750,7 +4761,7 @@ endif()


 target_include_directories(bad_client_test
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4791,7 +4802,7 @@ endif()


 target_include_directories(bad_ssl_test_server
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4892,7 +4903,7 @@ endif()


 target_include_directories(end2end_tests
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
@ -4993,7 +5004,7 @@ endif()


 target_include_directories(end2end_nosec_tests
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
  PRIVATE ${BORINGSSL_ROOT_DIR}/include
  PRIVATE ${PROTOBUF_ROOT_DIR}/src
--- a/15
+++ b/15
@ -3190,6 +3190,7 @@ LIBGRPC_SRC = \
    src/core/plugin_registry/grpc_plugin_registry.c \

 PUBLIC_HEADERS_C += \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -3496,6 +3497,7 @@ LIBGRPC_CRONET_SRC = \
    src/core/plugin_registry/grpc_cronet_plugin_registry.c \

 PUBLIC_HEADERS_C += \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -3773,6 +3775,7 @@ LIBGRPC_TEST_UTIL_SRC = \
    src/core/ext/filters/http/server/http_server_filter.c \

 PUBLIC_HEADERS_C += \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -4024,6 +4027,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
    src/core/ext/filters/http/server/http_server_filter.c \

 PUBLIC_HEADERS_C += \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -4296,6 +4300,7 @@ LIBGRPC_UNSECURE_SRC = \
    src/core/plugin_registry/grpc_unsecure_plugin_registry.c \

 PUBLIC_HEADERS_C += \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -4584,6 +4589,7 @@ PUBLIC_HEADERS_CXX += \
    include/grpc/slice_buffer.h \
    include/grpc/status.h \
    include/grpc/support/workaround_list.h \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -4594,6 +4600,7 @@ PUBLIC_HEADERS_CXX += \
    include/grpc/impl/codegen/status.h \
    include/grpc++/impl/codegen/async_stream.h \
    include/grpc++/impl/codegen/async_unary_call.h \
+    include/grpc++/impl/codegen/byte_buffer.h \
    include/grpc++/impl/codegen/call.h \
    include/grpc++/impl/codegen/call_hook.h \
    include/grpc++/impl/codegen/channel_interface.h \
@ -5077,6 +5084,7 @@ PUBLIC_HEADERS_CXX += \
    include/grpc/slice_buffer.h \
    include/grpc/status.h \
    include/grpc/support/workaround_list.h \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -5087,6 +5095,7 @@ PUBLIC_HEADERS_CXX += \
    include/grpc/impl/codegen/status.h \
    include/grpc++/impl/codegen/async_stream.h \
    include/grpc++/impl/codegen/async_unary_call.h \
+    include/grpc++/impl/codegen/byte_buffer.h \
    include/grpc++/impl/codegen/call.h \
    include/grpc++/impl/codegen/call_hook.h \
    include/grpc++/impl/codegen/channel_interface.h \
@ -5440,6 +5449,7 @@ LIBGRPC++_TEST_UTIL_SRC = \
 PUBLIC_HEADERS_CXX += \
    include/grpc++/impl/codegen/async_stream.h \
    include/grpc++/impl/codegen/async_unary_call.h \
+    include/grpc++/impl/codegen/byte_buffer.h \
    include/grpc++/impl/codegen/call.h \
    include/grpc++/impl/codegen/call_hook.h \
    include/grpc++/impl/codegen/channel_interface.h \
@ -5467,6 +5477,7 @@ PUBLIC_HEADERS_CXX += \
    include/grpc++/impl/codegen/stub_options.h \
    include/grpc++/impl/codegen/sync_stream.h \
    include/grpc++/impl/codegen/time.h \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -5555,6 +5566,7 @@ LIBGRPC++_TEST_UTIL_UNSECURE_SRC = \
 PUBLIC_HEADERS_CXX += \
    include/grpc++/impl/codegen/async_stream.h \
    include/grpc++/impl/codegen/async_unary_call.h \
+    include/grpc++/impl/codegen/byte_buffer.h \
    include/grpc++/impl/codegen/call.h \
    include/grpc++/impl/codegen/call_hook.h \
    include/grpc++/impl/codegen/channel_interface.h \
@ -5582,6 +5594,7 @@ PUBLIC_HEADERS_CXX += \
    include/grpc++/impl/codegen/stub_options.h \
    include/grpc++/impl/codegen/sync_stream.h \
    include/grpc++/impl/codegen/time.h \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -5789,6 +5802,7 @@ PUBLIC_HEADERS_CXX += \
    include/grpc/slice_buffer.h \
    include/grpc/status.h \
    include/grpc/support/workaround_list.h \
+    include/grpc/impl/codegen/byte_buffer.h \
    include/grpc/impl/codegen/byte_buffer_reader.h \
    include/grpc/impl/codegen/compression_types.h \
    include/grpc/impl/codegen/connectivity_state.h \
@ -5799,6 +5813,7 @@ PUBLIC_HEADERS_CXX += \
    include/grpc/impl/codegen/status.h \
    include/grpc++/impl/codegen/async_stream.h \
    include/grpc++/impl/codegen/async_unary_call.h \
+    include/grpc++/impl/codegen/byte_buffer.h \
    include/grpc++/impl/codegen/call.h \
    include/grpc++/impl/codegen/call_hook.h \
    include/grpc++/impl/codegen/channel_interface.h \
--- a/build.yaml
+++ b/build.yaml
@ -502,6 +502,7 @@ filegroups:
  - grpc_deadline_filter
 - name: grpc_codegen
  public_headers:
+  - include/grpc/impl/codegen/byte_buffer.h
  - include/grpc/impl/codegen/byte_buffer_reader.h
  - include/grpc/impl/codegen/compression_types.h
  - include/grpc/impl/codegen/connectivity_state.h
@ -969,6 +970,7 @@ filegroups:
  public_headers:
  - include/grpc++/impl/codegen/async_stream.h
  - include/grpc++/impl/codegen/async_unary_call.h
+  - include/grpc++/impl/codegen/byte_buffer.h
  - include/grpc++/impl/codegen/call.h
  - include/grpc++/impl/codegen/call_hook.h
  - include/grpc++/impl/codegen/channel_interface.h
--- a/doc/c-style-guide.md
+++ b/doc/c-style-guide.md
@ -32,14 +32,14 @@ Header Files
  # endif
  ```
 - Header files should be self-contained and end in .h.
- All header files should have a #define guard to prevent multiple inclusion.
+- All header files should have a `#define` guard to prevent multiple inclusion.
  To guarantee uniqueness they should be based on the file's path.

  For public headers: `include/grpc/grpc.h` → `GRPC_GRPC_H`

  For private headers:
-  `src/core/channel/channel_stack.h` →
-  `GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_STACK_H`
+  `src/core/lib/channel/channel_stack.h` →
+  `GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_H`

 Variable Initialization
 -----------------------
@ -72,8 +72,16 @@ Symbol Names

 - Non-static functions must be prefixed by `grpc_`
 - Static functions must *not* be prefixed by `grpc_`
+- Typenames of `struct`s , `union`s, and `enum`s must be prefixed by `grpc_` if
+  they are declared in a header file. They must not be prefixed by `grpc_` if
+  they are declared in a source file.
 - Enumeration values and `#define` names must be uppercase. All other values
  must be lowercase.
+- Enumeration values or `#define` names defined in a header file must be
+  prefixed with `GRPC_` (except for `#define` macros that are being used to
+  substitute functions; those should follow the general rules for
+  functions). Enumeration values or `#define`s defined in source files must not
+  be prefixed with `GRPC_`.
 - Multiple word identifiers use underscore as a delimiter, *never* camel
  case. E.g. `variable_name`.

--- a/examples/cpp/helloworld/CMakeLists.txt
+++ b/examples/cpp/helloworld/CMakeLists.txt
@ -2,7 +2,11 @@
 cmake_minimum_required(VERSION 2.8)

 # Project
-project(HelloWorld CXX)
+project(HelloWorld C CXX)
+
+if(NOT MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+endif()

 # Protobuf
 set(protobuf_MODULE_COMPATIBLE TRUE)
--- a/gRPC-Core.podspec
+++ b/gRPC-Core.podspec
@ -141,6 +141,7 @@ Pod::Spec.new do |s|
                      'include/grpc/impl/codegen/sync_generic.h',
                      'include/grpc/impl/codegen/sync_posix.h',
                      'include/grpc/impl/codegen/sync_windows.h',
+                      'include/grpc/impl/codegen/byte_buffer.h',
                      'include/grpc/impl/codegen/byte_buffer_reader.h',
                      'include/grpc/impl/codegen/compression_types.h',
                      'include/grpc/impl/codegen/connectivity_state.h',
--- a/grpc.def
+++ b/grpc.def
@ -1,14 +1,4 @@
 EXPORTS
-    grpc_raw_byte_buffer_create
-    grpc_raw_compressed_byte_buffer_create
-    grpc_byte_buffer_copy
-    grpc_byte_buffer_length
-    grpc_byte_buffer_destroy
-    grpc_byte_buffer_reader_init
-    grpc_byte_buffer_reader_destroy
-    grpc_byte_buffer_reader_next
-    grpc_byte_buffer_reader_readall
-    grpc_raw_byte_buffer_from_reader
    census_initialize
    census_shutdown
    census_supported
@ -145,6 +135,16 @@ EXPORTS
    grpc_server_add_secure_http2_port
    grpc_call_set_credentials
    grpc_server_credentials_set_auth_metadata_processor
+    grpc_raw_byte_buffer_create
+    grpc_raw_compressed_byte_buffer_create
+    grpc_byte_buffer_copy
+    grpc_byte_buffer_length
+    grpc_byte_buffer_destroy
+    grpc_byte_buffer_reader_init
+    grpc_byte_buffer_reader_destroy
+    grpc_byte_buffer_reader_next
+    grpc_byte_buffer_reader_readall
+    grpc_raw_byte_buffer_from_reader
    grpc_slice_ref
    grpc_slice_unref
    grpc_slice_copy
--- a/grpc.gemspec
+++ b/grpc.gemspec
@ -145,6 +145,7 @@ Gem::Specification.new do |s|
  s.files += %w( src/core/lib/support/tmpfile_posix.c )
  s.files += %w( src/core/lib/support/tmpfile_windows.c )
  s.files += %w( src/core/lib/support/wrap_memcpy.c )
+  s.files += %w( include/grpc/impl/codegen/byte_buffer.h )
  s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h )
  s.files += %w( include/grpc/impl/codegen/compression_types.h )
  s.files += %w( include/grpc/impl/codegen/connectivity_state.h )
--- a/include/grpc++/impl/codegen/byte_buffer.h
+++ b/include/grpc++/impl/codegen/byte_buffer.h
@ -0,0 +1,141 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H
+#define GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H
+
+#include <grpc/impl/codegen/byte_buffer.h>
+
+#include <grpc++/impl/codegen/config.h>
+#include <grpc++/impl/codegen/core_codegen_interface.h>
+#include <grpc++/impl/codegen/serialization_traits.h>
+#include <grpc++/impl/codegen/slice.h>
+#include <grpc++/impl/codegen/status.h>
+
+#include <vector>
+
+namespace grpc {
+
+template <class R>
+class CallOpRecvMessage;
+class MethodHandler;
+namespace internal {
+template <class M, class T>
+class MessageDeserializer;
+}
+
+/// A sequence of bytes.
+class ByteBuffer final {
+ public:
+  /// Constuct an empty buffer.
+  ByteBuffer() : buffer_(nullptr) {}
+
+  /// Construct buffer from \a slices, of which there are \a nslices.
+  ByteBuffer(const Slice* slices, size_t nslices);
+
+  /// Constuct a byte buffer by referencing elements of existing buffer
+  /// \a buf. Wrapper of core function grpc_byte_buffer_copy
+  ByteBuffer(const ByteBuffer& buf);
+
+  ~ByteBuffer() {
+    if (buffer_) {
+      g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_);
+    }
+  }
+
+  ByteBuffer& operator=(const ByteBuffer&);
+
+  /// Dump (read) the buffer contents into \a slices.
+  Status Dump(std::vector<Slice>* slices) const;
+
+  /// Remove all data.
+  void Clear() {
+    if (buffer_) {
+      g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_);
+      buffer_ = nullptr;
+    }
+  }
+
+  /// Make a duplicate copy of the internals of this byte
+  /// buffer so that we have our own owned version of it.
+  /// bbuf.Duplicate(); is equivalent to bbuf=bbuf; but is actually readable
+  void Duplicate() {
+    buffer_ = g_core_codegen_interface->grpc_byte_buffer_copy(buffer_);
+  }
+
+  /// Forget underlying byte buffer without destroying
+  /// Use this only for un-owned byte buffers
+  void Release() { buffer_ = nullptr; }
+
+  /// Buffer size in bytes.
+  size_t Length() const;
+
+  /// Swap the state of *this and *other.
+  void Swap(ByteBuffer* other);
+
+  /// Is this ByteBuffer valid?
+  bool Valid() const { return (buffer_ != nullptr); }
+
+ private:
+  friend class SerializationTraits<ByteBuffer, void>;
+  friend class CallOpSendMessage;
+  template <class R>
+  friend class CallOpRecvMessage;
+  friend class CallOpGenericRecvMessage;
+  friend class MethodHandler;
+  template <class M, class T>
+  friend class internal::MessageDeserializer;
+
+  // takes ownership
+  void set_buffer(grpc_byte_buffer* buf) {
+    if (buffer_) {
+      Clear();
+    }
+    buffer_ = buf;
+  }
+
+  grpc_byte_buffer* c_buffer() { return buffer_; }
+  grpc_byte_buffer** c_buffer_ptr() { return &buffer_; }
+
+  // DEPRECATED: Implicit conversion to transparently
+  // support deprecated SerializationTraits API
+  // No need to inline since deprecated
+  operator grpc_byte_buffer*();
+  operator const grpc_byte_buffer*() const;
+
+  grpc_byte_buffer* buffer_;
+};
+
+template <>
+class SerializationTraits<ByteBuffer, void> {
+ public:
+  static Status Deserialize(const ByteBuffer& byte_buffer, ByteBuffer* dest) {
+    dest->set_buffer(byte_buffer.buffer_);
+    return Status::OK;
+  }
+  static Status Serialize(const ByteBuffer& source, ByteBuffer* buffer,
+                          bool* own_buffer) {
+    *buffer = source;
+    *own_buffer = true;
+    return Status::OK;
+  }
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H
--- a/include/grpc++/impl/codegen/call.h
+++ b/include/grpc++/impl/codegen/call.h
@ -25,6 +25,7 @@
 #include <map>
 #include <memory>

+#include <grpc++/impl/codegen/byte_buffer.h>
 #include <grpc++/impl/codegen/call_hook.h>
 #include <grpc++/impl/codegen/client_context.h>
 #include <grpc++/impl/codegen/completion_queue_tag.h>
@ -39,8 +40,6 @@
 #include <grpc/impl/codegen/compression_types.h>
 #include <grpc/impl/codegen/grpc_types.h>

-struct grpc_byte_buffer;
-
 namespace grpc {

 class ByteBuffer;
@ -281,7 +280,7 @@ class CallOpSendInitialMetadata {

 class CallOpSendMessage {
 public:
-  CallOpSendMessage() : send_buf_(nullptr) {}
+  CallOpSendMessage() : send_buf_() {}

  /// Send \a message using \a options for the write. The \a options are cleared
  /// after use.
@ -294,33 +293,67 @@ class CallOpSendMessage {

 protected:
  void AddOp(grpc_op* ops, size_t* nops) {
-    if (send_buf_ == nullptr) return;
+    if (!send_buf_.Valid()) return;
    grpc_op* op = &ops[(*nops)++];
    op->op = GRPC_OP_SEND_MESSAGE;
    op->flags = write_options_.flags();
    op->reserved = NULL;
-    op->data.send_message.send_message = send_buf_;
+    op->data.send_message.send_message = send_buf_.c_buffer();
    // Flags are per-message: clear them after use.
    write_options_.Clear();
  }
-  void FinishOp(bool* status) {
-    g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_);
-    send_buf_ = nullptr;
-  }
+  void FinishOp(bool* status) { send_buf_.Clear(); }

 private:
-  grpc_byte_buffer* send_buf_;
+  template <class M, class T = void>
+  class MessageSerializer;
+
+  ByteBuffer send_buf_;
  WriteOptions write_options_;
 };

+namespace internal {
+template <class T>
+T Example();
+}  // namespace internal
+
+template <class M>
+class CallOpSendMessage::MessageSerializer<
+    M, typename std::enable_if<std::is_same<
+           ::grpc::Status, decltype(SerializationTraits<M>::Serialize(
+                               internal::Example<const M&>(),
+                               internal::Example<grpc_byte_buffer**>(),
+                               internal::Example<bool*>()))>::value>::type> {
+ public:
+  static Status SendMessageInternal(const M& message, ByteBuffer* bbuf,
+                                    bool* own_buf) {
+    return SerializationTraits<M>::Serialize(message, bbuf->c_buffer_ptr(),
+                                             own_buf);
+  }
+};
+
+template <class M>
+class CallOpSendMessage::MessageSerializer<
+    M, typename std::enable_if<std::is_same<
+           ::grpc::Status, decltype(SerializationTraits<M>::Serialize(
+                               internal::Example<const M&>(),
+                               internal::Example<::grpc::ByteBuffer*>(),
+                               internal::Example<bool*>()))>::value>::type> {
+ public:
+  static Status SendMessageInternal(const M& message, ByteBuffer* bbuf,
+                                    bool* own_buf) {
+    return SerializationTraits<M>::Serialize(message, bbuf, own_buf);
+  }
+};
+
 template <class M>
 Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) {
  write_options_ = options;
  bool own_buf;
  Status result =
-      SerializationTraits<M>::Serialize(message, &send_buf_, &own_buf);
+      MessageSerializer<M>::SendMessageInternal(message, &send_buf_, &own_buf);
  if (!own_buf) {
-    send_buf_ = g_core_codegen_interface->grpc_byte_buffer_copy(send_buf_);
+    send_buf_.Duplicate();
  }
  return result;
 }
@ -330,6 +363,36 @@ Status CallOpSendMessage::SendMessage(const M& message) {
  return SendMessage(message, WriteOptions());
 }

+namespace internal {
+template <class M, class T = void>
+class MessageDeserializer;
+
+template <class M>
+class MessageDeserializer<
+    M, typename std::enable_if<std::is_same<
+           ::grpc::Status, decltype(SerializationTraits<M>::Deserialize(
+                               internal::Example<const ::grpc::ByteBuffer&>(),
+                               internal::Example<M*>()))>::value>::type> {
+ public:
+  static Status Deserialize(const ByteBuffer& bbuf, M* message) {
+    return SerializationTraits<M>::Deserialize(bbuf, message);
+  }
+};
+
+template <class M>
+class MessageDeserializer<
+    M, typename std::enable_if<std::is_same<
+           ::grpc::Status, decltype(SerializationTraits<M>::Deserialize(
+                               internal::Example<grpc_byte_buffer*>(),
+                               internal::Example<M*>()))>::value>::type> {
+ public:
+  static Status Deserialize(const ByteBuffer& bbuf, M* message) {
+    return SerializationTraits<M>::Deserialize(
+        const_cast<ByteBuffer&>(bbuf).c_buffer(), message);
+  }
+};
+}  // namespace internal
+
 template <class R>
 class CallOpRecvMessage {
 public:
@ -352,18 +415,20 @@ class CallOpRecvMessage {
    op->op = GRPC_OP_RECV_MESSAGE;
    op->flags = 0;
    op->reserved = NULL;
-    op->data.recv_message.recv_message = &recv_buf_;
+    op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr();
  }

  void FinishOp(bool* status) {
    if (message_ == nullptr) return;
-    if (recv_buf_) {
+    if (recv_buf_.Valid()) {
      if (*status) {
        got_message = *status =
-            SerializationTraits<R>::Deserialize(recv_buf_, message_).ok();
+            internal::MessageDeserializer<R>::Deserialize(recv_buf_, message_)
+                .ok();
+        recv_buf_.Release();
      } else {
        got_message = false;
-        g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_);
+        recv_buf_.Clear();
      }
    } else {
      got_message = false;
@ -376,14 +441,14 @@ class CallOpRecvMessage {

 private:
  R* message_;
-  grpc_byte_buffer* recv_buf_;
+  ByteBuffer recv_buf_;
  bool allow_not_getting_message_;
 };

 namespace CallOpGenericRecvMessageHelper {
 class DeserializeFunc {
 public:
-  virtual Status Deserialize(grpc_byte_buffer* buf) = 0;
+  virtual Status Deserialize(const ByteBuffer& buf) = 0;
  virtual ~DeserializeFunc() {}
 };

@ -391,8 +456,8 @@ template <class R>
 class DeserializeFuncType final : public DeserializeFunc {
 public:
  DeserializeFuncType(R* message) : message_(message) {}
-  Status Deserialize(grpc_byte_buffer* buf) override {
-    return SerializationTraits<R>::Deserialize(buf, message_);
+  Status Deserialize(const ByteBuffer& buf) override {
+    return grpc::internal::MessageDeserializer<R>::Deserialize(buf, message_);
  }

  ~DeserializeFuncType() override {}
@ -428,18 +493,19 @@ class CallOpGenericRecvMessage {
    op->op = GRPC_OP_RECV_MESSAGE;
    op->flags = 0;
    op->reserved = NULL;
-    op->data.recv_message.recv_message = &recv_buf_;
+    op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr();
  }

  void FinishOp(bool* status) {
    if (!deserialize_) return;
-    if (recv_buf_) {
+    if (recv_buf_.Valid()) {
      if (*status) {
        got_message = true;
        *status = deserialize_->Deserialize(recv_buf_).ok();
+        recv_buf_.Release();
      } else {
        got_message = false;
-        g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_);
+        recv_buf_.Clear();
      }
    } else {
      got_message = false;
@ -452,7 +518,7 @@ class CallOpGenericRecvMessage {

 private:
  std::unique_ptr<CallOpGenericRecvMessageHelper::DeserializeFunc> deserialize_;
-  grpc_byte_buffer* recv_buf_;
+  ByteBuffer recv_buf_;
  bool allow_not_getting_message_;
 };

--- a/include/grpc++/impl/codegen/method_handler_impl.h
+++ b/include/grpc++/impl/codegen/method_handler_impl.h
@ -19,6 +19,7 @@
 #ifndef GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H
 #define GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H

+#include <grpc++/impl/codegen/byte_buffer.h>
 #include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/rpc_service_method.h>
 #include <grpc++/impl/codegen/sync_stream.h>
@ -37,8 +38,8 @@ class RpcMethodHandler : public MethodHandler {

  void RunHandler(const HandlerParameter& param) final {
    RequestType req;
-    Status status =
-        SerializationTraits<RequestType>::Deserialize(param.request, &req);
+    Status status = internal::MessageDeserializer<RequestType>::Deserialize(
+        param.request, &req);
    ResponseType rsp;
    if (status.ok()) {
      status = func_(service_, param.server_context, &req, &rsp);
@ -123,8 +124,8 @@ class ServerStreamingHandler : public MethodHandler {

  void RunHandler(const HandlerParameter& param) final {
    RequestType req;
-    Status status =
-        SerializationTraits<RequestType>::Deserialize(param.request, &req);
+    Status status = internal::MessageDeserializer<RequestType>::Deserialize(
+        param.request, &req);

    if (status.ok()) {
      ServerWriter<ResponseType> writer(param.call, param.server_context);
--- a/include/grpc++/impl/codegen/rpc_service_method.h
+++ b/include/grpc++/impl/codegen/rpc_service_method.h
@ -25,14 +25,11 @@
 #include <memory>
 #include <vector>

+#include <grpc++/impl/codegen/byte_buffer.h>
 #include <grpc++/impl/codegen/config.h>
 #include <grpc++/impl/codegen/rpc_method.h>
 #include <grpc++/impl/codegen/status.h>

-extern "C" {
-struct grpc_byte_buffer;
-}
-
 namespace grpc {
 class ServerContext;
 class StreamContextInterface;
@ -43,11 +40,14 @@ class MethodHandler {
  virtual ~MethodHandler() {}
  struct HandlerParameter {
    HandlerParameter(Call* c, ServerContext* context, grpc_byte_buffer* req)
-        : call(c), server_context(context), request(req) {}
+        : call(c), server_context(context) {
+      request.set_buffer(req);
+    }
+    ~HandlerParameter() { request.Release(); }
    Call* call;
    ServerContext* server_context;
-    // Handler required to grpc_byte_buffer_destroy this
-    grpc_byte_buffer* request;
+    // Handler required to destroy these contents
+    ByteBuffer request;
  };
  virtual void RunHandler(const HandlerParameter& param) = 0;
 };
--- a/include/grpc++/impl/codegen/serialization_traits.h
+++ b/include/grpc++/impl/codegen/serialization_traits.h
@ -26,15 +26,24 @@ namespace grpc {
 /// Used for hooking different message serialization API's into GRPC.
 /// Each SerializationTraits implementation must provide the following
 /// functions:
-///   static Status Serialize(const Message& msg,
-///                           grpc_byte_buffer** buffer,
-///                           bool* own_buffer);
-///   static Status Deserialize(grpc_byte_buffer* buffer,
-///                             Message* msg,
-///                             int max_receive_message_size);
+/// 1.  static Status Serialize(const Message& msg,
+///                             ByteBuffer* buffer,
+///                             bool* own_buffer);
+///     AND/OR
+///     static Status Serialize(const Message& msg,
+///                             grpc_byte_buffer** buffer,
+///                             bool* own_buffer);
+///     The former is preferred; the latter is deprecated
 ///
-/// Serialize is required to convert message to a grpc_byte_buffer, and
-/// to store a pointer to that byte buffer at *buffer. *own_buffer should
+/// 2.  static Status Deserialize(const ByteBuffer& buffer,
+///                               Message* msg);
+///     AND/OR
+///     static Status Deserialize(grpc_byte_buffer* buffer,
+///                               Message* msg);
+///     The former is preferred; the latter is deprecated
+///
+/// Serialize is required to convert message to a ByteBuffer, and
+/// return that byte buffer through *buffer. *own_buffer should
 /// be set to true if the caller owns said byte buffer, or false if
 /// ownership is retained elsewhere.
 ///
--- a/include/grpc++/impl/codegen/slice.h
+++ b/include/grpc++/impl/codegen/slice.h
@ -19,11 +19,89 @@
 #ifndef GRPCXX_IMPL_CODEGEN_SLICE_H
 #define GRPCXX_IMPL_CODEGEN_SLICE_H

+#include <grpc++/impl/codegen/config.h>
 #include <grpc++/impl/codegen/core_codegen_interface.h>
 #include <grpc++/impl/codegen/string_ref.h>

+#include <grpc/impl/codegen/slice.h>
+
 namespace grpc {

+/// A wrapper around \a grpc_slice.
+///
+/// A slice represents a contiguous reference counted array of bytes.
+/// It is cheap to take references to a slice, and it is cheap to create a
+/// slice pointing to a subset of another slice.
+class Slice final {
+ public:
+  /// Construct an empty slice.
+  Slice();
+  /// Destructor - drops one reference.
+  ~Slice();
+
+  enum AddRef { ADD_REF };
+  /// Construct a slice from \a slice, adding a reference.
+  Slice(grpc_slice slice, AddRef);
+
+  enum StealRef { STEAL_REF };
+  /// Construct a slice from \a slice, stealing a reference.
+  Slice(grpc_slice slice, StealRef);
+
+  /// Allocate a slice of specified size
+  Slice(size_t len);
+
+  /// Construct a slice from a copied buffer
+  Slice(const void* buf, size_t len);
+
+  /// Construct a slice from a copied string
+  Slice(const grpc::string& str);
+
+  enum StaticSlice { STATIC_SLICE };
+
+  /// Construct a slice from a static buffer
+  Slice(const void* buf, size_t len, StaticSlice);
+
+  /// Copy constructor, adds a reference.
+  Slice(const Slice& other);
+
+  /// Assignment, reference count is unchanged.
+  Slice& operator=(Slice other) {
+    std::swap(slice_, other.slice_);
+    return *this;
+  }
+
+  /// Create a slice pointing at some data. Calls malloc to allocate a refcount
+  /// for the object, and arranges that destroy will be called with the
+  /// user data pointer passed in at destruction. Can be the same as buf or
+  /// different (e.g., if data is part of a larger structure that must be
+  /// destroyed when the data is no longer needed)
+  Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data);
+
+  /// Specialization of above for common case where buf == user_data
+  Slice(void* buf, size_t len, void (*destroy)(void*))
+      : Slice(buf, len, destroy, buf) {}
+
+  /// Similar to the above but has a destroy that also takes slice length
+  Slice(void* buf, size_t len, void (*destroy)(void*, size_t));
+
+  /// Byte size.
+  size_t size() const { return GRPC_SLICE_LENGTH(slice_); }
+
+  /// Raw pointer to the beginning (first element) of the slice.
+  const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); }
+
+  /// Raw pointer to the end (one byte \em past the last element) of the slice.
+  const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); }
+
+  /// Raw C slice. Caller needs to call grpc_slice_unref when done.
+  grpc_slice c_slice() const;
+
+ private:
+  friend class ByteBuffer;
+
+  grpc_slice slice_;
+};
+
 inline grpc::string_ref StringRefFromSlice(const grpc_slice* slice) {
  return grpc::string_ref(
      reinterpret_cast<const char*>(GRPC_SLICE_START_PTR(*slice)),
--- a/include/grpc++/server_builder.h
+++ b/include/grpc++/server_builder.h
@ -136,8 +136,10 @@ class ServerBuilder {
  /// It can be invoked multiple times.
  ///
  /// \param addr_uri The address to try to bind to the server in URI form. If
-  /// the scheme name is omitted, "dns:///" is assumed. Valid values include
-  /// dns:///localhost:1234, / 192.168.1.1:31416, dns:///[::1]:27182, etc.).
+  /// the scheme name is omitted, "dns:///" is assumed. To bind to any address,
+  /// please use IPv6 any, i.e., [::]:<port>, which also accepts IPv4
+  /// connections.  Valid values include dns:///localhost:1234, /
+  /// 192.168.1.1:31416, dns:///[::1]:27182, etc.).
  /// \params creds The credentials associated with the server.
  /// \param selected_port[out] If not `nullptr`, gets populated with the port
  /// number bound to the \a grpc::Server for the corresponding endpoint after
--- a/include/grpc++/support/byte_buffer.h
+++ b/include/grpc++/support/byte_buffer.h
@ -19,6 +19,7 @@
 #ifndef GRPCXX_SUPPORT_BYTE_BUFFER_H
 #define GRPCXX_SUPPORT_BYTE_BUFFER_H

+#include <grpc++/impl/codegen/byte_buffer.h>
 #include <grpc++/impl/serialization_traits.h>
 #include <grpc++/support/config.h>
 #include <grpc++/support/slice.h>
@ -27,71 +28,4 @@
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>

-#include <vector>
-
-namespace grpc {
-
-/// A sequence of bytes.
-class ByteBuffer final {
- public:
-  /// Constuct an empty buffer.
-  ByteBuffer() : buffer_(nullptr) {}
-
-  /// Construct buffer from \a slices, of which there are \a nslices.
-  ByteBuffer(const Slice* slices, size_t nslices);
-
-  /// Constuct a byte buffer by referencing elements of existing buffer
-  /// \a buf. Wrapper of core function grpc_byte_buffer_copy
-  ByteBuffer(const ByteBuffer& buf);
-
-  ~ByteBuffer();
-
-  ByteBuffer& operator=(const ByteBuffer&);
-
-  /// Dump (read) the buffer contents into \a slices.
-  Status Dump(std::vector<Slice>* slices) const;
-
-  /// Remove all data.
-  void Clear();
-
-  /// Buffer size in bytes.
-  size_t Length() const;
-
-  /// Swap the state of *this and *other.
-  void Swap(ByteBuffer* other);
-
- private:
-  friend class SerializationTraits<ByteBuffer, void>;
-
-  // takes ownership
-  void set_buffer(grpc_byte_buffer* buf) {
-    if (buffer_) {
-      Clear();
-    }
-    buffer_ = buf;
-  }
-
-  // For \a SerializationTraits's usage.
-  grpc_byte_buffer* buffer() const { return buffer_; }
-
-  grpc_byte_buffer* buffer_;
-};
-
-template <>
-class SerializationTraits<ByteBuffer, void> {
- public:
-  static Status Deserialize(grpc_byte_buffer* byte_buffer, ByteBuffer* dest) {
-    dest->set_buffer(byte_buffer);
-    return Status::OK;
-  }
-  static Status Serialize(const ByteBuffer& source, grpc_byte_buffer** buffer,
-                          bool* own_buffer) {
-    *buffer = grpc_byte_buffer_copy(source.buffer());
-    *own_buffer = true;
-    return Status::OK;
-  }
-};
-
-}  // namespace grpc
-
 #endif  // GRPCXX_SUPPORT_BYTE_BUFFER_H
--- a/include/grpc++/support/channel_arguments.h
+++ b/include/grpc++/support/channel_arguments.h
@ -64,6 +64,12 @@ class ChannelArguments {
  /// Set the compression algorithm for the channel.
  void SetCompressionAlgorithm(grpc_compression_algorithm algorithm);

+  /// Set the grpclb fallback timeout (in ms) for the channel. If this amount
+  /// of time has passed but we have not gotten any non-empty \a serverlist from
+  /// the balancer, we will fall back to use the backend address(es) returned by
+  /// the resolver.
+  void SetGrpclbFallbackTimeout(int fallback_timeout);
+
  /// Set the socket mutator for the channel.
  void SetSocketMutator(grpc_socket_mutator* mutator);

--- a/include/grpc++/support/slice.h
+++ b/include/grpc++/support/slice.h
@ -19,86 +19,8 @@
 #ifndef GRPCXX_SUPPORT_SLICE_H
 #define GRPCXX_SUPPORT_SLICE_H

+#include <grpc++/impl/codegen/slice.h>
 #include <grpc++/support/config.h>
 #include <grpc/slice.h>

-namespace grpc {
-
-/// A wrapper around \a grpc_slice.
-///
-/// A slice represents a contiguous reference counted array of bytes.
-/// It is cheap to take references to a slice, and it is cheap to create a
-/// slice pointing to a subset of another slice.
-class Slice final {
- public:
-  /// Construct an empty slice.
-  Slice();
-  /// Destructor - drops one reference.
-  ~Slice();
-
-  enum AddRef { ADD_REF };
-  /// Construct a slice from \a slice, adding a reference.
-  Slice(grpc_slice slice, AddRef);
-
-  enum StealRef { STEAL_REF };
-  /// Construct a slice from \a slice, stealing a reference.
-  Slice(grpc_slice slice, StealRef);
-
-  /// Allocate a slice of specified size
-  Slice(size_t len);
-
-  /// Construct a slice from a copied buffer
-  Slice(const void* buf, size_t len);
-
-  /// Construct a slice from a copied string
-  Slice(const grpc::string& str);
-
-  enum StaticSlice { STATIC_SLICE };
-
-  /// Construct a slice from a static buffer
-  Slice(const void* buf, size_t len, StaticSlice);
-
-  /// Copy constructor, adds a reference.
-  Slice(const Slice& other);
-
-  /// Assignment, reference count is unchanged.
-  Slice& operator=(Slice other) {
-    std::swap(slice_, other.slice_);
-    return *this;
-  }
-
-  /// Create a slice pointing at some data. Calls malloc to allocate a refcount
-  /// for the object, and arranges that destroy will be called with the
-  /// user data pointer passed in at destruction. Can be the same as buf or
-  /// different (e.g., if data is part of a larger structure that must be
-  /// destroyed when the data is no longer needed)
-  Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data);
-
-  /// Specialization of above for common case where buf == user_data
-  Slice(void* buf, size_t len, void (*destroy)(void*))
-      : Slice(buf, len, destroy, buf) {}
-
-  /// Similar to the above but has a destroy that also takes slice length
-  Slice(void* buf, size_t len, void (*destroy)(void*, size_t));
-
-  /// Byte size.
-  size_t size() const { return GRPC_SLICE_LENGTH(slice_); }
-
-  /// Raw pointer to the beginning (first element) of the slice.
-  const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); }
-
-  /// Raw pointer to the end (one byte \em past the last element) of the slice.
-  const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); }
-
-  /// Raw C slice. Caller needs to call grpc_slice_unref when done.
-  grpc_slice c_slice() const { return grpc_slice_ref(slice_); }
-
- private:
-  friend class ByteBuffer;
-
-  grpc_slice slice_;
-};
-
-}  // namespace grpc
-
 #endif  // GRPCXX_SUPPORT_SLICE_H
--- a/include/grpc/byte_buffer.h
+++ b/include/grpc/byte_buffer.h
@ -19,69 +19,7 @@
 #ifndef GRPC_BYTE_BUFFER_H
 #define GRPC_BYTE_BUFFER_H

-#include <grpc/impl/codegen/grpc_types.h>
+#include <grpc/impl/codegen/byte_buffer.h>
 #include <grpc/slice_buffer.h>

-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/** Returns a RAW byte buffer instance over the given slices (up to \a nslices).
- *
- * Increases the reference count for all \a slices processed. The user is
- * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/
-GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices,
-                                                      size_t nslices);
-
-/** Returns a *compressed* RAW byte buffer instance over the given slices (up to
- * \a nslices). The \a compression argument defines the compression algorithm
- * used to generate the data in \a slices.
- *
- * Increases the reference count for all \a slices processed. The user is
- * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/
-GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create(
-    grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression);
-
-/** Copies input byte buffer \a bb.
- *
- * Increases the reference count of all the source slices. The user is
- * responsible for calling grpc_byte_buffer_destroy over the returned copy. */
-GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb);
-
-/** Returns the size of the given byte buffer, in bytes. */
-GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb);
-
-/** Destroys \a byte_buffer deallocating all its memory. */
-GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer);
-
-/** Reader for byte buffers. Iterates over slices in the byte buffer */
-struct grpc_byte_buffer_reader;
-typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader;
-
-/** Initialize \a reader to read over \a buffer.
- * Returns 1 upon success, 0 otherwise. */
-GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader,
-                                         grpc_byte_buffer *buffer);
-
-/** Cleanup and destroy \a reader */
-GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader);
-
-/** Updates \a slice with the next piece of data from from \a reader and returns
- * 1. Returns 0 at the end of the stream. Caller is responsible for calling
- * grpc_slice_unref on the result. */
-GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader,
-                                         grpc_slice *slice);
-
-/** Merge all data from \a reader into single slice */
-GRPCAPI grpc_slice
-grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader);
-
-/** Returns a RAW byte buffer instance from the output of \a reader. */
-GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader(
-    grpc_byte_buffer_reader *reader);
-
-#ifdef __cplusplus
-}
-#endif
-
 #endif /* GRPC_BYTE_BUFFER_H */
--- a/include/grpc/impl/codegen/byte_buffer.h
+++ b/include/grpc/impl/codegen/byte_buffer.h
@ -0,0 +1,86 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_IMPL_CODEGEN_BYTE_BUFFER_H
+#define GRPC_IMPL_CODEGEN_BYTE_BUFFER_H
+
+#include <grpc/impl/codegen/grpc_types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Returns a RAW byte buffer instance over the given slices (up to \a nslices).
+ *
+ * Increases the reference count for all \a slices processed. The user is
+ * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/
+GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices,
+                                                      size_t nslices);
+
+/** Returns a *compressed* RAW byte buffer instance over the given slices (up to
+ * \a nslices). The \a compression argument defines the compression algorithm
+ * used to generate the data in \a slices.
+ *
+ * Increases the reference count for all \a slices processed. The user is
+ * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/
+GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create(
+    grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression);
+
+/** Copies input byte buffer \a bb.
+ *
+ * Increases the reference count of all the source slices. The user is
+ * responsible for calling grpc_byte_buffer_destroy over the returned copy. */
+GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb);
+
+/** Returns the size of the given byte buffer, in bytes. */
+GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb);
+
+/** Destroys \a byte_buffer deallocating all its memory. */
+GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer);
+
+/** Reader for byte buffers. Iterates over slices in the byte buffer */
+struct grpc_byte_buffer_reader;
+typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader;
+
+/** Initialize \a reader to read over \a buffer.
+ * Returns 1 upon success, 0 otherwise. */
+GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader,
+                                         grpc_byte_buffer *buffer);
+
+/** Cleanup and destroy \a reader */
+GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader);
+
+/** Updates \a slice with the next piece of data from from \a reader and returns
+ * 1. Returns 0 at the end of the stream. Caller is responsible for calling
+ * grpc_slice_unref on the result. */
+GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader,
+                                         grpc_slice *slice);
+
+/** Merge all data from \a reader into single slice */
+GRPCAPI grpc_slice
+grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader);
+
+/** Returns a RAW byte buffer instance from the output of \a reader. */
+GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader(
+    grpc_byte_buffer_reader *reader);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GRPC_IMPL_CODEGEN_BYTE_BUFFER_H */
--- a/include/grpc/impl/codegen/grpc_types.h
+++ b/include/grpc/impl/codegen/grpc_types.h
@ -287,7 +287,11 @@ typedef struct {
  "grpc.experimental.tcp_max_read_chunk_size"
 /* Timeout in milliseconds to use for calls to the grpclb load balancer.
   If 0 or unset, the balancer calls will have no deadline. */
-#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_timeout_ms"
+#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms"
+/* Timeout in milliseconds to wait for the serverlist from the grpclb load
+   balancer before using fallback backend addresses from the resolver.
+   If 0, fallback will never be used. */
+#define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms"
 /** If non-zero, grpc server's cronet compression workaround will be enabled */
 #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \
  "grpc.workaround.cronet_compression"
--- a/package.xml
+++ b/package.xml
@ -155,6 +155,7 @@
    <file baseinstalldir="/" name="src/core/lib/support/tmpfile_posix.c" role="src" />
    <file baseinstalldir="/" name="src/core/lib/support/tmpfile_windows.c" role="src" />
    <file baseinstalldir="/" name="src/core/lib/support/wrap_memcpy.c" role="src" />
+    <file baseinstalldir="/" name="include/grpc/impl/codegen/byte_buffer.h" role="src" />
    <file baseinstalldir="/" name="include/grpc/impl/codegen/byte_buffer_reader.h" role="src" />
    <file baseinstalldir="/" name="include/grpc/impl/codegen/compression_types.h" role="src" />
    <file baseinstalldir="/" name="include/grpc/impl/codegen/connectivity_state.h" role="src" />
--- a/src/core/ext/filters/client_channel/client_channel.c
+++ b/src/core/ext/filters/client_channel/client_channel.c
@ -1016,13 +1016,11 @@ static void create_subchannel_call_locked(grpc_exec_ctx *exec_ctx,
  GRPC_ERROR_UNREF(error);
 }

-static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx,
-                                    grpc_call_element *elem,
-                                    grpc_error *error) {
+// Invoked when a pick is completed, on both success or failure.
+static void pick_done_locked(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+                             grpc_error *error) {
  call_data *calld = (call_data *)elem->call_data;
  channel_data *chand = (channel_data *)elem->channel_data;
-  grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent,
-                                           chand->interested_parties);
  if (calld->connected_subchannel == NULL) {
    // Failed to create subchannel.
    GRPC_ERROR_UNREF(calld->error);
@ -1044,12 +1042,116 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx,
  GRPC_ERROR_UNREF(error);
 }

-/** Return true if subchannel is available immediately (in which case
-    subchannel_ready_locked() should not be called), or false otherwise (in
-    which case subchannel_ready_locked() should be called when the subchannel
-    is available). */
-static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
-                                   grpc_call_element *elem);
+// A wrapper around pick_done_locked() that is used in cases where
+// either (a) the pick was deferred pending a resolver result or (b) the
+// pick was done asynchronously.  Removes the call's polling entity from
+// chand->interested_parties before invoking pick_done_locked().
+static void async_pick_done_locked(grpc_exec_ctx *exec_ctx,
+                                   grpc_call_element *elem, grpc_error *error) {
+  channel_data *chand = (channel_data *)elem->channel_data;
+  call_data *calld = (call_data *)elem->call_data;
+  grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent,
+                                           chand->interested_parties);
+  pick_done_locked(exec_ctx, elem, error);
+}
+
+// Note: This runs under the client_channel combiner, but will NOT be
+// holding the call combiner.
+static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  channel_data *chand = (channel_data *)elem->channel_data;
+  call_data *calld = (call_data *)elem->call_data;
+  if (calld->lb_policy != NULL) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
+              chand, calld, calld->lb_policy);
+    }
+    grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy,
+                                      &calld->connected_subchannel,
+                                      GRPC_ERROR_REF(error));
+  }
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_callback_cancel");
+}
+
+// Callback invoked by grpc_lb_policy_pick_locked() for async picks.
+// Unrefs the LB policy and invokes async_pick_done_locked().
+static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg,
+                                      grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  channel_data *chand = (channel_data *)elem->channel_data;
+  call_data *calld = (call_data *)elem->call_data;
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously",
+            chand, calld);
+  }
+  GPR_ASSERT(calld->lb_policy != NULL);
+  GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
+  calld->lb_policy = NULL;
+  async_pick_done_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
+}
+
+// Takes a ref to chand->lb_policy and calls grpc_lb_policy_pick_locked().
+// If the pick was completed synchronously, unrefs the LB policy and
+// returns true.
+static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx,
+                                       grpc_call_element *elem) {
+  channel_data *chand = (channel_data *)elem->channel_data;
+  call_data *calld = (call_data *)elem->call_data;
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p calld=%p: starting pick on lb_policy=%p",
+            chand, calld, chand->lb_policy);
+  }
+  apply_service_config_to_call_locked(exec_ctx, elem);
+  // If the application explicitly set wait_for_ready, use that.
+  // Otherwise, if the service config specified a value for this
+  // method, use that.
+  uint32_t initial_metadata_flags =
+      calld->initial_metadata_batch->payload->send_initial_metadata
+          .send_initial_metadata_flags;
+  const bool wait_for_ready_set_from_api =
+      initial_metadata_flags &
+      GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET;
+  const bool wait_for_ready_set_from_service_config =
+      calld->method_params != NULL &&
+      calld->method_params->wait_for_ready != WAIT_FOR_READY_UNSET;
+  if (!wait_for_ready_set_from_api && wait_for_ready_set_from_service_config) {
+    if (calld->method_params->wait_for_ready == WAIT_FOR_READY_TRUE) {
+      initial_metadata_flags |= GRPC_INITIAL_METADATA_WAIT_FOR_READY;
+    } else {
+      initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY;
+    }
+  }
+  const grpc_lb_policy_pick_args inputs = {
+      calld->initial_metadata_batch->payload->send_initial_metadata
+          .send_initial_metadata,
+      initial_metadata_flags, &calld->lb_token_mdelem};
+  // Keep a ref to the LB policy in calld while the pick is pending.
+  GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel");
+  calld->lb_policy = chand->lb_policy;
+  GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem,
+                    grpc_combiner_scheduler(chand->combiner));
+  const bool pick_done = grpc_lb_policy_pick_locked(
+      exec_ctx, chand->lb_policy, &inputs, &calld->connected_subchannel,
+      calld->subchannel_call_context, NULL, &calld->lb_pick_closure);
+  if (pick_done) {
+    /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously",
+              chand, calld);
+    }
+    GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
+    calld->lb_policy = NULL;
+  } else {
+    GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel");
+    grpc_call_combiner_set_notify_on_cancel(
+        exec_ctx, calld->call_combiner,
+        GRPC_CLOSURE_INIT(&calld->lb_pick_cancel_closure,
+                          pick_callback_cancel_locked, elem,
+                          grpc_combiner_scheduler(chand->combiner)));
+  }
+  return pick_done;
+}

 typedef struct {
  grpc_call_element *elem;
@ -1069,17 +1171,17 @@ static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx,
    gpr_free(args);
    return;
  }
-  args->finished = true;
-  grpc_call_element *elem = args->elem;
-  channel_data *chand = (channel_data *)elem->channel_data;
-  call_data *calld = (call_data *)elem->call_data;
  // If we don't yet have a resolver result, then a closure for
  // pick_after_resolver_result_done_locked() will have been added to
  // chand->waiting_for_resolver_result_closures, and it may not be invoked
  // until after this call has been destroyed.  We mark the operation as
  // finished, so that when pick_after_resolver_result_done_locked()
  // is called, it will be a no-op.  We also immediately invoke
-  // subchannel_ready_locked() to propagate the error back to the caller.
+  // async_pick_done_locked() to propagate the error back to the caller.
+  args->finished = true;
+  grpc_call_element *elem = args->elem;
+  channel_data *chand = (channel_data *)elem->channel_data;
+  call_data *calld = (call_data *)elem->call_data;
  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
    gpr_log(GPR_DEBUG,
            "chand=%p calld=%p: cancelling pick waiting for resolver result",
@ -1087,12 +1189,12 @@ static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx,
  }
  // Note: Although we are not in the call combiner here, we are
  // basically stealing the call combiner from the pending pick, so
-  // it's safe to call subchannel_ready_locked() here -- we are
+  // it's safe to call async_pick_done_locked() here -- we are
  // essentially calling it here instead of calling it in
  // pick_after_resolver_result_done_locked().
-  subchannel_ready_locked(exec_ctx, elem,
-                          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                              "Pick cancelled", &error, 1));
+  async_pick_done_locked(exec_ctx, elem,
+                         GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+                             "Pick cancelled", &error, 1));
 }

 static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx,
@ -1117,14 +1219,19 @@ static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx,
      gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver failed to return data",
              chand, calld);
    }
-    subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
+    async_pick_done_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
  } else {
    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
      gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver returned, doing pick",
              chand, calld);
    }
-    if (pick_subchannel_locked(exec_ctx, elem)) {
-      subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_NONE);
+    if (pick_callback_start_locked(exec_ctx, elem)) {
+      // Even if the LB policy returns a result synchronously, we have
+      // already added our polling entity to chand->interested_parties
+      // in order to wait for the resolver result, so we need to
+      // remove it here.  Therefore, we call async_pick_done_locked()
+      // instead of pick_done_locked().
+      async_pick_done_locked(exec_ctx, elem, GRPC_ERROR_NONE);
    }
  }
 }
@ -1152,154 +1259,38 @@ static void pick_after_resolver_result_start_locked(grpc_exec_ctx *exec_ctx,
                        grpc_combiner_scheduler(chand->combiner)));
 }

-// Note: This runs under the client_channel combiner, but will NOT be
-// holding the call combiner.
-static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg,
-                                        grpc_error *error) {
-  grpc_call_element *elem = (grpc_call_element *)arg;
-  channel_data *chand = (channel_data *)elem->channel_data;
-  call_data *calld = (call_data *)elem->call_data;
-  if (error != GRPC_ERROR_NONE && calld->lb_policy != NULL) {
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
-              chand, calld, calld->lb_policy);
-    }
-    grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy,
-                                      &calld->connected_subchannel,
-                                      GRPC_ERROR_REF(error));
-  }
-  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_callback_cancel");
-}
-
-// Callback invoked by grpc_lb_policy_pick_locked() for async picks.
-// Unrefs the LB policy and invokes subchannel_ready_locked().
-static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg,
-                                      grpc_error *error) {
+static void start_pick_locked(grpc_exec_ctx *exec_ctx, void *arg,
+                              grpc_error *ignored) {
  grpc_call_element *elem = (grpc_call_element *)arg;
-  channel_data *chand = (channel_data *)elem->channel_data;
  call_data *calld = (call_data *)elem->call_data;
-  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-    gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously",
-            chand, calld);
-  }
-  GPR_ASSERT(calld->lb_policy != NULL);
-  GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
-  calld->lb_policy = NULL;
-  subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
-}
-
-// Takes a ref to chand->lb_policy and calls grpc_lb_policy_pick_locked().
-// If the pick was completed synchronously, unrefs the LB policy and
-// returns true.
-static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx,
-                                       grpc_call_element *elem,
-                                       const grpc_lb_policy_pick_args *inputs) {
  channel_data *chand = (channel_data *)elem->channel_data;
-  call_data *calld = (call_data *)elem->call_data;
-  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-    gpr_log(GPR_DEBUG, "chand=%p calld=%p: starting pick on lb_policy=%p",
-            chand, calld, chand->lb_policy);
-  }
-  // Keep a ref to the LB policy in calld while the pick is pending.
-  GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel");
-  calld->lb_policy = chand->lb_policy;
-  GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem,
-                    grpc_combiner_scheduler(chand->combiner));
-  const bool pick_done = grpc_lb_policy_pick_locked(
-      exec_ctx, chand->lb_policy, inputs, &calld->connected_subchannel,
-      calld->subchannel_call_context, NULL, &calld->lb_pick_closure);
-  if (pick_done) {
-    /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */
-    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously",
-              chand, calld);
+  GPR_ASSERT(calld->connected_subchannel == NULL);
+  if (chand->lb_policy != NULL) {
+    // We already have an LB policy, so ask it for a pick.
+    if (pick_callback_start_locked(exec_ctx, elem)) {
+      // Pick completed synchronously.
+      pick_done_locked(exec_ctx, elem, GRPC_ERROR_NONE);
+      return;
    }
-    GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
-    calld->lb_policy = NULL;
  } else {
-    GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel");
-    grpc_call_combiner_set_notify_on_cancel(
-        exec_ctx, calld->call_combiner,
-        GRPC_CLOSURE_INIT(&calld->lb_pick_cancel_closure,
-                          pick_callback_cancel_locked, elem,
-                          grpc_combiner_scheduler(chand->combiner)));
-  }
-  return pick_done;
-}
-
-static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
-                                   grpc_call_element *elem) {
-  GPR_TIMER_BEGIN("pick_subchannel", 0);
-  channel_data *chand = (channel_data *)elem->channel_data;
-  call_data *calld = (call_data *)elem->call_data;
-  bool pick_done = false;
-  if (chand->lb_policy != NULL) {
-    apply_service_config_to_call_locked(exec_ctx, elem);
-    // If the application explicitly set wait_for_ready, use that.
-    // Otherwise, if the service config specified a value for this
-    // method, use that.
-    uint32_t initial_metadata_flags =
-        calld->initial_metadata_batch->payload->send_initial_metadata
-            .send_initial_metadata_flags;
-    const bool wait_for_ready_set_from_api =
-        initial_metadata_flags &
-        GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET;
-    const bool wait_for_ready_set_from_service_config =
-        calld->method_params != NULL &&
-        calld->method_params->wait_for_ready != WAIT_FOR_READY_UNSET;
-    if (!wait_for_ready_set_from_api &&
-        wait_for_ready_set_from_service_config) {
-      if (calld->method_params->wait_for_ready == WAIT_FOR_READY_TRUE) {
-        initial_metadata_flags |= GRPC_INITIAL_METADATA_WAIT_FOR_READY;
-      } else {
-        initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY;
-      }
+    // We do not yet have an LB policy, so wait for a resolver result.
+    if (chand->resolver == NULL) {
+      pick_done_locked(exec_ctx, elem,
+                       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Disconnected"));
+      return;
    }
-    const grpc_lb_policy_pick_args inputs = {
-        calld->initial_metadata_batch->payload->send_initial_metadata
-            .send_initial_metadata,
-        initial_metadata_flags, &calld->lb_token_mdelem};
-    pick_done = pick_callback_start_locked(exec_ctx, elem, &inputs);
-  } else if (chand->resolver != NULL) {
    if (!chand->started_resolving) {
      start_resolving_locked(exec_ctx, chand);
    }
    pick_after_resolver_result_start_locked(exec_ctx, elem);
-  } else {
-    subchannel_ready_locked(
-        exec_ctx, elem, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Disconnected"));
-  }
-  GPR_TIMER_END("pick_subchannel", 0);
-  return pick_done;
-}
-
-static void start_pick_locked(grpc_exec_ctx *exec_ctx, void *arg,
-                              grpc_error *error_ignored) {
-  GPR_TIMER_BEGIN("start_pick_locked", 0);
-  grpc_call_element *elem = (grpc_call_element *)arg;
-  call_data *calld = (call_data *)elem->call_data;
-  channel_data *chand = (channel_data *)elem->channel_data;
-  GPR_ASSERT(calld->connected_subchannel == NULL);
-  if (pick_subchannel_locked(exec_ctx, elem)) {
-    // Pick was returned synchronously.
-    if (calld->connected_subchannel == NULL) {
-      GRPC_ERROR_UNREF(calld->error);
-      calld->error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-          "Call dropped by load balancing policy");
-      waiting_for_pick_batches_fail(exec_ctx, elem,
-                                    GRPC_ERROR_REF(calld->error));
-    } else {
-      // Create subchannel call.
-      create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_NONE);
-    }
-  } else {
-    // Pick will be done asynchronously.  Add the call's polling entity to
-    // the channel's interested_parties, so that I/O for the resolver
-    // and LB policy can be done under it.
-    grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent,
-                                           chand->interested_parties);
  }
-  GPR_TIMER_END("start_pick_locked", 0);
+  // We need to wait for either a resolver result or for an async result
+  // from the LB policy.  Add the polling entity from call_data to the
+  // channel_data's interested_parties, so that the I/O of the LB policy
+  // and resolver can be done under it.  The polling entity will be
+  // removed in async_pick_done_locked().
+  grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent,
+                                         chand->interested_parties);
 }

 static void on_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
@ -1394,7 +1385,8 @@ static void cc_start_transport_stream_op_batch(
  // combiner to start a pick.
  if (batch->send_initial_metadata) {
    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-      gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering combiner", chand, calld);
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering client_channel combiner",
+              chand, calld);
    }
    GRPC_CLOSURE_SCHED(
        exec_ctx,
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c
@ -101,6 +101,7 @@
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/parse_address.h"
 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
+#include "src/core/ext/filters/client_channel/subchannel_index.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/iomgr/combiner.h"
@ -122,6 +123,7 @@
 #define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6
 #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120
 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2
+#define GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS 10000

 grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false, "glb");

@ -298,6 +300,10 @@ typedef struct glb_lb_policy {
  /** timeout in milliseconds for the LB call. 0 means no deadline. */
  int lb_call_timeout_ms;

+  /** timeout in milliseconds for before using fallback backend addresses.
+   * 0 means not using fallback. */
+  int lb_fallback_timeout_ms;
+
  /** for communicating with the LB server */
  grpc_channel *lb_channel;

@ -324,6 +330,9 @@ typedef struct glb_lb_policy {
   * Otherwise, we delegate to the RR policy. */
  size_t serverlist_index;

+  /** stores the backend addresses from the resolver */
+  grpc_lb_addresses *fallback_backend_addresses;
+
  /** list of picks that are waiting on RR's policy connectivity */
  pending_pick *pending_picks;

@ -344,6 +353,9 @@ typedef struct glb_lb_policy {
  /** is \a lb_call_retry_timer active? */
  bool retry_timer_active;

+  /** is \a lb_fallback_timer active? */
+  bool fallback_timer_active;
+
  /** called upon changes to the LB channel's connectivity. */
  grpc_closure lb_channel_on_connectivity_changed;

@ -366,6 +378,9 @@ typedef struct glb_lb_policy {
  /* LB call retry timer callback. */
  grpc_closure lb_on_call_retry;

+  /* LB fallback timer callback. */
+  grpc_closure lb_on_fallback;
+
  grpc_call *lb_call; /* streaming call to the LB server, */

  grpc_metadata_array lb_initial_metadata_recv; /* initial MD from LB server */
@ -389,6 +404,9 @@ typedef struct glb_lb_policy {
  /** LB call retry timer */
  grpc_timer lb_call_retry_timer;

+  /** LB fallback timer */
+  grpc_timer lb_fallback_timer;
+
  bool initial_request_sent;
  bool seen_initial_response;

@ -535,6 +553,32 @@ static grpc_lb_addresses *process_serverlist_locked(
  return lb_addresses;
 }

+/* Returns the backend addresses extracted from the given addresses */
+static grpc_lb_addresses *extract_backend_addresses_locked(
+    grpc_exec_ctx *exec_ctx, const grpc_lb_addresses *addresses) {
+  /* first pass: count the number of backend addresses */
+  size_t num_backends = 0;
+  for (size_t i = 0; i < addresses->num_addresses; ++i) {
+    if (!addresses->addresses[i].is_balancer) {
+      ++num_backends;
+    }
+  }
+  /* second pass: actually populate the addresses and (empty) LB tokens */
+  grpc_lb_addresses *backend_addresses =
+      grpc_lb_addresses_create(num_backends, &lb_token_vtable);
+  size_t num_copied = 0;
+  for (size_t i = 0; i < addresses->num_addresses; ++i) {
+    if (addresses->addresses[i].is_balancer) continue;
+    const grpc_resolved_address *addr = &addresses->addresses[i].address;
+    grpc_lb_addresses_set_address(backend_addresses, num_copied, &addr->addr,
+                                  addr->len, false /* is_balancer */,
+                                  NULL /* balancer_name */,
+                                  (void *)GRPC_MDELEM_LB_TOKEN_EMPTY.payload);
+    ++num_copied;
+  }
+  return backend_addresses;
+}
+
 static void update_lb_connectivity_status_locked(
    grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
    grpc_connectivity_state rr_state, grpc_error *rr_state_error) {
@ -602,35 +646,38 @@ static bool pick_from_internal_rr_locked(
    grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
    const grpc_lb_policy_pick_args *pick_args, bool force_async,
    grpc_connected_subchannel **target, wrapped_rr_closure_arg *wc_arg) {
-  // Look at the index into the serverlist to see if we should drop this call.
-  grpc_grpclb_server *server =
-      glb_policy->serverlist->servers[glb_policy->serverlist_index++];
-  if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) {
-    glb_policy->serverlist_index = 0;  // Wrap-around.
-  }
-  if (server->drop) {
-    // Not using the RR policy, so unref it.
-    if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
-      gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")",
-              (intptr_t)wc_arg->rr_policy);
+  // Check for drops if we are not using fallback backend addresses.
+  if (glb_policy->serverlist != NULL) {
+    // Look at the index into the serverlist to see if we should drop this call.
+    grpc_grpclb_server *server =
+        glb_policy->serverlist->servers[glb_policy->serverlist_index++];
+    if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) {
+      glb_policy->serverlist_index = 0;  // Wrap-around.
    }
-    GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync");
-    // Update client load reporting stats to indicate the number of
-    // dropped calls.  Note that we have to do this here instead of in
-    // the client_load_reporting filter, because we do not create a
-    // subchannel call (and therefore no client_load_reporting filter)
-    // for dropped calls.
-    grpc_grpclb_client_stats_add_call_dropped_locked(server->load_balance_token,
-                                                     wc_arg->client_stats);
-    grpc_grpclb_client_stats_unref(wc_arg->client_stats);
-    if (force_async) {
-      GPR_ASSERT(wc_arg->wrapped_closure != NULL);
-      GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE);
+    if (server->drop) {
+      // Not using the RR policy, so unref it.
+      if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
+        gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")",
+                (intptr_t)wc_arg->rr_policy);
+      }
+      GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync");
+      // Update client load reporting stats to indicate the number of
+      // dropped calls.  Note that we have to do this here instead of in
+      // the client_load_reporting filter, because we do not create a
+      // subchannel call (and therefore no client_load_reporting filter)
+      // for dropped calls.
+      grpc_grpclb_client_stats_add_call_dropped_locked(
+          server->load_balance_token, wc_arg->client_stats);
+      grpc_grpclb_client_stats_unref(wc_arg->client_stats);
+      if (force_async) {
+        GPR_ASSERT(wc_arg->wrapped_closure != NULL);
+        GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE);
+        gpr_free(wc_arg->free_when_done);
+        return false;
+      }
      gpr_free(wc_arg->free_when_done);
-      return false;
+      return true;
    }
-    gpr_free(wc_arg->free_when_done);
-    return true;
  }
  // Pick via the RR policy.
  const bool pick_done = grpc_lb_policy_pick_locked(
@ -668,8 +715,18 @@ static bool pick_from_internal_rr_locked(

 static grpc_lb_policy_args *lb_policy_args_create(grpc_exec_ctx *exec_ctx,
                                                  glb_lb_policy *glb_policy) {
-  grpc_lb_addresses *addresses =
-      process_serverlist_locked(exec_ctx, glb_policy->serverlist);
+  grpc_lb_addresses *addresses;
+  if (glb_policy->serverlist != NULL) {
+    GPR_ASSERT(glb_policy->serverlist->num_servers > 0);
+    addresses = process_serverlist_locked(exec_ctx, glb_policy->serverlist);
+  } else {
+    // If rr_handover_locked() is invoked when we haven't received any
+    // serverlist from the balancer, we use the fallback backends returned by
+    // the resolver. Note that the fallback backend list may be empty, in which
+    // case the new round_robin policy will keep the requested picks pending.
+    GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL);
+    addresses = grpc_lb_addresses_copy(glb_policy->fallback_backend_addresses);
+  }
  GPR_ASSERT(addresses != NULL);
  grpc_lb_policy_args *args = (grpc_lb_policy_args *)gpr_zalloc(sizeof(*args));
  args->client_channel_factory = glb_policy->cc_factory;
@ -775,8 +832,6 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
 /* glb_policy->rr_policy may be NULL (initial handover) */
 static void rr_handover_locked(grpc_exec_ctx *exec_ctx,
                               glb_lb_policy *glb_policy) {
-  GPR_ASSERT(glb_policy->serverlist != NULL &&
-             glb_policy->serverlist->num_servers > 0);
  if (glb_policy->shutting_down) return;
  grpc_lb_policy_args *args = lb_policy_args_create(exec_ctx, glb_policy);
  GPR_ASSERT(args != NULL);
@ -917,13 +972,7 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx,
 static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
                                  grpc_lb_policy_factory *factory,
                                  grpc_lb_policy_args *args) {
-  /* Count the number of gRPC-LB addresses. There must be at least one.
-   * TODO(roth): For now, we ignore non-balancer addresses, but in the
-   * future, we may change the behavior such that we fall back to using
-   * the non-balancer addresses if we cannot reach any balancers. In the
-   * fallback case, we should use the LB policy indicated by
-   * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is
-   * unset, we should default to pick_first). */
+  /* Count the number of gRPC-LB addresses. There must be at least one. */
  const grpc_arg *arg =
      grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
  if (arg == NULL || arg->type != GRPC_ARG_POINTER) {
@ -959,6 +1008,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
  glb_policy->lb_call_timeout_ms =
      grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX});

+  arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS);
+  glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer(
+      arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0,
+                                  INT_MAX});
+
  // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args,
  // since we use this to trigger the client_load_reporting filter.
  grpc_arg new_arg =
@ -967,6 +1021,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
  glb_policy->args = grpc_channel_args_copy_and_add_and_remove(
      args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1);

+  /* Extract the backend addresses (may be empty) from the resolver for
+   * fallback. */
+  glb_policy->fallback_backend_addresses =
+      extract_backend_addresses_locked(exec_ctx, addresses);
+
  /* Create a client channel over them to communicate with a LB service */
  glb_policy->response_generator =
      grpc_fake_resolver_response_generator_create();
@ -988,6 +1047,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
    gpr_free(glb_policy);
    return NULL;
  }
+  grpc_subchannel_index_ref();
  GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed,
                    glb_lb_channel_on_connectivity_changed_cb, glb_policy,
                    grpc_combiner_scheduler(args->combiner));
@ -1010,7 +1070,11 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
  if (glb_policy->serverlist != NULL) {
    grpc_grpclb_destroy_serverlist(glb_policy->serverlist);
  }
+  if (glb_policy->fallback_backend_addresses != NULL) {
+    grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses);
+  }
  grpc_fake_resolver_response_generator_unref(glb_policy->response_generator);
+  grpc_subchannel_index_unref();
  if (glb_policy->pending_update_args != NULL) {
    grpc_channel_args_destroy(exec_ctx, glb_policy->pending_update_args->args);
    gpr_free(glb_policy->pending_update_args);
@ -1150,10 +1214,28 @@ static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx,
  GRPC_ERROR_UNREF(error);
 }

+static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error);
 static void query_for_backends_locked(grpc_exec_ctx *exec_ctx,
                                      glb_lb_policy *glb_policy);
 static void start_picking_locked(grpc_exec_ctx *exec_ctx,
                                 glb_lb_policy *glb_policy) {
+  /* start a timer to fall back */
+  if (glb_policy->lb_fallback_timeout_ms > 0 &&
+      glb_policy->serverlist == NULL) {
+    gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
+    gpr_timespec deadline = gpr_time_add(
+        now,
+        gpr_time_from_millis(glb_policy->lb_fallback_timeout_ms, GPR_TIMESPAN));
+    GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer");
+    GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked,
+                      glb_policy,
+                      grpc_combiner_scheduler(glb_policy->base.combiner));
+    glb_policy->fallback_timer_active = true;
+    grpc_timer_init(exec_ctx, &glb_policy->lb_fallback_timer, deadline,
+                    &glb_policy->lb_on_fallback, now);
+  }
+
  glb_policy->started_picking = true;
  gpr_backoff_reset(&glb_policy->lb_call_backoff_state);
  query_for_backends_locked(exec_ctx, glb_policy);
@ -1607,6 +1689,15 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
            if (glb_policy->serverlist != NULL) {
              /* dispose of the old serverlist */
              grpc_grpclb_destroy_serverlist(glb_policy->serverlist);
+            } else {
+              /* or dispose of the fallback */
+              grpc_lb_addresses_destroy(exec_ctx,
+                                        glb_policy->fallback_backend_addresses);
+              glb_policy->fallback_backend_addresses = NULL;
+              if (glb_policy->fallback_timer_active) {
+                grpc_timer_cancel(exec_ctx, &glb_policy->lb_fallback_timer);
+                glb_policy->fallback_timer_active = false;
+              }
            }
            /* and update the copy in the glb_lb_policy instance. This
             * serverlist instance will be destroyed either upon the next
@ -1617,9 +1708,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
          }
        } else {
          if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
-            gpr_log(GPR_INFO,
-                    "Received empty server list. Picks will stay pending until "
-                    "a response with > 0 servers is received");
+            gpr_log(GPR_INFO, "Received empty server list, ignoring.");
          }
          grpc_grpclb_destroy_serverlist(serverlist);
        }
@ -1666,6 +1755,26 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
  GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer");
 }

+static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  glb_lb_policy *glb_policy = arg;
+  /* If we receive a serverlist after the timer fires but before this callback
+   * actually runs, don't do anything. */
+  if (glb_policy->serverlist != NULL) return;
+  glb_policy->fallback_timer_active = false;
+  if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) {
+    if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
+      gpr_log(GPR_INFO,
+              "Falling back to use backends from resolver (grpclb %p)",
+              (void *)glb_policy);
+    }
+    GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL);
+    rr_handover_locked(exec_ctx, glb_policy);
+  }
+  GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
+                            "grpclb_fallback_timer");
+}
+
 static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx,
                                                void *arg, grpc_error *error) {
  glb_lb_policy *glb_policy = (glb_lb_policy *)arg;
@ -1786,6 +1895,17 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
        &glb_policy->lb_channel_connectivity,
        &glb_policy->lb_channel_on_connectivity_changed, NULL);
  }
+
+  // Propagate update to fallback_backend_addresses if a non-empty serverlist
+  // hasn't been received from the balancer.
+  if (glb_policy->serverlist == NULL) {
+    grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses);
+    glb_policy->fallback_backend_addresses =
+        extract_backend_addresses_locked(exec_ctx, addresses);
+    if (glb_policy->rr_policy != NULL) {
+      rr_handover_locked(exec_ctx, glb_policy);
+    }
+  }
 }

 // Invoked as part of the update process. It continues watching the LB channel
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c
@ -89,6 +89,7 @@ static void pf_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
                                    "picked_first_destroy");
  }
  grpc_connectivity_state_destroy(exec_ctx, &p->state_tracker);
+  grpc_subchannel_index_unref();
  if (p->pending_update_args != NULL) {
    grpc_channel_args_destroy(exec_ctx, p->pending_update_args->args);
    gpr_free(p->pending_update_args);
@ -686,6 +687,7 @@ static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx,
  }
  pf_update_locked(exec_ctx, &p->base, args);
  grpc_lb_policy_init(&p->base, &pick_first_lb_policy_vtable, args->combiner);
+  grpc_subchannel_index_ref();
  GRPC_CLOSURE_INIT(&p->connectivity_changed, pf_connectivity_changed_locked, p,
                    grpc_combiner_scheduler(args->combiner));
  return &p->base;
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c
@ -30,6 +30,7 @@

 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
+#include "src/core/ext/filters/client_channel/subchannel_index.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/combiner.h"
@ -310,6 +311,7 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
            (void *)pol, (void *)pol);
  }
  grpc_connectivity_state_destroy(exec_ctx, &p->state_tracker);
+  grpc_subchannel_index_unref();
  gpr_free(p);
 }

@ -890,6 +892,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
  GPR_ASSERT(args->client_channel_factory != NULL);
  round_robin_lb_policy *p = (round_robin_lb_policy *)gpr_zalloc(sizeof(*p));
  grpc_lb_policy_init(&p->base, &round_robin_lb_policy_vtable, args->combiner);
+  grpc_subchannel_index_ref();
  grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE,
                               "round_robin");
  rr_update_locked(exec_ctx, &p->base, args);
--- a/src/core/ext/filters/client_channel/lb_policy_factory.c
+++ b/src/core/ext/filters/client_channel/lb_policy_factory.c
@ -56,7 +56,7 @@ grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses) {
 }

 void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index,
-                                   void* address, size_t address_len,
+                                   const void* address, size_t address_len,
                                   bool is_balancer, const char* balancer_name,
                                   void* user_data) {
  GPR_ASSERT(index < addresses->num_addresses);
--- a/src/core/ext/filters/client_channel/lb_policy_factory.h
+++ b/src/core/ext/filters/client_channel/lb_policy_factory.h
@ -73,7 +73,7 @@ grpc_lb_addresses *grpc_lb_addresses_copy(const grpc_lb_addresses *addresses);
 * \a address is a socket address of length \a address_len.
 * Takes ownership of \a balancer_name. */
 void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index,
-                                   void *address, size_t address_len,
+                                   const void *address, size_t address_len,
                                   bool is_balancer, const char *balancer_name,
                                   void *user_data);

--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c
@ -174,7 +174,7 @@ static void on_hostbyname_done_cb(void *arg, int status, int timeouts,
          grpc_lb_addresses_set_address(
              *lb_addresses, i, &addr, addr_len,
              hr->is_balancer /* is_balancer */,
-              hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */,
+              hr->is_balancer ? hr->host : NULL /* balancer_name */,
              NULL /* user_data */);
          char output[INET6_ADDRSTRLEN];
          ares_inet_ntop(AF_INET6, &addr.sin6_addr, output, INET6_ADDRSTRLEN);
@ -195,7 +195,7 @@ static void on_hostbyname_done_cb(void *arg, int status, int timeouts,
          grpc_lb_addresses_set_address(
              *lb_addresses, i, &addr, addr_len,
              hr->is_balancer /* is_balancer */,
-              hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */,
+              hr->is_balancer ? hr->host : NULL /* balancer_name */,
              NULL /* user_data */);
          char output[INET_ADDRSTRLEN];
          ares_inet_ntop(AF_INET, &addr.sin_addr, output, INET_ADDRSTRLEN);
--- a/src/core/ext/filters/client_channel/subchannel_index.c
+++ b/src/core/ext/filters/client_channel/subchannel_index.c
@ -34,6 +34,8 @@ static gpr_avl g_subchannel_index;

 static gpr_mu g_mu;

+static gpr_refcount g_refcount;
+
 struct grpc_subchannel_key {
  grpc_subchannel_args args;
 };
@ -119,15 +121,27 @@ static const gpr_avl_vtable subchannel_avl_vtable = {
 void grpc_subchannel_index_init(void) {
  g_subchannel_index = gpr_avl_create(&subchannel_avl_vtable);
  gpr_mu_init(&g_mu);
+  gpr_ref_init(&g_refcount, 1);
 }

 void grpc_subchannel_index_shutdown(void) {
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  gpr_mu_destroy(&g_mu);
-  gpr_avl_unref(g_subchannel_index, &exec_ctx);
-  grpc_exec_ctx_finish(&exec_ctx);
+  // TODO(juanlishen): This refcounting mechanism may lead to memory leackage.
+  // To solve that, we should force polling to flush any pending callbacks, then
+  // shutdown safely.
+  grpc_subchannel_index_unref();
+}
+
+void grpc_subchannel_index_unref(void) {
+  if (gpr_unref(&g_refcount)) {
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    gpr_mu_destroy(&g_mu);
+    gpr_avl_unref(g_subchannel_index, &exec_ctx);
+    grpc_exec_ctx_finish(&exec_ctx);
+  }
 }

+void grpc_subchannel_index_ref(void) { gpr_ref_non_zero(&g_refcount); }
+
 grpc_subchannel *grpc_subchannel_index_find(grpc_exec_ctx *exec_ctx,
                                            grpc_subchannel_key *key) {
  // Lock, and take a reference to the subchannel index.
--- a/src/core/ext/filters/client_channel/subchannel_index.h
+++ b/src/core/ext/filters/client_channel/subchannel_index.h
@ -59,6 +59,13 @@ void grpc_subchannel_index_init(void);
 /** Shutdown the subchannel index (global) */
 void grpc_subchannel_index_shutdown(void);

+/** Increment the refcount (non-zero) of subchannel index (global). */
+void grpc_subchannel_index_ref(void);
+
+/** Decrement the refcount of subchannel index (global). If the refcount drops
+    to zero, unref the subchannel index and destroy its mutex. */
+void grpc_subchannel_index_unref(void);
+
 /** \em TEST ONLY.
 * If \a force_creation is true, all key comparisons will be false, resulting in
 * new subchannels always being created. Otherwise, the keys will be compared as
--- a/src/core/ext/filters/max_age/max_age_filter.c
+++ b/src/core/ext/filters/max_age/max_age_filter.c
@ -402,7 +402,7 @@ static bool maybe_add_max_age_filter(grpc_exec_ctx* exec_ctx,
  bool enable =
      grpc_channel_arg_get_integer(
          grpc_channel_args_find(channel_args, GRPC_ARG_MAX_CONNECTION_AGE_MS),
-          MAX_CONNECTION_AGE_INTEGER_OPTIONS) != INT_MAX &&
+          MAX_CONNECTION_AGE_INTEGER_OPTIONS) != INT_MAX ||
      grpc_channel_arg_get_integer(
          grpc_channel_args_find(channel_args, GRPC_ARG_MAX_CONNECTION_IDLE_MS),
          MAX_CONNECTION_IDLE_INTEGER_OPTIONS) != INT_MAX;
--- a/src/core/ext/transport/chttp2/transport/flow_control.c
+++ b/src/core/ext/transport/chttp2/transport/flow_control.c
@ -18,6 +18,7 @@

 #include "src/core/ext/transport/chttp2/transport/internal.h"

+#include <limits.h>
 #include <math.h>
 #include <string.h>

@ -483,7 +484,8 @@ grpc_chttp2_flowctl_action grpc_chttp2_flowctl_get_bdp_action(
    if (grpc_bdp_estimator_get_bw(&tfc->bdp_estimator, &bw_dbl)) {
      // we target the max of BDP or bandwidth in microseconds.
      int32_t frame_size = (int32_t)GPR_CLAMP(
-          GPR_MAX((int32_t)bw_dbl / 1000, bdp), 16384, 16777215);
+          GPR_MAX((int32_t)GPR_CLAMP(bw_dbl, 0, INT_MAX) / 1000, bdp), 16384,
+          16777215);
      grpc_chttp2_flowctl_urgency frame_size_urgency = delta_is_significant(
          tfc, frame_size, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE);
      if (frame_size_urgency != GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED) {
--- a/src/core/ext/transport/chttp2/transport/writing.c
+++ b/src/core/ext/transport/chttp2/transport/writing.c
@ -391,6 +391,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
                                                    s->id, GRPC_HTTP2_NO_ERROR,
                                                    &s->stats.outgoing));
            }
+            grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1,
+                                           GRPC_ERROR_NONE);
          }
          result.early_results_scheduled |=
              update_list(exec_ctx, t, s,
@ -449,6 +451,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write(
              &t->outbuf, grpc_chttp2_rst_stream_create(
                              s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing));
        }
+        grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1,
+                                       GRPC_ERROR_NONE);
        now_writing = true;
        result.early_results_scheduled = true;
        grpc_chttp2_complete_closure_step(
@ -519,10 +523,6 @@ void grpc_chttp2_end_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
                  GRPC_ERROR_REF(error));
      s->sending_bytes = 0;
    }
-    if (s->sent_trailing_metadata) {
-      grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1,
-                                     GRPC_ERROR_REF(error));
-    }
    GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:end");
  }
  grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &t->outbuf);
--- a/src/core/lib/channel/channel_args.c
+++ b/src/core/lib/channel/channel_args.c
@ -212,7 +212,7 @@ void grpc_channel_args_destroy(grpc_exec_ctx *exec_ctx, grpc_channel_args *a) {
 grpc_compression_algorithm grpc_channel_args_get_compression_algorithm(
    const grpc_channel_args *a) {
  size_t i;
-  if (a == NULL) return 0;
+  if (a == NULL) return GRPC_COMPRESS_NONE;
  for (i = 0; i < a->num_args; ++i) {
    if (a->args[i].type == GRPC_ARG_INTEGER &&
        !strcmp(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, a->args[i].key)) {
@ -226,7 +226,7 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm(
 grpc_stream_compression_algorithm
 grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args *a) {
  size_t i;
-  if (a == NULL) return 0;
+  if (a == NULL) return GRPC_STREAM_COMPRESS_NONE;
  for (i = 0; i < a->num_args; ++i) {
    if (a->args[i].type == GRPC_ARG_INTEGER &&
        !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM,
--- a/src/core/lib/channel/channel_stack_builder.c
+++ b/src/core/lib/channel/channel_stack_builder.c
@ -214,13 +214,13 @@ bool grpc_channel_stack_builder_prepend_filter(
 static void add_after(filter_node *before, const grpc_channel_filter *filter,
                      grpc_post_filter_create_init_func post_init_func,
                      void *user_data) {
-  filter_node *new = (filter_node *)gpr_malloc(sizeof(*new));
-  new->next = before->next;
-  new->prev = before;
-  new->next->prev = new->prev->next = new;
-  new->filter = filter;
-  new->init = post_init_func;
-  new->init_arg = user_data;
+  filter_node *new_node = (filter_node *)gpr_malloc(sizeof(*new_node));
+  new_node->next = before->next;
+  new_node->prev = before;
+  new_node->next->prev = new_node->prev->next = new_node;
+  new_node->filter = filter;
+  new_node->init = post_init_func;
+  new_node->init_arg = user_data;
 }

 bool grpc_channel_stack_builder_add_filter_before(
@ -268,7 +268,7 @@ grpc_error *grpc_channel_stack_builder_finish(

  // create an array of filters
  const grpc_channel_filter **filters =
-      gpr_malloc(sizeof(*filters) * num_filters);
+      (const grpc_channel_filter **)gpr_malloc(sizeof(*filters) * num_filters);
  size_t i = 0;
  for (filter_node *p = builder->begin.next; p != &builder->end; p = p->next) {
    filters[i++] = p->filter;
--- a/src/core/lib/compression/stream_compression.c
+++ b/src/core/lib/compression/stream_compression.c
@ -159,7 +159,8 @@ bool grpc_stream_decompress(grpc_stream_compression_context *ctx,
 grpc_stream_compression_context *grpc_stream_compression_context_create(
    grpc_stream_compression_method method) {
  grpc_stream_compression_context *ctx =
-      gpr_zalloc(sizeof(grpc_stream_compression_context));
+      (grpc_stream_compression_context *)gpr_zalloc(
+          sizeof(grpc_stream_compression_context));
  int r;
  if (ctx == NULL) {
    return NULL;
--- a/src/core/lib/debug/stats_data.c
+++ b/src/core/lib/debug/stats_data.c
@ -58,6 +58,8 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
    "executor_push_retries",
    "executor_threads_created",
    "executor_threads_used",
+    "server_requested_calls",
+    "server_slowpath_requests_queued",
 };
 const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = {
    "Number of client side calls created by this process",
@ -102,6 +104,9 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = {
    "the executor",
    "Size of the backing thread pool for overflow gRPC Core work",
    "How many executor threads actually got used",
+    "How many calls were requested (not necessarily received) by the server",
+    "How many times was the server slow path taken (indicates too few "
+    "outstanding requests)",
 };
 const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = {
    "tcp_write_size",
@ -115,6 +120,7 @@ const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = {
    "http2_send_trailing_metadata_per_write",
    "http2_send_flowctl_per_write",
    "executor_closures_per_wakeup",
+    "server_cqs_checked",
 };
 const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = {
    "Number of bytes offered to each syscall_write",
@ -128,6 +134,8 @@ const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = {
    "Number of streams terminated per TCP write",
    "Number of flow control updates written per TCP write",
    "Number of closures executed each time an executor wakes up",
+    "How many completion queues were checked looking for a CQ that had "
+    "requested the incoming call",
 };
 const int grpc_stats_table_0[65] = {
    0,       1,       2,       3,       4,       6,       8,        11,
@ -158,6 +166,8 @@ const uint8_t grpc_stats_table_3[102] = {
    23, 24, 24, 24, 25, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32,
    32, 33, 33, 34, 35, 35, 36, 37, 37, 38, 38, 39, 39, 40, 40, 41, 41,
    42, 42, 43, 44, 44, 45, 46, 46, 47, 48, 48, 49, 49, 50, 50, 51, 51};
+const int grpc_stats_table_4[9] = {0, 1, 2, 4, 7, 13, 23, 39, 64};
+const uint8_t grpc_stats_table_5[9] = {0, 0, 1, 2, 2, 3, 4, 4, 5};
 void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) {
  value = GPR_CLAMP(value, 0, 16777216);
  if (value < 5) {
@ -451,16 +461,41 @@ void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx,
                           grpc_stats_histo_find_bucket_slow(
                               (exec_ctx), value, grpc_stats_table_2, 64));
 }
-const int grpc_stats_histo_buckets[11] = {64, 64, 64, 64, 64, 64,
-                                          64, 64, 64, 64, 64};
-const int grpc_stats_histo_start[11] = {0,   64,  128, 192, 256, 320,
-                                        384, 448, 512, 576, 640};
-const int *const grpc_stats_histo_bucket_boundaries[11] = {
+void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) {
+  value = GPR_CLAMP(value, 0, 64);
+  if (value < 3) {
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx),
+                             GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, value);
+    return;
+  }
+  union {
+    double dbl;
+    uint64_t uint;
+  } _val, _bkt;
+  _val.dbl = value;
+  if (_val.uint < 4625196817309499392ull) {
+    int bucket =
+        grpc_stats_table_5[((_val.uint - 4613937818241073152ull) >> 51)] + 3;
+    _bkt.dbl = grpc_stats_table_4[bucket];
+    bucket -= (_val.uint < _bkt.uint);
+    GRPC_STATS_INC_HISTOGRAM((exec_ctx),
+                             GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, bucket);
+    return;
+  }
+  GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED,
+                           grpc_stats_histo_find_bucket_slow(
+                               (exec_ctx), value, grpc_stats_table_4, 8));
+}
+const int grpc_stats_histo_buckets[12] = {64, 64, 64, 64, 64, 64,
+                                          64, 64, 64, 64, 64, 8};
+const int grpc_stats_histo_start[12] = {0,   64,  128, 192, 256, 320,
+                                        384, 448, 512, 576, 640, 704};
+const int *const grpc_stats_histo_bucket_boundaries[12] = {
    grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0,
    grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0,
    grpc_stats_table_2, grpc_stats_table_2, grpc_stats_table_2,
-    grpc_stats_table_2, grpc_stats_table_2};
-void (*const grpc_stats_inc_histogram[11])(grpc_exec_ctx *exec_ctx, int x) = {
+    grpc_stats_table_2, grpc_stats_table_2, grpc_stats_table_4};
+void (*const grpc_stats_inc_histogram[12])(grpc_exec_ctx *exec_ctx, int x) = {
    grpc_stats_inc_tcp_write_size,
    grpc_stats_inc_tcp_write_iov_size,
    grpc_stats_inc_tcp_read_size,
@ -471,4 +506,5 @@ void (*const grpc_stats_inc_histogram[11])(grpc_exec_ctx *exec_ctx, int x) = {
    grpc_stats_inc_http2_send_message_per_write,
    grpc_stats_inc_http2_send_trailing_metadata_per_write,
    grpc_stats_inc_http2_send_flowctl_per_write,
-    grpc_stats_inc_executor_closures_per_wakeup};
+    grpc_stats_inc_executor_closures_per_wakeup,
+    grpc_stats_inc_server_cqs_checked};
--- a/src/core/lib/debug/stats_data.h
+++ b/src/core/lib/debug/stats_data.h
@ -60,6 +60,8 @@ typedef enum {
  GRPC_STATS_COUNTER_EXECUTOR_PUSH_RETRIES,
  GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED,
  GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED,
+  GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS,
+  GRPC_STATS_COUNTER_SERVER_SLOWPATH_REQUESTS_QUEUED,
  GRPC_STATS_COUNTER_COUNT
 } grpc_stats_counters;
 extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];
@ -76,6 +78,7 @@ typedef enum {
  GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE,
  GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE,
  GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP,
+  GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED,
  GRPC_STATS_HISTOGRAM_COUNT
 } grpc_stats_histograms;
 extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT];
@ -103,7 +106,9 @@ typedef enum {
  GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_BUCKETS = 64,
  GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_FIRST_SLOT = 640,
  GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_BUCKETS = 64,
-  GRPC_STATS_HISTOGRAM_BUCKETS = 704
+  GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_FIRST_SLOT = 704,
+  GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_BUCKETS = 8,
+  GRPC_STATS_HISTOGRAM_BUCKETS = 712
 } grpc_stats_histogram_constants;
 #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \
  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED)
@ -189,6 +194,11 @@ typedef enum {
                         GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED)
 #define GRPC_STATS_INC_EXECUTOR_THREADS_USED(exec_ctx) \
  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED)
+#define GRPC_STATS_INC_SERVER_REQUESTED_CALLS(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS)
+#define GRPC_STATS_INC_SERVER_SLOWPATH_REQUESTS_QUEUED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx),                             \
+                         GRPC_STATS_COUNTER_SERVER_SLOWPATH_REQUESTS_QUEUED)
 #define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \
  grpc_stats_inc_tcp_write_size((exec_ctx), (int)(value))
 void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int x);
@ -228,10 +238,13 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx,
  grpc_stats_inc_executor_closures_per_wakeup((exec_ctx), (int)(value))
 void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx,
                                                 int x);
-extern const int grpc_stats_histo_buckets[11];
-extern const int grpc_stats_histo_start[11];
-extern const int *const grpc_stats_histo_bucket_boundaries[11];
-extern void (*const grpc_stats_inc_histogram[11])(grpc_exec_ctx *exec_ctx,
+#define GRPC_STATS_INC_SERVER_CQS_CHECKED(exec_ctx, value) \
+  grpc_stats_inc_server_cqs_checked((exec_ctx), (int)(value))
+void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int x);
+extern const int grpc_stats_histo_buckets[12];
+extern const int grpc_stats_histo_start[12];
+extern const int *const grpc_stats_histo_bucket_boundaries[12];
+extern void (*const grpc_stats_inc_histogram[12])(grpc_exec_ctx *exec_ctx,
                                                  int x);

 #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */
--- a/src/core/lib/debug/stats_data.yaml
+++ b/src/core/lib/debug/stats_data.yaml
@ -143,3 +143,14 @@
  max: 1024
  buckets: 64
  doc: Number of closures executed each time an executor wakes up
+# server
+- counter: server_requested_calls
+  doc: How many calls were requested (not necessarily received) by the server
+- histogram: server_cqs_checked
+  buckets: 8
+  max: 64
+  doc: How many completion queues were checked looking for a CQ that had
+       requested the incoming call
+- counter: server_slowpath_requests_queued
+  doc: How many times was the server slow path taken (indicates too few
+       outstanding requests)
--- a/src/core/lib/debug/stats_data_bq_schema.sql
+++ b/src/core/lib/debug/stats_data_bq_schema.sql
@ -0,0 +1,37 @@
+client_calls_created_per_iteration:FLOAT,
+server_calls_created_per_iteration:FLOAT,
+syscall_poll_per_iteration:FLOAT,
+syscall_wait_per_iteration:FLOAT,
+histogram_slow_lookups_per_iteration:FLOAT,
+syscall_write_per_iteration:FLOAT,
+syscall_read_per_iteration:FLOAT,
+tcp_backup_pollers_created_per_iteration:FLOAT,
+tcp_backup_poller_polls_per_iteration:FLOAT,
+http2_op_batches_per_iteration:FLOAT,
+http2_op_cancel_per_iteration:FLOAT,
+http2_op_send_initial_metadata_per_iteration:FLOAT,
+http2_op_send_message_per_iteration:FLOAT,
+http2_op_send_trailing_metadata_per_iteration:FLOAT,
+http2_op_recv_initial_metadata_per_iteration:FLOAT,
+http2_op_recv_message_per_iteration:FLOAT,
+http2_op_recv_trailing_metadata_per_iteration:FLOAT,
+http2_settings_writes_per_iteration:FLOAT,
+http2_pings_sent_per_iteration:FLOAT,
+http2_writes_begun_per_iteration:FLOAT,
+http2_writes_offloaded_per_iteration:FLOAT,
+http2_writes_continued_per_iteration:FLOAT,
+http2_partial_writes_per_iteration:FLOAT,
+combiner_locks_initiated_per_iteration:FLOAT,
+combiner_locks_scheduled_items_per_iteration:FLOAT,
+combiner_locks_scheduled_final_items_per_iteration:FLOAT,
+combiner_locks_offloaded_per_iteration:FLOAT,
+executor_scheduled_short_items_per_iteration:FLOAT,
+executor_scheduled_long_items_per_iteration:FLOAT,
+executor_scheduled_to_self_per_iteration:FLOAT,
+executor_wakeup_initiated_per_iteration:FLOAT,
+executor_queue_drained_per_iteration:FLOAT,
+executor_push_retries_per_iteration:FLOAT,
+executor_threads_created_per_iteration:FLOAT,
+executor_threads_used_per_iteration:FLOAT,
+server_requested_calls_per_iteration:FLOAT,
+server_slowpath_requests_queued_per_iteration:FLOAT
--- a/src/core/lib/iomgr/combiner.c
+++ b/src/core/lib/iomgr/combiner.c
@ -356,7 +356,8 @@ static void combiner_finally_exec(grpc_exec_ctx *exec_ctx,

 static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure,
                            grpc_error *error) {
-  combiner_finally_exec(exec_ctx, closure, GRPC_ERROR_REF(error));
+  combiner_finally_exec(exec_ctx, (grpc_closure *)closure,
+                        GRPC_ERROR_REF(error));
 }

 grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *combiner) {
--- a/src/core/lib/iomgr/error.c
+++ b/src/core/lib/iomgr/error.c
@ -278,13 +278,13 @@ static void internal_set_time(grpc_error **err, grpc_error_times which,
  memcpy((*err)->arena + slot, &value, sizeof(value));
 }

-static void internal_add_error(grpc_error **err, grpc_error *new) {
-  grpc_linked_error new_last = {new, UINT8_MAX};
+static void internal_add_error(grpc_error **err, grpc_error *new_err) {
+  grpc_linked_error new_last = {new_err, UINT8_MAX};
  uint8_t slot = get_placement(err, sizeof(grpc_linked_error));
  if (slot == UINT8_MAX) {
-    gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err, new,
-            grpc_error_string(new));
-    GRPC_ERROR_UNREF(new);
+    gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err,
+            new_err, grpc_error_string(new_err));
+    GRPC_ERROR_UNREF(new_err);
    return;
  }
  if ((*err)->first_err == UINT8_MAX) {
@ -321,8 +321,8 @@ grpc_error *grpc_error_create(const char *file, int line, grpc_slice desc,
  uint8_t initial_arena_capacity = (uint8_t)(
      DEFAULT_ERROR_CAPACITY +
      (uint8_t)(num_referencing * SLOTS_PER_LINKED_ERROR) + SURPLUS_CAPACITY);
-  grpc_error *err =
-      gpr_malloc(sizeof(*err) + initial_arena_capacity * sizeof(intptr_t));
+  grpc_error *err = (grpc_error *)gpr_malloc(
+      sizeof(*err) + initial_arena_capacity * sizeof(intptr_t));
  if (err == NULL) {  // TODO(ctiller): make gpr_malloc return NULL
    return GRPC_ERROR_OOM;
  }
@ -432,10 +432,10 @@ static grpc_error *copy_error_and_unref(grpc_error *in) {
 grpc_error *grpc_error_set_int(grpc_error *src, grpc_error_ints which,
                               intptr_t value) {
  GPR_TIMER_BEGIN("grpc_error_set_int", 0);
-  grpc_error *new = copy_error_and_unref(src);
-  internal_set_int(&new, which, value);
+  grpc_error *new_err = copy_error_and_unref(src);
+  internal_set_int(&new_err, which, value);
  GPR_TIMER_END("grpc_error_set_int", 0);
-  return new;
+  return new_err;
 }

 typedef struct {
@ -477,10 +477,10 @@ bool grpc_error_get_int(grpc_error *err, grpc_error_ints which, intptr_t *p) {
 grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which,
                               grpc_slice str) {
  GPR_TIMER_BEGIN("grpc_error_set_str", 0);
-  grpc_error *new = copy_error_and_unref(src);
-  internal_set_str(&new, which, str);
+  grpc_error *new_err = copy_error_and_unref(src);
+  internal_set_str(&new_err, which, str);
  GPR_TIMER_END("grpc_error_set_str", 0);
-  return new;
+  return new_err;
 }

 bool grpc_error_get_str(grpc_error *err, grpc_error_strs which,
@ -507,10 +507,10 @@ bool grpc_error_get_str(grpc_error *err, grpc_error_strs which,

 grpc_error *grpc_error_add_child(grpc_error *src, grpc_error *child) {
  GPR_TIMER_BEGIN("grpc_error_add_child", 0);
-  grpc_error *new = copy_error_and_unref(src);
-  internal_add_error(&new, child);
+  grpc_error *new_err = copy_error_and_unref(src);
+  internal_add_error(&new_err, child);
  GPR_TIMER_END("grpc_error_add_child", 0);
-  return new;
+  return new_err;
 }

 static const char *no_error_string = "\"No Error\"";
@ -733,7 +733,7 @@ const char *grpc_error_string(grpc_error *err) {
  void *p = (void *)gpr_atm_acq_load(&err->atomics.error_string);
  if (p != NULL) {
    GPR_TIMER_END("grpc_error_string", 0);
-    return p;
+    return (const char *)p;
  }

  kv_pairs kvs;
--- a/src/core/lib/iomgr/ev_epoll1_linux.c
+++ b/src/core/lib/iomgr/ev_epoll1_linux.c
@ -130,9 +130,9 @@ static void fd_global_shutdown(void);
 * Pollset Declarations
 */

-typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state;
+typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state_t;

-static const char *kick_state_string(kick_state st) {
+static const char *kick_state_string(kick_state_t st) {
  switch (st) {
    case UNKICKED:
      return "UNKICKED";
@ -145,7 +145,7 @@ static const char *kick_state_string(kick_state st) {
 }

 struct grpc_pollset_worker {
-  kick_state kick_state;
+  kick_state_t kick_state;
  int kick_state_mutator;  // which line of code last changed kick state
  bool initialized_cv;
  grpc_pollset_worker *next;
@ -160,18 +160,18 @@ struct grpc_pollset_worker {
    (worker)->kick_state_mutator = __LINE__; \
  } while (false)

-#define MAX_NEIGHBOURHOODS 1024
+#define MAX_NEIGHBORHOODS 1024

-typedef struct pollset_neighbourhood {
+typedef struct pollset_neighborhood {
  gpr_mu mu;
  grpc_pollset *active_root;
  char pad[GPR_CACHELINE_SIZE];
-} pollset_neighbourhood;
+} pollset_neighborhood;

 struct grpc_pollset {
  gpr_mu mu;
-  pollset_neighbourhood *neighbourhood;
-  bool reassigning_neighbourhood;
+  pollset_neighborhood *neighborhood;
+  bool reassigning_neighborhood;
  grpc_pollset_worker *root_worker;
  bool kicked_without_poller;

@ -384,8 +384,8 @@ GPR_TLS_DECL(g_current_thread_worker);
 /* The designated poller */
 static gpr_atm g_active_poller;

-static pollset_neighbourhood *g_neighbourhoods;
-static size_t g_num_neighbourhoods;
+static pollset_neighborhood *g_neighborhoods;
+static size_t g_num_neighborhoods;

 /* Return true if first in list */
 static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) {
@ -424,8 +424,8 @@ static worker_remove_result worker_remove(grpc_pollset *pollset,
  }
 }

-static size_t choose_neighbourhood(void) {
-  return (size_t)gpr_cpu_current_cpu() % g_num_neighbourhoods;
+static size_t choose_neighborhood(void) {
+  return (size_t)gpr_cpu_current_cpu() % g_num_neighborhoods;
 }

 static grpc_error *pollset_global_init(void) {
@ -441,11 +441,11 @@ static grpc_error *pollset_global_init(void) {
                &ev) != 0) {
    return GRPC_OS_ERROR(errno, "epoll_ctl");
  }
-  g_num_neighbourhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBOURHOODS);
-  g_neighbourhoods = (pollset_neighbourhood *)gpr_zalloc(
-      sizeof(*g_neighbourhoods) * g_num_neighbourhoods);
-  for (size_t i = 0; i < g_num_neighbourhoods; i++) {
-    gpr_mu_init(&g_neighbourhoods[i].mu);
+  g_num_neighborhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBORHOODS);
+  g_neighborhoods = (pollset_neighborhood *)gpr_zalloc(
+      sizeof(*g_neighborhoods) * g_num_neighborhoods);
+  for (size_t i = 0; i < g_num_neighborhoods; i++) {
+    gpr_mu_init(&g_neighborhoods[i].mu);
  }
  return GRPC_ERROR_NONE;
 }
@ -454,17 +454,17 @@ static void pollset_global_shutdown(void) {
  gpr_tls_destroy(&g_current_thread_pollset);
  gpr_tls_destroy(&g_current_thread_worker);
  if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd);
-  for (size_t i = 0; i < g_num_neighbourhoods; i++) {
-    gpr_mu_destroy(&g_neighbourhoods[i].mu);
+  for (size_t i = 0; i < g_num_neighborhoods; i++) {
+    gpr_mu_destroy(&g_neighborhoods[i].mu);
  }
-  gpr_free(g_neighbourhoods);
+  gpr_free(g_neighborhoods);
 }

 static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
  gpr_mu_init(&pollset->mu);
  *mu = &pollset->mu;
-  pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()];
-  pollset->reassigning_neighbourhood = false;
+  pollset->neighborhood = &g_neighborhoods[choose_neighborhood()];
+  pollset->reassigning_neighborhood = false;
  pollset->root_worker = NULL;
  pollset->kicked_without_poller = false;
  pollset->seen_inactive = true;
@ -477,26 +477,26 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
 static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
  gpr_mu_lock(&pollset->mu);
  if (!pollset->seen_inactive) {
-    pollset_neighbourhood *neighbourhood = pollset->neighbourhood;
+    pollset_neighborhood *neighborhood = pollset->neighborhood;
    gpr_mu_unlock(&pollset->mu);
-  retry_lock_neighbourhood:
-    gpr_mu_lock(&neighbourhood->mu);
+  retry_lock_neighborhood:
+    gpr_mu_lock(&neighborhood->mu);
    gpr_mu_lock(&pollset->mu);
    if (!pollset->seen_inactive) {
-      if (pollset->neighbourhood != neighbourhood) {
-        gpr_mu_unlock(&neighbourhood->mu);
-        neighbourhood = pollset->neighbourhood;
+      if (pollset->neighborhood != neighborhood) {
+        gpr_mu_unlock(&neighborhood->mu);
+        neighborhood = pollset->neighborhood;
        gpr_mu_unlock(&pollset->mu);
-        goto retry_lock_neighbourhood;
+        goto retry_lock_neighborhood;
      }
      pollset->prev->next = pollset->next;
      pollset->next->prev = pollset->prev;
-      if (pollset == pollset->neighbourhood->active_root) {
-        pollset->neighbourhood->active_root =
+      if (pollset == pollset->neighborhood->active_root) {
+        pollset->neighborhood->active_root =
            pollset->next == pollset ? NULL : pollset->next;
      }
    }
-    gpr_mu_unlock(&pollset->neighbourhood->mu);
+    gpr_mu_unlock(&pollset->neighborhood->mu);
  }
  gpr_mu_unlock(&pollset->mu);
  gpr_mu_destroy(&pollset->mu);
@ -675,16 +675,16 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
    // pollset has been observed to be inactive, we need to move back to the
    // active list
    bool is_reassigning = false;
-    if (!pollset->reassigning_neighbourhood) {
+    if (!pollset->reassigning_neighborhood) {
      is_reassigning = true;
-      pollset->reassigning_neighbourhood = true;
-      pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()];
+      pollset->reassigning_neighborhood = true;
+      pollset->neighborhood = &g_neighborhoods[choose_neighborhood()];
    }
-    pollset_neighbourhood *neighbourhood = pollset->neighbourhood;
+    pollset_neighborhood *neighborhood = pollset->neighborhood;
    gpr_mu_unlock(&pollset->mu);
  // pollset unlocked: state may change (even worker->kick_state)
-  retry_lock_neighbourhood:
-    gpr_mu_lock(&neighbourhood->mu);
+  retry_lock_neighborhood:
+    gpr_mu_lock(&neighborhood->mu);
    gpr_mu_lock(&pollset->mu);
    if (GRPC_TRACER_ON(grpc_polling_trace)) {
      gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d",
@ -692,17 +692,17 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
              is_reassigning);
    }
    if (pollset->seen_inactive) {
-      if (neighbourhood != pollset->neighbourhood) {
-        gpr_mu_unlock(&neighbourhood->mu);
-        neighbourhood = pollset->neighbourhood;
+      if (neighborhood != pollset->neighborhood) {
+        gpr_mu_unlock(&neighborhood->mu);
+        neighborhood = pollset->neighborhood;
        gpr_mu_unlock(&pollset->mu);
-        goto retry_lock_neighbourhood;
+        goto retry_lock_neighborhood;
      }

      /* In the brief time we released the pollset locks above, the worker MAY
         have been kicked. In this case, the worker should get out of this
         pollset ASAP and hence this should neither add the pollset to
-         neighbourhood nor mark the pollset as active.
+         neighborhood nor mark the pollset as active.

         On a side note, the only way a worker's kick state could have changed
         at this point is if it were "kicked specifically". Since the worker has
@ -710,25 +710,25 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
         not visible in the "kick any" path yet */
      if (worker->kick_state == UNKICKED) {
        pollset->seen_inactive = false;
-        if (neighbourhood->active_root == NULL) {
-          neighbourhood->active_root = pollset->next = pollset->prev = pollset;
+        if (neighborhood->active_root == NULL) {
+          neighborhood->active_root = pollset->next = pollset->prev = pollset;
          /* Make this the designated poller if there isn't one already */
          if (worker->kick_state == UNKICKED &&
              gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) {
            SET_KICK_STATE(worker, DESIGNATED_POLLER);
          }
        } else {
-          pollset->next = neighbourhood->active_root;
+          pollset->next = neighborhood->active_root;
          pollset->prev = pollset->next->prev;
          pollset->next->prev = pollset->prev->next = pollset;
        }
      }
    }
    if (is_reassigning) {
-      GPR_ASSERT(pollset->reassigning_neighbourhood);
-      pollset->reassigning_neighbourhood = false;
+      GPR_ASSERT(pollset->reassigning_neighborhood);
+      pollset->reassigning_neighborhood = false;
    }
-    gpr_mu_unlock(&neighbourhood->mu);
+    gpr_mu_unlock(&neighborhood->mu);
  }

  worker_insert(pollset, worker);
@ -763,7 +763,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
  }

  /* We release pollset lock in this function at a couple of places:
-   *   1. Briefly when assigning pollset to a neighbourhood
+   *   1. Briefly when assigning pollset to a neighborhood
   *   2. When doing gpr_cv_wait()
   * It is possible that 'kicked_without_poller' was set to true during (1) and
   * 'shutting_down' is set to true during (1) or (2). If either of them is
@ -781,12 +781,12 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
  return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down;
 }

-static bool check_neighbourhood_for_available_poller(
-    pollset_neighbourhood *neighbourhood) {
-  GPR_TIMER_BEGIN("check_neighbourhood_for_available_poller", 0);
+static bool check_neighborhood_for_available_poller(
+    pollset_neighborhood *neighborhood) {
+  GPR_TIMER_BEGIN("check_neighborhood_for_available_poller", 0);
  bool found_worker = false;
  do {
-    grpc_pollset *inspect = neighbourhood->active_root;
+    grpc_pollset *inspect = neighborhood->active_root;
    if (inspect == NULL) {
      break;
    }
@ -831,8 +831,8 @@ static bool check_neighbourhood_for_available_poller(
        gpr_log(GPR_DEBUG, " .. mark pollset %p inactive", inspect);
      }
      inspect->seen_inactive = true;
-      if (inspect == neighbourhood->active_root) {
-        neighbourhood->active_root =
+      if (inspect == neighborhood->active_root) {
+        neighborhood->active_root =
            inspect->next == inspect ? NULL : inspect->next;
      }
      inspect->next->prev = inspect->prev;
@ -841,7 +841,7 @@ static bool check_neighbourhood_for_available_poller(
    }
    gpr_mu_unlock(&inspect->mu);
  } while (!found_worker);
-  GPR_TIMER_END("check_neighbourhood_for_available_poller", 0);
+  GPR_TIMER_END("check_neighborhood_for_available_poller", 0);
  return found_worker;
 }

@ -873,32 +873,31 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
      }
    } else {
      gpr_atm_no_barrier_store(&g_active_poller, 0);
-      size_t poller_neighbourhood_idx =
-          (size_t)(pollset->neighbourhood - g_neighbourhoods);
+      size_t poller_neighborhood_idx =
+          (size_t)(pollset->neighborhood - g_neighborhoods);
      gpr_mu_unlock(&pollset->mu);
      bool found_worker = false;
-      bool scan_state[MAX_NEIGHBOURHOODS];
-      for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) {
-        pollset_neighbourhood *neighbourhood =
-            &g_neighbourhoods[(poller_neighbourhood_idx + i) %
-                              g_num_neighbourhoods];
-        if (gpr_mu_trylock(&neighbourhood->mu)) {
-          found_worker =
-              check_neighbourhood_for_available_poller(neighbourhood);
-          gpr_mu_unlock(&neighbourhood->mu);
+      bool scan_state[MAX_NEIGHBORHOODS];
+      for (size_t i = 0; !found_worker && i < g_num_neighborhoods; i++) {
+        pollset_neighborhood *neighborhood =
+            &g_neighborhoods[(poller_neighborhood_idx + i) %
+                             g_num_neighborhoods];
+        if (gpr_mu_trylock(&neighborhood->mu)) {
+          found_worker = check_neighborhood_for_available_poller(neighborhood);
+          gpr_mu_unlock(&neighborhood->mu);
          scan_state[i] = true;
        } else {
          scan_state[i] = false;
        }
      }
-      for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) {
+      for (size_t i = 0; !found_worker && i < g_num_neighborhoods; i++) {
        if (scan_state[i]) continue;
-        pollset_neighbourhood *neighbourhood =
-            &g_neighbourhoods[(poller_neighbourhood_idx + i) %
-                              g_num_neighbourhoods];
-        gpr_mu_lock(&neighbourhood->mu);
-        found_worker = check_neighbourhood_for_available_poller(neighbourhood);
-        gpr_mu_unlock(&neighbourhood->mu);
+        pollset_neighborhood *neighborhood =
+            &g_neighborhoods[(poller_neighborhood_idx + i) %
+                             g_num_neighborhoods];
+        gpr_mu_lock(&neighborhood->mu);
+        found_worker = check_neighborhood_for_available_poller(neighborhood);
+        gpr_mu_unlock(&neighborhood->mu);
      }
      grpc_exec_ctx_flush(exec_ctx);
      gpr_mu_lock(&pollset->mu);
--- a/src/core/lib/iomgr/ev_epollex_linux.c
+++ b/src/core/lib/iomgr/ev_epollex_linux.c
@ -97,12 +97,12 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg,
 * pollable Declarations
 */

-typedef struct pollable {
+typedef struct pollable_t {
  polling_obj po;
  int epfd;
  grpc_wakeup_fd wakeup;
  grpc_pollset_worker *root_worker;
-} pollable;
+} pollable_t;

 static const char *polling_obj_type_string(polling_obj_type t) {
  switch (t) {
@ -122,7 +122,7 @@ static const char *polling_obj_type_string(polling_obj_type t) {
  return "<invalid>";
 }

-static char *pollable_desc(pollable *p) {
+static char *pollable_desc(pollable_t *p) {
  char *out;
  gpr_asprintf(&out, "type=%s group=%p epfd=%d wakeup=%d",
               polling_obj_type_string(p->po.type), p->po.group, p->epfd,
@ -130,19 +130,19 @@ static char *pollable_desc(pollable *p) {
  return out;
 }

-static pollable g_empty_pollable;
+static pollable_t g_empty_pollable;

-static void pollable_init(pollable *p, polling_obj_type type);
-static void pollable_destroy(pollable *p);
+static void pollable_init(pollable_t *p, polling_obj_type type);
+static void pollable_destroy(pollable_t *p);
 /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */
-static grpc_error *pollable_materialize(pollable *p);
+static grpc_error *pollable_materialize(pollable_t *p);

 /*******************************************************************************
 * Fd Declarations
 */

 struct grpc_fd {
-  pollable pollable;
+  pollable_t pollable;
  int fd;
  /* refst format:
       bit 0    : 1=Active / 0=Orphaned
@ -193,15 +193,15 @@ struct grpc_pollset_worker {
  pollset_worker_link links[POLLSET_WORKER_LINK_COUNT];
  gpr_cv cv;
  grpc_pollset *pollset;
-  pollable *pollable;
+  pollable_t *pollable;
 };

 #define MAX_EPOLL_EVENTS 100
 #define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 5

 struct grpc_pollset {
-  pollable pollable;
-  pollable *current_pollable;
+  pollable_t pollable;
+  pollable_t *current_pollable;
  int kick_alls_pending;
  bool kicked_without_poller;
  grpc_closure *shutdown_closure;
@ -451,13 +451,13 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 * Pollable Definitions
 */

-static void pollable_init(pollable *p, polling_obj_type type) {
+static void pollable_init(pollable_t *p, polling_obj_type type) {
  po_init(&p->po, type);
  p->root_worker = NULL;
  p->epfd = -1;
 }

-static void pollable_destroy(pollable *p) {
+static void pollable_destroy(pollable_t *p) {
  po_destroy(&p->po);
  if (p->epfd != -1) {
    close(p->epfd);
@ -466,7 +466,7 @@ static void pollable_destroy(pollable *p) {
 }

 /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */
-static grpc_error *pollable_materialize(pollable *p) {
+static grpc_error *pollable_materialize(pollable_t *p) {
  if (p->epfd == -1) {
    int new_epfd = epoll_create1(EPOLL_CLOEXEC);
    if (new_epfd < 0) {
@ -492,7 +492,7 @@ static grpc_error *pollable_materialize(pollable *p) {
 }

 /* pollable must be materialized */
-static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) {
+static grpc_error *pollable_add_fd(pollable_t *p, grpc_fd *fd) {
  grpc_error *error = GRPC_ERROR_NONE;
  static const char *err_desc = "pollable_add_fd";
  const int epfd = p->epfd;
@ -599,7 +599,7 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
                     GRPC_ERROR_NONE);
 }

-static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p,
+static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p,
                                      grpc_pollset_worker *specific_worker) {
  if (GRPC_TRACER_ON(grpc_polling_trace)) {
    gpr_log(GPR_DEBUG,
@ -664,7 +664,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p,
 /* p->po.mu must be held before calling this function */
 static grpc_error *pollset_kick(grpc_pollset *pollset,
                                grpc_pollset_worker *specific_worker) {
-  pollable *p = pollset->current_pollable;
+  pollable_t *p = pollset->current_pollable;
  if (p != &pollset->pollable) {
    gpr_mu_lock(&p->po.mu);
  }
@ -744,7 +744,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
  pollset_maybe_finish_shutdown(exec_ctx, pollset);
 }

-static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) {
+static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable_t *p) {
  return p != &g_empty_pollable && p != &pollset->pollable;
 }

@ -762,8 +762,9 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx,
      if (GRPC_TRACER_ON(grpc_polling_trace)) {
        gpr_log(GPR_DEBUG, "PS:%p got pollset_wakeup %p", pollset, data_ptr);
      }
-      append_error(&error, grpc_wakeup_fd_consume_wakeup(
-                               (void *)((~(intptr_t)1) & (intptr_t)data_ptr)),
+      append_error(&error,
+                   grpc_wakeup_fd_consume_wakeup(
+                       (grpc_wakeup_fd *)((~(intptr_t)1) & (intptr_t)data_ptr)),
                   err_desc);
    } else {
      grpc_fd *fd = (grpc_fd *)data_ptr;
@ -800,7 +801,7 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
 }

 static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                                 pollable *p, gpr_timespec now,
+                                 pollable_t *p, gpr_timespec now,
                                 gpr_timespec deadline) {
  int timeout = poll_deadline_to_millis_timeout(deadline, now);

@ -1028,7 +1029,7 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx,
              "PS:%p add fd %p; transition pollable from empty to fd", pollset,
              fd);
    }
-    /* empty pollable --> single fd pollable */
+    /* empty pollable --> single fd pollable_t */
    pollset_kick_all(exec_ctx, pollset);
    pollset->current_pollable = &fd->pollable;
    if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu);
--- a/src/core/lib/iomgr/ev_poll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@ -1539,7 +1539,7 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
  for (i = 0; i < nfds; i++) {
    fds[i].revents = 0;
    if (fds[i].fd < 0 && (fds[i].events & POLLIN)) {
-      idx = FD_TO_IDX(fds[i].fd);
+      idx = GRPC_FD_TO_IDX(fds[i].fd);
      fd_cvs[i].cv = &pollcv_cv;
      fd_cvs[i].prev = NULL;
      fd_cvs[i].next = g_cvfds.cvfds[idx].cvs;
@ -1602,8 +1602,8 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
  idx = 0;
  for (i = 0; i < nfds; i++) {
    if (fds[i].fd < 0 && (fds[i].events & POLLIN)) {
-      remove_cvn(&g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs, &(fd_cvs[i]));
-      if (g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].is_set) {
+      remove_cvn(&g_cvfds.cvfds[GRPC_FD_TO_IDX(fds[i].fd)].cvs, &(fd_cvs[i]));
+      if (g_cvfds.cvfds[GRPC_FD_TO_IDX(fds[i].fd)].is_set) {
        fds[i].revents = POLLIN;
        if (res >= 0) res++;
      }
--- a/src/core/lib/iomgr/executor.c
+++ b/src/core/lib/iomgr/executor.c
@ -32,16 +32,14 @@
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/support/spinlock.h"

-#define MAX_DEPTH 2
-
 typedef struct {
  gpr_mu mu;
  gpr_cv cv;
  grpc_closure_list elems;
-  size_t depth;
  bool shutdown;
  bool queued_long_job;
  gpr_thd_id id;
+  grpc_closure_list local_elems;
 } thread_state;

 static thread_state *g_thread_state;
@ -56,34 +54,35 @@ static grpc_tracer_flag executor_trace =

 static void executor_thread(void *arg);

-static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) {
-  size_t n = 0;
+static void run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) {
+  int n = 0;  // number of closures executed

-  grpc_closure *c = list.head;
-  while (c != NULL) {
-    grpc_closure *next = c->next_data.next;
-    grpc_error *error = c->error_data.error;
-    if (GRPC_TRACER_ON(executor_trace)) {
+  while (!grpc_closure_list_empty(*list)) {
+    grpc_closure *c = list->head;
+    grpc_closure_list_init(list);
+    while (c != NULL) {
+      grpc_closure *next = c->next_data.next;
+      grpc_error *error = c->error_data.error;
+      if (GRPC_TRACER_ON(executor_trace)) {
 #ifndef NDEBUG
-      gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c,
-              c->file_created, c->line_created);
+        gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c,
+                c->file_created, c->line_created);
 #else
-      gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c);
+        gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c);
 #endif
-    }
+      }
 #ifndef NDEBUG
-    c->scheduled = false;
+      c->scheduled = false;
 #endif
-    c->cb(exec_ctx, c->cb_arg, error);
-    GRPC_ERROR_UNREF(error);
-    c = next;
-    n++;
-    grpc_exec_ctx_flush(exec_ctx);
+      n++;
+      c->cb(exec_ctx, c->cb_arg, error);
+      GRPC_ERROR_UNREF(error);
+      c = next;
+      grpc_exec_ctx_flush(exec_ctx);
+    }
  }

  GRPC_STATS_INC_EXECUTOR_CLOSURES_PER_WAKEUP(exec_ctx, n);
-
-  return n;
 }

 bool grpc_executor_is_threaded() {
@ -128,7 +127,7 @@ void grpc_executor_set_threading(grpc_exec_ctx *exec_ctx, bool threading) {
    for (size_t i = 0; i < g_max_threads; i++) {
      gpr_mu_destroy(&g_thread_state[i].mu);
      gpr_cv_destroy(&g_thread_state[i].cv);
-      run_closures(exec_ctx, g_thread_state[i].elems);
+      run_closures(exec_ctx, &g_thread_state[i].elems);
    }
    gpr_free(g_thread_state);
    gpr_tls_destroy(&g_this_thread_state);
@ -154,15 +153,12 @@ static void executor_thread(void *arg) {

  GRPC_STATS_INC_EXECUTOR_THREADS_CREATED(&exec_ctx);

-  size_t subtract_depth = 0;
  bool used = false;
  for (;;) {
    if (GRPC_TRACER_ON(executor_trace)) {
-      gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")",
-              (int)(ts - g_thread_state), subtract_depth);
+      gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step", (int)(ts - g_thread_state));
    }
    gpr_mu_lock(&ts->mu);
-    ts->depth -= subtract_depth;
    while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) {
      ts->queued_long_job = false;
      gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_REALTIME));
@ -180,14 +176,15 @@ static void executor_thread(void *arg) {
      used = true;
    }
    GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx);
-    grpc_closure_list exec = ts->elems;
+    GPR_ASSERT(grpc_closure_list_empty(ts->local_elems));
+    ts->local_elems = ts->elems;
    ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT;
    gpr_mu_unlock(&ts->mu);
    if (GRPC_TRACER_ON(executor_trace)) {
      gpr_log(GPR_DEBUG, "EXECUTOR[%d]: execute", (int)(ts - g_thread_state));
    }

-    subtract_depth = run_closures(&exec_ctx, exec);
+    run_closures(&exec_ctx, &ts->local_elems);
  }
  grpc_exec_ctx_finish(&exec_ctx);
 }
@ -220,6 +217,10 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
      ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)];
    } else {
      GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx);
+      if (!is_short) {
+        grpc_closure_list_append(&ts->local_elems, closure, error);
+        return;
+      }
    }
    thread_state *orig_ts = ts;

@ -259,8 +260,7 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
        gpr_cv_signal(&ts->cv);
      }
      grpc_closure_list_append(&ts->elems, closure, error);
-      ts->depth++;
-      try_new_thread = ts->depth > MAX_DEPTH &&
+      try_new_thread = ts->elems.head != closure &&
                       cur_thread_count < g_max_threads && !ts->shutdown;
      if (!is_short) ts->queued_long_job = true;
      gpr_mu_unlock(&ts->mu);
--- a/src/core/lib/iomgr/polling_entity.c
+++ b/src/core/lib/iomgr/polling_entity.c
@ -25,7 +25,7 @@ grpc_polling_entity grpc_polling_entity_create_from_pollset_set(
    grpc_pollset_set *pollset_set) {
  grpc_polling_entity pollent;
  pollent.pollent.pollset_set = pollset_set;
-  pollent.tag = POPS_POLLSET_SET;
+  pollent.tag = GRPC_POLLS_POLLSET_SET;
  return pollent;
 }

@ -33,12 +33,12 @@ grpc_polling_entity grpc_polling_entity_create_from_pollset(
    grpc_pollset *pollset) {
  grpc_polling_entity pollent;
  pollent.pollent.pollset = pollset;
-  pollent.tag = POPS_POLLSET;
+  pollent.tag = GRPC_POLLS_POLLSET;
  return pollent;
 }

 grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) {
-  if (pollent->tag == POPS_POLLSET) {
+  if (pollent->tag == GRPC_POLLS_POLLSET) {
    return pollent->pollent.pollset;
  }
  return NULL;
@ -46,23 +46,23 @@ grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) {

 grpc_pollset_set *grpc_polling_entity_pollset_set(
    grpc_polling_entity *pollent) {
-  if (pollent->tag == POPS_POLLSET_SET) {
+  if (pollent->tag == GRPC_POLLS_POLLSET_SET) {
    return pollent->pollent.pollset_set;
  }
  return NULL;
 }

 bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent) {
-  return pollent->tag == POPS_NONE;
+  return pollent->tag == GRPC_POLLS_NONE;
 }

 void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx,
                                            grpc_polling_entity *pollent,
                                            grpc_pollset_set *pss_dst) {
-  if (pollent->tag == POPS_POLLSET) {
+  if (pollent->tag == GRPC_POLLS_POLLSET) {
    GPR_ASSERT(pollent->pollent.pollset != NULL);
    grpc_pollset_set_add_pollset(exec_ctx, pss_dst, pollent->pollent.pollset);
-  } else if (pollent->tag == POPS_POLLSET_SET) {
+  } else if (pollent->tag == GRPC_POLLS_POLLSET_SET) {
    GPR_ASSERT(pollent->pollent.pollset_set != NULL);
    grpc_pollset_set_add_pollset_set(exec_ctx, pss_dst,
                                     pollent->pollent.pollset_set);
@ -75,10 +75,10 @@ void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx,
 void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx,
                                              grpc_polling_entity *pollent,
                                              grpc_pollset_set *pss_dst) {
-  if (pollent->tag == POPS_POLLSET) {
+  if (pollent->tag == GRPC_POLLS_POLLSET) {
    GPR_ASSERT(pollent->pollent.pollset != NULL);
    grpc_pollset_set_del_pollset(exec_ctx, pss_dst, pollent->pollent.pollset);
-  } else if (pollent->tag == POPS_POLLSET_SET) {
+  } else if (pollent->tag == GRPC_POLLS_POLLSET_SET) {
    GPR_ASSERT(pollent->pollent.pollset_set != NULL);
    grpc_pollset_set_del_pollset_set(exec_ctx, pss_dst,
                                     pollent->pollent.pollset_set);
--- a/src/core/lib/iomgr/polling_entity.h
+++ b/src/core/lib/iomgr/polling_entity.h
@ -22,6 +22,12 @@
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_set.h"

+typedef enum grpc_pollset_tag {
+  GRPC_POLLS_NONE,
+  GRPC_POLLS_POLLSET,
+  GRPC_POLLS_POLLSET_SET
+} grpc_pollset_tag;
+
 /* A grpc_polling_entity is a pollset-or-pollset_set container. It allows
 * functions that accept a pollset XOR a pollset_set to do so through an
 * abstract interface. No ownership is taken. */
@ -31,7 +37,7 @@ typedef struct grpc_polling_entity {
    grpc_pollset *pollset;
    grpc_pollset_set *pollset_set;
  } pollent;
-  enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } tag;
+  grpc_pollset_tag tag;
 } grpc_polling_entity;

 grpc_polling_entity grpc_polling_entity_create_from_pollset_set(
--- a/src/core/lib/iomgr/resource_quota.c
+++ b/src/core/lib/iomgr/resource_quota.c
@ -22,6 +22,7 @@
 #include <stdint.h>
 #include <string.h>

+#include <grpc/slice_buffer.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
@ -656,7 +657,7 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args(
    if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) {
      if (channel_args->args[i].type == GRPC_ARG_POINTER) {
        return grpc_resource_quota_ref_internal(
-            channel_args->args[i].value.pointer.p);
+            (grpc_resource_quota *)channel_args->args[i].value.pointer.p);
      } else {
        gpr_log(GPR_DEBUG, GRPC_ARG_RESOURCE_QUOTA " should be a pointer");
      }
@ -666,12 +667,12 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args(
 }

 static void *rq_copy(void *rq) {
-  grpc_resource_quota_ref(rq);
+  grpc_resource_quota_ref((grpc_resource_quota *)rq);
  return rq;
 }

 static void rq_destroy(grpc_exec_ctx *exec_ctx, void *rq) {
-  grpc_resource_quota_unref_internal(exec_ctx, rq);
+  grpc_resource_quota_unref_internal(exec_ctx, (grpc_resource_quota *)rq);
 }

 static int rq_cmp(void *a, void *b) { return GPR_ICMP(a, b); }
--- a/src/core/lib/iomgr/socket_factory_posix.c
+++ b/src/core/lib/iomgr/socket_factory_posix.c
@ -69,11 +69,11 @@ void grpc_socket_factory_unref(grpc_socket_factory *factory) {
 }

 static void *socket_factory_arg_copy(void *p) {
-  return grpc_socket_factory_ref(p);
+  return grpc_socket_factory_ref((grpc_socket_factory *)p);
 }

 static void socket_factory_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) {
-  grpc_socket_factory_unref(p);
+  grpc_socket_factory_unref((grpc_socket_factory *)p);
 }

 static int socket_factory_cmp(void *a, void *b) {
--- a/src/core/lib/iomgr/socket_mutator.c
+++ b/src/core/lib/iomgr/socket_mutator.c
@ -60,11 +60,11 @@ void grpc_socket_mutator_unref(grpc_socket_mutator *mutator) {
 }

 static void *socket_mutator_arg_copy(void *p) {
-  return grpc_socket_mutator_ref(p);
+  return grpc_socket_mutator_ref((grpc_socket_mutator *)p);
 }

 static void socket_mutator_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) {
-  grpc_socket_mutator_unref(p);
+  grpc_socket_mutator_unref((grpc_socket_mutator *)p);
 }

 static int socket_mutator_cmp(void *a, void *b) {
--- a/src/core/lib/iomgr/timer_manager.c
+++ b/src/core/lib/iomgr/timer_manager.c
@ -276,7 +276,7 @@ static void timer_thread(void *completed_thread_ptr) {
      GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL);
  timer_main_loop(&exec_ctx);
  grpc_exec_ctx_finish(&exec_ctx);
-  timer_thread_cleanup(completed_thread_ptr);
+  timer_thread_cleanup((completed_thread *)completed_thread_ptr);
 }

 static void start_threads(void) {
--- a/src/core/lib/iomgr/udp_server.c
+++ b/src/core/lib/iomgr/udp_server.c
@ -118,7 +118,7 @@ static grpc_socket_factory *get_socket_factory(const grpc_channel_args *args) {
    const grpc_arg *arg = grpc_channel_args_find(args, GRPC_ARG_SOCKET_FACTORY);
    if (arg) {
      GPR_ASSERT(arg->type == GRPC_ARG_POINTER);
-      return arg->value.pointer.p;
+      return (grpc_socket_factory *)arg->value.pointer.p;
    }
  }
  return NULL;
--- a/src/core/lib/iomgr/wakeup_fd_cv.c
+++ b/src/core/lib/iomgr/wakeup_fd_cv.c
@ -57,7 +57,7 @@ static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) {
  g_cvfds.free_fds = g_cvfds.free_fds->next_free;
  g_cvfds.cvfds[idx].cvs = NULL;
  g_cvfds.cvfds[idx].is_set = 0;
-  fd_info->read_fd = IDX_TO_FD(idx);
+  fd_info->read_fd = GRPC_IDX_TO_FD(idx);
  fd_info->write_fd = -1;
  gpr_mu_unlock(&g_cvfds.mu);
  return GRPC_ERROR_NONE;
@ -66,8 +66,8 @@ static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) {
 static grpc_error* cv_fd_wakeup(grpc_wakeup_fd* fd_info) {
  cv_node* cvn;
  gpr_mu_lock(&g_cvfds.mu);
-  g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].is_set = 1;
-  cvn = g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].cvs;
+  g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].is_set = 1;
+  cvn = g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].cvs;
  while (cvn) {
    gpr_cv_signal(cvn->cv);
    cvn = cvn->next;
@ -78,7 +78,7 @@ static grpc_error* cv_fd_wakeup(grpc_wakeup_fd* fd_info) {

 static grpc_error* cv_fd_consume(grpc_wakeup_fd* fd_info) {
  gpr_mu_lock(&g_cvfds.mu);
-  g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].is_set = 0;
+  g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].is_set = 0;
  gpr_mu_unlock(&g_cvfds.mu);
  return GRPC_ERROR_NONE;
 }
@ -89,9 +89,9 @@ static void cv_fd_destroy(grpc_wakeup_fd* fd_info) {
  }
  gpr_mu_lock(&g_cvfds.mu);
  // Assert that there are no active pollers
-  GPR_ASSERT(!g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].cvs);
-  g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].next_free = g_cvfds.free_fds;
-  g_cvfds.free_fds = &g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)];
+  GPR_ASSERT(!g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].cvs);
+  g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].next_free = g_cvfds.free_fds;
+  g_cvfds.free_fds = &g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)];
  gpr_mu_unlock(&g_cvfds.mu);
 }

--- a/src/core/lib/iomgr/wakeup_fd_cv.h
+++ b/src/core/lib/iomgr/wakeup_fd_cv.h
@ -37,8 +37,8 @@

 #include "src/core/lib/iomgr/ev_posix.h"

-#define FD_TO_IDX(fd) (-(fd)-1)
-#define IDX_TO_FD(idx) (-(idx)-1)
+#define GRPC_FD_TO_IDX(fd) (-(fd)-1)
+#define GRPC_IDX_TO_FD(idx) (-(idx)-1)

 typedef struct cv_node {
  gpr_cv* cv;
--- a/src/core/lib/slice/slice.c
+++ b/src/core/lib/slice/slice.c
@ -174,8 +174,8 @@ static const grpc_slice_refcount_vtable new_with_len_vtable = {
 grpc_slice grpc_slice_new_with_len(void *p, size_t len,
                                   void (*destroy)(void *, size_t)) {
  grpc_slice slice;
-  new_with_len_slice_refcount *rc =
-      gpr_malloc(sizeof(new_with_len_slice_refcount));
+  new_with_len_slice_refcount *rc = (new_with_len_slice_refcount *)gpr_malloc(
+      sizeof(new_with_len_slice_refcount));
  gpr_ref_init(&rc->refs, 1);
  rc->rc.vtable = &new_with_len_vtable;
  rc->rc.sub_refcount = &rc->rc;
--- a/src/core/lib/surface/call.c
+++ b/src/core/lib/surface/call.c
@ -135,7 +135,7 @@ typedef struct batch_control {
 typedef struct {
  gpr_mu child_list_mu;
  grpc_call *first_child;
-} parent_call;
+} parent_call_t;

 typedef struct {
  grpc_call *parent;
@ -144,7 +144,7 @@ typedef struct {
      parent->mu */
  grpc_call *sibling_next;
  grpc_call *sibling_prev;
-} child_call;
+} child_call_t;

 #define RECV_NONE ((gpr_atm)0)
 #define RECV_INITIAL_METADATA_FIRST ((gpr_atm)1)
@ -157,8 +157,8 @@ struct grpc_call {
  grpc_polling_entity pollent;
  grpc_channel *channel;
  gpr_timespec start_time;
-  /* parent_call* */ gpr_atm parent_call_atm;
-  child_call *child_call;
+  /* parent_call_t* */ gpr_atm parent_call_atm;
+  child_call_t *child_call;

  /* client or server call */
  bool is_client;
@ -293,32 +293,32 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, batch_control *bctl);
 static void add_batch_error(grpc_exec_ctx *exec_ctx, batch_control *bctl,
                            grpc_error *error, bool has_cancelled);

-static void add_init_error(grpc_error **composite, grpc_error *new) {
-  if (new == GRPC_ERROR_NONE) return;
+static void add_init_error(grpc_error **composite, grpc_error *new_err) {
+  if (new_err == GRPC_ERROR_NONE) return;
  if (*composite == GRPC_ERROR_NONE)
    *composite = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Call creation failed");
-  *composite = grpc_error_add_child(*composite, new);
+  *composite = grpc_error_add_child(*composite, new_err);
 }

 void *grpc_call_arena_alloc(grpc_call *call, size_t size) {
  return gpr_arena_alloc(call->arena, size);
 }

-static parent_call *get_or_create_parent_call(grpc_call *call) {
-  parent_call *p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm);
+static parent_call_t *get_or_create_parent_call(grpc_call *call) {
+  parent_call_t *p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm);
  if (p == NULL) {
-    p = (parent_call *)gpr_arena_alloc(call->arena, sizeof(*p));
+    p = (parent_call_t *)gpr_arena_alloc(call->arena, sizeof(*p));
    gpr_mu_init(&p->child_list_mu);
    if (!gpr_atm_rel_cas(&call->parent_call_atm, (gpr_atm)NULL, (gpr_atm)p)) {
      gpr_mu_destroy(&p->child_list_mu);
-      p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm);
+      p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm);
    }
  }
  return p;
 }

-static parent_call *get_parent_call(grpc_call *call) {
-  return (parent_call *)gpr_atm_acq_load(&call->parent_call_atm);
+static parent_call_t *get_parent_call(grpc_call *call) {
+  return (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm);
 }

 grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx,
@ -378,15 +378,15 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx,
  bool immediately_cancel = false;

  if (args->parent_call != NULL) {
-    child_call *cc = call->child_call =
-        gpr_arena_alloc(arena, sizeof(child_call));
+    child_call_t *cc = call->child_call =
+        (child_call_t *)gpr_arena_alloc(arena, sizeof(child_call_t));
    call->child_call->parent = args->parent_call;

    GRPC_CALL_INTERNAL_REF(args->parent_call, "child");
    GPR_ASSERT(call->is_client);
    GPR_ASSERT(!args->parent_call->is_client);

-    parent_call *pc = get_or_create_parent_call(args->parent_call);
+    parent_call_t *pc = get_or_create_parent_call(args->parent_call);

    gpr_mu_lock(&pc->child_list_mu);

@ -533,7 +533,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call,
  if (c->receiving_stream != NULL) {
    grpc_byte_stream_destroy(exec_ctx, c->receiving_stream);
  }
-  parent_call *pc = get_parent_call(c);
+  parent_call_t *pc = get_parent_call(c);
  if (pc != NULL) {
    gpr_mu_destroy(&pc->child_list_mu);
  }
@ -549,7 +549,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call,
    GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind");
  }

-  get_final_status(call, set_status_value_directly, &c->final_info.final_status,
+  get_final_status(c, set_status_value_directly, &c->final_info.final_status,
                   NULL);
  c->final_info.stats.latency =
      gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time);
@ -570,14 +570,14 @@ void grpc_call_ref(grpc_call *c) { gpr_ref(&c->ext_ref); }
 void grpc_call_unref(grpc_call *c) {
  if (!gpr_unref(&c->ext_ref)) return;

-  child_call *cc = c->child_call;
+  child_call_t *cc = c->child_call;
  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;

  GPR_TIMER_BEGIN("grpc_call_unref", 0);
  GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (c));

  if (cc) {
-    parent_call *pc = get_parent_call(cc->parent);
+    parent_call_t *pc = get_parent_call(cc->parent);
    gpr_mu_lock(&pc->child_list_mu);
    if (c == pc->first_child) {
      pc->first_child = cc->sibling_next;
@ -1309,7 +1309,7 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx,

    /* propagate cancellation to any interested children */
    gpr_atm_rel_store(&call->received_final_op_atm, 1);
-    parent_call *pc = get_parent_call(call);
+    parent_call_t *pc = get_parent_call(call);
    if (pc != NULL) {
      grpc_call *child;
      gpr_mu_lock(&pc->child_list_mu);
@ -1345,7 +1345,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx,
  if (bctl->completion_data.notify_tag.is_closure) {
    /* unrefs bctl->error */
    bctl->call = NULL;
-    GRPC_CLOSURE_RUN(exec_ctx, bctl->completion_data.notify_tag.tag, error);
+    GRPC_CLOSURE_RUN(
+        exec_ctx, (grpc_closure *)bctl->completion_data.notify_tag.tag, error);
    GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completion");
  } else {
    /* unrefs bctl->error */
@ -1474,7 +1475,7 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
   * acq_load is in receiving_initial_metadata_ready() */
  if (error != GRPC_ERROR_NONE || call->receiving_stream == NULL ||
      !gpr_atm_rel_cas(&call->recv_state, RECV_NONE, (gpr_atm)bctlp)) {
-    process_data_after_md(exec_ctx, bctlp);
+    process_data_after_md(exec_ctx, bctl);
  }
 }

@ -1679,11 +1680,12 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
  if (nops == 0) {
    if (!is_notify_tag_closure) {
      GPR_ASSERT(grpc_cq_begin_op(call->cq, notify_tag));
-      grpc_cq_end_op(exec_ctx, call->cq, notify_tag, GRPC_ERROR_NONE,
-                     free_no_op_completion, NULL,
-                     gpr_malloc(sizeof(grpc_cq_completion)));
+      grpc_cq_end_op(
+          exec_ctx, call->cq, notify_tag, GRPC_ERROR_NONE,
+          free_no_op_completion, NULL,
+          (grpc_cq_completion *)gpr_malloc(sizeof(grpc_cq_completion)));
    } else {
-      GRPC_CLOSURE_SCHED(exec_ctx, notify_tag, GRPC_ERROR_NONE);
+      GRPC_CLOSURE_SCHED(exec_ctx, (grpc_closure *)notify_tag, GRPC_ERROR_NONE);
    }
    error = GRPC_CALL_OK;
    goto done;
--- a/src/core/lib/surface/init.c
+++ b/src/core/lib/surface/init.c
@ -36,6 +36,7 @@
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/iomgr/resource_quota.h"
+#include "src/core/lib/iomgr/timer_manager.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/alarm_internal.h"
@ -179,14 +180,16 @@ void grpc_shutdown(void) {
      GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL);
  gpr_mu_lock(&g_init_mu);
  if (--g_initializations == 0) {
-    grpc_iomgr_shutdown(&exec_ctx);
-    gpr_timers_global_destroy();
-    grpc_tracer_shutdown();
+    grpc_executor_shutdown(&exec_ctx);
+    grpc_timer_manager_set_threading(false);  // shutdown timer_manager thread
    for (i = g_number_of_plugins; i >= 0; i--) {
      if (g_all_of_the_plugins[i].destroy != NULL) {
        g_all_of_the_plugins[i].destroy();
      }
    }
+    grpc_iomgr_shutdown(&exec_ctx);
+    gpr_timers_global_destroy();
+    grpc_tracer_shutdown();
    grpc_mdctx_global_shutdown(&exec_ctx);
    grpc_handshaker_factory_registry_shutdown(&exec_ctx);
    grpc_slice_intern_shutdown();
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@ -29,6 +29,7 @@

 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/slice/slice_internal.h"
@ -540,6 +541,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg,
    if (request_id == -1) {
      continue;
    } else {
+      GRPC_STATS_INC_SERVER_CQS_CHECKED(exec_ctx, i);
      gpr_mu_lock(&calld->mu_state);
      calld->state = ACTIVATED;
      gpr_mu_unlock(&calld->mu_state);
@ -550,6 +552,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg,
  }

  /* no cq to take the request found: queue it on the slow list */
+  GRPC_STATS_INC_SERVER_SLOWPATH_REQUESTS_QUEUED(exec_ctx);
  gpr_mu_lock(&server->mu_call);
  gpr_mu_lock(&calld->mu_state);
  calld->state = PENDING;
@ -1432,6 +1435,7 @@ grpc_call_error grpc_server_request_call(
  grpc_call_error error;
  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
  requested_call *rc = (requested_call *)gpr_malloc(sizeof(*rc));
+  GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx);
  GRPC_API_TRACE(
      "grpc_server_request_call("
      "server=%p, call=%p, details=%p, initial_metadata=%p, "
@ -1478,6 +1482,7 @@ grpc_call_error grpc_server_request_registered_call(
  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
  requested_call *rc = (requested_call *)gpr_malloc(sizeof(*rc));
  registered_method *rm = (registered_method *)rmp;
+  GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx);
  GRPC_API_TRACE(
      "grpc_server_request_registered_call("
      "server=%p, rmp=%p, call=%p, deadline=%p, initial_metadata=%p, "
--- a/src/cpp/common/channel_arguments.cc
+++ b/src/cpp/common/channel_arguments.cc
@ -86,6 +86,10 @@ void ChannelArguments::SetCompressionAlgorithm(
  SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, algorithm);
 }

+void ChannelArguments::SetGrpclbFallbackTimeout(int fallback_timeout) {
+  SetInt(GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS, fallback_timeout);
+}
+
 void ChannelArguments::SetSocketMutator(grpc_socket_mutator* mutator) {
  if (!mutator) {
    return;
--- a/src/cpp/server/health/default_health_check_service.cc
+++ b/src/cpp/server/health/default_health_check_service.cc
@ -20,6 +20,7 @@
 #include <mutex>

 #include <grpc++/impl/codegen/method_handler_impl.h>
+#include <grpc/slice.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>

--- a/src/cpp/util/byte_buffer_cc.cc
+++ b/src/cpp/util/byte_buffer_cc.cc
@ -16,11 +16,15 @@
 *
 */

+#include <grpc++/impl/grpc_library.h>
 #include <grpc++/support/byte_buffer.h>
+#include <grpc/byte_buffer.h>
 #include <grpc/byte_buffer_reader.h>

 namespace grpc {

+static internal::GrpcLibraryInitializer g_gli_initializer;
+
 ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) {
  // The following assertions check that the representation of a grpc::Slice is
  // identical to that of a grpc_slice:  it has a grpc_slice field, and nothing
@ -29,6 +33,7 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) {
                "Slice must have same representation as grpc_slice");
  static_assert(sizeof(Slice) == sizeof(grpc_slice),
                "Slice must have same representation as grpc_slice");
+  g_gli_initializer.summon();  // Make sure that initializer linked in
  // The const_cast is legal if grpc_raw_byte_buffer_create() does no more
  // than its advertised side effect of increasing the reference count of the
  // slices it processes, and such an increase does not affect the semantics
@ -37,19 +42,6 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) {
      reinterpret_cast<grpc_slice*>(const_cast<Slice*>(slices)), nslices);
 }

-ByteBuffer::~ByteBuffer() {
-  if (buffer_) {
-    grpc_byte_buffer_destroy(buffer_);
-  }
-}
-
-void ByteBuffer::Clear() {
-  if (buffer_) {
-    grpc_byte_buffer_destroy(buffer_);
-    buffer_ = nullptr;
-  }
-}
-
 Status ByteBuffer::Dump(std::vector<Slice>* slices) const {
  slices->clear();
  if (!buffer_) {
@ -80,7 +72,9 @@ ByteBuffer::ByteBuffer(const ByteBuffer& buf)
    : buffer_(grpc_byte_buffer_copy(buf.buffer_)) {}

 ByteBuffer& ByteBuffer::operator=(const ByteBuffer& buf) {
-  Clear();  // first remove existing data
+  if (this != &buf) {
+    Clear();  // first remove existing data
+  }
  if (buf.buffer_) {
    buffer_ = grpc_byte_buffer_copy(buf.buffer_);  // then copy
  }
@ -93,4 +87,19 @@ void ByteBuffer::Swap(ByteBuffer* other) {
  buffer_ = tmp;
 }

+ByteBuffer::operator grpc_byte_buffer*() {
+  // The following assertions check that the representation of a ByteBuffer is
+  // identical to grpc_byte_buffer*:  it has a grpc_byte_buffer* field,
+  // and nothing else.
+  static_assert(std::is_same<decltype(buffer_), grpc_byte_buffer*>::value,
+                "ByteBuffer must have same representation as "
+                "grpc_byte_buffer*");
+  static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*),
+                "ByteBuffer must have same representation as "
+                "grpc_byte_buffer*");
+  return buffer_;
+}
+
+ByteBuffer::operator const grpc_byte_buffer*() const { return buffer_; }
+
 }  // namespace grpc
--- a/src/cpp/util/slice_cc.cc
+++ b/src/cpp/util/slice_cc.cc
@ -50,4 +50,6 @@ Slice::Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data)
 Slice::Slice(void* buf, size_t len, void (*destroy)(void*, size_t))
    : slice_(grpc_slice_new_with_len(buf, len, destroy)) {}

+grpc_slice Slice::c_slice() const { return grpc_slice_ref(slice_); }
+
 }  // namespace grpc
--- a/src/python/grpcio_tests/tests/tests.json
+++ b/src/python/grpcio_tests/tests/tests.json
@ -26,6 +26,8 @@
  "unit._credentials_test.CredentialsTest",
  "unit._cython._cancel_many_calls_test.CancelManyCallsTest",
  "unit._cython._channel_test.ChannelTest",
+  "unit._cython._no_messages_server_completion_queue_per_call_test.Test",
+  "unit._cython._no_messages_single_server_completion_queue_test.Test",
  "unit._cython._read_some_but_not_all_responses_test.ReadSomeButNotAllResponsesTest",
  "unit._cython.cygrpc_test.InsecureServerInsecureClient",
  "unit._cython.cygrpc_test.SecureServerSecureClient",
--- a/src/python/grpcio_tests/tests/unit/_cython/_common.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/_common.py
@ -0,0 +1,118 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Common utilities for tests of the Cython layer of gRPC Python."""
+
+import collections
+import threading
+
+from grpc._cython import cygrpc
+
+RPC_COUNT = 4000
+
+INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
+EMPTY_FLAGS = 0
+
+INVOCATION_METADATA = cygrpc.Metadata(
+    (cygrpc.Metadatum(b'client-md-key', b'client-md-key'),
+     cygrpc.Metadatum(b'client-md-key-bin', b'\x00\x01' * 3000),))
+
+INITIAL_METADATA = cygrpc.Metadata(
+    (cygrpc.Metadatum(b'server-initial-md-key', b'server-initial-md-value'),
+     cygrpc.Metadatum(b'server-initial-md-key-bin', b'\x00\x02' * 3000),))
+
+TRAILING_METADATA = cygrpc.Metadata(
+    (cygrpc.Metadatum(b'server-trailing-md-key', b'server-trailing-md-value'),
+     cygrpc.Metadatum(b'server-trailing-md-key-bin', b'\x00\x03' * 3000),))
+
+
+class QueueDriver(object):
+
+    def __init__(self, condition, completion_queue):
+        self._condition = condition
+        self._completion_queue = completion_queue
+        self._due = collections.defaultdict(int)
+        self._events = collections.defaultdict(list)
+
+    def add_due(self, tags):
+        if not self._due:
+
+            def in_thread():
+                while True:
+                    event = self._completion_queue.poll()
+                    with self._condition:
+                        self._events[event.tag].append(event)
+                        self._due[event.tag] -= 1
+                        self._condition.notify_all()
+                        if self._due[event.tag] <= 0:
+                            self._due.pop(event.tag)
+                            if not self._due:
+                                return
+
+            thread = threading.Thread(target=in_thread)
+            thread.start()
+        for tag in tags:
+            self._due[tag] += 1
+
+    def event_with_tag(self, tag):
+        with self._condition:
+            while True:
+                if self._events[tag]:
+                    return self._events[tag].pop(0)
+                else:
+                    self._condition.wait()
+
+
+def execute_many_times(behavior):
+    return tuple(behavior() for _ in range(RPC_COUNT))
+
+
+class OperationResult(
+        collections.namedtuple('OperationResult', (
+            'start_batch_result', 'completion_type', 'success',))):
+    pass
+
+
+SUCCESSFUL_OPERATION_RESULT = OperationResult(
+    cygrpc.CallError.ok, cygrpc.CompletionType.operation_complete, True)
+
+
+class RpcTest(object):
+
+    def setUp(self):
+        self.server_completion_queue = cygrpc.CompletionQueue()
+        self.server = cygrpc.Server(cygrpc.ChannelArgs([]))
+        self.server.register_completion_queue(self.server_completion_queue)
+        port = self.server.add_http2_port(b'[::]:0')
+        self.server.start()
+        self.channel = cygrpc.Channel('localhost:{}'.format(port).encode(),
+                                      cygrpc.ChannelArgs([]))
+
+        self._server_shutdown_tag = 'server_shutdown_tag'
+        self.server_condition = threading.Condition()
+        self.server_driver = QueueDriver(self.server_condition,
+                                         self.server_completion_queue)
+        with self.server_condition:
+            self.server_driver.add_due({
+                self._server_shutdown_tag,
+            })
+
+        self.client_condition = threading.Condition()
+        self.client_completion_queue = cygrpc.CompletionQueue()
+        self.client_driver = QueueDriver(self.client_condition,
+                                         self.client_completion_queue)
+
+    def tearDown(self):
+        self.server.shutdown(self.server_completion_queue,
+                             self._server_shutdown_tag)
+        self.server.cancel_all_calls()
--- a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py
@ -0,0 +1,131 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test a corner-case at the level of the Cython API."""
+
+import threading
+import unittest
+
+from grpc._cython import cygrpc
+
+from tests.unit._cython import _common
+
+
+class Test(_common.RpcTest, unittest.TestCase):
+
+    def _do_rpcs(self):
+        server_call_condition = threading.Condition()
+        server_call_completion_queue = cygrpc.CompletionQueue()
+        server_call_driver = _common.QueueDriver(server_call_condition,
+                                                 server_call_completion_queue)
+
+        server_request_call_tag = 'server_request_call_tag'
+        server_send_initial_metadata_tag = 'server_send_initial_metadata_tag'
+        server_complete_rpc_tag = 'server_complete_rpc_tag'
+
+        with self.server_condition:
+            server_request_call_start_batch_result = self.server.request_call(
+                server_call_completion_queue, self.server_completion_queue,
+                server_request_call_tag)
+            self.server_driver.add_due({
+                server_request_call_tag,
+            })
+
+        client_call = self.channel.create_call(
+            None, _common.EMPTY_FLAGS, self.client_completion_queue,
+            b'/twinkies', None, _common.INFINITE_FUTURE)
+        client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag'
+        client_complete_rpc_tag = 'client_complete_rpc_tag'
+        with self.client_condition:
+            client_receive_initial_metadata_start_batch_result = (
+                client_call.start_client_batch(
+                    cygrpc.Operations([
+                        cygrpc.operation_receive_initial_metadata(
+                            _common.EMPTY_FLAGS),
+                    ]), client_receive_initial_metadata_tag))
+            client_complete_rpc_start_batch_result = client_call.start_client_batch(
+                cygrpc.Operations([
+                    cygrpc.operation_send_initial_metadata(
+                        _common.INVOCATION_METADATA, _common.EMPTY_FLAGS),
+                    cygrpc.operation_send_close_from_client(
+                        _common.EMPTY_FLAGS),
+                    cygrpc.operation_receive_status_on_client(
+                        _common.EMPTY_FLAGS),
+                ]), client_complete_rpc_tag)
+            self.client_driver.add_due({
+                client_receive_initial_metadata_tag,
+                client_complete_rpc_tag,
+            })
+
+        server_request_call_event = self.server_driver.event_with_tag(
+            server_request_call_tag)
+
+        with server_call_condition:
+            server_send_initial_metadata_start_batch_result = (
+                server_request_call_event.operation_call.start_server_batch([
+                    cygrpc.operation_send_initial_metadata(
+                        _common.INITIAL_METADATA, _common.EMPTY_FLAGS),
+                ], server_send_initial_metadata_tag))
+            server_call_driver.add_due({
+                server_send_initial_metadata_tag,
+            })
+        server_send_initial_metadata_event = server_call_driver.event_with_tag(
+            server_send_initial_metadata_tag)
+
+        with server_call_condition:
+            server_complete_rpc_start_batch_result = (
+                server_request_call_event.operation_call.start_server_batch([
+                    cygrpc.operation_receive_close_on_server(
+                        _common.EMPTY_FLAGS),
+                    cygrpc.operation_send_status_from_server(
+                        _common.TRAILING_METADATA, cygrpc.StatusCode.ok,
+                        b'test details', _common.EMPTY_FLAGS),
+                ], server_complete_rpc_tag))
+            server_call_driver.add_due({
+                server_complete_rpc_tag,
+            })
+        server_complete_rpc_event = server_call_driver.event_with_tag(
+            server_complete_rpc_tag)
+
+        client_receive_initial_metadata_event = self.client_driver.event_with_tag(
+            client_receive_initial_metadata_tag)
+        client_complete_rpc_event = self.client_driver.event_with_tag(
+            client_complete_rpc_tag)
+
+        return (_common.OperationResult(server_request_call_start_batch_result,
+                                        server_request_call_event.type,
+                                        server_request_call_event.success),
+                _common.OperationResult(
+                    client_receive_initial_metadata_start_batch_result,
+                    client_receive_initial_metadata_event.type,
+                    client_receive_initial_metadata_event.success),
+                _common.OperationResult(client_complete_rpc_start_batch_result,
+                                        client_complete_rpc_event.type,
+                                        client_complete_rpc_event.success),
+                _common.OperationResult(
+                    server_send_initial_metadata_start_batch_result,
+                    server_send_initial_metadata_event.type,
+                    server_send_initial_metadata_event.success),
+                _common.OperationResult(server_complete_rpc_start_batch_result,
+                                        server_complete_rpc_event.type,
+                                        server_complete_rpc_event.success),)
+
+    def test_rpcs(self):
+        expecteds = [(_common.SUCCESSFUL_OPERATION_RESULT,) *
+                     5] * _common.RPC_COUNT
+        actuallys = _common.execute_many_times(self._do_rpcs)
+        self.assertSequenceEqual(expecteds, actuallys)
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)
--- a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py
+++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py
@ -0,0 +1,126 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test a corner-case at the level of the Cython API."""
+
+import threading
+import unittest
+
+from grpc._cython import cygrpc
+
+from tests.unit._cython import _common
+
+
+class Test(_common.RpcTest, unittest.TestCase):
+
+    def _do_rpcs(self):
+        server_request_call_tag = 'server_request_call_tag'
+        server_send_initial_metadata_tag = 'server_send_initial_metadata_tag'
+        server_complete_rpc_tag = 'server_complete_rpc_tag'
+
+        with self.server_condition:
+            server_request_call_start_batch_result = self.server.request_call(
+                self.server_completion_queue, self.server_completion_queue,
+                server_request_call_tag)
+            self.server_driver.add_due({
+                server_request_call_tag,
+            })
+
+        client_call = self.channel.create_call(
+            None, _common.EMPTY_FLAGS, self.client_completion_queue,
+            b'/twinkies', None, _common.INFINITE_FUTURE)
+        client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag'
+        client_complete_rpc_tag = 'client_complete_rpc_tag'
+        with self.client_condition:
+            client_receive_initial_metadata_start_batch_result = (
+                client_call.start_client_batch(
+                    cygrpc.Operations([
+                        cygrpc.operation_receive_initial_metadata(
+                            _common.EMPTY_FLAGS),
+                    ]), client_receive_initial_metadata_tag))
+            client_complete_rpc_start_batch_result = client_call.start_client_batch(
+                cygrpc.Operations([
+                    cygrpc.operation_send_initial_metadata(
+                        _common.INVOCATION_METADATA, _common.EMPTY_FLAGS),
+                    cygrpc.operation_send_close_from_client(
+                        _common.EMPTY_FLAGS),
+                    cygrpc.operation_receive_status_on_client(
+                        _common.EMPTY_FLAGS),
+                ]), client_complete_rpc_tag)
+            self.client_driver.add_due({
+                client_receive_initial_metadata_tag,
+                client_complete_rpc_tag,
+            })
+
+        server_request_call_event = self.server_driver.event_with_tag(
+            server_request_call_tag)
+
+        with self.server_condition:
+            server_send_initial_metadata_start_batch_result = (
+                server_request_call_event.operation_call.start_server_batch([
+                    cygrpc.operation_send_initial_metadata(
+                        _common.INITIAL_METADATA, _common.EMPTY_FLAGS),
+                ], server_send_initial_metadata_tag))
+            self.server_driver.add_due({
+                server_send_initial_metadata_tag,
+            })
+        server_send_initial_metadata_event = self.server_driver.event_with_tag(
+            server_send_initial_metadata_tag)
+
+        with self.server_condition:
+            server_complete_rpc_start_batch_result = (
+                server_request_call_event.operation_call.start_server_batch([
+                    cygrpc.operation_receive_close_on_server(
+                        _common.EMPTY_FLAGS),
+                    cygrpc.operation_send_status_from_server(
+                        _common.TRAILING_METADATA, cygrpc.StatusCode.ok,
+                        b'test details', _common.EMPTY_FLAGS),
+                ], server_complete_rpc_tag))
+            self.server_driver.add_due({
+                server_complete_rpc_tag,
+            })
+        server_complete_rpc_event = self.server_driver.event_with_tag(
+            server_complete_rpc_tag)
+
+        client_receive_initial_metadata_event = self.client_driver.event_with_tag(
+            client_receive_initial_metadata_tag)
+        client_complete_rpc_event = self.client_driver.event_with_tag(
+            client_complete_rpc_tag)
+
+        return (_common.OperationResult(server_request_call_start_batch_result,
+                                        server_request_call_event.type,
+                                        server_request_call_event.success),
+                _common.OperationResult(
+                    client_receive_initial_metadata_start_batch_result,
+                    client_receive_initial_metadata_event.type,
+                    client_receive_initial_metadata_event.success),
+                _common.OperationResult(client_complete_rpc_start_batch_result,
+                                        client_complete_rpc_event.type,
+                                        client_complete_rpc_event.success),
+                _common.OperationResult(
+                    server_send_initial_metadata_start_batch_result,
+                    server_send_initial_metadata_event.type,
+                    server_send_initial_metadata_event.success),
+                _common.OperationResult(server_complete_rpc_start_batch_result,
+                                        server_complete_rpc_event.type,
+                                        server_complete_rpc_event.success),)
+
+    def test_rpcs(self):
+        expecteds = [(_common.SUCCESSFUL_OPERATION_RESULT,) *
+                     5] * _common.RPC_COUNT
+        actuallys = _common.execute_many_times(self._do_rpcs)
+        self.assertSequenceEqual(expecteds, actuallys)
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
@ -22,16 +22,6 @@

 #include "rb_grpc_imports.generated.h"

-grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import;
-grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import;
-grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import;
-grpc_byte_buffer_length_type grpc_byte_buffer_length_import;
-grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import;
-grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import;
-grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import;
-grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import;
-grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import;
-grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
 census_initialize_type census_initialize_import;
 census_shutdown_type census_shutdown_import;
 census_supported_type census_supported_import;
@ -168,6 +158,16 @@ grpc_ssl_server_credentials_create_ex_type grpc_ssl_server_credentials_create_ex
 grpc_server_add_secure_http2_port_type grpc_server_add_secure_http2_port_import;
 grpc_call_set_credentials_type grpc_call_set_credentials_import;
 grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import;
+grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import;
+grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import;
+grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import;
+grpc_byte_buffer_length_type grpc_byte_buffer_length_import;
+grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import;
+grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import;
+grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import;
+grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import;
+grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import;
+grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
 grpc_slice_ref_type grpc_slice_ref_import;
 grpc_slice_unref_type grpc_slice_unref_import;
 grpc_slice_copy_type grpc_slice_copy_import;
@ -330,16 +330,6 @@ gpr_sleep_until_type gpr_sleep_until_import;
 gpr_timespec_to_micros_type gpr_timespec_to_micros_import;

 void grpc_rb_load_imports(HMODULE library) {
-  grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create");
-  grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create");
-  grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy");
-  grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length");
-  grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy");
-  grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init");
-  grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy");
-  grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next");
-  grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall");
-  grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader");
  census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize");
  census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown");
  census_supported_import = (census_supported_type) GetProcAddress(library, "census_supported");
@ -476,6 +466,16 @@ void grpc_rb_load_imports(HMODULE library) {
  grpc_server_add_secure_http2_port_import = (grpc_server_add_secure_http2_port_type) GetProcAddress(library, "grpc_server_add_secure_http2_port");
  grpc_call_set_credentials_import = (grpc_call_set_credentials_type) GetProcAddress(library, "grpc_call_set_credentials");
  grpc_server_credentials_set_auth_metadata_processor_import = (grpc_server_credentials_set_auth_metadata_processor_type) GetProcAddress(library, "grpc_server_credentials_set_auth_metadata_processor");
+  grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create");
+  grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create");
+  grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy");
+  grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length");
+  grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy");
+  grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init");
+  grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy");
+  grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next");
+  grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall");
+  grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader");
  grpc_slice_ref_import = (grpc_slice_ref_type) GetProcAddress(library, "grpc_slice_ref");
  grpc_slice_unref_import = (grpc_slice_unref_type) GetProcAddress(library, "grpc_slice_unref");
  grpc_slice_copy_import = (grpc_slice_copy_type) GetProcAddress(library, "grpc_slice_copy");
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
@ -25,12 +25,12 @@

 #include <windows.h>

-#include <grpc/byte_buffer.h>
 #include <grpc/census.h>
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
 #include <grpc/grpc_posix.h>
 #include <grpc/grpc_security.h>
+#include <grpc/impl/codegen/byte_buffer.h>
 #include <grpc/slice.h>
 #include <grpc/slice_buffer.h>
 #include <grpc/support/alloc.h>
@ -47,36 +47,6 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>

-typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices);
-extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import;
-#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import
-typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression);
-extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import;
-#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import
-typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb);
-extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import;
-#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import
-typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb);
-extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import;
-#define grpc_byte_buffer_length grpc_byte_buffer_length_import
-typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer);
-extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import;
-#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import
-typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer);
-extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import;
-#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import
-typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader);
-extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import;
-#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import
-typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice);
-extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import;
-#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import
-typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader);
-extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import;
-#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import
-typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader);
-extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
-#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import
 typedef int(*census_initialize_type)(int features);
 extern census_initialize_type census_initialize_import;
 #define census_initialize census_initialize_import
@ -485,6 +455,36 @@ extern grpc_call_set_credentials_type grpc_call_set_credentials_import;
 typedef void(*grpc_server_credentials_set_auth_metadata_processor_type)(grpc_server_credentials *creds, grpc_auth_metadata_processor processor);
 extern grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import;
 #define grpc_server_credentials_set_auth_metadata_processor grpc_server_credentials_set_auth_metadata_processor_import
+typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices);
+extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import;
+#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import
+typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression);
+extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import;
+#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import
+typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb);
+extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import;
+#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import
+typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb);
+extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import;
+#define grpc_byte_buffer_length grpc_byte_buffer_length_import
+typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer);
+extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import;
+#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import
+typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer);
+extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import;
+#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import
+typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader);
+extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import;
+#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import
+typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice);
+extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import;
+#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import
+typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader);
+extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import;
+#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import
+typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader);
+extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
+#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import
 typedef grpc_slice(*grpc_slice_ref_type)(grpc_slice s);
 extern grpc_slice_ref_type grpc_slice_ref_import;
 #define grpc_slice_ref grpc_slice_ref_import
--- a/templates/CMakeLists.txt.template
+++ b/templates/CMakeLists.txt.template
@ -168,10 +168,8 @@
      set(gRPC_INSTALL FALSE)
    endif()
  elseif("<%text>${gRPC_ZLIB_PROVIDER}</%text>" STREQUAL "package")
-    find_package(ZLIB)
-    if(TARGET ZLIB::ZLIB)
-      set(_gRPC_ZLIB_LIBRARIES ZLIB::ZLIB)
-    endif()
+    find_package(ZLIB REQUIRED)
+    set(_gRPC_ZLIB_LIBRARIES <%text>${ZLIB_LIBRARIES}</%text>)
    set(_gRPC_FIND_ZLIB "if(NOT ZLIB_FOUND)\n  find_package(ZLIB)\nendif()")
  endif()

@ -190,7 +188,7 @@
      set(gRPC_INSTALL FALSE)
    endif()
  elseif("<%text>${gRPC_CARES_PROVIDER}</%text>" STREQUAL "package")
-    find_package(c-ares CONFIG)
+    find_package(c-ares REQUIRED CONFIG)
    if(TARGET c-ares::cares)
      set(_gRPC_CARES_LIBRARIES c-ares::cares)
    endif()
@ -234,7 +232,7 @@
      set(gRPC_INSTALL FALSE)
    endif()
  elseif("<%text>${gRPC_PROTOBUF_PROVIDER}</%text>" STREQUAL "package")
-    find_package(Protobuf <%text>${gRPC_PROTOBUF_PACKAGE_TYPE}</%text>)
+    find_package(Protobuf REQUIRED <%text>${gRPC_PROTOBUF_PACKAGE_TYPE}</%text>)
    if(Protobuf_FOUND OR PROTOBUF_FOUND)
      if(TARGET protobuf::<%text>${_gRPC_PROTOBUF_LIBRARY_NAME}</%text>)
        set(_gRPC_PROTOBUF_LIBRARIES protobuf::<%text>${_gRPC_PROTOBUF_LIBRARY_NAME}</%text>)
@ -279,11 +277,9 @@
      set(gRPC_INSTALL FALSE)
    endif()
  elseif("<%text>${gRPC_SSL_PROVIDER}</%text>" STREQUAL "package")
-    find_package(OpenSSL)
-    if(TARGET OpenSSL::SSL)
-      set(_gRPC_SSL_LIBRARIES OpenSSL::SSL)
-    endif()
-    set(_gRPC_FIND_SSL "if(NOT OpenSSL_FOUND)\n  find_package(OpenSSL)\nendif()")
+    find_package(OpenSSL REQUIRED)
+    set(_gRPC_SSL_LIBRARIES <%text>${OPENSSL_LIBRARIES}</%text>)
+    set(_gRPC_FIND_SSL "if(NOT OPENSSL_FOUND)\n  find_package(OpenSSL)\nendif()")
  endif()

  if("<%text>${gRPC_GFLAGS_PROVIDER}</%text>" STREQUAL "module")
@ -515,7 +511,7 @@
  % endfor

  target_include_directories(${lib.name}
-    PUBLIC <%text>$<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include></%text>
+    PUBLIC <%text>$<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include></%text>
    PRIVATE <%text>${CMAKE_CURRENT_SOURCE_DIR}</%text>
    PRIVATE <%text>${BORINGSSL_ROOT_DIR}</%text>/include
    PRIVATE <%text>${PROTOBUF_ROOT_DIR}</%text>/src
--- a/test/core/surface/public_headers_must_be_c89.c
+++ b/test/core/surface/public_headers_must_be_c89.c
@ -24,6 +24,7 @@
 #include <grpc/grpc_security.h>
 #include <grpc/grpc_security_constants.h>
 #include <grpc/impl/codegen/atm.h>
+#include <grpc/impl/codegen/byte_buffer.h>
 #include <grpc/impl/codegen/byte_buffer_reader.h>
 #include <grpc/impl/codegen/compression_types.h>
 #include <grpc/impl/codegen/connectivity_state.h>
--- a/test/cpp/end2end/grpclb_end2end_test.cc
+++ b/test/cpp/end2end/grpclb_end2end_test.cc
@ -368,8 +368,9 @@ class GrpclbEnd2endTest : public ::testing::Test {
    grpc_fake_resolver_response_generator_unref(response_generator_);
  }

-  void ResetStub() {
+  void ResetStub(int fallback_timeout = 0) {
    ChannelArguments args;
+    args.SetGrpclbFallbackTimeout(fallback_timeout);
    args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR,
                    response_generator_);
    std::ostringstream uri;
@ -441,10 +442,10 @@ class GrpclbEnd2endTest : public ::testing::Test {
    grpc_exec_ctx_finish(&exec_ctx);
  }

-  const std::vector<int> GetBackendPorts() const {
+  const std::vector<int> GetBackendPorts(const size_t start_index = 0) const {
    std::vector<int> backend_ports;
-    for (const auto& bs : backend_servers_) {
-      backend_ports.push_back(bs.port_);
+    for (size_t i = start_index; i < backend_servers_.size(); ++i) {
+      backend_ports.push_back(backend_servers_[i].port_);
    }
    return backend_ports;
  }
@ -615,6 +616,71 @@ TEST_F(SingleBalancerTest, InitiallyEmptyServerlist) {
  EXPECT_EQ("grpclb", channel_->GetLoadBalancingPolicyName());
 }

+TEST_F(SingleBalancerTest, Fallback) {
+  const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor();
+  const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor();
+  const size_t kNumBackendInResolution = backends_.size() / 2;
+
+  ResetStub(kFallbackTimeoutMs);
+  std::vector<AddressData> addresses;
+  addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""});
+  for (size_t i = 0; i < kNumBackendInResolution; ++i) {
+    addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""});
+  }
+  SetNextResolution(addresses);
+
+  // Send non-empty serverlist only after kServerlistDelayMs
+  ScheduleResponseForBalancer(
+      0, BalancerServiceImpl::BuildResponseForBackends(
+             GetBackendPorts(kNumBackendInResolution /* start_index */), {}),
+      kServerlistDelayMs);
+
+  // The first request. The client will block while it's still trying to
+  // contact the balancer.
+  gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
+  CheckRpcSendOk(kNumBackendInResolution);
+  gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
+
+  // Fallback is used: each backend returned by the resolver should have
+  // gotten one request.
+  for (size_t i = 0; i < kNumBackendInResolution; ++i) {
+    EXPECT_EQ(1U, backend_servers_[i].service_->request_count());
+  }
+  for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) {
+    EXPECT_EQ(0U, backend_servers_[i].service_->request_count());
+  }
+
+  // Wait until update has been processed, as signaled by the backend returned
+  // by the balancer receiving a request.
+  do {
+    CheckRpcSendOk(1);
+  } while (
+      backend_servers_[kNumBackendInResolution].service_->request_count() == 0);
+  for (size_t i = 0; i < backends_.size(); ++i) {
+    backend_servers_[i].service_->ResetCounters();
+  }
+
+  // Send out the second request.
+  gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
+  CheckRpcSendOk(backends_.size() - kNumBackendInResolution);
+  gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
+
+  // Serverlist is used: each backend returned by the balancer should
+  // have gotten one request.
+  for (size_t i = 0; i < kNumBackendInResolution; ++i) {
+    EXPECT_EQ(0U, backend_servers_[i].service_->request_count());
+  }
+  for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) {
+    EXPECT_EQ(1U, backend_servers_[i].service_->request_count());
+  }
+
+  balancers_[0]->NotifyDoneWithServerlists();
+  // The balancer got a single request.
+  EXPECT_EQ(1U, balancer_servers_[0].service_->request_count());
+  // and sent a single response.
+  EXPECT_EQ(1U, balancer_servers_[0].service_->response_count());
+}
+
 TEST_F(SingleBalancerTest, BackendsRestart) {
  const size_t kNumRpcsPerAddress = 100;
  ScheduleResponseForBalancer(
--- a/test/cpp/util/byte_buffer_test.cc
+++ b/test/cpp/util/byte_buffer_test.cc
@ -93,7 +93,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) {
  std::vector<Slice> slices;
  slices.push_back(Slice(hello, Slice::STEAL_REF));
  slices.push_back(Slice(world, Slice::STEAL_REF));
-  grpc_byte_buffer* send_buffer = nullptr;
+  ByteBuffer send_buffer;
  bool owned = false;
  ByteBuffer buffer(&slices[0], 2);
  slices.clear();
@ -101,8 +101,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) {
      buffer, &send_buffer, &owned);
  EXPECT_TRUE(status.ok());
  EXPECT_TRUE(owned);
-  EXPECT_TRUE(send_buffer != nullptr);
-  grpc_byte_buffer_destroy(send_buffer);
+  EXPECT_TRUE(send_buffer.Valid());
 }

 }  // namespace
--- a/test/distrib/cpp/run_distrib_test_cmake.sh
+++ b/test/distrib/cpp/run_distrib_test_cmake.sh
@ -0,0 +1,67 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list
+apt-get update
+#apt-get install -t jessie-backports -y libc-ares-dev  # we need specifically version 1.12
+apt-get install -t jessie-backports -y libssl-dev
+
+# Install c-ares
+cd third_party/cares/cares
+git fetch origin
+git checkout cares-1_13_0
+mkdir -p cmake/build
+cd cmake/build
+cmake -DCMAKE_BUILD_TYPE=Release ../..
+make -j4 install
+cd ../../../../..
+rm -rf third_party/cares/cares  # wipe out to prevent influencing the grpc build
+
+# Install zlib
+cd third_party/zlib
+mkdir -p cmake/build
+cd cmake/build
+cmake -DCMAKE_BUILD_TYPE=Release ../..
+make -j4 install
+cd ../../../..
+rm -rf third_party/zlib  # wipe out to prevent influencing the grpc build
+
+# Install protobuf
+cd third_party/protobuf
+mkdir -p cmake/build
+cd cmake/build
+cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release ..
+make -j4 install
+cd ../../../..
+rm -rf third_party/protobuf  # wipe out to prevent influencing the grpc build
+
+# Install gRPC
+mkdir -p cmake/build
+cd cmake/build
+cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DCMAKE_BUILD_TYPE=Release ../..
+make -j4 install
+cd ../..
+
+# Build helloworld example using cmake
+cd examples/cpp/helloworld
+mkdir -p cmake/build
+cd cmake/build
+cmake ../..
+make
+
--- a/test/distrib/cpp/run_distrib_test_routeguide.sh
+++ b/test/distrib/cpp/run_distrib_test_routeguide.sh
--- a/tools/buildgen/generate_projects.py
+++ b/tools/buildgen/generate_projects.py
@ -34,6 +34,7 @@ argp.add_argument('build_files', nargs='+', default=[])
 argp.add_argument('--templates', nargs='+', default=[])
 argp.add_argument('--output_merged', default=None, type=str)
 argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int)
+argp.add_argument('--base', default='.', type=str)
 args = argp.parse_args()

 json = args.build_files
@ -69,7 +70,7 @@ jobs = []
 for template in reversed(sorted(templates)):
  root, f = os.path.split(template)
  if os.path.splitext(f)[1] == '.template':
-    out_dir = '.' + root[len('templates'):]
+    out_dir = args.base + root[len('templates'):]
    out = out_dir + '/' + os.path.splitext(f)[0]
    if not os.path.exists(out_dir):
      os.makedirs(out_dir)
@ -84,7 +85,7 @@ for template in reversed(sorted(templates)):
      test[out] = tf[1]
      os.close(tf[0])
      cmd.append(test[out])
-    cmd.append(root + '/' + f)
+    cmd.append(args.base + '/' + root + '/' + f)
    jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None))

 jobset.run(pre_jobs, maxjobs=args.jobs)
--- a/tools/codegen/core/gen_stats_data.py
+++ b/tools/codegen/core/gen_stats_data.py
@ -313,3 +313,98 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C:
      len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries))
  print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % (
      len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram']))
+
+# patch qps_test bigquery schema
+RECORD_EXPLICIT_PERCENTILES = [50, 95, 99]
+
+with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f:
+  qps_schema = json.loads(f.read())
+
+def FindNamed(js, name):
+  for el in js:
+    if el['name'] == name:
+      return el
+
+def RemoveCoreFields(js):
+  new_fields = []
+  for field in js['fields']:
+    if not field['name'].startswith('core_'):
+      new_fields.append(field)
+  js['fields'] = new_fields
+
+RemoveCoreFields(FindNamed(qps_schema, 'clientStats'))
+RemoveCoreFields(FindNamed(qps_schema, 'serverStats'))
+
+def AddCoreFields(js):
+  for counter in inst_map['Counter']:
+    js['fields'].append({
+      'name': 'core_%s' % counter.name,
+      'type': 'INTEGER',
+      'mode': 'NULLABLE'
+    })
+  for histogram in inst_map['Histogram']:
+    js['fields'].append({
+      'name': 'core_%s' % histogram.name,
+      'type': 'STRING',
+      'mode': 'NULLABLE'
+    })
+    js['fields'].append({
+      'name': 'core_%s_bkts' % histogram.name,
+      'type': 'STRING',
+      'mode': 'NULLABLE'
+    })
+    for pctl in RECORD_EXPLICIT_PERCENTILES:
+      js['fields'].append({
+        'name': 'core_%s_%dp' % (histogram.name, pctl),
+        'type': 'FLOAT',
+        'mode': 'NULLABLE'
+      })
+
+AddCoreFields(FindNamed(qps_schema, 'clientStats'))
+AddCoreFields(FindNamed(qps_schema, 'serverStats'))
+
+with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f:
+  f.write(json.dumps(qps_schema, indent=2, sort_keys=True))
+
+# and generate a helper script to massage scenario results into the format we'd
+# like to query
+with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P:
+  with open(sys.argv[0]) as my_source:
+    for line in my_source:
+      if line[0] != '#': break
+    for line in my_source:
+      if line[0] == '#':
+        print >>P, line.rstrip()
+        break
+    for line in my_source:
+      if line[0] != '#':
+        break
+      print >>P, line.rstrip()
+
+  print >>P
+  print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py'
+  print >>P
+
+  print >>P, 'import massage_qps_stats_helpers'
+
+  print >>P, 'def massage_qps_stats(scenario_result):'
+  print >>P, '  for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:'
+  print >>P, '    if "coreStats" not in stats: return'
+  print >>P, '    core_stats = stats["coreStats"]'
+  print >>P, '    del stats["coreStats"]'
+  for counter in inst_map['Counter']:
+    print >>P, '    stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name)
+  for i, histogram in enumerate(inst_map['Histogram']):
+    print >>P, '    h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name
+    print >>P, '    stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name
+    print >>P, '    stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name
+    for pctl in RECORD_EXPLICIT_PERCENTILES:
+      print >>P, '    stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % (
+          histogram.name, pctl, pctl)
+
+with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S:
+  columns = []
+  for counter in inst_map['Counter']:
+    columns.append(('%s_per_iteration' % counter.name, 'FLOAT'))
+  print >>S, ',\n'.join('%s:%s' % x for x in columns)
+
--- a/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile
@ -27,4 +27,6 @@ RUN apt-get update && apt-get install -y \
      pkg-config \
      unzip && apt-get clean

+RUN apt-get update && apt-get install -y cmake golang && apt-get clean
+
 CMD ["bash"]
--- a/tools/doxygen/Doxyfile.c++
+++ b/tools/doxygen/Doxyfile.c++
@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \
 include/grpc++/impl/client_unary_call.h \
 include/grpc++/impl/codegen/async_stream.h \
 include/grpc++/impl/codegen/async_unary_call.h \
+include/grpc++/impl/codegen/byte_buffer.h \
 include/grpc++/impl/codegen/call.h \
 include/grpc++/impl/codegen/call_hook.h \
 include/grpc++/impl/codegen/channel_interface.h \
@ -881,6 +882,7 @@ include/grpc/impl/codegen/atm.h \
 include/grpc/impl/codegen/atm_gcc_atomic.h \
 include/grpc/impl/codegen/atm_gcc_sync.h \
 include/grpc/impl/codegen/atm_windows.h \
+include/grpc/impl/codegen/byte_buffer.h \
 include/grpc/impl/codegen/byte_buffer_reader.h \
 include/grpc/impl/codegen/compression_types.h \
 include/grpc/impl/codegen/connectivity_state.h \
--- a/tools/doxygen/Doxyfile.c++.internal
+++ b/tools/doxygen/Doxyfile.c++.internal
@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \
 include/grpc++/impl/client_unary_call.h \
 include/grpc++/impl/codegen/async_stream.h \
 include/grpc++/impl/codegen/async_unary_call.h \
+include/grpc++/impl/codegen/byte_buffer.h \
 include/grpc++/impl/codegen/call.h \
 include/grpc++/impl/codegen/call_hook.h \
 include/grpc++/impl/codegen/channel_interface.h \
@ -882,6 +883,7 @@ include/grpc/impl/codegen/atm.h \
 include/grpc/impl/codegen/atm_gcc_atomic.h \
 include/grpc/impl/codegen/atm_gcc_sync.h \
 include/grpc/impl/codegen/atm_windows.h \
+include/grpc/impl/codegen/byte_buffer.h \
 include/grpc/impl/codegen/byte_buffer_reader.h \
 include/grpc/impl/codegen/compression_types.h \
 include/grpc/impl/codegen/connectivity_state.h \
--- a/tools/doxygen/Doxyfile.core
+++ b/tools/doxygen/Doxyfile.core
@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \
 include/grpc/impl/codegen/atm_gcc_sync.h \
 include/grpc/impl/codegen/atm_windows.h \
 include/grpc/impl/codegen/atm_windows.h \
+include/grpc/impl/codegen/byte_buffer.h \
 include/grpc/impl/codegen/byte_buffer_reader.h \
 include/grpc/impl/codegen/compression_types.h \
 include/grpc/impl/codegen/connectivity_state.h \
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \
 include/grpc/impl/codegen/atm_gcc_sync.h \
 include/grpc/impl/codegen/atm_windows.h \
 include/grpc/impl/codegen/atm_windows.h \
+include/grpc/impl/codegen/byte_buffer.h \
 include/grpc/impl/codegen/byte_buffer_reader.h \
 include/grpc/impl/codegen/compression_types.h \
 include/grpc/impl/codegen/connectivity_state.h \
--- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc
@ -26,6 +26,7 @@ sudo service docker restart

 # Populate xdg-cache-home to workaround https://github.com/grpc/grpc/issues/11968
 sudo mkdir -p /tmp/xdg-cache-home
+PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install setuptools wheel
 PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install coverage==4.4 pylint==1.6.5

 # Download Docker images from DockerHub
--- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc
@ -31,23 +31,16 @@ ulimit -a
 pip install google-api-python-client --user python
 export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json

-# required to build protobuf
-brew install gflags
-
 set +ex  # rvm script is very verbose and exits with errorcode
 source $HOME/.rvm/scripts/rvm
 set -e  # rvm commands are very verbose
-rvm install ruby-2.4
+rvm use ruby-2.4
 rvm osx-ssl-certs status all
 rvm osx-ssl-certs update all
 set -ex

-gem install bundler
-
 # cocoapods
 export LANG=en_US.UTF-8
-gem install cocoapods
-gem install xcpretty
 pod repo update  # needed by python

 # python
@ -56,10 +49,6 @@ pip install virtualenv --user python
 pip install -U six tox setuptools --user python
 export PYTHONPATH=/Library/Python/3.4/site-packages

-# python 3.4
-wget -q https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg
-sudo installer -pkg python-3.4.4-macosx10.6.pkg -target /
-
 # set xcode version for Obj-C tests
 sudo xcode-select -switch /Applications/Xcode_8.2.1.app/Contents/Developer

--- a/tools/internal_ci/macos/grpc_basictests_dbg.cfg
+++ b/tools/internal_ci/macos/grpc_basictests_dbg.cfg
@ -27,5 +27,5 @@ action {

 env_vars {
  key: "RUN_TESTS_FLAGS"
-  value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4"
+  value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results"
 }
--- a/tools/internal_ci/macos/grpc_basictests_opt.cfg
+++ b/tools/internal_ci/macos/grpc_basictests_opt.cfg
@ -27,5 +27,5 @@ action {

 env_vars {
  key: "RUN_TESTS_FLAGS"
-  value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4"
+  value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results"
 }
--- a/tools/internal_ci/macos/grpc_build_artifacts.sh
+++ b/tools/internal_ci/macos/grpc_build_artifacts.sh
@ -20,25 +20,13 @@ cd $(dirname $0)/../../..

 source tools/internal_ci/helper_scripts/prepare_build_macos_rc

-# python 3.5
-wget -q https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg
-sudo installer -pkg python-3.5.2-macosx10.6.pkg -target /
-
 # install cython for all python versions
 python2.7 -m pip install cython setuptools wheel
 python3.4 -m pip install cython setuptools wheel
 python3.5 -m pip install cython setuptools wheel
 python3.6 -m pip install cython setuptools wheel

-# node-gyp (needed for node artifacts)
-npm install -g node-gyp
-
-# php dependencies: pecl
-curl -O http://pear.php.net/go-pear.phar
-sudo php -d detect_unicode=0 go-pear.phar
-
 # needed to build ruby artifacts
-gem install rake-compiler
 wget https://raw.githubusercontent.com/grpc/grpc/master/tools/distrib/build_ruby_environment_macos.sh
 bash build_ruby_environment_macos.sh

--- a/tools/internal_ci/windows/grpc_build_artifacts.bat
+++ b/tools/internal_ci/windows/grpc_build_artifacts.bat
@ -19,8 +19,6 @@ rename C:\Python34_32bit Python34_32bits
 rename C:\Python35_32bit Python35_32bits
 rename C:\Python36_32bit Python36_32bits

-pacman -S --noconfirm mingw64/mingw-w64-x86_64-gcc mingw32/mingw-w64-i686-gcc
-
@rem enter repo root
 cd /d %~dp0\..\..\..

--- a/tools/profiling/microbenchmarks/bm2bq.py
+++ b/tools/profiling/microbenchmarks/bm2bq.py
@ -22,42 +22,15 @@ import sys
 import json
 import csv
 import bm_json
+import json
+import subprocess
+
+columns = []

-columns = [
-  ('jenkins_build', 'integer'),
-  ('jenkins_job', 'string'),
-  ('date', 'timestamp'),
-  ('cpu_scaling_enabled', 'boolean'),
-  ('num_cpus', 'integer'),
-  ('mhz_per_cpu', 'integer'),
-  ('library_build_type', 'string'),
-  ('name', 'string'),
-  ('fixture', 'string'),
-  ('client_mutator', 'string'),
-  ('server_mutator', 'string'),
-  ('request_size', 'integer'),
-  ('response_size', 'integer'),
-  ('request_count', 'integer'),
-  ('iterations', 'integer'),
-  ('time_unit', 'string'),
-  ('real_time', 'integer'),
-  ('cpu_time', 'integer'),
-  ('bytes_per_second', 'float'),
-  ('allocs_per_iteration', 'float'),
-  ('locks_per_iteration', 'float'),
-  ('writes_per_iteration', 'float'),
-  ('bandwidth_kilobits', 'integer'),
-  ('cli_transport_stalls_per_iteration', 'float'),
-  ('cli_stream_stalls_per_iteration', 'float'),
-  ('svr_transport_stalls_per_iteration', 'float'),
-  ('svr_stream_stalls_per_iteration', 'float'),
-  ('atm_cas_per_iteration', 'float'),
-  ('atm_add_per_iteration', 'float'),
-  ('end_of_stream', 'boolean'),
-  ('header_bytes_per_iteration', 'float'),
-  ('framing_bytes_per_iteration', 'float'),
-  ('nows_per_iteration', 'float'),
-]
+for row in json.loads(
+    subprocess.check_output([
+      'bq','--format=json','show','microbenchmarks.microbenchmarks']))['schema']['fields']:
+  columns.append((row['name'], row['type'].lower()))

 SANITIZE = {
  'integer': int,
--- a/tools/run_tests/artifacts/distribtest_targets.py
+++ b/tools/run_tests/artifacts/distribtest_targets.py
@ -255,12 +255,13 @@ class PHPDistribTest(object):
 class CppDistribTest(object):
  """Tests Cpp make intall by building examples."""

-  def __init__(self, platform, arch, docker_suffix=None):
-    self.name = 'cpp_%s_%s_%s' % (platform, arch, docker_suffix)
+  def __init__(self, platform, arch, docker_suffix=None, testcase=None):
+    self.name = 'cpp_%s_%s_%s_%s' % (platform, arch, docker_suffix, testcase)
    self.platform = platform
    self.arch = arch
    self.docker_suffix = docker_suffix
-    self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix]
+    self.testcase = testcase
+    self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix, testcase]

  def pre_build_jobspecs(self):
    return []
@ -271,7 +272,7 @@ class CppDistribTest(object):
                                   'tools/dockerfile/distribtest/cpp_%s_%s' % (
                                       self.docker_suffix,
                                       self.arch),
-                                   'test/distrib/cpp/run_distrib_test.sh')
+                                   'test/distrib/cpp/run_distrib_test_%s.sh' % self.testcase)
    else:
      raise Exception("Not supported yet.")

@ -281,7 +282,8 @@ class CppDistribTest(object):

 def targets():
  """Gets list of supported targets"""
-  return [CppDistribTest('linux', 'x64', 'jessie'),
+  return [CppDistribTest('linux', 'x64', 'jessie', 'routeguide'),
+          CppDistribTest('linux', 'x64', 'jessie', 'cmake'),
          CSharpDistribTest('linux', 'x64', 'wheezy'),
          CSharpDistribTest('linux', 'x64', 'jessie'),
          CSharpDistribTest('linux', 'x86', 'jessie'),
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@ -8395,6 +8395,7 @@
      "gpr_codegen"
    ], 
    "headers": [
+      "include/grpc/impl/codegen/byte_buffer.h", 
      "include/grpc/impl/codegen/byte_buffer_reader.h", 
      "include/grpc/impl/codegen/compression_types.h", 
      "include/grpc/impl/codegen/connectivity_state.h", 
@ -8408,6 +8409,7 @@
    "language": "c", 
    "name": "grpc_codegen", 
    "src": [
+      "include/grpc/impl/codegen/byte_buffer.h", 
      "include/grpc/impl/codegen/byte_buffer_reader.h", 
      "include/grpc/impl/codegen/compression_types.h", 
      "include/grpc/impl/codegen/connectivity_state.h", 
@ -9286,6 +9288,7 @@
    "headers": [
      "include/grpc++/impl/codegen/async_stream.h", 
      "include/grpc++/impl/codegen/async_unary_call.h", 
+      "include/grpc++/impl/codegen/byte_buffer.h", 
      "include/grpc++/impl/codegen/call.h", 
      "include/grpc++/impl/codegen/call_hook.h", 
      "include/grpc++/impl/codegen/channel_interface.h", 
@ -9320,6 +9323,7 @@
    "src": [
      "include/grpc++/impl/codegen/async_stream.h", 
      "include/grpc++/impl/codegen/async_unary_call.h", 
+      "include/grpc++/impl/codegen/byte_buffer.h", 
      "include/grpc++/impl/codegen/call.h", 
      "include/grpc++/impl/codegen/call_hook.h", 
      "include/grpc++/impl/codegen/channel_interface.h", 
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@ -49905,7 +49905,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -49955,7 +49955,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50005,7 +50005,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50105,7 +50105,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50205,7 +50205,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50255,7 +50255,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50305,7 +50305,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50355,7 +50355,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50759,7 +50759,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50809,7 +50809,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50859,7 +50859,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -50959,7 +50959,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -51059,7 +51059,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -51109,7 +51109,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -51159,7 +51159,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -51209,7 +51209,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -51884,7 +51884,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -51960,7 +51960,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -52036,7 +52036,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -52188,7 +52188,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -52340,7 +52340,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -52416,7 +52416,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -52492,7 +52492,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -52568,7 +52568,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -53180,7 +53180,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -53256,7 +53256,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -53332,7 +53332,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -53484,7 +53484,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -53636,7 +53636,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -53712,7 +53712,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -53788,7 +53788,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
@ -53864,7 +53864,7 @@
  {
    "args": [
      "--scenarios_json", 
-      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
+      "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}"
    ], 
    "boringssl": true, 
    "ci_platforms": [
--- a/tools/run_tests/performance/bq_upload_result.py
+++ b/tools/run_tests/performance/bq_upload_result.py
@ -24,6 +24,7 @@ import os
 import sys
 import time
 import uuid
+import massage_qps_stats


 gcp_utils_dir = os.path.abspath(os.path.join(
@ -117,6 +118,7 @@ def _flatten_result_inplace(scenario_result):
  scenario_result['serverCpuUsage'] = scenario_result['summary'].pop('serverCpuUsage', None)
  scenario_result['summary'].pop('successfulRequestsPerSecond', None)
  scenario_result['summary'].pop('failedRequestsPerSecond', None)
+  massage_qps_stats.massage_qps_stats(scenario_result)


 def _populate_metadata_inplace(scenario_result):
--- a/tools/run_tests/performance/massage_qps_stats.py
+++ b/tools/run_tests/performance/massage_qps_stats.py
@ -0,0 +1,131 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Autogenerated by tools/codegen/core/gen_stats_data.py
+
+import massage_qps_stats_helpers
+def massage_qps_stats(scenario_result):
+  for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:
+    if "coreStats" not in stats: return
+    core_stats = stats["coreStats"]
+    del stats["coreStats"]
+    stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created")
+    stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created")
+    stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll")
+    stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait")
+    stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups")
+    stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write")
+    stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read")
+    stats["core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_pollers_created")
+    stats["core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_poller_polls")
+    stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_batches")
+    stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_cancel")
+    stats["core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_initial_metadata")
+    stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_message")
+    stats["core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_trailing_metadata")
+    stats["core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_initial_metadata")
+    stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_message")
+    stats["core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_trailing_metadata")
+    stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_settings_writes")
+    stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_pings_sent")
+    stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_begun")
+    stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded")
+    stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued")
+    stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes")
+    stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated")
+    stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items")
+    stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items")
+    stats["core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_offloaded")
+    stats["core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_short_items")
+    stats["core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_long_items")
+    stats["core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_to_self")
+    stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated")
+    stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained")
+    stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries")
+    stats["core_executor_threads_created"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_created")
+    stats["core_executor_threads_used"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_used")
+    stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls")
+    stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued")
+    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size")
+    stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size")
+    stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size")
+    stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer")
+    stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size")
+    stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size")
+    stats["core_http2_send_message_size"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write")
+    stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write")
+    stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write")
+    stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write")
+    stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "executor_closures_per_wakeup")
+    stats["core_executor_closures_per_wakeup"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_executor_closures_per_wakeup_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_executor_closures_per_wakeup_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_executor_closures_per_wakeup_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_executor_closures_per_wakeup_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked")
+    stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
--- a/tools/run_tests/performance/massage_qps_stats_helpers.py
+++ b/tools/run_tests/performance/massage_qps_stats_helpers.py
@ -0,0 +1,57 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+
+def _threshold_for_count_below(buckets, boundaries, count_below):
+  count_so_far = 0
+  for lower_idx in range(0, len(buckets)):
+    count_so_far += buckets[lower_idx]
+    if count_so_far >= count_below:
+      break
+  if count_so_far == count_below:
+    # this bucket hits the threshold exactly... we should be midway through
+    # any run of zero values following the bucket
+    for upper_idx in range(lower_idx + 1, len(buckets)):
+      if buckets[upper_idx] != 0:
+        break
+    return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0
+  else:
+    # treat values as uniform throughout the bucket, and find where this value
+    # should lie
+    lower_bound = boundaries[lower_idx]
+    upper_bound = boundaries[lower_idx + 1]
+    return (upper_bound -
+           (upper_bound - lower_bound) * (count_so_far - count_below) /
+               float(buckets[lower_idx]))
+
+def percentile(buckets, pctl, boundaries):
+  return _threshold_for_count_below(
+      buckets, boundaries, sum(buckets) * pctl / 100.0)
+
+def counter(core_stats, name):
+  for stat in core_stats['metrics']:
+    if stat['name'] == name:
+      return int(stat.get('count', 0))
+
+Histogram = collections.namedtuple('Histogram', 'buckets boundaries')
+def histogram(core_stats, name):
+  for stat in core_stats['metrics']:
+    if stat['name'] == name:
+      buckets = []
+      boundaries = []
+      for b in stat['histogram']['buckets']:
+        buckets.append(int(b.get('count', 0)))
+        boundaries.append(int(b.get('start', 0)))
+  return Histogram(buckets=buckets, boundaries=boundaries)
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@ -453,6 +453,8 @@ class CXXLanguage:
              unconstrained_client=synchronicity,
              secure=secure,
              minimal_stack=not secure,
+              server_threads_per_cq=3,
+              client_threads_per_cq=3,
              categories=smoketest_categories+[SCALABLE])

          # TODO(vjpai): Re-enable this test. It has a lot of timeouts
--- a/tools/run_tests/performance/scenario_result_schema.json
+++ b/tools/run_tests/performance/scenario_result_schema.json
--- a/Show More
+++ b/Show More