Merge branch 'master' of github.com:grpc/grpc into grpclb_v0

.gitignore

@@ -96,3 +96,6 @@ Pods/
 
 # Artifacts directory
 artifacts/
+
+# Git generated files for conflicting
+*.orig

.travis.yml

@@ -8,33 +8,37 @@ env:
     - TEST=objc
     - JOBS=1
   matrix:
-    - SCHEME="RxLibraryUnitTests" WORKSPACE="Tests.xcworkspace"
+    - SCHEME="RxLibraryUnitTests"
-      TEST_PATH="src/objective-c/tests" BUILD_ONLY="false"
+      WORKSPACE="Tests.xcworkspace" TEST_PATH="src/objective-c/tests" BUILD_ONLY="false"
       INTEROP_SERVER="false"
-    - SCHEME="InteropTestsLocalSSL" WORKSPACE="Tests.xcworkspace"
+    - SCHEME="InteropTestsLocalSSL"
-      TEST_PATH="src/objective-c/tests" BUILD_ONLY="false" INTEROP_SERVER="true"
+      WORKSPACE="Tests.xcworkspace" TEST_PATH="src/objective-c/tests" BUILD_ONLY="false"
-    - SCHEME="InteropTestsLocalCleartext" WORKSPACE="Tests.xcworkspace"
+      INTEROP_SERVER="true"
-      TEST_PATH="src/objective-c/tests"  BUILD_ONLY="false"
+    - SCHEME="InteropTestsLocalCleartext"
+      WORKSPACE="Tests.xcworkspace" TEST_PATH="src/objective-c/tests"  BUILD_ONLY="false"
       INTEROP_SERVER="true"
     # TODO(jcanizales): Make tests an app project (instead of library), so the following will work.
-    # - SCHEME="InteropTestsRemote" WORKSPACE="Tests.xcworkspace"
+    # - SCHEME="InteropTestsRemote"
-    #   TEST_PATH="src/objective-c/tests" BUILD_ONLY="false"
+    #   WORKSPACE="Tests.xcworkspace" TEST_PATH="src/objective-c/tests" BUILD_ONLY="false"
     #   INTEROP_SERVER="true"
-    - SCHEME="HelloWorld" WORKSPACE="HelloWorld.xcworkspace"
+    - SCHEME="HelloWorld"
-      TEST_PATH="examples/objective-c/helloworld" BUILD_ONLY="true"
+      WORKSPACE="HelloWorld.xcworkspace" TEST_PATH="examples/objective-c/helloworld"
-      INTEROP_SERVER="false"
+      BUILD_ONLY="true" INTEROP_SERVER="false"
-    - SCHEME="RouteGuideClient" WORKSPACE="RouteGuideClient.xcworkspace"
+    - SCHEME="RouteGuideClient"
-      TEST_PATH="examples/objective-c/route_guide" BUILD_ONLY="true"
+      WORKSPACE="RouteGuideClient.xcworkspace" TEST_PATH="examples/objective-c/route_guide"
-      INTEROP_SERVER="false"
+      BUILD_ONLY="true" INTEROP_SERVER="false"
-    - SCHEME="AuthSample" WORKSPACE="AuthSample.xcworkspace"
+    - SCHEME="AuthSample"
-      TEST_PATH="examples/objective-c/auth_sample" BUILD_ONLY="true"
+      WORKSPACE="AuthSample.xcworkspace" TEST_PATH="examples/objective-c/auth_sample"
-      INTEROP_SERVER="false"
+      BUILD_ONLY="true" INTEROP_SERVER="false"
-    - SCHEME="Sample" WORKSPACE="Sample.xcworkspace"
+    - SCHEME="Sample"
-      TEST_PATH="src/objective-c/examples/Sample" BUILD_ONLY="true"
+      WORKSPACE="Sample.xcworkspace" TEST_PATH="src/objective-c/examples/Sample" BUILD_ONLY="true"
-      INTEROP_SERVER="false"
-    - SCHEME="SwiftSample" WORKSPACE="SwiftSample.xcworkspace"
-      TEST_PATH="src/objective-c/examples/SwiftSample" BUILD_ONLY="true"
       INTEROP_SERVER="false"
+    - SCHEME="Sample"
+      WORKSPACE="Sample.xcworkspace" TEST_PATH="src/objective-c/examples/Sample" BUILD_ONLY="true"
+      INTEROP_SERVER="false" FRAMEWORKS="YES"
+    - SCHEME="SwiftSample"
+      WORKSPACE="SwiftSample.xcworkspace" TEST_PATH="src/objective-c/examples/SwiftSample"
+      BUILD_ONLY="true" INTEROP_SERVER="false"
 before_install:
   # Until Travis upgrades from Cocoapods 0.39, we need to do it here.
   - pod --version

BUILD

@@ -165,6 +165,7 @@ cc_library(
     "src/core/lib/channel/compress_filter.h",
     "src/core/lib/channel/connected_channel.h",
     "src/core/lib/channel/context.h",
+    "src/core/lib/channel/handshaker.h",
     "src/core/lib/channel/http_client_filter.h",
     "src/core/lib/channel/http_server_filter.h",
     "src/core/lib/compression/algorithm_metadata.h",
@@ -317,6 +318,7 @@ cc_library(
     "src/core/lib/channel/channel_stack_builder.c",
     "src/core/lib/channel/compress_filter.c",
     "src/core/lib/channel/connected_channel.c",
+    "src/core/lib/channel/handshaker.c",
     "src/core/lib/channel/http_client_filter.c",
     "src/core/lib/channel/http_server_filter.c",
     "src/core/lib/compression/compression.c",
@@ -554,6 +556,7 @@ cc_library(
     "src/core/lib/channel/compress_filter.h",
     "src/core/lib/channel/connected_channel.h",
     "src/core/lib/channel/context.h",
+    "src/core/lib/channel/handshaker.h",
     "src/core/lib/channel/http_client_filter.h",
     "src/core/lib/channel/http_server_filter.h",
     "src/core/lib/compression/algorithm_metadata.h",
@@ -695,6 +698,7 @@ cc_library(
     "src/core/lib/channel/channel_stack_builder.c",
     "src/core/lib/channel/compress_filter.c",
     "src/core/lib/channel/connected_channel.c",
+    "src/core/lib/channel/handshaker.c",
     "src/core/lib/channel/http_client_filter.c",
     "src/core/lib/channel/http_server_filter.c",
     "src/core/lib/compression/compression.c",
@@ -906,6 +910,7 @@ cc_library(
     "src/core/lib/channel/compress_filter.h",
     "src/core/lib/channel/connected_channel.h",
     "src/core/lib/channel/context.h",
+    "src/core/lib/channel/handshaker.h",
     "src/core/lib/channel/http_client_filter.h",
     "src/core/lib/channel/http_server_filter.h",
     "src/core/lib/compression/algorithm_metadata.h",
@@ -1035,6 +1040,7 @@ cc_library(
     "src/core/lib/channel/channel_stack_builder.c",
     "src/core/lib/channel/compress_filter.c",
     "src/core/lib/channel/connected_channel.c",
+    "src/core/lib/channel/handshaker.c",
     "src/core/lib/channel/http_client_filter.c",
     "src/core/lib/channel/http_server_filter.c",
     "src/core/lib/compression/compression.c",
@@ -1795,6 +1801,7 @@ objc_library(
     "src/core/lib/channel/channel_stack_builder.c",
     "src/core/lib/channel/compress_filter.c",
     "src/core/lib/channel/connected_channel.c",
+    "src/core/lib/channel/handshaker.c",
     "src/core/lib/channel/http_client_filter.c",
     "src/core/lib/channel/http_server_filter.c",
     "src/core/lib/compression/compression.c",
@@ -2011,6 +2018,7 @@ objc_library(
     "src/core/lib/channel/compress_filter.h",
     "src/core/lib/channel/connected_channel.h",
     "src/core/lib/channel/context.h",
+    "src/core/lib/channel/handshaker.h",
     "src/core/lib/channel/http_client_filter.h",
     "src/core/lib/channel/http_server_filter.h",
     "src/core/lib/compression/algorithm_metadata.h",

Makefile

@@ -405,6 +405,29 @@ LIBS = m pthread ws2_32
 LDFLAGS += -pthread
 endif
 
+#
+# The steps for cross-compiling are as follows:
+# First, clone and make install of grpc using the native compilers for the host.
+# Also, install protoc (e.g., from a package like apt-get)
+# Then clone a fresh grpc for the actual cross-compiled build
+# Set the environment variable GRPC_CROSS_COMPILE to true
+# Set CC, CXX, LD, LDXX, AR, and STRIP to the cross-compiling binaries
+# Also set PROTOBUF_CONFIG_OPTS to indicate cross-compilation to protobuf (e.g.,
+#  PROTOBUF_CONFIG_OPTS="--host=arm-linux --with-protoc=/usr/local/bin/protoc" )
+# Set HAS_PKG_CONFIG=false
+# To build tests, go to third_party/gflags and follow its ccmake instructions
+# Make sure that you enable building shared libraries and set your prefix to
+# something useful like /usr/local/cross
+# You will also need to set GRPC_CROSS_LDOPTS and GRPC_CROSS_AROPTS to hold
+# additional required arguments for LD and AR (examples below)
+# Then you can do a make from the cross-compiling fresh clone!
+#
+ifeq ($(GRPC_CROSS_COMPILE),true)
+LDFLAGS += $(GRPC_CROSS_LDOPTS) # e.g. -L/usr/local/lib -L/usr/local/cross/lib
+AROPTS = $(GRPC_CROSS_AROPTS) # e.g., rc --target=elf32-little
+USE_BUILT_PROTOC = false
+endif
+
 GTEST_LIB = -Ithird_party/googletest/include -Ithird_party/googletest third_party/googletest/src/gtest-all.cc
 GTEST_LIB += -lgflags
 ifeq ($(V),1)
@@ -709,6 +732,9 @@ PC_LIBS_GRPCXX =
 
 CPPFLAGS := -Ithird_party/googletest/include $(CPPFLAGS)
 
+PROTOC_PLUGINS_ALL = $(BINDIR)/$(CONFIG)/grpc_cpp_plugin $(BINDIR)/$(CONFIG)/grpc_csharp_plugin $(BINDIR)/$(CONFIG)/grpc_node_plugin $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin $(BINDIR)/$(CONFIG)/grpc_python_plugin $(BINDIR)/$(CONFIG)/grpc_ruby_plugin
+PROTOC_PLUGINS_DIR = $(BINDIR)/$(CONFIG)
+
 ifeq ($(HAS_SYSTEM_PROTOBUF),true)
 ifeq ($(HAS_PKG_CONFIG),true)
 PROTOBUF_PKG_CONFIG = true
@@ -723,12 +749,19 @@ endif
 else
 PC_LIBS_GRPCXX = -lprotobuf
 endif
+PROTOC_PLUGINS = $(PROTOC_PLUGINS_ALL)
 else
 ifeq ($(HAS_EMBEDDED_PROTOBUF),true)
 PROTOBUF_DEP = $(LIBDIR)/$(CONFIG)/protobuf/libprotobuf.a
 CPPFLAGS := -Ithird_party/protobuf/src $(CPPFLAGS)
 LDFLAGS := -L$(LIBDIR)/$(CONFIG)/protobuf $(LDFLAGS)
+ifneq ($(USE_BUILT_PROTOC),false)
 PROTOC = $(BINDIR)/$(CONFIG)/protobuf/protoc
+PROTOC_PLUGINS = $(PROTOC_PLUGINS_ALL)
+else
+PROTOC_PLUGINS =
+PROTOC_PLUGINS_DIR = $(prefix)/bin
+endif
 else
 NO_PROTOBUF = true
 endif
@@ -776,7 +809,6 @@ endif
 
 .SECONDARY = %.pb.h %.pb.cc
 
-PROTOC_PLUGINS = $(BINDIR)/$(CONFIG)/grpc_cpp_plugin $(BINDIR)/$(CONFIG)/grpc_csharp_plugin $(BINDIR)/$(CONFIG)/grpc_node_plugin $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin $(BINDIR)/$(CONFIG)/grpc_python_plugin $(BINDIR)/$(CONFIG)/grpc_ruby_plugin
 ifeq ($(DEP_MISSING),)
 all: static shared plugins
 dep_error:
@@ -991,7 +1023,6 @@ transport_security_test: $(BINDIR)/$(CONFIG)/transport_security_test
 udp_server_test: $(BINDIR)/$(CONFIG)/udp_server_test
 uri_fuzzer_test: $(BINDIR)/$(CONFIG)/uri_fuzzer_test
 uri_parser_test: $(BINDIR)/$(CONFIG)/uri_parser_test
-workqueue_test: $(BINDIR)/$(CONFIG)/workqueue_test
 alarm_cpp_test: $(BINDIR)/$(CONFIG)/alarm_cpp_test
 async_end2end_test: $(BINDIR)/$(CONFIG)/async_end2end_test
 auth_property_iterator_test: $(BINDIR)/$(CONFIG)/auth_property_iterator_test
@@ -1149,7 +1180,7 @@ third_party/protobuf/configure:
 
 $(LIBDIR)/$(CONFIG)/protobuf/libprotobuf.a: third_party/protobuf/configure
 	$(E) "[MAKE]    Building protobuf"
-	$(Q)(cd third_party/protobuf ; CC="$(CC)" CXX="$(CXX)" LDFLAGS="$(LDFLAGS_$(CONFIG)) -g $(PROTOBUF_LDFLAGS_EXTRA)" CPPFLAGS="$(PIC_CPPFLAGS) $(CPPFLAGS_$(CONFIG)) -g $(PROTOBUF_CPPFLAGS_EXTRA)" ./configure --disable-shared --enable-static)
+	$(Q)(cd third_party/protobuf ; CC="$(CC)" CXX="$(CXX)" LDFLAGS="$(LDFLAGS_$(CONFIG)) -g $(PROTOBUF_LDFLAGS_EXTRA)" CPPFLAGS="$(PIC_CPPFLAGS) $(CPPFLAGS_$(CONFIG)) -g $(PROTOBUF_CPPFLAGS_EXTRA)" ./configure --disable-shared --enable-static $(PROTOBUF_CONFIG_OPTS))
 	$(Q)$(MAKE) -C third_party/protobuf clean
 	$(Q)$(MAKE) -C third_party/protobuf
 	$(Q)mkdir -p $(LIBDIR)/$(CONFIG)/protobuf
@@ -1186,9 +1217,9 @@ pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc
 pc_cxx_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc
 
 ifeq ($(EMBED_OPENSSL),true)
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_reflection_codegen.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a
 else
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_reflection_codegen.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a
 endif
 
 
@@ -1296,7 +1327,6 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/transport_security_test \
   $(BINDIR)/$(CONFIG)/udp_server_test \
   $(BINDIR)/$(CONFIG)/uri_parser_test \
-  $(BINDIR)/$(CONFIG)/workqueue_test \
   $(BINDIR)/$(CONFIG)/public_headers_must_be_c89 \
   $(BINDIR)/$(CONFIG)/badreq_bad_client_test \
   $(BINDIR)/$(CONFIG)/connection_prefix_bad_client_test \
@@ -1677,8 +1707,6 @@ test_c: buildtests_c
 	$(Q) $(BINDIR)/$(CONFIG)/udp_server_test || ( echo test udp_server_test failed ; exit 1 )
 	$(E) "[RUN]     Testing uri_parser_test"
 	$(Q) $(BINDIR)/$(CONFIG)/uri_parser_test || ( echo test uri_parser_test failed ; exit 1 )
-	$(E) "[RUN]     Testing workqueue_test"
-	$(Q) $(BINDIR)/$(CONFIG)/workqueue_test || ( echo test workqueue_test failed ; exit 1 )
 	$(E) "[RUN]     Testing public_headers_must_be_c89"
 	$(Q) $(BINDIR)/$(CONFIG)/public_headers_must_be_c89 || ( echo test public_headers_must_be_c89 failed ; exit 1 )
 	$(E) "[RUN]     Testing badreq_bad_client_test"
@@ -1895,7 +1923,22 @@ $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.pb.cc: src/proto/grpc/lb/v1/load_ba
 $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc: src/proto/grpc/lb/v1/load_balancer.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
+endif
+
+ifeq ($(NO_PROTOC),true)
+$(GENDIR)/src/proto/grpc/reflection/v1alpha/reflection.pb.cc: protoc_dep_error
+$(GENDIR)/src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.cc: protoc_dep_error
+else
+$(GENDIR)/src/proto/grpc/reflection/v1alpha/reflection.pb.cc: src/proto/grpc/reflection/v1alpha/reflection.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
+	$(E) "[PROTOC]  Generating protobuf CC file from $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --cpp_out=$(GENDIR) $<
+
+$(GENDIR)/src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.cc: src/proto/grpc/reflection/v1alpha/reflection.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
+	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -1910,7 +1953,7 @@ $(GENDIR)/src/proto/grpc/testing/compiler_test.pb.cc: src/proto/grpc/testing/com
 $(GENDIR)/src/proto/grpc/testing/compiler_test.grpc.pb.cc: src/proto/grpc/testing/compiler_test.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -1925,7 +1968,7 @@ $(GENDIR)/src/proto/grpc/testing/control.pb.cc: src/proto/grpc/testing/control.p
 $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc: src/proto/grpc/testing/control.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -1940,7 +1983,7 @@ $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.pb.cc: src/proto/grpc/
 $(GENDIR)/src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.cc: src/proto/grpc/testing/duplicate/echo_duplicate.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -1955,7 +1998,7 @@ $(GENDIR)/src/proto/grpc/testing/echo.pb.cc: src/proto/grpc/testing/echo.proto $
 $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc: src/proto/grpc/testing/echo.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -1970,7 +2013,7 @@ $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc: src/proto/grpc/testing/ech
 $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc: src/proto/grpc/testing/echo_messages.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -1985,7 +2028,7 @@ $(GENDIR)/src/proto/grpc/testing/empty.pb.cc: src/proto/grpc/testing/empty.proto
 $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc: src/proto/grpc/testing/empty.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -2000,7 +2043,7 @@ $(GENDIR)/src/proto/grpc/testing/messages.pb.cc: src/proto/grpc/testing/messages
 $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc: src/proto/grpc/testing/messages.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -2015,7 +2058,7 @@ $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc: src/proto/grpc/testing/metrics.p
 $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc: src/proto/grpc/testing/metrics.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -2030,7 +2073,7 @@ $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc: src/proto/grpc/testing/payloads
 $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc: src/proto/grpc/testing/payloads.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -2045,7 +2088,7 @@ $(GENDIR)/src/proto/grpc/testing/services.pb.cc: src/proto/grpc/testing/services
 $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc: src/proto/grpc/testing/services.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -2060,7 +2103,7 @@ $(GENDIR)/src/proto/grpc/testing/stats.pb.cc: src/proto/grpc/testing/stats.proto
 $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc: src/proto/grpc/testing/stats.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) 
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 ifeq ($(NO_PROTOC),true)
@@ -2075,7 +2118,7 @@ $(GENDIR)/src/proto/grpc/testing/test.pb.cc: src/proto/grpc/testing/test.proto $
 $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc: src/proto/grpc/testing/test.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc
 	$(E) "[GRPC]    Generating gRPC's protobuf service CC file from $<"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $<
+	$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin $<
 endif
 
 
@@ -2422,7 +2465,7 @@ $(LIBDIR)/$(CONFIG)/libgpr.a: $(ZLIB_DEP)  $(LIBGPR_OBJS)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBGPR_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBGPR_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgpr.a
 endif
@@ -2464,7 +2507,7 @@ $(LIBDIR)/$(CONFIG)/libgpr_test_util.a: $(ZLIB_DEP)  $(LIBGPR_TEST_UTIL_OBJS)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgpr_test_util.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBGPR_TEST_UTIL_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBGPR_TEST_UTIL_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgpr_test_util.a
 endif
@@ -2484,6 +2527,7 @@ LIBGRPC_SRC = \
     src/core/lib/channel/channel_stack_builder.c \
     src/core/lib/channel/compress_filter.c \
     src/core/lib/channel/connected_channel.c \
+    src/core/lib/channel/handshaker.c \
     src/core/lib/channel/http_client_filter.c \
     src/core/lib/channel/http_server_filter.c \
     src/core/lib/compression/compression.c \
@@ -2716,7 +2760,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBGRPC_OBJS)  $(LIB
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBGRPC_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS)  $(OPENSSL_MERGE_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBGRPC_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS)  $(OPENSSL_MERGE_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc.a
 endif
@@ -2757,6 +2801,7 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/channel/channel_stack_builder.c \
     src/core/lib/channel/compress_filter.c \
     src/core/lib/channel/connected_channel.c \
+    src/core/lib/channel/handshaker.c \
     src/core/lib/channel/http_client_filter.c \
     src/core/lib/channel/http_server_filter.c \
     src/core/lib/compression/compression.c \
@@ -2965,7 +3010,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_cronet.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBGRPC_CRONE
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_cronet.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc_cronet.a $(LIBGRPC_CRONET_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS)  $(OPENSSL_MERGE_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_cronet.a $(LIBGRPC_CRONET_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS)  $(OPENSSL_MERGE_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_cronet.a
 endif
@@ -3151,7 +3196,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBGRPC_TE
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBGRPC_TEST_UTIL_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBGRPC_TEST_UTIL_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a
 endif
@@ -3191,7 +3236,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a: $(ZLIB_DEP)  $(LIBGRPC_TEST_UT
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBGRPC_TEST_UTIL_UNSECURE_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBGRPC_TEST_UTIL_UNSECURE_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a
 endif
@@ -3212,6 +3257,7 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/lib/channel/channel_stack_builder.c \
     src/core/lib/channel/compress_filter.c \
     src/core/lib/channel/connected_channel.c \
+    src/core/lib/channel/handshaker.c \
     src/core/lib/channel/http_client_filter.c \
     src/core/lib/channel/http_server_filter.c \
     src/core/lib/compression/compression.c \
@@ -3401,7 +3447,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a: $(ZLIB_DEP)  $(LIBGRPC_UNSECURE_OBJS)  $
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBGRPC_UNSECURE_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBGRPC_UNSECURE_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a
 endif
@@ -3453,7 +3499,7 @@ $(LIBDIR)/$(CONFIG)/libreconnect_server.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBRECON
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libreconnect_server.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libreconnect_server.a $(LIBRECONNECT_SERVER_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libreconnect_server.a $(LIBRECONNECT_SERVER_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libreconnect_server.a
 endif
@@ -3492,7 +3538,7 @@ $(LIBDIR)/$(CONFIG)/libtest_tcp_server.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBTEST_T
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libtest_tcp_server.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libtest_tcp_server.a $(LIBTEST_TCP_SERVER_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libtest_tcp_server.a $(LIBTEST_TCP_SERVER_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libtest_tcp_server.a
 endif
@@ -3671,7 +3717,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc++.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LI
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBGRPC++_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBGRPC++_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc++.a
 endif
@@ -3798,7 +3844,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBU
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBGRPC++_REFLECTION_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBGRPC++_REFLECTION_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a
 endif
@@ -3834,6 +3880,55 @@ endif
 endif
 
 
+LIBGRPC++_REFLECTION_CODEGEN_SRC = \
+    $(GENDIR)/src/proto/grpc/reflection/v1alpha/reflection.pb.cc $(GENDIR)/src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.cc \
+
+PUBLIC_HEADERS_CXX += \
+
+LIBGRPC++_REFLECTION_CODEGEN_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_REFLECTION_CODEGEN_SRC))))
+
+
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure libraries if you don't have OpenSSL.
+
+$(LIBDIR)/$(CONFIG)/libgrpc++_reflection_codegen.a: openssl_dep_error
+
+
+else
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay.
+
+$(LIBDIR)/$(CONFIG)/libgrpc++_reflection_codegen.a: protobuf_dep_error
+
+
+else
+
+$(LIBDIR)/$(CONFIG)/libgrpc++_reflection_codegen.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_REFLECTION_CODEGEN_OBJS) 
+	$(E) "[AR]      Creating $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_reflection_codegen.a
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc++_reflection_codegen.a $(LIBGRPC++_REFLECTION_CODEGEN_OBJS) 
+ifeq ($(SYSTEM),Darwin)
+	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc++_reflection_codegen.a
+endif
+
+
+
+
+endif
+
+endif
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(LIBGRPC++_REFLECTION_CODEGEN_OBJS:.o=.dep)
+endif
+endif
+
+
 LIBGRPC++_TEST_CONFIG_SRC = \
     test/cpp/util/test_config.cc \
 
@@ -3864,7 +3959,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOB
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBGRPC++_TEST_CONFIG_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBGRPC++_TEST_CONFIG_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 endif
@@ -3975,7 +4070,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBGRPC++_TEST_UTIL_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBGRPC++_TEST_UTIL_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a
 endif
@@ -4148,7 +4243,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(LIBGRPC
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBGRPC++_UNSECURE_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBGRPC++_UNSECURE_OBJS)  $(LIBGPR_OBJS)  $(ZLIB_MERGE_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a
 endif
@@ -4183,6 +4278,7 @@ endif
 LIBGRPC_CLI_LIBS_SRC = \
     test/cpp/util/cli_call.cc \
     test/cpp/util/proto_file_parser.cc \
+    test/cpp/util/proto_reflection_descriptor_database.cc \
 
 PUBLIC_HEADERS_CXX += \
 
@@ -4211,7 +4307,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DE
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBGRPC_CLI_LIBS_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBGRPC_CLI_LIBS_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a
 endif
@@ -4257,7 +4353,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_plugin_support.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(LIB
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_plugin_support.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc_plugin_support.a $(LIBGRPC_PLUGIN_SUPPORT_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_plugin_support.a $(LIBGRPC_PLUGIN_SUPPORT_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_plugin_support.a
 endif
@@ -4303,7 +4399,7 @@ $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PRO
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBINTEROP_CLIENT_HELPER_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBINTEROP_CLIENT_HELPER_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a
 endif
@@ -4357,7 +4453,7 @@ $(LIBDIR)/$(CONFIG)/libinterop_client_main.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTO
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libinterop_client_main.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBINTEROP_CLIENT_MAIN_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBINTEROP_CLIENT_MAIN_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libinterop_client_main.a
 endif
@@ -4408,7 +4504,7 @@ $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PRO
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBINTEROP_SERVER_HELPER_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBINTEROP_SERVER_HELPER_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a
 endif
@@ -4460,7 +4556,7 @@ $(LIBDIR)/$(CONFIG)/libinterop_server_main.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTO
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libinterop_server_main.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBINTEROP_SERVER_MAIN_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBINTEROP_SERVER_MAIN_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libinterop_server_main.a
 endif
@@ -4525,7 +4621,7 @@ $(LIBDIR)/$(CONFIG)/libqps.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBQP
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libqps.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBQPS_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBQPS_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libqps.a
 endif
@@ -4578,7 +4674,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBGRPC_C
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a $(LIBGRPC_CSHARP_EXT_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a $(LIBGRPC_CSHARP_EXT_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a
 endif
@@ -4922,7 +5018,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl.a: $(ZLIB_DEP)  $(LIBBORINGSSL_OBJS)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl.a $(LIBBORINGSSL_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl.a $(LIBBORINGSSL_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl.a
 endif
@@ -4960,7 +5056,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(LIB
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBBORINGSSL_TEST_UTIL_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBBORINGSSL_TEST_UTIL_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a
 endif
@@ -4998,7 +5094,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBBORINGSSL_AES_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBBORINGSSL_AES_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a
 endif
@@ -5036,7 +5132,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBBORINGSSL_ASN1_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBBORINGSSL_ASN1_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a
 endif
@@ -5074,7 +5170,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBBORINGSSL_BASE64_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBBORINGSSL_BASE64_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a
 endif
@@ -5112,7 +5208,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBBORINGSSL_BIO_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBBORINGSSL_BIO_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a
 endif
@@ -5150,7 +5246,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(L
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBBORINGSSL_BN_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBBORINGSSL_BN_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a
 endif
@@ -5188,7 +5284,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBBORINGSSL_BYTESTRING_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBBORINGSSL_BYTESTRING_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a
 endif
@@ -5226,7 +5322,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBBORINGSSL_AEAD_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBBORINGSSL_AEAD_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a
 endif
@@ -5264,7 +5360,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBBORINGSSL_CIPHER_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBBORINGSSL_CIPHER_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a
 endif
@@ -5302,7 +5398,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBBORINGSSL_CMAC_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBBORINGSSL_CMAC_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a
 endif
@@ -5331,7 +5427,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_constant_time_test_lib.a: $(ZLIB_DEP)  $(LIBBOR
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_constant_time_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_constant_time_test_lib.a $(LIBBORINGSSL_CONSTANT_TIME_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_constant_time_test_lib.a $(LIBBORINGSSL_CONSTANT_TIME_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_constant_time_test_lib.a
 endif
@@ -5367,7 +5463,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBBORINGSSL_ED25519_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBBORINGSSL_ED25519_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a
 endif
@@ -5405,7 +5501,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBBORINGSSL_X25519_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBBORINGSSL_X25519_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a
 endif
@@ -5443,7 +5539,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(L
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBBORINGSSL_DH_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBBORINGSSL_DH_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a
 endif
@@ -5481,7 +5577,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBBORINGSSL_DIGEST_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBBORINGSSL_DIGEST_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a
 endif
@@ -5510,7 +5606,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_dsa_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_DSA
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_dsa_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_dsa_test_lib.a $(LIBBORINGSSL_DSA_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_dsa_test_lib.a $(LIBBORINGSSL_DSA_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_dsa_test_lib.a
 endif
@@ -5546,7 +5642,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(L
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBBORINGSSL_EC_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBBORINGSSL_EC_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a
 endif
@@ -5575,7 +5671,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_example_mul_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_example_mul_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_example_mul_lib.a $(LIBBORINGSSL_EXAMPLE_MUL_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_example_mul_lib.a $(LIBBORINGSSL_EXAMPLE_MUL_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_example_mul_lib.a
 endif
@@ -5611,7 +5707,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBBORINGSSL_ECDSA_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBBORINGSSL_ECDSA_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a
 endif
@@ -5649,7 +5745,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBBORINGSSL_ERR_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBBORINGSSL_ERR_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a
 endif
@@ -5687,7 +5783,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_D
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBBORINGSSL_EVP_EXTRA_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBBORINGSSL_EVP_EXTRA_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a
 endif
@@ -5725,7 +5821,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBBORINGSSL_EVP_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBBORINGSSL_EVP_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a
 endif
@@ -5763,7 +5859,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBBORINGSSL_PBKDF_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBBORINGSSL_PBKDF_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a
 endif
@@ -5792,7 +5888,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_hkdf_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_HK
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_hkdf_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_hkdf_test_lib.a $(LIBBORINGSSL_HKDF_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_hkdf_test_lib.a $(LIBBORINGSSL_HKDF_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_hkdf_test_lib.a
 endif
@@ -5828,7 +5924,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBBORINGSSL_HMAC_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBBORINGSSL_HMAC_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a
 endif
@@ -5857,7 +5953,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_lhash_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_L
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_lhash_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_lhash_test_lib.a $(LIBBORINGSSL_LHASH_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_lhash_test_lib.a $(LIBBORINGSSL_LHASH_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_lhash_test_lib.a
 endif
@@ -5884,7 +5980,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_gcm_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_GCM
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_gcm_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_gcm_test_lib.a $(LIBBORINGSSL_GCM_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_gcm_test_lib.a $(LIBBORINGSSL_GCM_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_gcm_test_lib.a
 endif
@@ -5920,7 +6016,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBBORINGSSL_PKCS12_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBBORINGSSL_PKCS12_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a
 endif
@@ -5958,7 +6054,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBBORINGSSL_PKCS8_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBBORINGSSL_PKCS8_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a
 endif
@@ -5996,7 +6092,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DE
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBBORINGSSL_POLY1305_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBBORINGSSL_POLY1305_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a
 endif
@@ -6025,7 +6121,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_refcount_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSS
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_refcount_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_refcount_test_lib.a $(LIBBORINGSSL_REFCOUNT_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_refcount_test_lib.a $(LIBBORINGSSL_REFCOUNT_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_refcount_test_lib.a
 endif
@@ -6061,7 +6157,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBBORINGSSL_RSA_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBBORINGSSL_RSA_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a
 endif
@@ -6090,7 +6186,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_thread_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_thread_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_thread_test_lib.a $(LIBBORINGSSL_THREAD_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_thread_test_lib.a $(LIBBORINGSSL_THREAD_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_thread_test_lib.a
 endif
@@ -6117,7 +6213,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_pkcs7_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_P
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_pkcs7_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_pkcs7_test_lib.a $(LIBBORINGSSL_PKCS7_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_pkcs7_test_lib.a $(LIBBORINGSSL_PKCS7_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_pkcs7_test_lib.a
 endif
@@ -6153,7 +6249,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBBORINGSSL_X509_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBBORINGSSL_X509_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a
 endif
@@ -6182,7 +6278,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_tab_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_TAB
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_tab_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_tab_test_lib.a $(LIBBORINGSSL_TAB_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_tab_test_lib.a $(LIBBORINGSSL_TAB_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_tab_test_lib.a
 endif
@@ -6209,7 +6305,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_v3name_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_v3name_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_v3name_test_lib.a $(LIBBORINGSSL_V3NAME_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_v3name_test_lib.a $(LIBBORINGSSL_V3NAME_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_v3name_test_lib.a
 endif
@@ -6236,7 +6332,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_pqueue_test_lib.a: $(ZLIB_DEP)  $(LIBBORINGSSL_
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_pqueue_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_pqueue_test_lib.a $(LIBBORINGSSL_PQUEUE_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_pqueue_test_lib.a $(LIBBORINGSSL_PQUEUE_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_pqueue_test_lib.a
 endif
@@ -6272,7 +6368,7 @@ $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a $(LIBBORINGSSL_SSL_TEST_LIB_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a $(LIBBORINGSSL_SSL_TEST_LIB_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a
 endif
@@ -6314,7 +6410,7 @@ $(LIBDIR)/$(CONFIG)/libz.a:  $(LIBZ_OBJS)
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libz.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libz.a $(LIBZ_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libz.a $(LIBZ_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libz.a
 endif
@@ -6349,7 +6445,7 @@ $(LIBDIR)/$(CONFIG)/libbad_client_test.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBBAD_CL
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libbad_client_test.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libbad_client_test.a $(LIBBAD_CLIENT_TEST_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libbad_client_test.a $(LIBBAD_CLIENT_TEST_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libbad_client_test.a
 endif
@@ -6388,7 +6484,7 @@ $(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBBA
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a $(LIBBAD_SSL_TEST_SERVER_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a $(LIBBAD_SSL_TEST_SERVER_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a
 endif
@@ -6467,7 +6563,7 @@ $(LIBDIR)/$(CONFIG)/libend2end_tests.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBEND2END_
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libend2end_tests.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libend2end_tests.a $(LIBEND2END_TESTS_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libend2end_tests.a $(LIBEND2END_TESTS_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libend2end_tests.a
 endif
@@ -6535,7 +6631,7 @@ $(LIBDIR)/$(CONFIG)/libend2end_nosec_tests.a: $(ZLIB_DEP)  $(LIBEND2END_NOSEC_TE
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
 	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libend2end_nosec_tests.a
-	$(Q) $(AR) $(LIBDIR)/$(CONFIG)/libend2end_nosec_tests.a $(LIBEND2END_NOSEC_TESTS_OBJS) 
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libend2end_nosec_tests.a $(LIBEND2END_NOSEC_TESTS_OBJS) 
 ifeq ($(SYSTEM),Darwin)
 	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libend2end_nosec_tests.a
 endif
@@ -10296,38 +10392,6 @@ endif
 endif
 
 
-WORKQUEUE_TEST_SRC = \
-    test/core/iomgr/workqueue_test.c \
-
-WORKQUEUE_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(WORKQUEUE_TEST_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/workqueue_test: openssl_dep_error
-
-else
-
-
-
-$(BINDIR)/$(CONFIG)/workqueue_test: $(WORKQUEUE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LD) $(LDFLAGS) $(WORKQUEUE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/workqueue_test
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/core/iomgr/workqueue_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-deps_workqueue_test: $(WORKQUEUE_TEST_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(WORKQUEUE_TEST_OBJS:.o=.dep)
-endif
-endif
-
-
 ALARM_CPP_TEST_SRC = \
     test/cpp/common/alarm_cpp_test.cc \
 
@@ -11122,16 +11186,16 @@ $(BINDIR)/$(CONFIG)/grpc_cli: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli
+	$(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o:  $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o:  $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_grpc_cli: $(GRPC_CLI_OBJS:.o=.dep)
 
@@ -15092,6 +15156,7 @@ test/cpp/util/byte_buffer_proto_helper.cc: $(OPENSSL_DEP)
 test/cpp/util/cli_call.cc: $(OPENSSL_DEP)
 test/cpp/util/create_test_channel.cc: $(OPENSSL_DEP)
 test/cpp/util/proto_file_parser.cc: $(OPENSSL_DEP)
+test/cpp/util/proto_reflection_descriptor_database.cc: $(OPENSSL_DEP)
 test/cpp/util/string_ref_helper.cc: $(OPENSSL_DEP)
 test/cpp/util/subprocess.cc: $(OPENSSL_DEP)
 test/cpp/util/test_config.cc: $(OPENSSL_DEP)

PYTHON-MANIFEST.in

@@ -7,7 +7,7 @@ graft include/grpc
 graft third_party/boringssl
 graft third_party/nanopb
 graft third_party/zlib
-include src/python/grpcio/build.py
+include src/python/grpcio/_unixccompiler_patch.py
 include src/python/grpcio/commands.py
 include src/python/grpcio/grpc_version.py
 include src/python/grpcio/grpc_core_dependencies.py

Rakefile

@@ -83,7 +83,7 @@ task 'dlls' do
   env += 'EMBED_ZLIB=true '
   env += 'BUILDDIR=/tmp '
   env += "V=#{verbose} "
-  out = '/tmp/libs/opt/grpc-0.dll'
+  out = '/tmp/libs/opt/grpc-1.dll'
 
   w64 = { cross: 'x86_64-w64-mingw32', out: 'grpc_c.64.ruby' }
   w32 = { cross: 'i686-w64-mingw32', out: 'grpc_c.32.ruby' }

binding.gyp

@@ -568,6 +568,7 @@
         'src/core/lib/channel/channel_stack_builder.c',
         'src/core/lib/channel/compress_filter.c',
         'src/core/lib/channel/connected_channel.c',
+        'src/core/lib/channel/handshaker.c',
         'src/core/lib/channel/http_client_filter.c',
         'src/core/lib/channel/http_server_filter.c',
         'src/core/lib/compression/compression.c',

build.yaml

@@ -156,6 +156,7 @@ filegroups:
   - src/core/lib/channel/compress_filter.h
   - src/core/lib/channel/connected_channel.h
   - src/core/lib/channel/context.h
+  - src/core/lib/channel/handshaker.h
   - src/core/lib/channel/http_client_filter.h
   - src/core/lib/channel/http_server_filter.h
   - src/core/lib/compression/algorithm_metadata.h
@@ -234,6 +235,7 @@ filegroups:
   - src/core/lib/channel/channel_stack_builder.c
   - src/core/lib/channel/compress_filter.c
   - src/core/lib/channel/connected_channel.c
+  - src/core/lib/channel/handshaker.c
   - src/core/lib/channel/http_client_filter.c
   - src/core/lib/channel/http_server_filter.c
   - src/core/lib/compression/compression.c
@@ -771,6 +773,16 @@ filegroups:
   language: c++
   public_headers:
   - include/grpc++/impl/codegen/config_protobuf.h
+- name: grpc++_reflection_proto
+  language: c++
+  public_headers:
+  - include/grpc++/ext/reflection.grpc.pb.h
+  - include/grpc++/ext/reflection.pb.h
+  src:
+  - src/cpp/ext/reflection.grpc.pb.cc
+  - src/cpp/ext/reflection.pb.cc
+  uses:
+  - grpc++_codegen_proto
 libs:
 - name: gpr
   build: all
@@ -966,19 +978,20 @@ libs:
   language: c++
   public_headers:
   - include/grpc++/ext/proto_server_reflection_plugin.h
-  - include/grpc++/ext/reflection.grpc.pb.h
-  - include/grpc++/ext/reflection.pb.h
   headers:
   - src/cpp/ext/proto_server_reflection.h
   src:
   - src/cpp/ext/proto_server_reflection.cc
   - src/cpp/ext/proto_server_reflection_plugin.cc
-  - src/cpp/ext/reflection.grpc.pb.cc
-  - src/cpp/ext/reflection.pb.cc
   deps:
   - grpc++
   filegroups:
-  - grpc++_codegen_proto
+  - grpc++_reflection_proto
+- name: grpc++_reflection_codegen
+  build: private
+  language: c++
+  src:
+  - src/proto/grpc/reflection/v1alpha/reflection.proto
 - name: grpc++_test_config
   build: private
   language: c++
@@ -1036,10 +1049,13 @@ libs:
   headers:
   - test/cpp/util/cli_call.h
   - test/cpp/util/proto_file_parser.h
+  - test/cpp/util/proto_reflection_descriptor_database.h
   src:
   - test/cpp/util/cli_call.cc
   - test/cpp/util/proto_file_parser.cc
+  - test/cpp/util/proto_reflection_descriptor_database.cc
   deps:
+  - grpc++_reflection
   - grpc++
   - grpc_plugin_support
 - name: grpc_plugin_support
@@ -2436,20 +2452,6 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
-- name: workqueue_test
-  build: test
-  language: c
-  src:
-  - test/core/iomgr/workqueue_test.c
-  deps:
-  - grpc_test_util
-  - grpc
-  - gpr_test_util
-  - gpr
-  platforms:
-  - mac
-  - linux
-  - posix
 - name: alarm_cpp_test
   gtest: true
   build: test
@@ -2677,6 +2679,7 @@ targets:
   - grpc_cli_libs
   - grpc++_test_util
   - grpc_test_util
+  - grpc++_reflection
   - grpc++
   - grpc
   - gpr_test_util
@@ -3387,6 +3390,7 @@ php_config_m4:
   - src/php/ext/grpc/channel.h
   - src/php/ext/grpc/channel_credentials.h
   - src/php/ext/grpc/completion_queue.h
+  - src/php/ext/grpc/php7_wrapper.h
   - src/php/ext/grpc/php_grpc.h
   - src/php/ext/grpc/server.h
   - src/php/ext/grpc/server_credentials.h

composer.json

@@ -7,7 +7,7 @@
   "license": "BSD-3-Clause",
   "require": {
     "php": ">=5.5.0",
-    "stanley-cheung/protobuf-php": "dev-master"
+    "stanley-cheung/protobuf-php": "v0.6"
   },
   "require-dev": {
     "google/auth": "v0.9"

config.m4

@@ -87,6 +87,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/channel/channel_stack_builder.c \
     src/core/lib/channel/compress_filter.c \
     src/core/lib/channel/connected_channel.c \
+    src/core/lib/channel/handshaker.c \
     src/core/lib/channel/http_client_filter.c \
     src/core/lib/channel/http_server_filter.c \
     src/core/lib/compression/compression.c \

doc/.gitignore

@@ -0,0 +1,2 @@
+build/
+src/

doc/c-style-guide.md

@@ -9,16 +9,17 @@ Here we document style rules for C usage in the gRPC Core library.
 General
 -------
 
-- Layout rules are defined by clang-format, and all code should be passed through
+- Layout rules are defined by clang-format, and all code should be passed
-  clang-format. A (docker-based) script to do so is included in 
+  through clang-format. A (docker-based) script to do so is included in
-  [tools/distrib/clang\_format\_code.sh] (../tools/distrib/clang_format_code.sh).
+  [tools/distrib/clang\_format\_code.sh](../tools/distrib/clang_format_code.sh).
 
 Header Files
 ------------
 
-- Public header files (those in the include/grpc tree) should compile as pedantic C89
+- Public header files (those in the include/grpc tree) should compile as
-- Public header files should be includable from C++ programs. That is, they should 
+  pedantic C89.
-  include the following:
+- Public header files should be includable from C++ programs. That is, they
+  should include the following:
   ```c
   #ifdef __cplusplus
   extern "C" {
@@ -34,24 +35,34 @@ Header Files
 - All header files should have a #define guard to prevent multiple inclusion.
   To guarantee uniqueness they should be based on the file's path.
 
-  For public headers: include/grpc/grpc.h --> GRPC_GRPC_H
+  For public headers: `include/grpc/grpc.h` → `GRPC_GRPC_H`
+
+  For private headers:
+  `src/core/channel/channel_stack.h` →
+  `GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_STACK_H`
+
+Variable Initialization
+-----------------------
+
+When declaring a (non-static) pointer variable, always initialize it to `NULL`.
+Even in the case of static pointer variables, it's recommended to explicitly
+initialize them to `NULL`.
 
-  For private headers: 
-  src/core/channel/channel_stack.h --> GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_STACK_H
 
 C99 Features
 ------------
 
-- Variable sized arrays are not allowed
+- Variable sized arrays are not allowed.
-- Do not use the 'inline' keyword
+- Do not use the 'inline' keyword.
-- Flexible array members are allowed (https://en.wikipedia.org/wiki/Flexible_array_member)
+- Flexible array members are allowed
+  (https://en.wikipedia.org/wiki/Flexible_array_member).
 
 Comments
 --------
 
 Within public header files, only `/* */` comments are allowed.
 
-Within implementation files and private headers, either single line `//` 
+Within implementation files and private headers, either single line `//`
 or multi line `/* */` comments are allowed. Only one comment style per file is
 allowed however (i.e. if single line comments are used anywhere within a file,
 ALL comments within that file must be single line comments).
@@ -59,7 +70,15 @@ ALL comments within that file must be single line comments).
 Symbol Names
 ------------
 
-- Non-static functions must be prefixed by grpc_
+- Non-static functions must be prefixed by `grpc_`
-- static functions must not be prefixed by grpc_
+- Static functions must *not* be prefixed by `grpc_`
-- enumeration values and #define names are uppercased, all others are lowercased
+- Enumeration values and `#define` names must be uppercase. All other values
-- Multiple word identifiers use underscore as a delimiter (NEVER camel casing)
+  must be lowercase.
+- Multiple word identifiers use underscore as a delimiter, *never* camel
+  case. E.g. `variable_name`.
+
+Functions
+----------
+
+- The use of [`atexit()`](http://man7.org/linux/man-pages/man3/atexit.3.html) is
+  in forbidden in libgrpc.

doc/command_line_tool.md

@@ -15,6 +15,7 @@ The command line tool can do the following things:
 - Send unary rpc.
 - Attach metadata and display received metadata.
 - Handle common authentication to server.
+- Infer request/response types from server reflection result.
 - Find the request/response types from a given proto file.
 - Read proto request in text form.
 - Read request in wire form (for protobuf messages, this means serialized binary form).
@@ -24,7 +25,6 @@ The command line tool can do the following things:
 The command line tool should support the following things:
 
 - List server services and methods through server reflection.
-- Infer request/response types from server reflection result.
 - Fine-grained auth control (such as, use this oauth token to talk to the server).
 - Send streaming rpc.
 
@@ -46,24 +46,35 @@ https://github.com/grpc/grpc/blob/master/test/cpp/util/grpc_cli.cc
 Send a rpc to a helloworld server at `localhost:50051`:
 
 ```
-$ bins/opt/grpc_cli call localhost:50051 SayHello examples/protos/helloworld.proto \
+$ bins/opt/grpc_cli call localhost:50051 SayHello "name: 'world'" \
-    "name: 'world'"  --enable_ssl=false
+    --enable_ssl=false
 ```
 
 On success, the tool will print out
 
 ```
 Rpc succeeded with OK status
-Response: 
+Response:
  message: "Hello world"
 ```
 
 The `localhost:50051` part indicates the server you are connecting to. `SayHello` is (part of) the
-gRPC method string. Then there is the path to the proto file containing the service definition,
+gRPC method string. Then `"name: 'world'"` is the text format of the request proto message. We are
-if it is not under current directory, you can use `--proto_path` to specify a new search root.
+not using ssl here by `--enable_ssl=false`. For information on more flags, look at the comments of `grpc_cli.cc`.
-`"name: 'world'"` is the text format of the request proto message. 
+
-We are not using ssl here by `--enable_ssl=false`. For information on more
+### Use local proto files
-flags, look at the comments of `grpc_cli.cc`.
+
+If the server does not have the server reflection service, you will need to provide local proto
+files containing the service definition. The tool will try to find request/response types from
+them.
+
+```
+$ bins/opt/grpc_cli call localhost:50051 SayHello "name: 'world'" \
+    --protofiles=examples/protos/helloworld.proto --enable_ssl=false
+```
+
+If the proto files is not under current directory, you can use `--proto_path` to specify a new
+search root.
 
 ### Send non-proto rpc
 

doc/load-balancing.md

@@ -4,7 +4,7 @@ Load Balancing in gRPC
 # Objective
 
 To design a load balancing API between a gRPC client and a Load Balancer to
-instruct the client how to send load to multiple backend servers. 
+instruct the client how to send load to multiple backend servers.
 
 # Background
 
@@ -19,7 +19,7 @@ have temporary copies of the RPC request and response. This model also increases
 latency to the RPCs.
 
 The proxy model was deemed inefficient when considering request heavy services
-like storage. 
+like storage.
 
 ### Balancing-aware Client
 
@@ -28,7 +28,7 @@ example, the client could contain many load balancing policies (Round Robin,
 Random, etc) used to select servers from a list. In this model, a list of
 servers would be either statically configured in the client, provided by the
 name resolution system, an external load balancer, etc. In any case, the client
-is responsible for choosing the preferred server from the list. 
+is responsible for choosing the preferred server from the list.
 
 One of the drawbacks of this approach is writing and maintaining the load
 balancing policies in multiple languages and/or versions of the clients. These
@@ -53,14 +53,69 @@ unavailability or health issues. The load balancer will make any necessary
 complex decisions and inform the client. The load balancer may communicate with
 the backend servers to collect load and health information.
 
+
+## Requirements
+
+#### Simple API and client
+
+The gRPC client load balancing code must be simple and portable. The client
+should only contain simple algorithms (ie Round Robin) for server selection. For
+complex algorithms, the client should rely on a load balancer to provide load
+balancing configuration and the list of servers to which the client should send
+requests. The balancer will update the server list as needed to balance the load
+as well as handle server unavailability or health issues. The load balancer will
+make any necessary complex decisions and inform the client. The load balancer
+may communicate with the backend servers to collect load and health information.
+
+#### Security
+
+The load balancer may be separate from the actual server backends and a
+compromise of the load balancer should only lead to a compromise of the
+loadbalancing functionality. In other words, a compromised load balancer should
+not be able to cause a client to trust a (potentially malicious) backend server
+any more than in a comparable situation without loadbalancing.
+
 # Proposed Architecture
 
-The gRPC load balancing approach follows the third approach, by having an
+The gRPC load balancing implements the external load balancing server approach:
-external load balancer which provides simple clients with a list of servers.
+an external load balancer provides simple clients with an up-to-date list of
+servers.
+
+![image](images/load_balancing_design.png)
+
+1. On startup, the gRPC client issues a name resolution request for the service.
+   The name will resolve to one or more IP addresses to gRPC servers, a hint on
+   whether the IP address(es) point to a load balancer or not, and also return a
+   client config.
+2. The gRPC client connects to a gRPC Server.
+   1. If the name resolution has hinted that the endpoint is a load balancer,
+      the client's gRPC LB policy will attempt to open a stream to the load
+      balancer service. The server may respond in only one of the following
+      ways.
+      1. `status::UNIMPLEMENTED`. There is no loadbalancing in use. The client
+         call will fail.
+      2. "I am a Load Balancer and here is the server list." (Goto Step 4.)
+      3. "Please contact Load Balancer X" (See Step 3.) The client will close
+         this connection and cancel the stream.
+      4. If the server fails to respond, the client will wait for some timeout
+         and then re-resolve the name (process to Step 1 above).
+   2. If the name resolution has not hinted that the endpoint is a load
+      balancer, the client connects directly to the service it wants to talk to.
+3. The gRPC client's gRPC LB policy opens a separate connection to the Load
+   Balancer. If this fails, it will go back to step 1 and try another address.
+   1. During channel initialization to the Load Balancer, the client will
+      attempt to open a stream to the Load Balancer service.
+   2. The Load Balancer will return a server list to the gRPC client. If the
+      server list is empty, the call will wait until a non-empty one is
+      received. Optional: The Load Balancer will also open channels to the gRPC
+      servers if load reporting is needed.
+4. The gRPC client will send RPCs to the gRPC servers contained in the server
+   list from the Load Balancer.
+5. Optional: The gRPC servers may periodically report load to the Load Balancer.
 
 ## Client
 
-When establishing a gRPC stream to the balancer, the client will send an initial
+When establishing a gRPC _stream_ to the balancer, the client will send an initial
 request to the load balancer (via a regular gRPC message). The load balancer
 will respond with client config (including, for example, settings for flow
 control, RPC deadlines, etc.) or a redirect to another load balancer. If the
@@ -87,11 +142,3 @@ balancer in order to compute the next list of servers.
 The gRPC Server is responsible for answering RPC requests and providing
 responses to the client. The server will also report load to the load balancer
 if a reporting stream was opened for this purpose.
-
-### Security 
-
-The load balancer may be separate from the actual server backends and a
-compromise of the load balancer should only lead to a compromise of the
-loadbalancing functionality. In other words, a compromised load balancer should
-not be able to cause a client to trust a (potentially malicious) backend server
-any more than in a comparable situation without loadbalancing. 

doc/statuscodes.md

@@ -18,6 +18,7 @@ Only a subset of the pre-defined status codes are generated by the gRPC librarie
 | Could not decompress, but compression algorithm supported (Server -> Client)	| INTERNAL | Client |
 | Compression mechanism used by client not supported at server	| UNIMPLEMENTED | Server |
 | Server temporarily out of resources (e.g., Flow-control resource limits reached) |	RESOURCE_EXHAUSTED | Server|
+| Client does not have enough memory to hold the server response | RESOURCE_EXHAUSTED | Client |
 | Flow-control protocol violation |	INTERNAL | Both |
 | Error parsing returned status	| UNKNOWN | Client |
 | Incorrect Auth metadata ( Credentials failed to get metadata, Incompatible credentials set on channel and call, Invalid host set in `:authority` metadata, etc.) | UNAUTHENTICATED | Both |

examples/cpp/helloworld/CMakeLists.txt

@@ -0,0 +1,49 @@
+# Minimum CMake required
+cmake_minimum_required(VERSION 2.8)
+
+# Project
+project(HelloWorld CXX)
+
+# Protobuf
+set(protobuf_MODULE_COMPATIBLE TRUE)
+find_package(protobuf CONFIG REQUIRED)
+message(STATUS "Using protobuf ${protobuf_VERSION}")
+
+# gRPC
+find_package(gRPC CONFIG REQUIRED)
+message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+# gRPC C++ plugin
+get_target_property(gRPC_CPP_PLUGIN_EXECUTABLE gRPC::grpc_cpp_plugin
+    IMPORTED_LOCATION_RELEASE)
+
+# Proto file
+get_filename_component(hw_proto "../../protos/helloworld.proto" ABSOLUTE)
+get_filename_component(hw_proto_path "${hw_proto}" PATH)
+
+# Generated sources
+protobuf_generate_cpp(hw_proto_srcs hw_proto_hdrs "${hw_proto}")
+set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.grpc.pb.cc")
+set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.grpc.pb.h")
+add_custom_command(
+      OUTPUT "${hw_grpc_srcs}" "${hw_grpc_hdrs}"
+      COMMAND protobuf::protoc
+      ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" -I "${hw_proto_path}"
+        --plugin=protoc-gen-grpc="${gRPC_CPP_PLUGIN_EXECUTABLE}"
+        "${hw_proto}"
+      DEPENDS "${hw_proto}")
+
+# Generated include directory
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+
+# Targets greeter_[async_](client|server)
+foreach(_target
+  greeter_client greeter_server
+  greeter_async_client greeter_async_server)
+  add_executable(${_target} "${_target}.cc"
+    ${hw_proto_srcs}
+    ${hw_grpc_srcs})
+  target_link_libraries(${_target}
+    protobuf::libprotobuf
+    gRPC::grpc++_unsecure)
+endforeach()

examples/cpp/helloworld/Makefile

@@ -29,10 +29,20 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
+HOST_SYSTEM = $(shell uname | cut -f 1 -d_)
+SYSTEM ?= $(HOST_SYSTEM)
 CXX = g++
 CPPFLAGS += -I/usr/local/include -pthread
 CXXFLAGS += -std=c++11
-LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++ grpc` -lprotobuf -lpthread -ldl
+ifeq ($(SYSTEM),Darwin)
+LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++ grpc`       \
+           -lgrpc++_reflection \
+           -lprotobuf -lpthread -ldl
+else
+LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++ grpc`       \
+           -Wl,--no-as-needed -lgrpc++_reflection -Wl,--as-needed \
+           -lprotobuf -lpthread -ldl
+endif
 PROTOC = protoc
 GRPC_CPP_PLUGIN = grpc_cpp_plugin
 GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)`

examples/cpp/route_guide/Makefile

@@ -29,10 +29,20 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
+HOST_SYSTEM = $(shell uname | cut -f 1 -d_)
+SYSTEM ?= $(HOST_SYSTEM)
 CXX = g++
 CPPFLAGS += -I/usr/local/include -pthread
 CXXFLAGS += -std=c++11
-LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++` -lprotobuf -lpthread -ldl
+ifeq ($(SYSTEM),Darwin)
+LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++`            \
+           -lgrpc++_reflection \
+           -lprotobuf -lpthread -ldl
+else
+LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++`            \
+           -Wl,--no-as-needed -lgrpc++_reflection -Wl,--as-needed \
+           -lprotobuf -lpthread -ldl
+endif
 PROTOC = protoc
 GRPC_CPP_PLUGIN = grpc_cpp_plugin
 GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)`

examples/objective-c/auth_sample/AuthTestService.podspec

@@ -14,7 +14,7 @@ Pod::Spec.new do |s|
   src = "../../protos"
 
   # Run protoc with the Objective-C and gRPC plugins to generate protocol messages and gRPC clients.
-  s.dependency "!ProtoCompiler-gRPCPlugin", "~> 0.14"
+  s.dependency "!ProtoCompiler-gRPCPlugin", "~> 1.0.0-pre1"
 
   # Pods directory corresponding to this app's Podfile, relative to the location of this podspec.
   pods_root = 'Pods'

examples/objective-c/auth_sample/Misc/GoogleService-Info.plist

@@ -2,9 +2,39 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
+	<key>AD_UNIT_ID_FOR_BANNER_TEST</key>
+	<string>redacted</string>
+	<key>AD_UNIT_ID_FOR_INTERSTITIAL_TEST</key>
+	<string>redacted</string>
 	<key>CLIENT_ID</key>
 	<string>15087385131-lh9bpkiai9nls53uadju0if6k7un3uih.apps.googleusercontent.com</string>
 	<key>REVERSED_CLIENT_ID</key>
 	<string>com.googleusercontent.apps.15087385131-lh9bpkiai9nls53uadju0if6k7un3uih</string>
+	<key>API_KEY</key>
+	<string>redacted</string>
+	<key>GCM_SENDER_ID</key>
+	<string>redacted</string>
+	<key>PLIST_VERSION</key>
+	<string>1</string>
+	<key>BUNDLE_ID</key>
+	<string>io.grpc.AuthSample</string>
+	<key>PROJECT_ID</key>
+	<string>grpc-authsample</string>
+	<key>STORAGE_BUCKET</key>
+	<string>grpc-authsample.appspot.com</string>
+	<key>IS_ADS_ENABLED</key>
+	<false/>
+	<key>IS_ANALYTICS_ENABLED</key>
+	<false/>
+	<key>IS_APPINVITE_ENABLED</key>
+	<false/>
+	<key>IS_GCM_ENABLED</key>
+	<false/>
+	<key>IS_SIGNIN_ENABLED</key>
+	<true/>
+	<key>GOOGLE_APP_ID</key>
+	<string>1:15087385131:ios:d547168abe3c362f</string>
+	<key>DATABASE_URL</key>
+	<string>https://grpc-authsample.firebaseio.com</string>
 </dict>
-</plist>
+</plist>

examples/objective-c/helloworld/HelloWorld.podspec

@@ -14,7 +14,7 @@ Pod::Spec.new do |s|
   src = "../../protos"
 
   # Run protoc with the Objective-C and gRPC plugins to generate protocol messages and gRPC clients.
-  s.dependency "!ProtoCompiler-gRPCPlugin", "~> 0.14"
+  s.dependency "!ProtoCompiler-gRPCPlugin", "~> 1.0.0-pre1"
 
   # Pods directory corresponding to this app's Podfile, relative to the location of this podspec.
   pods_root = 'Pods'

examples/objective-c/route_guide/RouteGuide.podspec

@@ -14,7 +14,7 @@ Pod::Spec.new do |s|
   src = "../../protos"
 
   # Run protoc with the Objective-C and gRPC plugins to generate protocol messages and gRPC clients.
-  s.dependency "!ProtoCompiler-gRPCPlugin", "~> 0.14"
+  s.dependency "!ProtoCompiler-gRPCPlugin", "~> 1.0.0-pre1"
 
   # Pods directory corresponding to this app's Podfile, relative to the location of this podspec.
   pods_root = 'Pods'

examples/php/composer.json

@@ -3,6 +3,6 @@
   "description": "gRPC example for PHP",
   "minimum-stability": "dev",
   "require": {
-    "grpc/grpc": "v0.15.0"
+    "grpc/grpc": "v0.15.2"
   }
 }

examples/python/helloworld/greeter_client.py

@@ -31,17 +31,15 @@
 
 from __future__ import print_function
 
-from grpc.beta import implementations
+import grpc
 
 import helloworld_pb2
 
-_TIMEOUT_SECONDS = 10
-
 
 def run():
-  channel = implementations.insecure_channel('localhost', 50051)
+  channel = grpc.insecure_channel('localhost:50051')
-  stub = helloworld_pb2.beta_create_Greeter_stub(channel)
+  stub = helloworld_pb2.GreeterStub(channel)
-  response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'), _TIMEOUT_SECONDS)
+  response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
   print("Greeter client received: " + response.message)
 
 

examples/python/helloworld/greeter_server.py

@@ -29,21 +29,25 @@
 
 """The Python implementation of the GRPC helloworld.Greeter server."""
 
+from concurrent import futures
 import time
 
+import grpc
+
 import helloworld_pb2
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 
-class Greeter(helloworld_pb2.BetaGreeterServicer):
+class Greeter(helloworld_pb2.GreeterServicer):
 
   def SayHello(self, request, context):
     return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)
 
 
 def serve():
-  server = helloworld_pb2.beta_create_Greeter_server(Greeter())
+  server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+  helloworld_pb2.add_GreeterServicer_to_server(Greeter(), server)
   server.add_insecure_port('[::]:50051')
   server.start()
   try:

examples/python/helloworld/helloworld_pb2.py

@@ -107,13 +107,55 @@ _sym_db.RegisterMessage(HelloReply)
 
 DESCRIPTOR.has_options = True
 DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW'))
-import abc
+import grpc
-import six
 from grpc.beta import implementations as beta_implementations
 from grpc.beta import interfaces as beta_interfaces
 from grpc.framework.common import cardinality
 from grpc.framework.interfaces.face import utilities as face_utilities
 
+
+class GreeterStub(object):
+  """The greeting service definition.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.SayHello = channel.unary_unary(
+        '/helloworld.Greeter/SayHello',
+        request_serializer=HelloRequest.SerializeToString,
+        response_deserializer=HelloReply.FromString,
+        )
+
+
+class GreeterServicer(object):
+  """The greeting service definition.
+  """
+
+  def SayHello(self, request, context):
+    """Sends a greeting
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_GreeterServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'SayHello': grpc.unary_unary_rpc_method_handler(
+          servicer.SayHello,
+          request_deserializer=HelloRequest.FromString,
+          response_serializer=HelloReply.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'helloworld.Greeter', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))
+
+
 class BetaGreeterServicer(object):
   """The greeting service definition.
   """
@@ -122,23 +164,23 @@ class BetaGreeterServicer(object):
     """
     context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
 
+
 class BetaGreeterStub(object):
   """The greeting service definition.
   """
-  def SayHello(self, request, timeout):
+  def SayHello(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
     """Sends a greeting
     """
     raise NotImplementedError()
   SayHello.future = None
 
+
 def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-  import helloworld_pb2
-  import helloworld_pb2
   request_deserializers = {
-    ('helloworld.Greeter', 'SayHello'): helloworld_pb2.HelloRequest.FromString,
+    ('helloworld.Greeter', 'SayHello'): HelloRequest.FromString,
   }
   response_serializers = {
-    ('helloworld.Greeter', 'SayHello'): helloworld_pb2.HelloReply.SerializeToString,
+    ('helloworld.Greeter', 'SayHello'): HelloReply.SerializeToString,
   }
   method_implementations = {
     ('helloworld.Greeter', 'SayHello'): face_utilities.unary_unary_inline(servicer.SayHello),
@@ -146,14 +188,13 @@ def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_time
   server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
   return beta_implementations.server(method_implementations, options=server_options)
 
+
 def beta_create_Greeter_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-  import helloworld_pb2
-  import helloworld_pb2
   request_serializers = {
-    ('helloworld.Greeter', 'SayHello'): helloworld_pb2.HelloRequest.SerializeToString,
+    ('helloworld.Greeter', 'SayHello'): HelloRequest.SerializeToString,
   }
   response_deserializers = {
-    ('helloworld.Greeter', 'SayHello'): helloworld_pb2.HelloReply.FromString,
+    ('helloworld.Greeter', 'SayHello'): HelloReply.FromString,
   }
   cardinalities = {
     'SayHello': cardinality.Cardinality.UNARY_UNARY,

examples/python/helloworld/run_codegen.py

@@ -1,4 +1,3 @@
-#!/bin/bash
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -28,5 +27,16 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# Runs the protoc with gRPC plugin to generate protocol messages and gRPC stubs.
+"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
-python -m grpc.tools.protoc -I../../protos --python_out=. --grpc_python_out=. ../../protos/route_guide.proto
+
+from grpc.tools import protoc
+
+protoc.main(
+    (
+	'',
+	'-I../../protos',
+	'--python_out=.',
+	'--grpc_python_out=.',
+	'../../protos/helloworld.proto',
+    )
+)

examples/python/multiplex/.gitignore

@@ -0,0 +1 @@
+*.pyc

examples/python/multiplex/README.md

@@ -0,0 +1 @@
+An example showing two stubs sharing a channel and two servicers sharing a server.

examples/python/multiplex/helloworld_pb2.py

@@ -0,0 +1,204 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: helloworld.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='helloworld.proto',
+  package='helloworld',
+  syntax='proto3',
+  serialized_pb=_b('\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2I\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x42\x36\n\x1bio.grpc.examples.helloworldB\x0fHelloWorldProtoP\x01\xa2\x02\x03HLWb\x06proto3')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_HELLOREQUEST = _descriptor.Descriptor(
+  name='HelloRequest',
+  full_name='helloworld.HelloRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='name', full_name='helloworld.HelloRequest.name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=32,
+  serialized_end=60,
+)
+
+
+_HELLOREPLY = _descriptor.Descriptor(
+  name='HelloReply',
+  full_name='helloworld.HelloReply',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='message', full_name='helloworld.HelloReply.message', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=62,
+  serialized_end=91,
+)
+
+DESCRIPTOR.message_types_by_name['HelloRequest'] = _HELLOREQUEST
+DESCRIPTOR.message_types_by_name['HelloReply'] = _HELLOREPLY
+
+HelloRequest = _reflection.GeneratedProtocolMessageType('HelloRequest', (_message.Message,), dict(
+  DESCRIPTOR = _HELLOREQUEST,
+  __module__ = 'helloworld_pb2'
+  # @@protoc_insertion_point(class_scope:helloworld.HelloRequest)
+  ))
+_sym_db.RegisterMessage(HelloRequest)
+
+HelloReply = _reflection.GeneratedProtocolMessageType('HelloReply', (_message.Message,), dict(
+  DESCRIPTOR = _HELLOREPLY,
+  __module__ = 'helloworld_pb2'
+  # @@protoc_insertion_point(class_scope:helloworld.HelloReply)
+  ))
+_sym_db.RegisterMessage(HelloReply)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW'))
+import grpc
+from grpc.beta import implementations as beta_implementations
+from grpc.beta import interfaces as beta_interfaces
+from grpc.framework.common import cardinality
+from grpc.framework.interfaces.face import utilities as face_utilities
+
+
+class GreeterStub(object):
+  """The greeting service definition.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.SayHello = channel.unary_unary(
+        '/helloworld.Greeter/SayHello',
+        request_serializer=HelloRequest.SerializeToString,
+        response_deserializer=HelloReply.FromString,
+        )
+
+
+class GreeterServicer(object):
+  """The greeting service definition.
+  """
+
+  def SayHello(self, request, context):
+    """Sends a greeting
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_GreeterServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'SayHello': grpc.unary_unary_rpc_method_handler(
+          servicer.SayHello,
+          request_deserializer=HelloRequest.FromString,
+          response_serializer=HelloReply.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'helloworld.Greeter', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))
+
+
+class BetaGreeterServicer(object):
+  """The greeting service definition.
+  """
+  def SayHello(self, request, context):
+    """Sends a greeting
+    """
+    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+
+
+class BetaGreeterStub(object):
+  """The greeting service definition.
+  """
+  def SayHello(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+    """Sends a greeting
+    """
+    raise NotImplementedError()
+  SayHello.future = None
+
+
+def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+  request_deserializers = {
+    ('helloworld.Greeter', 'SayHello'): HelloRequest.FromString,
+  }
+  response_serializers = {
+    ('helloworld.Greeter', 'SayHello'): HelloReply.SerializeToString,
+  }
+  method_implementations = {
+    ('helloworld.Greeter', 'SayHello'): face_utilities.unary_unary_inline(servicer.SayHello),
+  }
+  server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
+  return beta_implementations.server(method_implementations, options=server_options)
+
+
+def beta_create_Greeter_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+  request_serializers = {
+    ('helloworld.Greeter', 'SayHello'): HelloRequest.SerializeToString,
+  }
+  response_deserializers = {
+    ('helloworld.Greeter', 'SayHello'): HelloReply.FromString,
+  }
+  cardinalities = {
+    'SayHello': cardinality.Cardinality.UNARY_UNARY,
+  }
+  stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
+  return beta_implementations.dynamic_stub(channel, 'helloworld.Greeter', cardinalities, options=stub_options)
+# @@protoc_insertion_point(module_scope)

examples/python/multiplex/multiplex_client.py

@@ -0,0 +1,139 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A client that makes both Greeter and RouteGuide RPCs."""
+
+from __future__ import print_function
+
+import random
+import time
+
+import grpc
+
+import helloworld_pb2
+import route_guide_pb2
+import route_guide_resources
+
+
+def make_route_note(message, latitude, longitude):
+  return route_guide_pb2.RouteNote(
+      message=message,
+      location=route_guide_pb2.Point(latitude=latitude, longitude=longitude))
+
+
+def guide_get_one_feature(route_guide_stub, point):
+  feature = route_guide_stub.GetFeature(point)
+  if not feature.location:
+    print("Server returned incomplete feature")
+    return
+
+  if feature.name:
+    print("Feature called %s at %s" % (feature.name, feature.location))
+  else:
+    print("Found no feature at %s" % feature.location)
+
+
+def guide_get_feature(route_guide_stub):
+  guide_get_one_feature(
+      route_guide_stub,
+      route_guide_pb2.Point(latitude=409146138, longitude=-746188906))
+  guide_get_one_feature(
+      route_guide_stub, route_guide_pb2.Point(latitude=0, longitude=0))
+
+
+def guide_list_features(route_guide_stub):
+  rectangle = route_guide_pb2.Rectangle(
+      lo=route_guide_pb2.Point(latitude=400000000, longitude=-750000000),
+      hi=route_guide_pb2.Point(latitude=420000000, longitude=-730000000))
+  print("Looking for features between 40, -75 and 42, -73")
+
+  features = route_guide_stub.ListFeatures(rectangle)
+
+  for feature in features:
+    print("Feature called %s at %s" % (feature.name, feature.location))
+
+
+def generate_route(feature_list):
+  for _ in range(0, 10):
+    random_feature = feature_list[random.randint(0, len(feature_list) - 1)]
+    print("Visiting point %s" % random_feature.location)
+    yield random_feature.location
+    time.sleep(random.uniform(0.5, 1.5))
+
+
+def guide_record_route(route_guide_stub):
+  feature_list = route_guide_resources.read_route_guide_database()
+
+  route_iterator = generate_route(feature_list)
+  route_summary = route_guide_stub.RecordRoute(route_iterator)
+  print("Finished trip with %s points " % route_summary.point_count)
+  print("Passed %s features " % route_summary.feature_count)
+  print("Travelled %s meters " % route_summary.distance)
+  print("It took %s seconds " % route_summary.elapsed_time)
+
+
+def generate_messages():
+  messages = [
+      make_route_note("First message", 0, 0),
+      make_route_note("Second message", 0, 1),
+      make_route_note("Third message", 1, 0),
+      make_route_note("Fourth message", 0, 0),
+      make_route_note("Fifth message", 1, 0),
+  ]
+  for msg in messages:
+    print("Sending %s at %s" % (msg.message, msg.location))
+    yield msg
+    time.sleep(random.uniform(0.5, 1.0))
+
+
+def guide_route_chat(route_guide_stub):
+  responses = route_guide_stub.RouteChat(generate_messages())
+  for response in responses:
+    print("Received message %s at %s" % (response.message, response.location))
+
+
+def run():
+  channel = grpc.insecure_channel('localhost:50051')
+  greeter_stub = helloworld_pb2.GreeterStub(channel)
+  route_guide_stub = route_guide_pb2.RouteGuideStub(channel)
+  greeter_response = greeter_stub.SayHello(
+      helloworld_pb2.HelloRequest(name='you'))
+  print("Greeter client received: " + greeter_response.message)
+  print("-------------- GetFeature --------------")
+  guide_get_feature(route_guide_stub)
+  print("-------------- ListFeatures --------------")
+  guide_list_features(route_guide_stub)
+  print("-------------- RecordRoute --------------")
+  guide_record_route(route_guide_stub)
+  print("-------------- RouteChat --------------")
+  guide_route_chat(route_guide_stub)
+
+
+if __name__ == '__main__':
+  run()

examples/python/multiplex/multiplex_server.py

@@ -0,0 +1,149 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A gRPC server servicing both Greeter and RouteGuide RPCs."""
+
+from concurrent import futures
+import time
+import math
+
+import grpc
+
+import helloworld_pb2
+import route_guide_pb2
+import route_guide_resources
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+
+def _get_feature(feature_db, point):
+  """Returns Feature at given location or None."""
+  for feature in feature_db:
+    if feature.location == point:
+      return feature
+  return None
+
+
+def _get_distance(start, end):
+  """Distance between two points."""
+  coord_factor = 10000000.0
+  lat_1 = start.latitude / coord_factor
+  lat_2 = end.latitude / coord_factor
+  lon_1 = start.longitude / coord_factor
+  lon_2 = end.longitude / coord_factor
+  lat_rad_1 = math.radians(lat_1)
+  lat_rad_2 = math.radians(lat_2)
+  delta_lat_rad = math.radians(lat_2 - lat_1)
+  delta_lon_rad = math.radians(lon_2 - lon_1)
+
+  a = (pow(math.sin(delta_lat_rad / 2), 2) +
+       (math.cos(lat_rad_1) * math.cos(lat_rad_2) *
+        pow(math.sin(delta_lon_rad / 2), 2)))
+  c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+  R = 6371000; # metres
+  return R * c;
+
+
+class _GreeterServicer(helloworld_pb2.GreeterServicer):
+
+  def SayHello(self, request, context):
+    return helloworld_pb2.HelloReply(message='Hello, {}!'.format(request.name))
+
+
+class _RouteGuideServicer(route_guide_pb2.RouteGuideServicer):
+  """Provides methods that implement functionality of route guide server."""
+
+  def __init__(self):
+    self.db = route_guide_resources.read_route_guide_database()
+
+  def GetFeature(self, request, context):
+    feature = _get_feature(self.db, request)
+    if feature is None:
+      return route_guide_pb2.Feature(name="", location=request)
+    else:
+      return feature
+
+  def ListFeatures(self, request, context):
+    left = min(request.lo.longitude, request.hi.longitude)
+    right = max(request.lo.longitude, request.hi.longitude)
+    top = max(request.lo.latitude, request.hi.latitude)
+    bottom = min(request.lo.latitude, request.hi.latitude)
+    for feature in self.db:
+      if (feature.location.longitude >= left and
+          feature.location.longitude <= right and
+          feature.location.latitude >= bottom and
+          feature.location.latitude <= top):
+        yield feature
+
+  def RecordRoute(self, request_iterator, context):
+    point_count = 0
+    feature_count = 0
+    distance = 0.0
+    prev_point = None
+
+    start_time = time.time()
+    for point in request_iterator:
+      point_count += 1
+      if _get_feature(self.db, point):
+        feature_count += 1
+      if prev_point:
+        distance += _get_distance(prev_point, point)
+      prev_point = point
+
+    elapsed_time = time.time() - start_time
+    return route_guide_pb2.RouteSummary(point_count=point_count,
+                                        feature_count=feature_count,
+                                        distance=int(distance),
+                                        elapsed_time=int(elapsed_time))
+
+  def RouteChat(self, request_iterator, context):
+    prev_notes = []
+    for new_note in request_iterator:
+      for prev_note in prev_notes:
+        if prev_note.location == new_note.location:
+          yield prev_note
+      prev_notes.append(new_note)
+
+
+def serve():
+  server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+  helloworld_pb2.add_GreeterServicer_to_server(_GreeterServicer(), server)
+  route_guide_pb2.add_RouteGuideServicer_to_server(
+      _RouteGuideServicer(), server)
+  server.add_insecure_port('[::]:50051')
+  server.start()
+  try:
+    while True:
+      time.sleep(_ONE_DAY_IN_SECONDS)
+  except KeyboardInterrupt:
+    server.stop(0)
+
+
+if __name__ == '__main__':
+  serve()

examples/python/multiplex/route_guide_db.json

@@ -0,0 +1,601 @@
+[{
+    "location": {
+        "latitude": 407838351,
+        "longitude": -746143763
+    },
+    "name": "Patriots Path, Mendham, NJ 07945, USA"
+}, {
+    "location": {
+        "latitude": 408122808,
+        "longitude": -743999179
+    },
+    "name": "101 New Jersey 10, Whippany, NJ 07981, USA"
+}, {
+    "location": {
+        "latitude": 413628156,
+        "longitude": -749015468
+    },
+    "name": "U.S. 6, Shohola, PA 18458, USA"
+}, {
+    "location": {
+        "latitude": 419999544,
+        "longitude": -740371136
+    },
+    "name": "5 Conners Road, Kingston, NY 12401, USA"
+}, {
+    "location": {
+        "latitude": 414008389,
+        "longitude": -743951297
+    },
+    "name": "Mid Hudson Psychiatric Center, New Hampton, NY 10958, USA"
+}, {
+    "location": {
+        "latitude": 419611318,
+        "longitude": -746524769
+    },
+    "name": "287 Flugertown Road, Livingston Manor, NY 12758, USA"
+}, {
+    "location": {
+        "latitude": 406109563,
+        "longitude": -742186778
+    },
+    "name": "4001 Tremley Point Road, Linden, NJ 07036, USA"
+}, {
+    "location": {
+        "latitude": 416802456,
+        "longitude": -742370183
+    },
+    "name": "352 South Mountain Road, Wallkill, NY 12589, USA"
+}, {
+    "location": {
+        "latitude": 412950425,
+        "longitude": -741077389
+    },
+    "name": "Bailey Turn Road, Harriman, NY 10926, USA"
+}, {
+    "location": {
+        "latitude": 412144655,
+        "longitude": -743949739
+    },
+    "name": "193-199 Wawayanda Road, Hewitt, NJ 07421, USA"
+}, {
+    "location": {
+        "latitude": 415736605,
+        "longitude": -742847522
+    },
+    "name": "406-496 Ward Avenue, Pine Bush, NY 12566, USA"
+}, {
+    "location": {
+        "latitude": 413843930,
+        "longitude": -740501726
+    },
+    "name": "162 Merrill Road, Highland Mills, NY 10930, USA"
+}, {
+    "location": {
+        "latitude": 410873075,
+        "longitude": -744459023
+    },
+    "name": "Clinton Road, West Milford, NJ 07480, USA"
+}, {
+    "location": {
+        "latitude": 412346009,
+        "longitude": -744026814
+    },
+    "name": "16 Old Brook Lane, Warwick, NY 10990, USA"
+}, {
+    "location": {
+        "latitude": 402948455,
+        "longitude": -747903913
+    },
+    "name": "3 Drake Lane, Pennington, NJ 08534, USA"
+}, {
+    "location": {
+        "latitude": 406337092,
+        "longitude": -740122226
+    },
+    "name": "6324 8th Avenue, Brooklyn, NY 11220, USA"
+}, {
+    "location": {
+        "latitude": 406421967,
+        "longitude": -747727624
+    },
+    "name": "1 Merck Access Road, Whitehouse Station, NJ 08889, USA"
+}, {
+    "location": {
+        "latitude": 416318082,
+        "longitude": -749677716
+    },
+    "name": "78-98 Schalck Road, Narrowsburg, NY 12764, USA"
+}, {
+    "location": {
+        "latitude": 415301720,
+        "longitude": -748416257
+    },
+    "name": "282 Lakeview Drive Road, Highland Lake, NY 12743, USA"
+}, {
+    "location": {
+        "latitude": 402647019,
+        "longitude": -747071791
+    },
+    "name": "330 Evelyn Avenue, Hamilton Township, NJ 08619, USA"
+}, {
+    "location": {
+        "latitude": 412567807,
+        "longitude": -741058078
+    },
+    "name": "New York State Reference Route 987E, Southfields, NY 10975, USA"
+}, {
+    "location": {
+        "latitude": 416855156,
+        "longitude": -744420597
+    },
+    "name": "103-271 Tempaloni Road, Ellenville, NY 12428, USA"
+}, {
+    "location": {
+        "latitude": 404663628,
+        "longitude": -744820157
+    },
+    "name": "1300 Airport Road, North Brunswick Township, NJ 08902, USA"
+}, {
+    "location": {
+        "latitude": 407113723,
+        "longitude": -749746483
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 402133926,
+        "longitude": -743613249
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 400273442,
+        "longitude": -741220915
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 411236786,
+        "longitude": -744070769
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 411633782,
+        "longitude": -746784970
+    },
+    "name": "211-225 Plains Road, Augusta, NJ 07822, USA"
+}, {
+    "location": {
+        "latitude": 415830701,
+        "longitude": -742952812
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 413447164,
+        "longitude": -748712898
+    },
+    "name": "165 Pedersen Ridge Road, Milford, PA 18337, USA"
+}, {
+    "location": {
+        "latitude": 405047245,
+        "longitude": -749800722
+    },
+    "name": "100-122 Locktown Road, Frenchtown, NJ 08825, USA"
+}, {
+    "location": {
+        "latitude": 418858923,
+        "longitude": -746156790
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 417951888,
+        "longitude": -748484944
+    },
+    "name": "650-652 Willi Hill Road, Swan Lake, NY 12783, USA"
+}, {
+    "location": {
+        "latitude": 407033786,
+        "longitude": -743977337
+    },
+    "name": "26 East 3rd Street, New Providence, NJ 07974, USA"
+}, {
+    "location": {
+        "latitude": 417548014,
+        "longitude": -740075041
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 410395868,
+        "longitude": -744972325
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 404615353,
+        "longitude": -745129803
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 406589790,
+        "longitude": -743560121
+    },
+    "name": "611 Lawrence Avenue, Westfield, NJ 07090, USA"
+}, {
+    "location": {
+        "latitude": 414653148,
+        "longitude": -740477477
+    },
+    "name": "18 Lannis Avenue, New Windsor, NY 12553, USA"
+}, {
+    "location": {
+        "latitude": 405957808,
+        "longitude": -743255336
+    },
+    "name": "82-104 Amherst Avenue, Colonia, NJ 07067, USA"
+}, {
+    "location": {
+        "latitude": 411733589,
+        "longitude": -741648093
+    },
+    "name": "170 Seven Lakes Drive, Sloatsburg, NY 10974, USA"
+}, {
+    "location": {
+        "latitude": 412676291,
+        "longitude": -742606606
+    },
+    "name": "1270 Lakes Road, Monroe, NY 10950, USA"
+}, {
+    "location": {
+        "latitude": 409224445,
+        "longitude": -748286738
+    },
+    "name": "509-535 Alphano Road, Great Meadows, NJ 07838, USA"
+}, {
+    "location": {
+        "latitude": 406523420,
+        "longitude": -742135517
+    },
+    "name": "652 Garden Street, Elizabeth, NJ 07202, USA"
+}, {
+    "location": {
+        "latitude": 401827388,
+        "longitude": -740294537
+    },
+    "name": "349 Sea Spray Court, Neptune City, NJ 07753, USA"
+}, {
+    "location": {
+        "latitude": 410564152,
+        "longitude": -743685054
+    },
+    "name": "13-17 Stanley Street, West Milford, NJ 07480, USA"
+}, {
+    "location": {
+        "latitude": 408472324,
+        "longitude": -740726046
+    },
+    "name": "47 Industrial Avenue, Teterboro, NJ 07608, USA"
+}, {
+    "location": {
+        "latitude": 412452168,
+        "longitude": -740214052
+    },
+    "name": "5 White Oak Lane, Stony Point, NY 10980, USA"
+}, {
+    "location": {
+        "latitude": 409146138,
+        "longitude": -746188906
+    },
+    "name": "Berkshire Valley Management Area Trail, Jefferson, NJ, USA"
+}, {
+    "location": {
+        "latitude": 404701380,
+        "longitude": -744781745
+    },
+    "name": "1007 Jersey Avenue, New Brunswick, NJ 08901, USA"
+}, {
+    "location": {
+        "latitude": 409642566,
+        "longitude": -746017679
+    },
+    "name": "6 East Emerald Isle Drive, Lake Hopatcong, NJ 07849, USA"
+}, {
+    "location": {
+        "latitude": 408031728,
+        "longitude": -748645385
+    },
+    "name": "1358-1474 New Jersey 57, Port Murray, NJ 07865, USA"
+}, {
+    "location": {
+        "latitude": 413700272,
+        "longitude": -742135189
+    },
+    "name": "367 Prospect Road, Chester, NY 10918, USA"
+}, {
+    "location": {
+        "latitude": 404310607,
+        "longitude": -740282632
+    },
+    "name": "10 Simon Lake Drive, Atlantic Highlands, NJ 07716, USA"
+}, {
+    "location": {
+        "latitude": 409319800,
+        "longitude": -746201391
+    },
+    "name": "11 Ward Street, Mount Arlington, NJ 07856, USA"
+}, {
+    "location": {
+        "latitude": 406685311,
+        "longitude": -742108603
+    },
+    "name": "300-398 Jefferson Avenue, Elizabeth, NJ 07201, USA"
+}, {
+    "location": {
+        "latitude": 419018117,
+        "longitude": -749142781
+    },
+    "name": "43 Dreher Road, Roscoe, NY 12776, USA"
+}, {
+    "location": {
+        "latitude": 412856162,
+        "longitude": -745148837
+    },
+    "name": "Swan Street, Pine Island, NY 10969, USA"
+}, {
+    "location": {
+        "latitude": 416560744,
+        "longitude": -746721964
+    },
+    "name": "66 Pleasantview Avenue, Monticello, NY 12701, USA"
+}, {
+    "location": {
+        "latitude": 405314270,
+        "longitude": -749836354
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 414219548,
+        "longitude": -743327440
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 415534177,
+        "longitude": -742900616
+    },
+    "name": "565 Winding Hills Road, Montgomery, NY 12549, USA"
+}, {
+    "location": {
+        "latitude": 406898530,
+        "longitude": -749127080
+    },
+    "name": "231 Rocky Run Road, Glen Gardner, NJ 08826, USA"
+}, {
+    "location": {
+        "latitude": 407586880,
+        "longitude": -741670168
+    },
+    "name": "100 Mount Pleasant Avenue, Newark, NJ 07104, USA"
+}, {
+    "location": {
+        "latitude": 400106455,
+        "longitude": -742870190
+    },
+    "name": "517-521 Huntington Drive, Manchester Township, NJ 08759, USA"
+}, {
+    "location": {
+        "latitude": 400066188,
+        "longitude": -746793294
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 418803880,
+        "longitude": -744102673
+    },
+    "name": "40 Mountain Road, Napanoch, NY 12458, USA"
+}, {
+    "location": {
+        "latitude": 414204288,
+        "longitude": -747895140
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 414777405,
+        "longitude": -740615601
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 415464475,
+        "longitude": -747175374
+    },
+    "name": "48 North Road, Forestburgh, NY 12777, USA"
+}, {
+    "location": {
+        "latitude": 404062378,
+        "longitude": -746376177
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 405688272,
+        "longitude": -749285130
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 400342070,
+        "longitude": -748788996
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 401809022,
+        "longitude": -744157964
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 404226644,
+        "longitude": -740517141
+    },
+    "name": "9 Thompson Avenue, Leonardo, NJ 07737, USA"
+}, {
+    "location": {
+        "latitude": 410322033,
+        "longitude": -747871659
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 407100674,
+        "longitude": -747742727
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 418811433,
+        "longitude": -741718005
+    },
+    "name": "213 Bush Road, Stone Ridge, NY 12484, USA"
+}, {
+    "location": {
+        "latitude": 415034302,
+        "longitude": -743850945
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 411349992,
+        "longitude": -743694161
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 404839914,
+        "longitude": -744759616
+    },
+    "name": "1-17 Bergen Court, New Brunswick, NJ 08901, USA"
+}, {
+    "location": {
+        "latitude": 414638017,
+        "longitude": -745957854
+    },
+    "name": "35 Oakland Valley Road, Cuddebackville, NY 12729, USA"
+}, {
+    "location": {
+        "latitude": 412127800,
+        "longitude": -740173578
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 401263460,
+        "longitude": -747964303
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 412843391,
+        "longitude": -749086026
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 418512773,
+        "longitude": -743067823
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 404318328,
+        "longitude": -740835638
+    },
+    "name": "42-102 Main Street, Belford, NJ 07718, USA"
+}, {
+    "location": {
+        "latitude": 419020746,
+        "longitude": -741172328
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 404080723,
+        "longitude": -746119569
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 401012643,
+        "longitude": -744035134
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 404306372,
+        "longitude": -741079661
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 403966326,
+        "longitude": -748519297
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 405002031,
+        "longitude": -748407866
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 409532885,
+        "longitude": -742200683
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 416851321,
+        "longitude": -742674555
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 406411633,
+        "longitude": -741722051
+    },
+    "name": "3387 Richmond Terrace, Staten Island, NY 10303, USA"
+}, {
+    "location": {
+        "latitude": 413069058,
+        "longitude": -744597778
+    },
+    "name": "261 Van Sickle Road, Goshen, NY 10924, USA"
+}, {
+    "location": {
+        "latitude": 418465462,
+        "longitude": -746859398
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 411733222,
+        "longitude": -744228360
+    },
+    "name": ""
+}, {
+    "location": {
+        "latitude": 410248224,
+        "longitude": -747127767
+    },
+    "name": "3 Hasta Way, Newton, NJ 07860, USA"
+}]

examples/python/multiplex/route_guide_pb2.py

@@ -0,0 +1,516 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: route_guide.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='route_guide.proto',
+  package='routeguide',
+  syntax='proto3',
+  serialized_pb=_b('\n\x11route_guide.proto\x12\nrouteguide\",\n\x05Point\x12\x10\n\x08latitude\x18\x01 \x01(\x05\x12\x11\n\tlongitude\x18\x02 \x01(\x05\"I\n\tRectangle\x12\x1d\n\x02lo\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x1d\n\x02hi\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"<\n\x07\x46\x65\x61ture\x12\x0c\n\x04name\x18\x01 \x01(\t\x12#\n\x08location\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"A\n\tRouteNote\x12#\n\x08location\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x0f\n\x07message\x18\x02 \x01(\t\"b\n\x0cRouteSummary\x12\x13\n\x0bpoint_count\x18\x01 \x01(\x05\x12\x15\n\rfeature_count\x18\x02 \x01(\x05\x12\x10\n\x08\x64istance\x18\x03 \x01(\x05\x12\x14\n\x0c\x65lapsed_time\x18\x04 \x01(\x05\x32\x85\x02\n\nRouteGuide\x12\x36\n\nGetFeature\x12\x11.routeguide.Point\x1a\x13.routeguide.Feature\"\x00\x12>\n\x0cListFeatures\x12\x15.routeguide.Rectangle\x1a\x13.routeguide.Feature\"\x00\x30\x01\x12>\n\x0bRecordRoute\x12\x11.routeguide.Point\x1a\x18.routeguide.RouteSummary\"\x00(\x01\x12?\n\tRouteChat\x12\x15.routeguide.RouteNote\x1a\x15.routeguide.RouteNote\"\x00(\x01\x30\x01\x42\x36\n\x1bio.grpc.examples.routeguideB\x0fRouteGuideProtoP\x01\xa2\x02\x03RTGb\x06proto3')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_POINT = _descriptor.Descriptor(
+  name='Point',
+  full_name='routeguide.Point',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='latitude', full_name='routeguide.Point.latitude', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='longitude', full_name='routeguide.Point.longitude', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=33,
+  serialized_end=77,
+)
+
+
+_RECTANGLE = _descriptor.Descriptor(
+  name='Rectangle',
+  full_name='routeguide.Rectangle',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='lo', full_name='routeguide.Rectangle.lo', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='hi', full_name='routeguide.Rectangle.hi', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=79,
+  serialized_end=152,
+)
+
+
+_FEATURE = _descriptor.Descriptor(
+  name='Feature',
+  full_name='routeguide.Feature',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='name', full_name='routeguide.Feature.name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='location', full_name='routeguide.Feature.location', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=154,
+  serialized_end=214,
+)
+
+
+_ROUTENOTE = _descriptor.Descriptor(
+  name='RouteNote',
+  full_name='routeguide.RouteNote',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='location', full_name='routeguide.RouteNote.location', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='message', full_name='routeguide.RouteNote.message', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=216,
+  serialized_end=281,
+)
+
+
+_ROUTESUMMARY = _descriptor.Descriptor(
+  name='RouteSummary',
+  full_name='routeguide.RouteSummary',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='point_count', full_name='routeguide.RouteSummary.point_count', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='feature_count', full_name='routeguide.RouteSummary.feature_count', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='distance', full_name='routeguide.RouteSummary.distance', index=2,
+      number=3, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='elapsed_time', full_name='routeguide.RouteSummary.elapsed_time', index=3,
+      number=4, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=283,
+  serialized_end=381,
+)
+
+_RECTANGLE.fields_by_name['lo'].message_type = _POINT
+_RECTANGLE.fields_by_name['hi'].message_type = _POINT
+_FEATURE.fields_by_name['location'].message_type = _POINT
+_ROUTENOTE.fields_by_name['location'].message_type = _POINT
+DESCRIPTOR.message_types_by_name['Point'] = _POINT
+DESCRIPTOR.message_types_by_name['Rectangle'] = _RECTANGLE
+DESCRIPTOR.message_types_by_name['Feature'] = _FEATURE
+DESCRIPTOR.message_types_by_name['RouteNote'] = _ROUTENOTE
+DESCRIPTOR.message_types_by_name['RouteSummary'] = _ROUTESUMMARY
+
+Point = _reflection.GeneratedProtocolMessageType('Point', (_message.Message,), dict(
+  DESCRIPTOR = _POINT,
+  __module__ = 'route_guide_pb2'
+  # @@protoc_insertion_point(class_scope:routeguide.Point)
+  ))
+_sym_db.RegisterMessage(Point)
+
+Rectangle = _reflection.GeneratedProtocolMessageType('Rectangle', (_message.Message,), dict(
+  DESCRIPTOR = _RECTANGLE,
+  __module__ = 'route_guide_pb2'
+  # @@protoc_insertion_point(class_scope:routeguide.Rectangle)
+  ))
+_sym_db.RegisterMessage(Rectangle)
+
+Feature = _reflection.GeneratedProtocolMessageType('Feature', (_message.Message,), dict(
+  DESCRIPTOR = _FEATURE,
+  __module__ = 'route_guide_pb2'
+  # @@protoc_insertion_point(class_scope:routeguide.Feature)
+  ))
+_sym_db.RegisterMessage(Feature)
+
+RouteNote = _reflection.GeneratedProtocolMessageType('RouteNote', (_message.Message,), dict(
+  DESCRIPTOR = _ROUTENOTE,
+  __module__ = 'route_guide_pb2'
+  # @@protoc_insertion_point(class_scope:routeguide.RouteNote)
+  ))
+_sym_db.RegisterMessage(RouteNote)
+
+RouteSummary = _reflection.GeneratedProtocolMessageType('RouteSummary', (_message.Message,), dict(
+  DESCRIPTOR = _ROUTESUMMARY,
+  __module__ = 'route_guide_pb2'
+  # @@protoc_insertion_point(class_scope:routeguide.RouteSummary)
+  ))
+_sym_db.RegisterMessage(RouteSummary)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.routeguideB\017RouteGuideProtoP\001\242\002\003RTG'))
+import grpc
+from grpc.beta import implementations as beta_implementations
+from grpc.beta import interfaces as beta_interfaces
+from grpc.framework.common import cardinality
+from grpc.framework.interfaces.face import utilities as face_utilities
+
+
+class RouteGuideStub(object):
+  """Interface exported by the server.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.GetFeature = channel.unary_unary(
+        '/routeguide.RouteGuide/GetFeature',
+        request_serializer=Point.SerializeToString,
+        response_deserializer=Feature.FromString,
+        )
+    self.ListFeatures = channel.unary_stream(
+        '/routeguide.RouteGuide/ListFeatures',
+        request_serializer=Rectangle.SerializeToString,
+        response_deserializer=Feature.FromString,
+        )
+    self.RecordRoute = channel.stream_unary(
+        '/routeguide.RouteGuide/RecordRoute',
+        request_serializer=Point.SerializeToString,
+        response_deserializer=RouteSummary.FromString,
+        )
+    self.RouteChat = channel.stream_stream(
+        '/routeguide.RouteGuide/RouteChat',
+        request_serializer=RouteNote.SerializeToString,
+        response_deserializer=RouteNote.FromString,
+        )
+
+
+class RouteGuideServicer(object):
+  """Interface exported by the server.
+  """
+
+  def GetFeature(self, request, context):
+    """A simple RPC.
+
+    Obtains the feature at a given position.
+
+    A feature with an empty name is returned if there's no feature at the given
+    position.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def ListFeatures(self, request, context):
+    """A server-to-client streaming RPC.
+
+    Obtains the Features available within the given Rectangle.  Results are
+    streamed rather than returned at once (e.g. in a response message with a
+    repeated field), as the rectangle may cover a large area and contain a
+    huge number of features.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def RecordRoute(self, request_iterator, context):
+    """A client-to-server streaming RPC.
+
+    Accepts a stream of Points on a route being traversed, returning a
+    RouteSummary when traversal is completed.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def RouteChat(self, request_iterator, context):
+    """A Bidirectional streaming RPC.
+
+    Accepts a stream of RouteNotes sent while a route is being traversed,
+    while receiving other RouteNotes (e.g. from other users).
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_RouteGuideServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'GetFeature': grpc.unary_unary_rpc_method_handler(
+          servicer.GetFeature,
+          request_deserializer=Point.FromString,
+          response_serializer=Feature.SerializeToString,
+      ),
+      'ListFeatures': grpc.unary_stream_rpc_method_handler(
+          servicer.ListFeatures,
+          request_deserializer=Rectangle.FromString,
+          response_serializer=Feature.SerializeToString,
+      ),
+      'RecordRoute': grpc.stream_unary_rpc_method_handler(
+          servicer.RecordRoute,
+          request_deserializer=Point.FromString,
+          response_serializer=RouteSummary.SerializeToString,
+      ),
+      'RouteChat': grpc.stream_stream_rpc_method_handler(
+          servicer.RouteChat,
+          request_deserializer=RouteNote.FromString,
+          response_serializer=RouteNote.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'routeguide.RouteGuide', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))
+
+
+class BetaRouteGuideServicer(object):
+  """Interface exported by the server.
+  """
+  def GetFeature(self, request, context):
+    """A simple RPC.
+
+    Obtains the feature at a given position.
+
+    A feature with an empty name is returned if there's no feature at the given
+    position.
+    """
+    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+  def ListFeatures(self, request, context):
+    """A server-to-client streaming RPC.
+
+    Obtains the Features available within the given Rectangle.  Results are
+    streamed rather than returned at once (e.g. in a response message with a
+    repeated field), as the rectangle may cover a large area and contain a
+    huge number of features.
+    """
+    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+  def RecordRoute(self, request_iterator, context):
+    """A client-to-server streaming RPC.
+
+    Accepts a stream of Points on a route being traversed, returning a
+    RouteSummary when traversal is completed.
+    """
+    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+  def RouteChat(self, request_iterator, context):
+    """A Bidirectional streaming RPC.
+
+    Accepts a stream of RouteNotes sent while a route is being traversed,
+    while receiving other RouteNotes (e.g. from other users).
+    """
+    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+
+
+class BetaRouteGuideStub(object):
+  """Interface exported by the server.
+  """
+  def GetFeature(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+    """A simple RPC.
+
+    Obtains the feature at a given position.
+
+    A feature with an empty name is returned if there's no feature at the given
+    position.
+    """
+    raise NotImplementedError()
+  GetFeature.future = None
+  def ListFeatures(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+    """A server-to-client streaming RPC.
+
+    Obtains the Features available within the given Rectangle.  Results are
+    streamed rather than returned at once (e.g. in a response message with a
+    repeated field), as the rectangle may cover a large area and contain a
+    huge number of features.
+    """
+    raise NotImplementedError()
+  def RecordRoute(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
+    """A client-to-server streaming RPC.
+
+    Accepts a stream of Points on a route being traversed, returning a
+    RouteSummary when traversal is completed.
+    """
+    raise NotImplementedError()
+  RecordRoute.future = None
+  def RouteChat(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
+    """A Bidirectional streaming RPC.
+
+    Accepts a stream of RouteNotes sent while a route is being traversed,
+    while receiving other RouteNotes (e.g. from other users).
+    """
+    raise NotImplementedError()
+
+
+def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+  request_deserializers = {
+    ('routeguide.RouteGuide', 'GetFeature'): Point.FromString,
+    ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.FromString,
+    ('routeguide.RouteGuide', 'RecordRoute'): Point.FromString,
+    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
+  }
+  response_serializers = {
+    ('routeguide.RouteGuide', 'GetFeature'): Feature.SerializeToString,
+    ('routeguide.RouteGuide', 'ListFeatures'): Feature.SerializeToString,
+    ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.SerializeToString,
+    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
+  }
+  method_implementations = {
+    ('routeguide.RouteGuide', 'GetFeature'): face_utilities.unary_unary_inline(servicer.GetFeature),
+    ('routeguide.RouteGuide', 'ListFeatures'): face_utilities.unary_stream_inline(servicer.ListFeatures),
+    ('routeguide.RouteGuide', 'RecordRoute'): face_utilities.stream_unary_inline(servicer.RecordRoute),
+    ('routeguide.RouteGuide', 'RouteChat'): face_utilities.stream_stream_inline(servicer.RouteChat),
+  }
+  server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
+  return beta_implementations.server(method_implementations, options=server_options)
+
+
+def beta_create_RouteGuide_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+  request_serializers = {
+    ('routeguide.RouteGuide', 'GetFeature'): Point.SerializeToString,
+    ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.SerializeToString,
+    ('routeguide.RouteGuide', 'RecordRoute'): Point.SerializeToString,
+    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
+  }
+  response_deserializers = {
+    ('routeguide.RouteGuide', 'GetFeature'): Feature.FromString,
+    ('routeguide.RouteGuide', 'ListFeatures'): Feature.FromString,
+    ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.FromString,
+    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
+  }
+  cardinalities = {
+    'GetFeature': cardinality.Cardinality.UNARY_UNARY,
+    'ListFeatures': cardinality.Cardinality.UNARY_STREAM,
+    'RecordRoute': cardinality.Cardinality.STREAM_UNARY,
+    'RouteChat': cardinality.Cardinality.STREAM_STREAM,
+  }
+  stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
+  return beta_implementations.dynamic_stub(channel, 'routeguide.RouteGuide', cardinalities, options=stub_options)
+# @@protoc_insertion_point(module_scope)

examples/python/multiplex/route_guide_resources.py

@@ -0,0 +1,53 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Common resources used in the gRPC route guide example."""
+
+import json
+
+import route_guide_pb2
+
+
+def read_route_guide_database():
+  """Reads the route guide database.
+
+  Returns:
+    The full contents of the route guide database as a sequence of
+      route_guide_pb2.Features.
+  """
+  feature_list = []
+  with open("route_guide_db.json") as route_guide_db_file:
+    for item in json.load(route_guide_db_file):
+      feature = route_guide_pb2.Feature(
+          name=item["name"],
+          location=route_guide_pb2.Point(
+              latitude=item["location"]["latitude"],
+              longitude=item["location"]["longitude"]))
+      feature_list.append(feature)
+  return feature_list

examples/python/multiplex/run_codegen.py

@@ -0,0 +1,51 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Generates protocol messages and gRPC stubs."""
+
+from grpc.tools import protoc
+
+protoc.main(
+    (
+        '',
+        '-I../../protos',
+        '--python_out=.',
+        '--grpc_python_out=.',
+        '../../protos/helloworld.proto',
+    )
+)
+protoc.main(
+    (
+        '',
+        '-I../../protos',
+        '--python_out=.',
+        '--grpc_python_out=.',
+        '../../protos/route_guide.proto',
+    )
+)

examples/python/route_guide/route_guide_client.py

@@ -34,13 +34,11 @@ from __future__ import print_function
 import random
 import time
 
-from grpc.beta import implementations
+import grpc
 
 import route_guide_pb2
 import route_guide_resources
 
-_TIMEOUT_SECONDS = 30
-
 
 def make_route_note(message, latitude, longitude):
   return route_guide_pb2.RouteNote(
@@ -49,7 +47,7 @@ def make_route_note(message, latitude, longitude):
 
 
 def guide_get_one_feature(stub, point):
-  feature = stub.GetFeature(point, _TIMEOUT_SECONDS)
+  feature = stub.GetFeature(point)
   if not feature.location:
     print("Server returned incomplete feature")
     return
@@ -66,14 +64,12 @@ def guide_get_feature(stub):
 
 
 def guide_list_features(stub):
-  rect = route_guide_pb2.Rectangle(
+  rectangle = route_guide_pb2.Rectangle(
-      lo=route_guide_pb2.Point(
+      lo=route_guide_pb2.Point(latitude=400000000, longitude=-750000000),
-          latitude=400000000, longitude = -750000000),
+      hi=route_guide_pb2.Point(latitude=420000000, longitude=-730000000))
-      hi=route_guide_pb2.Point(
-          latitude = 420000000, longitude = -730000000))
   print("Looking for features between 40, -75 and 42, -73")
 
-  features = stub.ListFeatures(rect, _TIMEOUT_SECONDS)
+  features = stub.ListFeatures(rectangle)
 
   for feature in features:
     print("Feature called %s at %s" % (feature.name, feature.location))
@@ -90,8 +86,8 @@ def generate_route(feature_list):
 def guide_record_route(stub):
   feature_list = route_guide_resources.read_route_guide_database()
 
-  route_iter = generate_route(feature_list)
+  route_iterator = generate_route(feature_list)
-  route_summary = stub.RecordRoute(route_iter, _TIMEOUT_SECONDS)
+  route_summary = stub.RecordRoute(route_iterator)
   print("Finished trip with %s points " % route_summary.point_count)
   print("Passed %s features " % route_summary.feature_count)
   print("Travelled %s meters " % route_summary.distance)
@@ -113,14 +109,14 @@ def generate_messages():
 
 
 def guide_route_chat(stub):
-  responses = stub.RouteChat(generate_messages(), _TIMEOUT_SECONDS)
+  responses = stub.RouteChat(generate_messages())
   for response in responses:
     print("Received message %s at %s" % (response.message, response.location))
 
 
 def run():
-  channel = implementations.insecure_channel('localhost', 50051)
+  channel = grpc.insecure_channel('localhost:50051')
-  stub = route_guide_pb2.beta_create_RouteGuide_stub(channel)
+  stub = route_guide_pb2.RouteGuideStub(channel)
   print("-------------- GetFeature --------------")
   guide_get_feature(stub)
   print("-------------- ListFeatures --------------")

examples/python/route_guide/route_guide_pb2.py

@@ -277,13 +277,122 @@ _sym_db.RegisterMessage(RouteSummary)
 
 DESCRIPTOR.has_options = True
 DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.routeguideB\017RouteGuideProtoP\001\242\002\003RTG'))
-import abc
+import grpc
-import six
 from grpc.beta import implementations as beta_implementations
 from grpc.beta import interfaces as beta_interfaces
 from grpc.framework.common import cardinality
 from grpc.framework.interfaces.face import utilities as face_utilities
 
+
+class RouteGuideStub(object):
+  """Interface exported by the server.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.GetFeature = channel.unary_unary(
+        '/routeguide.RouteGuide/GetFeature',
+        request_serializer=Point.SerializeToString,
+        response_deserializer=Feature.FromString,
+        )
+    self.ListFeatures = channel.unary_stream(
+        '/routeguide.RouteGuide/ListFeatures',
+        request_serializer=Rectangle.SerializeToString,
+        response_deserializer=Feature.FromString,
+        )
+    self.RecordRoute = channel.stream_unary(
+        '/routeguide.RouteGuide/RecordRoute',
+        request_serializer=Point.SerializeToString,
+        response_deserializer=RouteSummary.FromString,
+        )
+    self.RouteChat = channel.stream_stream(
+        '/routeguide.RouteGuide/RouteChat',
+        request_serializer=RouteNote.SerializeToString,
+        response_deserializer=RouteNote.FromString,
+        )
+
+
+class RouteGuideServicer(object):
+  """Interface exported by the server.
+  """
+
+  def GetFeature(self, request, context):
+    """A simple RPC.
+
+    Obtains the feature at a given position.
+
+    A feature with an empty name is returned if there's no feature at the given
+    position.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def ListFeatures(self, request, context):
+    """A server-to-client streaming RPC.
+
+    Obtains the Features available within the given Rectangle.  Results are
+    streamed rather than returned at once (e.g. in a response message with a
+    repeated field), as the rectangle may cover a large area and contain a
+    huge number of features.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def RecordRoute(self, request_iterator, context):
+    """A client-to-server streaming RPC.
+
+    Accepts a stream of Points on a route being traversed, returning a
+    RouteSummary when traversal is completed.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def RouteChat(self, request_iterator, context):
+    """A Bidirectional streaming RPC.
+
+    Accepts a stream of RouteNotes sent while a route is being traversed,
+    while receiving other RouteNotes (e.g. from other users).
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_RouteGuideServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'GetFeature': grpc.unary_unary_rpc_method_handler(
+          servicer.GetFeature,
+          request_deserializer=Point.FromString,
+          response_serializer=Feature.SerializeToString,
+      ),
+      'ListFeatures': grpc.unary_stream_rpc_method_handler(
+          servicer.ListFeatures,
+          request_deserializer=Rectangle.FromString,
+          response_serializer=Feature.SerializeToString,
+      ),
+      'RecordRoute': grpc.stream_unary_rpc_method_handler(
+          servicer.RecordRoute,
+          request_deserializer=Point.FromString,
+          response_serializer=RouteSummary.SerializeToString,
+      ),
+      'RouteChat': grpc.stream_stream_rpc_method_handler(
+          servicer.RouteChat,
+          request_deserializer=RouteNote.FromString,
+          response_serializer=RouteNote.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'routeguide.RouteGuide', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))
+
+
 class BetaRouteGuideServicer(object):
   """Interface exported by the server.
   """
@@ -320,10 +429,11 @@ class BetaRouteGuideServicer(object):
     """
     context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
 
+
 class BetaRouteGuideStub(object):
   """Interface exported by the server.
   """
-  def GetFeature(self, request, timeout):
+  def GetFeature(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
     """A simple RPC.
 
     Obtains the feature at a given position.
@@ -333,7 +443,7 @@ class BetaRouteGuideStub(object):
     """
     raise NotImplementedError()
   GetFeature.future = None
-  def ListFeatures(self, request, timeout):
+  def ListFeatures(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
     """A server-to-client streaming RPC.
 
     Obtains the Features available within the given Rectangle.  Results are
@@ -342,7 +452,7 @@ class BetaRouteGuideStub(object):
     huge number of features.
     """
     raise NotImplementedError()
-  def RecordRoute(self, request_iterator, timeout):
+  def RecordRoute(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
     """A client-to-server streaming RPC.
 
     Accepts a stream of Points on a route being traversed, returning a
@@ -350,7 +460,7 @@ class BetaRouteGuideStub(object):
     """
     raise NotImplementedError()
   RecordRoute.future = None
-  def RouteChat(self, request_iterator, timeout):
+  def RouteChat(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
     """A Bidirectional streaming RPC.
 
     Accepts a stream of RouteNotes sent while a route is being traversed,
@@ -358,26 +468,19 @@ class BetaRouteGuideStub(object):
     """
     raise NotImplementedError()
 
+
 def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
   request_deserializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): route_guide_pb2.Point.FromString,
+    ('routeguide.RouteGuide', 'GetFeature'): Point.FromString,
-    ('routeguide.RouteGuide', 'ListFeatures'): route_guide_pb2.Rectangle.FromString,
+    ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.FromString,
-    ('routeguide.RouteGuide', 'RecordRoute'): route_guide_pb2.Point.FromString,
+    ('routeguide.RouteGuide', 'RecordRoute'): Point.FromString,
-    ('routeguide.RouteGuide', 'RouteChat'): route_guide_pb2.RouteNote.FromString,
+    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
   }
   response_serializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): route_guide_pb2.Feature.SerializeToString,
+    ('routeguide.RouteGuide', 'GetFeature'): Feature.SerializeToString,
-    ('routeguide.RouteGuide', 'ListFeatures'): route_guide_pb2.Feature.SerializeToString,
+    ('routeguide.RouteGuide', 'ListFeatures'): Feature.SerializeToString,
-    ('routeguide.RouteGuide', 'RecordRoute'): route_guide_pb2.RouteSummary.SerializeToString,
+    ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.SerializeToString,
-    ('routeguide.RouteGuide', 'RouteChat'): route_guide_pb2.RouteNote.SerializeToString,
+    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
   }
   method_implementations = {
     ('routeguide.RouteGuide', 'GetFeature'): face_utilities.unary_unary_inline(servicer.GetFeature),
@@ -388,26 +491,19 @@ def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_t
   server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
   return beta_implementations.server(method_implementations, options=server_options)
 
+
 def beta_create_RouteGuide_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
-  import route_guide_pb2
   request_serializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): route_guide_pb2.Point.SerializeToString,
+    ('routeguide.RouteGuide', 'GetFeature'): Point.SerializeToString,
-    ('routeguide.RouteGuide', 'ListFeatures'): route_guide_pb2.Rectangle.SerializeToString,
+    ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.SerializeToString,
-    ('routeguide.RouteGuide', 'RecordRoute'): route_guide_pb2.Point.SerializeToString,
+    ('routeguide.RouteGuide', 'RecordRoute'): Point.SerializeToString,
-    ('routeguide.RouteGuide', 'RouteChat'): route_guide_pb2.RouteNote.SerializeToString,
+    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
   }
   response_deserializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): route_guide_pb2.Feature.FromString,
+    ('routeguide.RouteGuide', 'GetFeature'): Feature.FromString,
-    ('routeguide.RouteGuide', 'ListFeatures'): route_guide_pb2.Feature.FromString,
+    ('routeguide.RouteGuide', 'ListFeatures'): Feature.FromString,
-    ('routeguide.RouteGuide', 'RecordRoute'): route_guide_pb2.RouteSummary.FromString,
+    ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.FromString,
-    ('routeguide.RouteGuide', 'RouteChat'): route_guide_pb2.RouteNote.FromString,
+    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
   }
   cardinalities = {
     'GetFeature': cardinality.Cardinality.UNARY_UNARY,

examples/python/route_guide/route_guide_server.py

@@ -29,9 +29,12 @@
 
 """The Python implementation of the gRPC route guide server."""
 
+from concurrent import futures
 import time
 import math
 
+import grpc
+
 import route_guide_pb2
 import route_guide_resources
 
@@ -121,7 +124,9 @@ class RouteGuideServicer(route_guide_pb2.BetaRouteGuideServicer):
 
 
 def serve():
-  server = route_guide_pb2.beta_create_RouteGuide_server(RouteGuideServicer())
+  server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+  route_guide_pb2.add_RouteGuideServicer_to_server(
+      RouteGuideServicer(), server)
   server.add_insecure_port('[::]:50051')
   server.start()
   try:

examples/python/route_guide/run_codegen.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env ruby
+# Copyright 2015, Google Inc.
-# Copyright 2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -28,14 +27,16 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-require 'rbconfig'
+"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
 
-require_relative '../os_check'
+from grpc.tools import protoc
 
-protoc_name = 'protoc' + RbConfig::CONFIG['EXEEXT']
+protoc.main(
-
+    (
-protoc_path = File.join(File.dirname(__FILE__),
+	'',
-                        RbConfig::CONFIG['host_cpu'] + '-' + OS.os_name,
+	'-I../../protos',
-                        protoc_name)
+	'--python_out=.',
-
+	'--grpc_python_out=.',
-exec([ protoc_path, protoc_path ], *ARGV)
+	'../../protos/route_guide.proto',
+    )
+)

gRPC-Core.podspec

@@ -35,7 +35,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-Core'
-  version = '0.14.0'
+  version = '1.0.0-pre1'
   s.version  = version
   s.summary  = 'Core cross-platform gRPC library, written in C'
   s.homepage = 'http://www.grpc.io'
@@ -44,7 +44,7 @@ Pod::Spec.new do |s|
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
-    :tag => "release-#{version.gsub(/\./, '_')}-objectivec-#{version}",
+    :tag => "objective-c-v#{version}",
     # TODO(jcanizales): Depend explicitly on the nanopb pod, and disable submodules.
     :submodules => true,
   }
@@ -101,6 +101,8 @@ Pod::Spec.new do |s|
     'ALWAYS_SEARCH_USER_PATHS' => 'NO',
   }
 
+  s.default_subspecs = 'Interface', 'Implementation'
+
   # Like many other C libraries, gRPC-Core has its public headers under `include/<libname>/` and its
   # sources and private headers in other directories outside `include/`. Cocoapods' linter doesn't
   # allow any header to be listed outside the `header_mappings_dir` (even though doing so works in
@@ -191,7 +193,7 @@ Pod::Spec.new do |s|
     ss.header_mappings_dir = '.'
     ss.libraries = 'z'
     ss.dependency "#{s.name}/Interface", version
-    ss.dependency 'BoringSSL', '~> 4.0'
+    ss.dependency 'BoringSSL', '~> 5.0'
 
     # To save you from scrolling, this is the last part of the podspec.
     ss.source_files = 'src/core/lib/profiling/timers.h',
@@ -256,6 +258,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/channel/compress_filter.h',
                       'src/core/lib/channel/connected_channel.h',
                       'src/core/lib/channel/context.h',
+                      'src/core/lib/channel/handshaker.h',
                       'src/core/lib/channel/http_client_filter.h',
                       'src/core/lib/channel/http_server_filter.h',
                       'src/core/lib/compression/algorithm_metadata.h',
@@ -412,6 +415,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/channel/channel_stack_builder.c',
                       'src/core/lib/channel/compress_filter.c',
                       'src/core/lib/channel/connected_channel.c',
+                      'src/core/lib/channel/handshaker.c',
                       'src/core/lib/channel/http_client_filter.c',
                       'src/core/lib/channel/http_server_filter.c',
                       'src/core/lib/compression/compression.c',
@@ -611,6 +615,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/channel/compress_filter.h',
                               'src/core/lib/channel/connected_channel.h',
                               'src/core/lib/channel/context.h',
+                              'src/core/lib/channel/handshaker.h',
                               'src/core/lib/channel/http_client_filter.h',
                               'src/core/lib/channel/http_server_filter.h',
                               'src/core/lib/compression/algorithm_metadata.h',
@@ -762,4 +767,26 @@ Pod::Spec.new do |s|
                               'src/core/ext/census/mlog.h',
                               'src/core/ext/census/rpc_metric_id.h'
   end
+
+  s.subspec 'Cronet-Interface' do |ss|
+    ss.header_mappings_dir = 'include/grpc'
+    ss.source_files = 'include/grpc/grpc_cronet.h'
+  end
+
+  s.subspec 'Cronet-Tests' do |ss|
+    ss.header_mappings_dir = '.'
+
+    ss.source_files = 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c',
+                      'src/core/ext/transport/cronet/transport/cronet_transport.c',
+                      'test/core/end2end/cq_verifier.{c,h}',
+                      'test/core/end2end/end2end_tests.{c,h}',
+                      'test/core/end2end/tests/*.{c,h}',
+                      'test/core/end2end/data/*.{c,h}',
+                      'test/core/util/test_config.{c,h}',
+                      'test/core/util/port.h',
+                      'test/core/util/port_posix.c',
+                      'test/core/util/port_server_client.{c,h}'
+
+    ss.dependency 'CronetFramework'
+  end
 end

gRPC-ProtoRPC.podspec

@@ -36,7 +36,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-ProtoRPC'
-  version = '0.14.0'
+  version = '1.0.0-pre1'
   s.version  = version
   s.summary  = 'RPC library for Protocol Buffers, based on gRPC'
   s.homepage = 'http://www.grpc.io'
@@ -45,7 +45,7 @@ Pod::Spec.new do |s|
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
-    :tag => "release-#{version.gsub(/\./, '_')}-objectivec-#{version}",
+    :tag => "objective-c-v#{version}",
   }
 
   s.ios.deployment_target = '7.1'

gRPC-RxLibrary.podspec

@@ -36,7 +36,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-RxLibrary'
-  version = '0.14.0'
+  version = '1.0.0-pre1'
   s.version  = version
   s.summary  = 'Reactive Extensions library for iOS/OSX.'
   s.homepage = 'http://www.grpc.io'
@@ -45,7 +45,7 @@ Pod::Spec.new do |s|
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
-    :tag => "release-#{version.gsub(/\./, '_')}-objectivec-#{version}",
+    :tag => "objective-c-v#{version}",
   }
 
   s.ios.deployment_target = '7.1'

gRPC.podspec

@@ -36,7 +36,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC'
-  version = '0.14.0'
+  version = '1.0.0-pre1'
   s.version  = version
   s.summary  = 'gRPC client library for iOS/OSX'
   s.homepage = 'http://www.grpc.io'
@@ -45,7 +45,7 @@ Pod::Spec.new do |s|
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
-    :tag => "release-#{version.gsub(/\./, '_')}-objectivec-#{version}",
+    :tag => "objective-c-v#{version}",
   }
 
   s.ios.deployment_target = '7.1'

grpc.def

@@ -148,6 +148,7 @@ EXPORTS
     gpr_slice_ref
     gpr_slice_unref
     gpr_slice_new
+    gpr_slice_new_with_user_data
     gpr_slice_new_with_len
     gpr_slice_malloc
     gpr_slice_from_copied_string

grpc.gemspec

@@ -177,6 +177,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/channel/compress_filter.h )
   s.files += %w( src/core/lib/channel/connected_channel.h )
   s.files += %w( src/core/lib/channel/context.h )
+  s.files += %w( src/core/lib/channel/handshaker.h )
   s.files += %w( src/core/lib/channel/http_client_filter.h )
   s.files += %w( src/core/lib/channel/http_server_filter.h )
   s.files += %w( src/core/lib/compression/algorithm_metadata.h )
@@ -333,6 +334,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/channel/channel_stack_builder.c )
   s.files += %w( src/core/lib/channel/compress_filter.c )
   s.files += %w( src/core/lib/channel/connected_channel.c )
+  s.files += %w( src/core/lib/channel/handshaker.c )
   s.files += %w( src/core/lib/channel/http_client_filter.c )
   s.files += %w( src/core/lib/channel/http_server_filter.c )
   s.files += %w( src/core/lib/compression/compression.c )

include/grpc++/impl/codegen/async_stream.h

@@ -330,6 +330,9 @@ class ServerAsyncReader GRPC_FINAL : public ServerAsyncReaderInterface<W, R> {
     meta_ops_.set_output_tag(tag);
     meta_ops_.SendInitialMetadata(ctx_->initial_metadata_,
                                   ctx_->initial_metadata_flags());
+    if (ctx_->compression_level_set()) {
+      meta_ops_.set_compression_level(ctx_->compression_level());
+    }
     ctx_->sent_initial_metadata_ = true;
     call_.PerformOps(&meta_ops_);
   }
@@ -345,6 +348,9 @@ class ServerAsyncReader GRPC_FINAL : public ServerAsyncReaderInterface<W, R> {
     if (!ctx_->sent_initial_metadata_) {
       finish_ops_.SendInitialMetadata(ctx_->initial_metadata_,
                                       ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        finish_ops_.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     // The response is dropped if the status is not OK.
@@ -363,6 +369,9 @@ class ServerAsyncReader GRPC_FINAL : public ServerAsyncReaderInterface<W, R> {
     if (!ctx_->sent_initial_metadata_) {
       finish_ops_.SendInitialMetadata(ctx_->initial_metadata_,
                                       ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        finish_ops_.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
@@ -400,6 +409,9 @@ class ServerAsyncWriter GRPC_FINAL : public ServerAsyncWriterInterface<W> {
     meta_ops_.set_output_tag(tag);
     meta_ops_.SendInitialMetadata(ctx_->initial_metadata_,
                                   ctx_->initial_metadata_flags());
+    if (ctx_->compression_level_set()) {
+      meta_ops_.set_compression_level(ctx_->compression_level());
+    }
     ctx_->sent_initial_metadata_ = true;
     call_.PerformOps(&meta_ops_);
   }
@@ -409,6 +421,9 @@ class ServerAsyncWriter GRPC_FINAL : public ServerAsyncWriterInterface<W> {
     if (!ctx_->sent_initial_metadata_) {
       write_ops_.SendInitialMetadata(ctx_->initial_metadata_,
                                      ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        write_ops_.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     // TODO(ctiller): don't assert
@@ -421,6 +436,9 @@ class ServerAsyncWriter GRPC_FINAL : public ServerAsyncWriterInterface<W> {
     if (!ctx_->sent_initial_metadata_) {
       finish_ops_.SendInitialMetadata(ctx_->initial_metadata_,
                                       ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        finish_ops_.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
@@ -459,6 +477,9 @@ class ServerAsyncReaderWriter GRPC_FINAL
     meta_ops_.set_output_tag(tag);
     meta_ops_.SendInitialMetadata(ctx_->initial_metadata_,
                                   ctx_->initial_metadata_flags());
+    if (ctx_->compression_level_set()) {
+      meta_ops_.set_compression_level(ctx_->compression_level());
+    }
     ctx_->sent_initial_metadata_ = true;
     call_.PerformOps(&meta_ops_);
   }
@@ -474,6 +495,9 @@ class ServerAsyncReaderWriter GRPC_FINAL
     if (!ctx_->sent_initial_metadata_) {
       write_ops_.SendInitialMetadata(ctx_->initial_metadata_,
                                      ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        write_ops_.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     // TODO(ctiller): don't assert
@@ -486,6 +510,9 @@ class ServerAsyncReaderWriter GRPC_FINAL
     if (!ctx_->sent_initial_metadata_) {
       finish_ops_.SendInitialMetadata(ctx_->initial_metadata_,
                                       ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        finish_ops_.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);

include/grpc++/impl/codegen/async_unary_call.h

@@ -65,7 +65,7 @@ class ClientAsyncResponseReader GRPC_FINAL
                             const W& request)
       : context_(context),
         call_(channel->CreateCall(method, context, cq)),
-        collection_(new CallOpSetCollection) {
+        collection_(std::make_shared<CallOpSetCollection>()) {
     collection_->init_buf_.SetCollection(collection_);
     collection_->init_buf_.SendInitialMetadata(
         context->send_initial_metadata_, context->initial_metadata_flags());
@@ -126,6 +126,9 @@ class ServerAsyncResponseWriter GRPC_FINAL
     meta_buf_.set_output_tag(tag);
     meta_buf_.SendInitialMetadata(ctx_->initial_metadata_,
                                   ctx_->initial_metadata_flags());
+    if (ctx_->compression_level_set()) {
+      meta_buf_.set_compression_level(ctx_->compression_level());
+    }
     ctx_->sent_initial_metadata_ = true;
     call_.PerformOps(&meta_buf_);
   }
@@ -135,6 +138,9 @@ class ServerAsyncResponseWriter GRPC_FINAL
     if (!ctx_->sent_initial_metadata_) {
       finish_buf_.SendInitialMetadata(ctx_->initial_metadata_,
                                       ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        finish_buf_.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     // The response is dropped if the status is not OK.
@@ -153,6 +159,9 @@ class ServerAsyncResponseWriter GRPC_FINAL
     if (!ctx_->sent_initial_metadata_) {
       finish_buf_.SendInitialMetadata(ctx_->initial_metadata_,
                                       ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        finish_buf_.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     finish_buf_.ServerSendStatus(ctx_->trailing_metadata_, status);

include/grpc++/impl/codegen/call.h

@@ -180,17 +180,23 @@ class CallNoOp {
 
 class CallOpSendInitialMetadata {
  public:
-  CallOpSendInitialMetadata() : send_(false) {}
+  CallOpSendInitialMetadata() : send_(false) {
+    maybe_compression_level_.is_set = false;
+  }
 
   void SendInitialMetadata(
       const std::multimap<grpc::string, grpc::string>& metadata,
       uint32_t flags) {
+    maybe_compression_level_.is_set = false;
     send_ = true;
     flags_ = flags;
     initial_metadata_count_ = metadata.size();
     initial_metadata_ = FillMetadataArray(metadata);
-    // TODO(dgq): expose compression level in API so it can be properly set.
+  }
-    maybe_compression_level_.is_set = false;
+
+  void set_compression_level(grpc_compression_level level) {
+    maybe_compression_level_.is_set = true;
+    maybe_compression_level_.level = level;
   }
 
  protected:

include/grpc++/impl/codegen/client_context.h

@@ -41,7 +41,7 @@
 ///
 /// Context settings are only relevant to the call they are invoked with, that
 /// is to say, they aren't sticky. Some of these settings, such as the
-/// compression options, can be made persistant at channel construction time
+/// compression options, can be made persistent at channel construction time
 /// (see \a grpc::CreateCustomChannel).
 ///
 /// \warning ClientContext instances should \em not be reused across rpcs.

include/grpc++/impl/codegen/config_protobuf.h

@@ -40,16 +40,21 @@
 #endif
 
 #ifndef GRPC_CUSTOM_MESSAGE
+#ifdef GRPC_USE_PROTO_LITE
+#include <google/protobuf/message_lite.h>
+#define GRPC_CUSTOM_MESSAGE ::google::protobuf::MessageLite
+#else
 #include <google/protobuf/message.h>
 #define GRPC_CUSTOM_MESSAGE ::google::protobuf::Message
 #endif
+#endif
 
 #ifndef GRPC_CUSTOM_DESCRIPTOR
 #include <google/protobuf/descriptor.h>
 #include <google/protobuf/descriptor.pb.h>
 #define GRPC_CUSTOM_DESCRIPTOR ::google::protobuf::Descriptor
 #define GRPC_CUSTOM_DESCRIPTORPOOL ::google::protobuf::DescriptorPool
-#define GPRC_CUSTOM_FIELDDESCRIPTOR ::google::protobuf::FieldDescriptor
+#define GRPC_CUSTOM_FIELDDESCRIPTOR ::google::protobuf::FieldDescriptor
 #define GRPC_CUSTOM_FILEDESCRIPTOR ::google::protobuf::FileDescriptor
 #define GRPC_CUSTOM_FILEDESCRIPTORPROTO ::google::protobuf::FileDescriptorProto
 #define GRPC_CUSTOM_METHODDESCRIPTOR ::google::protobuf::MethodDescriptor
@@ -57,6 +62,13 @@
 #define GRPC_CUSTOM_SOURCELOCATION ::google::protobuf::SourceLocation
 #endif
 
+#ifndef GRPC_CUSTOM_DESCRIPTORDATABASE
+#include <google/protobuf/descriptor_database.h>
+#define GRPC_CUSTOM_DESCRIPTORDATABASE ::google::protobuf::DescriptorDatabase
+#define GRPC_CUSTOM_SIMPLEDESCRIPTORDATABASE \
+  ::google::protobuf::SimpleDescriptorDatabase
+#endif
+
 #ifndef GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/io/zero_copy_stream.h>
@@ -75,11 +87,13 @@ typedef GRPC_CUSTOM_PROTOBUF_INT64 int64;
 
 typedef GRPC_CUSTOM_DESCRIPTOR Descriptor;
 typedef GRPC_CUSTOM_DESCRIPTORPOOL DescriptorPool;
-typedef GPRC_CUSTOM_FIELDDESCRIPTOR FieldDescriptor;
+typedef GRPC_CUSTOM_DESCRIPTORDATABASE DescriptorDatabase;
+typedef GRPC_CUSTOM_FIELDDESCRIPTOR FieldDescriptor;
 typedef GRPC_CUSTOM_FILEDESCRIPTOR FileDescriptor;
 typedef GRPC_CUSTOM_FILEDESCRIPTORPROTO FileDescriptorProto;
 typedef GRPC_CUSTOM_METHODDESCRIPTOR MethodDescriptor;
 typedef GRPC_CUSTOM_SERVICEDESCRIPTOR ServiceDescriptor;
+typedef GRPC_CUSTOM_SIMPLEDESCRIPTORDATABASE SimpleDescriptorDatabase;
 typedef GRPC_CUSTOM_SOURCELOCATION SourceLocation;
 
 namespace io {

include/grpc++/impl/codegen/impl/async_stream.h

@@ -1,486 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef GRPCXX_IMPL_CODEGEN_IMPL_ASYNC_STREAM_H
-#define GRPCXX_IMPL_CODEGEN_IMPL_ASYNC_STREAM_H
-
-#include <grpc++/impl/codegen/call.h>
-#include <grpc++/impl/codegen/channel_interface.h>
-#include <grpc++/impl/codegen/core_codegen_interface.h>
-#include <grpc++/impl/codegen/server_context.h>
-#include <grpc++/impl/codegen/service_type.h>
-#include <grpc++/impl/codegen/status.h>
-
-namespace grpc {
-
-class CompletionQueue;
-
-/// Common interface for all client side asynchronous streaming.
-class ClientAsyncStreamingInterface {
- public:
-  virtual ~ClientAsyncStreamingInterface() {}
-
-  /// Request notification of the reading of the initial metadata. Completion
-  /// will be notified by \a tag on the associated completion queue.
-  ///
-  /// \param[in] tag Tag identifying this request.
-  virtual void ReadInitialMetadata(void* tag) = 0;
-
-  /// Request notification completion.
-  ///
-  /// \param[out] status To be updated with the operation status.
-  /// \param[in] tag Tag identifying this request.
-  virtual void Finish(Status* status, void* tag) = 0;
-};
-
-/// An interface that yields a sequence of messages of type \a R.
-template <class R>
-class AsyncReaderInterface {
- public:
-  virtual ~AsyncReaderInterface() {}
-
-  /// Read a message of type \a R into \a msg. Completion will be notified by \a
-  /// tag on the associated completion queue.
-  ///
-  /// \param[out] msg Where to eventually store the read message.
-  /// \param[in] tag The tag identifying the operation.
-  virtual void Read(R* msg, void* tag) = 0;
-};
-
-/// An interface that can be fed a sequence of messages of type \a W.
-template <class W>
-class AsyncWriterInterface {
- public:
-  virtual ~AsyncWriterInterface() {}
-
-  /// Request the writing of \a msg with identifying tag \a tag.
-  ///
-  /// Only one write may be outstanding at any given time. This means that
-  /// after calling Write, one must wait to receive \a tag from the completion
-  /// queue BEFORE calling Write again.
-  ///
-  /// \param[in] msg The message to be written.
-  /// \param[in] tag The tag identifying the operation.
-  virtual void Write(const W& msg, void* tag) = 0;
-};
-
-template <class R>
-class ClientAsyncReaderInterface : public ClientAsyncStreamingInterface,
-                                   public AsyncReaderInterface<R> {};
-
-template <class R>
-class ClientAsyncReader GRPC_FINAL : public ClientAsyncReaderInterface<R> {
- public:
-  /// Create a stream and write the first request out.
-  template <class W>
-  ClientAsyncReader(ChannelInterface* channel, CompletionQueue* cq,
-                    const RpcMethod& method, ClientContext* context,
-                    const W& request, void* tag)
-      : context_(context), call_(channel->CreateCall(method, context, cq)) {
-    init_ops_.set_output_tag(tag);
-    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
-    // TODO(ctiller): don't assert
-    GPR_CODEGEN_ASSERT(init_ops_.SendMessage(request).ok());
-    init_ops_.ClientSendClose();
-    call_.PerformOps(&init_ops_);
-  }
-
-  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.RecvInitialMetadata(context_);
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      read_ops_.RecvInitialMetadata(context_);
-    }
-    read_ops_.RecvMessage(msg);
-    call_.PerformOps(&read_ops_);
-  }
-
-  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
-    finish_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      finish_ops_.RecvInitialMetadata(context_);
-    }
-    finish_ops_.ClientRecvStatus(context_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  ClientContext* context_;
-  Call call_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose>
-      init_ops_;
-  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
-};
-
-/// Common interface for client side asynchronous writing.
-template <class W>
-class ClientAsyncWriterInterface : public ClientAsyncStreamingInterface,
-                                   public AsyncWriterInterface<W> {
- public:
-  /// Signal the client is done with the writes.
-  ///
-  /// \param[in] tag The tag identifying the operation.
-  virtual void WritesDone(void* tag) = 0;
-};
-
-template <class W>
-class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
- public:
-  template <class R>
-  ClientAsyncWriter(ChannelInterface* channel, CompletionQueue* cq,
-                    const RpcMethod& method, ClientContext* context,
-                    R* response, void* tag)
-      : context_(context), call_(channel->CreateCall(method, context, cq)) {
-    finish_ops_.RecvMessage(response);
-
-    init_ops_.set_output_tag(tag);
-    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
-    call_.PerformOps(&init_ops_);
-  }
-
-  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.RecvInitialMetadata(context_);
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_ops_.set_output_tag(tag);
-    // TODO(ctiller): don't assert
-    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
-    call_.PerformOps(&write_ops_);
-  }
-
-  void WritesDone(void* tag) GRPC_OVERRIDE {
-    writes_done_ops_.set_output_tag(tag);
-    writes_done_ops_.ClientSendClose();
-    call_.PerformOps(&writes_done_ops_);
-  }
-
-  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
-    finish_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      finish_ops_.RecvInitialMetadata(context_);
-    }
-    finish_ops_.ClientRecvStatus(context_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  ClientContext* context_;
-  Call call_;
-  CallOpSet<CallOpSendInitialMetadata> init_ops_;
-  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
-  CallOpSet<CallOpSendMessage> write_ops_;
-  CallOpSet<CallOpClientSendClose> writes_done_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpGenericRecvMessage,
-            CallOpClientRecvStatus>
-      finish_ops_;
-};
-
-/// Client-side interface for asynchronous bi-directional streaming.
-template <class W, class R>
-class ClientAsyncReaderWriterInterface : public ClientAsyncStreamingInterface,
-                                         public AsyncWriterInterface<W>,
-                                         public AsyncReaderInterface<R> {
- public:
-  /// Signal the client is done with the writes.
-  ///
-  /// \param[in] tag The tag identifying the operation.
-  virtual void WritesDone(void* tag) = 0;
-};
-
-template <class W, class R>
-class ClientAsyncReaderWriter GRPC_FINAL
-    : public ClientAsyncReaderWriterInterface<W, R> {
- public:
-  ClientAsyncReaderWriter(ChannelInterface* channel, CompletionQueue* cq,
-                          const RpcMethod& method, ClientContext* context,
-                          void* tag)
-      : context_(context), call_(channel->CreateCall(method, context, cq)) {
-    init_ops_.set_output_tag(tag);
-    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
-    call_.PerformOps(&init_ops_);
-  }
-
-  void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.RecvInitialMetadata(context_);
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      read_ops_.RecvInitialMetadata(context_);
-    }
-    read_ops_.RecvMessage(msg);
-    call_.PerformOps(&read_ops_);
-  }
-
-  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_ops_.set_output_tag(tag);
-    // TODO(ctiller): don't assert
-    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
-    call_.PerformOps(&write_ops_);
-  }
-
-  void WritesDone(void* tag) GRPC_OVERRIDE {
-    writes_done_ops_.set_output_tag(tag);
-    writes_done_ops_.ClientSendClose();
-    call_.PerformOps(&writes_done_ops_);
-  }
-
-  void Finish(Status* status, void* tag) GRPC_OVERRIDE {
-    finish_ops_.set_output_tag(tag);
-    if (!context_->initial_metadata_received_) {
-      finish_ops_.RecvInitialMetadata(context_);
-    }
-    finish_ops_.ClientRecvStatus(context_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  ClientContext* context_;
-  Call call_;
-  CallOpSet<CallOpSendInitialMetadata> init_ops_;
-  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
-  CallOpSet<CallOpSendMessage> write_ops_;
-  CallOpSet<CallOpClientSendClose> writes_done_ops_;
-  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
-};
-
-template <class W, class R>
-class ServerAsyncReaderInterface : public ServerAsyncStreamingInterface,
-                                   public AsyncReaderInterface<R> {
- public:
-  virtual void Finish(const W& msg, const Status& status, void* tag) = 0;
-
-  virtual void FinishWithError(const Status& status, void* tag) = 0;
-};
-
-template <class W, class R>
-class ServerAsyncReader GRPC_FINAL : public ServerAsyncReaderInterface<W, R> {
- public:
-  explicit ServerAsyncReader(ServerContext* ctx)
-      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
-
-  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-    ctx_->sent_initial_metadata_ = true;
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_ops_.set_output_tag(tag);
-    read_ops_.RecvMessage(msg);
-    call_.PerformOps(&read_ops_);
-  }
-
-  void Finish(const W& msg, const Status& status, void* tag) GRPC_OVERRIDE {
-    finish_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    // The response is dropped if the status is not OK.
-    if (status.ok()) {
-      finish_ops_.ServerSendStatus(ctx_->trailing_metadata_,
-                                   finish_ops_.SendMessage(msg));
-    } else {
-      finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
-    }
-    call_.PerformOps(&finish_ops_);
-  }
-
-  void FinishWithError(const Status& status, void* tag) GRPC_OVERRIDE {
-    GPR_CODEGEN_ASSERT(!status.ok());
-    finish_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
-
-  Call call_;
-  ServerContext* ctx_;
-  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
-  CallOpSet<CallOpRecvMessage<R>> read_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
-            CallOpServerSendStatus>
-      finish_ops_;
-};
-
-template <class W>
-class ServerAsyncWriterInterface : public ServerAsyncStreamingInterface,
-                                   public AsyncWriterInterface<W> {
- public:
-  virtual void Finish(const Status& status, void* tag) = 0;
-};
-
-template <class W>
-class ServerAsyncWriter GRPC_FINAL : public ServerAsyncWriterInterface<W> {
- public:
-  explicit ServerAsyncWriter(ServerContext* ctx)
-      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
-
-  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-    ctx_->sent_initial_metadata_ = true;
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    // TODO(ctiller): don't assert
-    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
-    call_.PerformOps(&write_ops_);
-  }
-
-  void Finish(const Status& status, void* tag) GRPC_OVERRIDE {
-    finish_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
-
-  Call call_;
-  ServerContext* ctx_;
-  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
-};
-
-/// Server-side interface for asynchronous bi-directional streaming.
-template <class W, class R>
-class ServerAsyncReaderWriterInterface : public ServerAsyncStreamingInterface,
-                                         public AsyncWriterInterface<W>,
-                                         public AsyncReaderInterface<R> {
- public:
-  virtual void Finish(const Status& status, void* tag) = 0;
-};
-
-template <class W, class R>
-class ServerAsyncReaderWriter GRPC_FINAL
-    : public ServerAsyncReaderWriterInterface<W, R> {
- public:
-  explicit ServerAsyncReaderWriter(ServerContext* ctx)
-      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
-
-  void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
-    GPR_CODEGEN_ASSERT(!ctx_->sent_initial_metadata_);
-
-    meta_ops_.set_output_tag(tag);
-    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-    ctx_->sent_initial_metadata_ = true;
-    call_.PerformOps(&meta_ops_);
-  }
-
-  void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_ops_.set_output_tag(tag);
-    read_ops_.RecvMessage(msg);
-    call_.PerformOps(&read_ops_);
-  }
-
-  void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    // TODO(ctiller): don't assert
-    GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok());
-    call_.PerformOps(&write_ops_);
-  }
-
-  void Finish(const Status& status, void* tag) GRPC_OVERRIDE {
-    finish_ops_.set_output_tag(tag);
-    if (!ctx_->sent_initial_metadata_) {
-      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
-      ctx_->sent_initial_metadata_ = true;
-    }
-    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_ops_);
-  }
-
- private:
-  friend class ::grpc::Server;
-
-  void BindCall(Call* call) GRPC_OVERRIDE { call_ = *call; }
-
-  Call call_;
-  ServerContext* ctx_;
-  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
-  CallOpSet<CallOpRecvMessage<R>> read_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
-  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
-};
-
-}  // namespace grpc
-
-#endif  // GRPCXX_IMPL_CODEGEN_IMPL_ASYNC_STREAM_H

include/grpc++/impl/codegen/method_handler_impl.h

@@ -65,6 +65,9 @@ class RpcMethodHandler : public MethodHandler {
         ops;
     ops.SendInitialMetadata(param.server_context->initial_metadata_,
                             param.server_context->initial_metadata_flags());
+    if (param.server_context->compression_level_set()) {
+      ops.set_compression_level(param.server_context->compression_level());
+    }
     if (status.ok()) {
       status = ops.SendMessage(rsp);
     }
@@ -104,6 +107,9 @@ class ClientStreamingHandler : public MethodHandler {
         ops;
     ops.SendInitialMetadata(param.server_context->initial_metadata_,
                             param.server_context->initial_metadata_flags());
+    if (param.server_context->compression_level_set()) {
+      ops.set_compression_level(param.server_context->compression_level());
+    }
     if (status.ok()) {
       status = ops.SendMessage(rsp);
     }
@@ -144,6 +150,9 @@ class ServerStreamingHandler : public MethodHandler {
     if (!param.server_context->sent_initial_metadata_) {
       ops.SendInitialMetadata(param.server_context->initial_metadata_,
                               param.server_context->initial_metadata_flags());
+      if (param.server_context->compression_level_set()) {
+        ops.set_compression_level(param.server_context->compression_level());
+      }
     }
     ops.ServerSendStatus(param.server_context->trailing_metadata_, status);
     param.call->PerformOps(&ops);
@@ -177,6 +186,9 @@ class BidiStreamingHandler : public MethodHandler {
     if (!param.server_context->sent_initial_metadata_) {
       ops.SendInitialMetadata(param.server_context->initial_metadata_,
                               param.server_context->initial_metadata_flags());
+      if (param.server_context->compression_level_set()) {
+        ops.set_compression_level(param.server_context->compression_level());
+      }
     }
     ops.ServerSendStatus(param.server_context->trailing_metadata_, status);
     param.call->PerformOps(&ops);
@@ -199,6 +211,9 @@ class UnknownMethodHandler : public MethodHandler {
     if (!context->sent_initial_metadata_) {
       ops->SendInitialMetadata(context->initial_metadata_,
                                context->initial_metadata_flags());
+      if (context->compression_level_set()) {
+        ops->set_compression_level(context->compression_level());
+      }
       context->sent_initial_metadata_ = true;
     }
     ops->ServerSendStatus(context->trailing_metadata_, status);

include/grpc++/impl/codegen/server_context.h

@@ -130,7 +130,13 @@ class ServerContext {
   grpc_compression_level compression_level() const {
     return compression_level_;
   }
-  void set_compression_level(grpc_compression_level level);
+
+  void set_compression_level(grpc_compression_level level) {
+    compression_level_set_ = true;
+    compression_level_ = level;
+  }
+
+  bool compression_level_set() const { return compression_level_set_; }
 
   grpc_compression_algorithm compression_algorithm() const {
     return compression_algorithm_;
@@ -217,6 +223,7 @@ class ServerContext {
   std::multimap<grpc::string, grpc::string> initial_metadata_;
   std::multimap<grpc::string, grpc::string> trailing_metadata_;
 
+  bool compression_level_set_;
   grpc_compression_level compression_level_;
   grpc_compression_algorithm compression_algorithm_;
 };

include/grpc++/impl/codegen/sync_stream.h

@@ -347,6 +347,9 @@ class ServerReader GRPC_FINAL : public ReaderInterface<R> {
     CallOpSet<CallOpSendInitialMetadata> ops;
     ops.SendInitialMetadata(ctx_->initial_metadata_,
                             ctx_->initial_metadata_flags());
+    if (ctx_->compression_level_set()) {
+      ops.set_compression_level(ctx_->compression_level());
+    }
     ctx_->sent_initial_metadata_ = true;
     call_->PerformOps(&ops);
     call_->cq()->Pluck(&ops);
@@ -375,6 +378,9 @@ class ServerWriter GRPC_FINAL : public WriterInterface<W> {
     CallOpSet<CallOpSendInitialMetadata> ops;
     ops.SendInitialMetadata(ctx_->initial_metadata_,
                             ctx_->initial_metadata_flags());
+    if (ctx_->compression_level_set()) {
+      ops.set_compression_level(ctx_->compression_level());
+    }
     ctx_->sent_initial_metadata_ = true;
     call_->PerformOps(&ops);
     call_->cq()->Pluck(&ops);
@@ -389,6 +395,9 @@ class ServerWriter GRPC_FINAL : public WriterInterface<W> {
     if (!ctx_->sent_initial_metadata_) {
       ops.SendInitialMetadata(ctx_->initial_metadata_,
                               ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        ops.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     call_->PerformOps(&ops);
@@ -413,6 +422,9 @@ class ServerReaderWriter GRPC_FINAL : public WriterInterface<W>,
     CallOpSet<CallOpSendInitialMetadata> ops;
     ops.SendInitialMetadata(ctx_->initial_metadata_,
                             ctx_->initial_metadata_flags());
+    if (ctx_->compression_level_set()) {
+      ops.set_compression_level(ctx_->compression_level());
+    }
     ctx_->sent_initial_metadata_ = true;
     call_->PerformOps(&ops);
     call_->cq()->Pluck(&ops);
@@ -434,6 +446,9 @@ class ServerReaderWriter GRPC_FINAL : public WriterInterface<W>,
     if (!ctx_->sent_initial_metadata_) {
       ops.SendInitialMetadata(ctx_->initial_metadata_,
                               ctx_->initial_metadata_flags());
+      if (ctx_->compression_level_set()) {
+        ops.set_compression_level(ctx_->compression_level());
+      }
       ctx_->sent_initial_metadata_ = true;
     }
     call_->PerformOps(&ops);

include/grpc++/server.h

@@ -179,10 +179,13 @@ class Server GRPC_FINAL : public ServerInterface, private GrpcLibraryCodegen {
   grpc::mutex mu_;
   bool started_;
   bool shutdown_;
+  bool shutdown_notified_;
   // The number of threads which are running callbacks.
   int num_running_cb_;
   grpc::condition_variable callback_cv_;
 
+  grpc::condition_variable shutdown_cv_;
+
   std::shared_ptr<GlobalCallbacks> global_callbacks_;
 
   std::list<SyncRequest>* sync_methods_;

include/grpc/impl/codegen/compression_types.h

@@ -46,12 +46,27 @@ extern "C" {
 #define GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY \
   "grpc-internal-encoding-request"
 
-/** To be used in channel arguments */
+/** To be used in channel arguments.
+ *
+ * \addtogroup grpc_arg_keys
+ * \{ */
+/** Default compression algorithm for the channel.
+ * Its value is an int from the \a grpc_compression_algorithm enum. */
 #define GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM \
   "grpc.default_compression_algorithm"
+/** Default compression level for the channel.
+ * Its value is an int from the \a grpc_compression_level enum. */
 #define GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL "grpc.default_compression_level"
+/** Compression algorithms supported by the channel.
+ * Its value is a bitset (an int). Bits correspond to algorithms in \a
+ * grpc_compression_algorithm. For example, its LSB corresponds to
+ * GRPC_COMPRESS_NONE, the next bit to GRPC_COMPRESS_DEFLATE, etc.
+ * Unset bits disable support for the algorithm. By default all algorithms are
+ * supported. It's not possible to disable GRPC_COMPRESS_NONE (the attempt will
+ * be ignored). */
 #define GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET \
   "grpc.compression_enabled_algorithms_bitset"
+/** \} */
 
 /* The various compression algorithms supported by gRPC */
 typedef enum {

include/grpc/impl/codegen/grpc_types.h

@@ -106,58 +106,66 @@ typedef struct {
     by grpc_arg; keys are strings to allow easy backwards-compatible extension
     by arbitrary parties.
     All evaluation is performed at channel creation time (i.e. the values in
-    this structure need only live through the creation invocation). */
+    this structure need only live through the creation invocation).
+
+    See the description of the \ref grpc_arg_keys "available args" for more
+    details. */
 typedef struct {
   size_t num_args;
   grpc_arg *args;
 } grpc_channel_args;
 
-/* Channel argument keys: */
+/** \defgroup grpc_arg_keys
-/** Enable census for tracing and stats collection */
+ * Channel argument keys.
+ * \{
+ */
+/** If non-zero, enable census for tracing and stats collection. */
 #define GRPC_ARG_ENABLE_CENSUS "grpc.census"
-/** Enable load reporting */
+/** If non-zero, enable load reporting. */
 #define GRPC_ARG_ENABLE_LOAD_REPORTING "grpc.loadreporting"
 /** Maximum number of concurrent incoming streams to allow on a http2
-    connection */
+    connection. Int valued. */
 #define GRPC_ARG_MAX_CONCURRENT_STREAMS "grpc.max_concurrent_streams"
-/** Maximum message length that the channel can receive */
+/** Maximum message length that the channel can receive. Int valued, bytes. */
 #define GRPC_ARG_MAX_MESSAGE_LENGTH "grpc.max_message_length"
-/** Initial sequence number for http2 transports */
+/** Initial sequence number for http2 transports. Int valued. */
 #define GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER \
   "grpc.http2.initial_sequence_number"
 /** Amount to read ahead on individual streams. Defaults to 64kb, larger
     values can help throughput on high-latency connections.
     NOTE: at some point we'd like to auto-tune this, and this parameter
-    will become a no-op. */
+    will become a no-op. Int valued, bytes. */
 #define GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES "grpc.http2.lookahead_bytes"
-/** How much memory to use for hpack decoding */
+/** How much memory to use for hpack decoding. Int valued, bytes. */
 #define GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER \
   "grpc.http2.hpack_table_size.decoder"
-/** How much memory to use for hpack encoding */
+/** How much memory to use for hpack encoding. Int valued, bytes. */
 #define GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER \
   "grpc.http2.hpack_table_size.encoder"
-/** Default authority to pass if none specified on call construction */
+/** Default authority to pass if none specified on call construction. A string.
+ * */
 #define GRPC_ARG_DEFAULT_AUTHORITY "grpc.default_authority"
 /** Primary user agent: goes at the start of the user-agent metadata
-    sent on each request */
+    sent on each request. A string. */
 #define GRPC_ARG_PRIMARY_USER_AGENT_STRING "grpc.primary_user_agent"
 /** Secondary user agent: goes at the end of the user-agent metadata
-    sent on each request */
+    sent on each request. A string. */
 #define GRPC_ARG_SECONDARY_USER_AGENT_STRING "grpc.secondary_user_agent"
 /** The maximum time between subsequent connection attempts, in ms */
 #define GRPC_ARG_MAX_RECONNECT_BACKOFF_MS "grpc.max_reconnect_backoff_ms"
 /* The caller of the secure_channel_create functions may override the target
    name used for SSL host name checking using this channel argument which is of
-   type GRPC_ARG_STRING. This *should* be used for testing only.
+   type \a GRPC_ARG_STRING. This *should* be used for testing only.
    If this argument is not specified, the name used for SSL host name checking
    will be the target parameter (assuming that the secure channel is an SSL
    channel). If this parameter is specified and the underlying is not an SSL
    channel, it will just be ignored. */
 #define GRPC_SSL_TARGET_NAME_OVERRIDE_ARG "grpc.ssl_target_name_override"
-/* Maximum metadata size */
+/* Maximum metadata size, in bytes. */
 #define GRPC_ARG_MAX_METADATA_SIZE "grpc.max_metadata_size"
 /** If non-zero, allow the use of SO_REUSEPORT if it's available (default 1) */
 #define GRPC_ARG_ALLOW_REUSEPORT "grpc.so_reuseport"
+/** \} */
 
 /** Result of a grpc call. If the caller satisfies the prerequisites of a
     particular operation, the grpc_call_error returned will be GRPC_CALL_OK.

include/grpc/impl/codegen/port_platform.h

@@ -119,7 +119,7 @@
 // libraries; it should be integrated with the `__linux__` definitions below.
 #define GPR_PLATFORM_STRING "manylinux"
 #define GPR_POSIX_CRASH_HANDLER 1
-#define GPR_CPU_LINUX 1
+#define GPR_CPU_POSIX 1
 #define GPR_GCC_ATOMIC 1
 #define GPR_GCC_TLS 1
 #define GPR_LINUX 1

include/grpc/impl/codegen/slice.h

@@ -120,6 +120,14 @@ GPRAPI void gpr_slice_unref(gpr_slice s);
    passed in at destruction. */
 GPRAPI gpr_slice gpr_slice_new(void *p, size_t len, void (*destroy)(void *));
 
+/* Equivalent to gpr_slice_new, but with a separate pointer that is
+   passed to the destroy function.  This function can be useful when
+   the data is part of a larger structure that must be destroyed when
+   the data is no longer needed. */
+GPRAPI gpr_slice gpr_slice_new_with_user_data(void *p, size_t len,
+                                              void (*destroy)(void *),
+                                              void *user_data);
+
 /* Equivalent to gpr_slice_new, but with a two argument destroy function that
    also takes the slice length. */
 GPRAPI gpr_slice gpr_slice_new_with_len(void *p, size_t len,

include/grpc/module.modulemap

@@ -1,5 +1,15 @@
 framework module grpc {
   umbrella header "grpc.h"
+
+  header "byte_buffer_reader.h"
+  header "grpc_security.h"
+  header "grpc_security_constants.h"
+  header "impl/codegen/alloc.h"
+  header "impl/codegen/byte_buffer_reader.h"
+  header "support/alloc.h"
+  header "support/port_platform.h"
+  header "support/string_util.h"
+
   export *
   module * { export * }
 }

package.xml

@@ -10,7 +10,7 @@
   <email>grpc-packages@google.com</email>
   <active>yes</active>
  </lead>
- <date>2016-07-13</date>
+ <date>2016-07-28</date>
  <time>16:06:07</time>
  <version>
   <release>1.1.0</release>
@@ -22,8 +22,7 @@
  </stability>
  <license>BSD</license>
  <notes>
-- GA release
+- PHP7 Support continued, reduce code duplication #7543
-- Fix shutdown hang problem #4017
  </notes>
  <contents>
   <dir baseinstalldir="/" name="/">
@@ -47,6 +46,7 @@
     <file baseinstalldir="/" name="src/php/ext/grpc/channel.h" role="src" />
     <file baseinstalldir="/" name="src/php/ext/grpc/channel_credentials.h" role="src" />
     <file baseinstalldir="/" name="src/php/ext/grpc/completion_queue.h" role="src" />
+    <file baseinstalldir="/" name="src/php/ext/grpc/php7_wrapper.h" role="src" />
     <file baseinstalldir="/" name="src/php/ext/grpc/php_grpc.h" role="src" />
     <file baseinstalldir="/" name="src/php/ext/grpc/server.h" role="src" />
     <file baseinstalldir="/" name="src/php/ext/grpc/server_credentials.h" role="src" />
@@ -185,6 +185,7 @@
     <file baseinstalldir="/" name="src/core/lib/channel/compress_filter.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/connected_channel.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/context.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/channel/handshaker.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/http_client_filter.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/http_server_filter.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/algorithm_metadata.h" role="src" />
@@ -341,6 +342,7 @@
     <file baseinstalldir="/" name="src/core/lib/channel/channel_stack_builder.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/compress_filter.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/connected_channel.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/channel/handshaker.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/http_client_filter.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/http_server_filter.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/compression.c" role="src" />
@@ -1089,8 +1091,8 @@ Update to wrap gRPC C Core version 0.10.0
   </release>
   <release>
    <version>
-    <release>1.0.0</release>
+    <release>1.0.0RC1</release>
-    <api>1.0.0</api>
+    <api>1.0.0RC1</api>
    </version>
    <stability>
     <release>stable</release>
@@ -1103,5 +1105,35 @@ Update to wrap gRPC C Core version 0.10.0
 - Fix shutdown hang problem #4017
    </notes>
   </release>
+  <release>
+   <version>
+    <release>1.0.0RC2</release>
+    <api>1.0.0RC2</api>
+   </version>
+   <stability>
+    <release>stable</release>
+    <api>stable</api>
+   </stability>
+   <date>2016-07-21</date>
+   <license>BSD</license>
+   <notes>
+- PHP7 Support #7464
+   </notes>
+  </release>
+  <release>
+   <version>
+    <release>1.0.0RC3</release>
+    <api>1.0.0RC3</api>
+   </version>
+   <stability>
+    <release>stable</release>
+    <api>stable</api>
+   </stability>
+   <date>2016-07-28</date>
+   <license>BSD</license>
+   <notes>
+- PHP7 Support continued, reduce code duplication #7543
+   </notes>
+  </release>
  </changelog>
 </package>

setup.cfg

@@ -9,5 +9,5 @@ build_base=python_build
 [build_ext]
 inplace=1
 
-[build_proto_modules]
+[build_package_protos]
 exclude=.*protoc_plugin/protoc_plugin_test\.proto$

setup.py

@@ -29,17 +29,18 @@
 
 """A setup module for the GRPC Python package."""
 
+from distutils import extension as _extension
+from distutils import util
 import os
 import os.path
+import pkg_resources
 import platform
+import re
 import shlex
 import shutil
 import sys
 import sysconfig
 
-from distutils import core as _core
-from distutils import extension as _extension
-import pkg_resources
 import setuptools
 from setuptools.command import egg_info
 
@@ -47,10 +48,10 @@ from setuptools.command import egg_info
 egg_info.manifest_maker.template = 'PYTHON-MANIFEST.in'
 
 PY3 = sys.version_info.major == 3
-PYTHON_STEM = './src/python/grpcio'
+PYTHON_STEM = os.path.join('src', 'python', 'grpcio')
-CORE_INCLUDE = ('./include', '.',)
+CORE_INCLUDE = ('include', '.',)
-BORINGSSL_INCLUDE = ('./third_party/boringssl/include',)
+BORINGSSL_INCLUDE = (os.path.join('third_party', 'boringssl', 'include'),)
-ZLIB_INCLUDE = ('./third_party/zlib',)
+ZLIB_INCLUDE = (os.path.join('third_party', 'zlib'),)
 
 # Ensure we're in the proper directory whether or not we're being used by pip.
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
@@ -84,18 +85,46 @@ ENABLE_CYTHON_TRACING = os.environ.get(
 # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
 # We use these environment variables to thus get around that without locking
 # ourselves in w.r.t. the multitude of operating systems this ought to build on.
-# By default we assume a GCC-like compiler.
+# We can also use these variables as a way to inject environment-specific
-EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', ''))
+# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
-EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', ''))
+# reasonable default.
+EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
+EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
+if EXTRA_ENV_COMPILE_ARGS is None:
+  EXTRA_ENV_COMPILE_ARGS = '-fno-wrapv'
+  if 'win32' in sys.platform:
+    # We use define flags here and don't directly add to DEFINE_MACROS below to
+    # ensure that the expert user/builder has a way of turning it off (via the
+    # envvars) without adding yet more GRPC-specific envvars.
+    # See https://sourceforge.net/p/mingw-w64/bugs/363/
+    if '32' in platform.architecture()[0]:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s'
+    else:
+      EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64'
+  elif "linux" in sys.platform or "darwin" in sys.platform:
+    EXTRA_ENV_COMPILE_ARGS += ' -fvisibility=hidden'
+if EXTRA_ENV_LINK_ARGS is None:
+  EXTRA_ENV_LINK_ARGS = '-lpthread'
+  if 'win32' in sys.platform:
+    # TODO(atash) check if this is actually safe to just import and call on
+    # non-Windows (to avoid breaking import style)
+    from distutils.cygwinccompiler import get_msvcr
+    msvcr = get_msvcr()[0]
+    # TODO(atash) sift through the GCC specs to see if libstdc++ can have any
+    # influence on the linkage outcome on MinGW for non-C++ programs.
+    EXTRA_ENV_LINK_ARGS += (
+        ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} '
+        '-static'.format(msvcr=msvcr))
+  elif "linux" in sys.platform:
+    EXTRA_ENV_LINK_ARGS += ' -Wl,-wrap,memcpy'
+EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
+EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
 
 CYTHON_EXTENSION_PACKAGE_NAMES = ()
 
 CYTHON_EXTENSION_MODULE_NAMES = ('grpc._cython.cygrpc',)
 
-CYTHON_HELPER_C_FILES = (
+CYTHON_HELPER_C_FILES = ()
-    os.path.join(PYTHON_STEM, 'grpc/_cython/loader.c'),
-    os.path.join(PYTHON_STEM, 'grpc/_cython/imports.generated.c'),
-)
 
 CORE_C_FILES = tuple(grpc_core_dependencies.CORE_SOURCE_FILES)
 
@@ -110,7 +139,9 @@ if not "win32" in sys.platform:
 if "win32" in sys.platform:
   EXTENSION_LIBRARIES += ('ws2_32',)
 
-DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),)
+DEFINE_MACROS = (
+    ('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600),
+    ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),)
 if "win32" in sys.platform:
   DEFINE_MACROS += (('OPENSSL_WINDOWS', 1), ('WIN32_LEAN_AND_MEAN', 1),)
   if '64bit' in platform.architecture()[0]:
@@ -118,13 +149,8 @@ if "win32" in sys.platform:
 
 LDFLAGS = tuple(EXTRA_LINK_ARGS)
 CFLAGS = tuple(EXTRA_COMPILE_ARGS)
-if "linux" in sys.platform:
-  LDFLAGS += ('-Wl,-wrap,memcpy',)
 if "linux" in sys.platform or "darwin" in sys.platform:
-  CFLAGS += ('-fvisibility=hidden',)
-
   pymodinit_type = 'PyObject*' if PY3 else 'void'
-
   pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type)
   DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),)
 
@@ -137,55 +163,32 @@ if 'darwin' in sys.platform and PY3:
   if mac_target and (pkg_resources.parse_version(mac_target) <
                      pkg_resources.parse_version('10.7.0')):
     os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
-
+    os.environ['_PYTHON_HOST_PLATFORM'] = re.sub(
-
+        r'macosx-[0-9]+\.[0-9]+-(.+)',
-def cython_extensions(module_names, extra_sources, include_dirs,
+        r'macosx-10.7-\1',
-                      libraries, define_macros, build_with_cython=False):
+        util.get_platform())
-  # Set compiler directives linetrace argument only if we care about tracing;
+
-  # this is due to Cython having different behavior between linetrace being
+def cython_extensions_and_necessity():
-  # False and linetrace being unset. See issue #5689.
+  cython_module_files = [os.path.join(PYTHON_STEM,
-  cython_compiler_directives = {}
+                               name.replace('.', '/') + '.pyx')
-  if ENABLE_CYTHON_TRACING:
+                  for name in CYTHON_EXTENSION_MODULE_NAMES]
-    define_macros = define_macros + [('CYTHON_TRACE_NOGIL', 1)]
-    cython_compiler_directives['linetrace'] = True
-  pyx_module_files = [os.path.join(PYTHON_STEM,
-                                   name.replace('.', '/') + '.pyx')
-                      for name in module_names]
-  c_module_files = [os.path.join(PYTHON_STEM,
-                                 name.replace('.', '/') + '.c')
-                    for name in module_names]
-  if not build_with_cython:
-    for module_file in c_module_files:
-      if not os.path.isfile(module_file):
-        sys.stderr.write('Cython-generated files are missing; '
-                         'forcing Cython build...\n')
-        build_with_cython = True
-        break
-  module_files = pyx_module_files if build_with_cython else c_module_files
   extensions = [
       _extension.Extension(
           name=module_name,
-          sources=[module_file] + extra_sources,
+          sources=[module_file] + list(CYTHON_HELPER_C_FILES) + list(CORE_C_FILES),
-          include_dirs=include_dirs, libraries=libraries,
+          include_dirs=list(EXTENSION_INCLUDE_DIRECTORIES),
-          define_macros=define_macros,
+          libraries=list(EXTENSION_LIBRARIES),
+          define_macros=list(DEFINE_MACROS),
           extra_compile_args=list(CFLAGS),
           extra_link_args=list(LDFLAGS),
-      ) for (module_name, module_file) in zip(module_names, module_files)
+      ) for (module_name, module_file) in zip(list(CYTHON_EXTENSION_MODULE_NAMES), cython_module_files)
   ]
-  if build_with_cython:
+  need_cython = BUILD_WITH_CYTHON
-    import Cython.Build
+  if not BUILD_WITH_CYTHON:
-    return Cython.Build.cythonize(
+    need_cython = need_cython or not commands.check_and_update_cythonization(extensions)
-        extensions,
+  return commands.try_cythonize(extensions, linetracing=ENABLE_CYTHON_TRACING, mandatory=BUILD_WITH_CYTHON), need_cython
-        include_path=include_dirs,
+
-        compiler_directives=cython_compiler_directives)
+CYTHON_EXTENSION_MODULES, need_cython = cython_extensions_and_necessity()
-  else:
-    return extensions
-
-CYTHON_EXTENSION_MODULES = cython_extensions(
-    list(CYTHON_EXTENSION_MODULE_NAMES),
-    list(CYTHON_HELPER_C_FILES) + list(CORE_C_FILES),
-    list(EXTENSION_INCLUDE_DIRECTORIES), list(EXTENSION_LIBRARIES),
-    list(DEFINE_MACROS), bool(BUILD_WITH_CYTHON))
 
 PACKAGE_DIRECTORIES = {
     '': PYTHON_STEM,
@@ -205,6 +208,15 @@ SETUP_REQUIRES = INSTALL_REQUIRES + (
     'sphinx_rtd_theme>=0.1.8',
     'six>=1.10',
 )
+if BUILD_WITH_CYTHON:
+  sys.stderr.write(
+    "You requested a Cython build via GRPC_PYTHON_BUILD_WITH_CYTHON, "
+    "but do not have Cython installed. We won't stop you from using "
+    "other commands, but the extension files will fail to build.\n")
+elif need_cython:
+  sys.stderr.write(
+      'We could not find Cython. Setup may take 10-20 minutes.\n')
+  SETUP_REQUIRES += ('cython>=0.23',)
 
 COMMAND_CLASS = {
     'doc': commands.SphinxDocumentation,
@@ -215,12 +227,13 @@ COMMAND_CLASS = {
 }
 
 # Ensure that package data is copied over before any commands have been run:
-credentials_dir = os.path.join(PYTHON_STEM, 'grpc/_cython/_credentials')
+credentials_dir = os.path.join(PYTHON_STEM, 'grpc', '_cython', '_credentials')
 try:
   os.mkdir(credentials_dir)
 except OSError:
   pass
-shutil.copyfile('etc/roots.pem', os.path.join(credentials_dir, 'roots.pem'))
+shutil.copyfile(os.path.join('etc', 'roots.pem'),
+                os.path.join(credentials_dir, 'roots.pem'))
 
 PACKAGE_DATA = {
     # Binaries that may or may not be present in the final installation, but are

src/compiler/ruby_generator_helpers-inl.h

@@ -48,7 +48,7 @@ inline bool ServicesFilename(const grpc::protobuf::FileDescriptor *file,
       file->name().find_last_of(".proto") == file->name().size() - 1) {
     *file_name_or_error =
         file->name().substr(0, file->name().size() - proto_suffix_length) +
-        "_services.rb";
+        "_services_pb.rb";
     return true;
   } else {
     *file_name_or_error = "Invalid proto file name:  must end with .proto";
@@ -58,7 +58,7 @@ inline bool ServicesFilename(const grpc::protobuf::FileDescriptor *file,
 
 inline grpc::string MessagesRequireName(
     const grpc::protobuf::FileDescriptor *file) {
-  return Replace(file->name(), ".proto", "");
+  return Replace(file->name(), ".proto", "_pb");
 }
 
 // Get leading or trailing comments in a string. Comment lines start with "# ".

src/core/ext/client_config/channel_connectivity.c

@@ -59,7 +59,7 @@ grpc_connectivity_state grpc_channel_check_connectivity_state(
   }
   gpr_log(GPR_ERROR,
           "grpc_channel_check_connectivity_state called on something that is "
-          "not a (u)client channel, but '%s'",
+          "not a client channel, but '%s'",
           client_channel_elem->filter->name);
   grpc_exec_ctx_finish(&exec_ctx);
   return GRPC_CHANNEL_SHUTDOWN;

src/core/ext/transport/chttp2/client/insecure/channel_create.c

@@ -45,6 +45,7 @@
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/compress_filter.h"
+#include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/channel/http_client_filter.h"
 #include "src/core/lib/iomgr/tcp_client.h"
 #include "src/core/lib/surface/api_trace.h"
@@ -63,6 +64,8 @@ typedef struct {
   grpc_endpoint *tcp;
 
   grpc_closure connected;
+
+  grpc_handshake_manager *handshake_mgr;
 } connector;
 
 static void connector_ref(grpc_connector *con) {
@@ -74,6 +77,7 @@ static void connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *con) {
   connector *c = (connector *)con;
   if (gpr_unref(&c->refs)) {
     /* c->initial_string_buffer does not need to be destroyed */
+    grpc_handshake_manager_destroy(exec_ctx, c->handshake_mgr);
     gpr_free(c);
   }
 }
@@ -83,9 +87,22 @@ static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
   connector_unref(exec_ctx, arg);
 }
 
+static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
+                              grpc_channel_args *args, void *user_data,
+                              grpc_error *error) {
+  connector *c = user_data;
+  c->result->transport =
+      grpc_create_chttp2_transport(exec_ctx, args, endpoint, 1);
+  GPR_ASSERT(c->result->transport);
+  grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL, 0);
+  c->result->channel_args = args;
+  grpc_closure *notify = c->notify;
+  c->notify = NULL;
+  grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
+}
+
 static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   connector *c = arg;
-  grpc_closure *notify;
   grpc_endpoint *tcp = c->tcp;
   if (tcp != NULL) {
     if (!GPR_SLICE_IS_EMPTY(c->args.initial_connect_string)) {
@@ -97,19 +114,17 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
       connector_ref(arg);
       grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer,
                           &c->initial_string_sent);
+    } else {
+      grpc_handshake_manager_do_handshake(
+          exec_ctx, c->handshake_mgr, tcp, c->args.channel_args,
+          c->args.deadline, NULL /* acceptor */, on_handshake_done, c);
     }
-    c->result->transport =
-        grpc_create_chttp2_transport(exec_ctx, c->args.channel_args, tcp, 1);
-    grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL,
-                                        0);
-    GPR_ASSERT(c->result->transport);
-    c->result->channel_args = grpc_channel_args_copy(c->args.channel_args);
   } else {
     memset(c->result, 0, sizeof(*c->result));
+    grpc_closure *notify = c->notify;
+    c->notify = NULL;
+    grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
   }
-  notify = c->notify;
-  c->notify = NULL;
-  grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
 }
 
 static void connector_shutdown(grpc_exec_ctx *exec_ctx, grpc_connector *con) {}
@@ -171,6 +186,7 @@ static grpc_subchannel *client_channel_factory_create_subchannel(
   memset(c, 0, sizeof(*c));
   c->base.vtable = &connector_vtable;
   gpr_ref_init(&c->refs, 1);
+  c->handshake_mgr = grpc_handshake_manager_create();
   args->args = final_args;
   s = grpc_subchannel_create(exec_ctx, &c->base, args);
   grpc_connector_unref(exec_ctx, &c->base);

src/core/ext/transport/chttp2/client/secure/secure_channel_create.c

@@ -44,6 +44,7 @@
 #include "src/core/ext/client_config/resolver_registry.h"
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/iomgr/tcp_client.h"
 #include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/credentials/credentials.h"
@@ -69,6 +70,11 @@ typedef struct {
   grpc_endpoint *newly_connecting_endpoint;
 
   grpc_closure connected_closure;
+
+  grpc_handshake_manager *handshake_mgr;
+
+  // TODO(roth): Remove once we eliminate on_secure_handshake_done().
+  grpc_channel_args *tmp_args;
 } connector;
 
 static void connector_ref(grpc_connector *con) {
@@ -80,6 +86,8 @@ static void connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *con) {
   connector *c = (connector *)con;
   if (gpr_unref(&c->refs)) {
     /* c->initial_string_buffer does not need to be destroyed */
+    grpc_channel_args_destroy(c->tmp_args);
+    grpc_handshake_manager_destroy(exec_ctx, c->handshake_mgr);
     gpr_free(c);
   }
 }
@@ -89,13 +97,14 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
                                      grpc_endpoint *secure_endpoint,
                                      grpc_auth_context *auth_context) {
   connector *c = arg;
-  grpc_closure *notify;
   gpr_mu_lock(&c->mu);
+  grpc_error *error = GRPC_ERROR_NONE;
   if (c->connecting_endpoint == NULL) {
     memset(c->result, 0, sizeof(*c->result));
     gpr_mu_unlock(&c->mu);
   } else if (status != GRPC_SECURITY_OK) {
-    gpr_log(GPR_ERROR, "Secure handshake failed with error %d.", status);
+    error = grpc_error_set_int(GRPC_ERROR_CREATE("Secure handshake failed"),
+                               GRPC_ERROR_INT_SECURITY_STATUS, status);
     memset(c->result, 0, sizeof(*c->result));
     c->connecting_endpoint = NULL;
     gpr_mu_unlock(&c->mu);
@@ -108,25 +117,43 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
     grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL,
                                         0);
     auth_context_arg = grpc_auth_context_to_arg(auth_context);
-    c->result->channel_args = grpc_channel_args_copy_and_add(
+    c->result->channel_args =
-        c->args.channel_args, &auth_context_arg, 1);
+        grpc_channel_args_copy_and_add(c->tmp_args, &auth_context_arg, 1);
   }
-  notify = c->notify;
+  grpc_closure *notify = c->notify;
   c->notify = NULL;
-  grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_NONE, NULL);
+  grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
+}
+
+static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
+                              grpc_channel_args *args, void *user_data,
+                              grpc_error *error) {
+  connector *c = user_data;
+  if (error != GRPC_ERROR_NONE) {
+    grpc_closure *notify = c->notify;
+    c->notify = NULL;
+    grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
+  } else {
+    // TODO(roth, jboeuf): Convert security connector handshaking to use new
+    // handshake API, and then move the code from on_secure_handshake_done()
+    // into this function.
+    c->tmp_args = args;
+    grpc_channel_security_connector_do_handshake(
+        exec_ctx, c->security_connector, endpoint, c->args.deadline,
+        on_secure_handshake_done, c);
+  }
 }
 
 static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
                                            grpc_error *error) {
   connector *c = arg;
-  grpc_channel_security_connector_do_handshake(
+  grpc_handshake_manager_do_handshake(
-      exec_ctx, c->security_connector, c->connecting_endpoint, c->args.deadline,
+      exec_ctx, c->handshake_mgr, c->connecting_endpoint, c->args.channel_args,
-      on_secure_handshake_done, c);
+      c->args.deadline, NULL /* acceptor */, on_handshake_done, c);
 }
 
 static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   connector *c = arg;
-  grpc_closure *notify;
   grpc_endpoint *tcp = c->newly_connecting_endpoint;
   if (tcp != NULL) {
     gpr_mu_lock(&c->mu);
@@ -142,13 +169,13 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
       grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer,
                           &c->initial_string_sent);
     } else {
-      grpc_channel_security_connector_do_handshake(
+      grpc_handshake_manager_do_handshake(
-          exec_ctx, c->security_connector, tcp, c->args.deadline,
+          exec_ctx, c->handshake_mgr, tcp, c->args.channel_args,
-          on_secure_handshake_done, c);
+          c->args.deadline, NULL /* acceptor */, on_handshake_done, c);
     }
   } else {
     memset(c->result, 0, sizeof(*c->result));
-    notify = c->notify;
+    grpc_closure *notify = c->notify;
     c->notify = NULL;
     grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
   }
@@ -227,6 +254,7 @@ static grpc_subchannel *client_channel_factory_create_subchannel(
   memset(c, 0, sizeof(*c));
   c->base.vtable = &connector_vtable;
   c->security_connector = f->security_connector;
+  c->handshake_mgr = grpc_handshake_manager_create();
   gpr_mu_init(&c->mu);
   gpr_ref_init(&c->refs, 1);
   args->args = final_args;

src/core/ext/transport/chttp2/server/insecure/server_chttp2.c

@@ -37,35 +37,74 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
+
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/channel/http_server_filter.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/tcp_server.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-static void new_transport(grpc_exec_ctx *exec_ctx, void *server,
+typedef struct server_connect_state {
-                          grpc_endpoint *tcp, grpc_pollset *accepting_pollset,
+  grpc_server *server;
-                          grpc_tcp_server_acceptor *acceptor) {
+  grpc_pollset *accepting_pollset;
-  /*
+  grpc_tcp_server_acceptor *acceptor;
-   * Beware that the call to grpc_create_chttp2_transport() has to happen before
+  grpc_handshake_manager *handshake_mgr;
-   * grpc_tcp_server_destroy(). This is fine here, but similar code
+} server_connect_state;
-   * asynchronously doing a handshake instead of calling grpc_tcp_server_start()
+
-   * (as in server_secure_chttp2.c) needs to add synchronization to avoid this
+static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
-   * case.
+                              grpc_channel_args *args, void *user_data,
-   */
+                              grpc_error *error) {
-  grpc_transport *transport = grpc_create_chttp2_transport(
+  server_connect_state *state = user_data;
-      exec_ctx, grpc_server_get_channel_args(server), tcp, 0);
+  if (error != GRPC_ERROR_NONE) {
-  grpc_server_setup_transport(exec_ctx, server, transport, accepting_pollset,
+    const char *error_str = grpc_error_string(error);
-                              grpc_server_get_channel_args(server));
+    gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str);
-  grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0);
+    grpc_error_free_string(error_str);
+    GRPC_ERROR_UNREF(error);
+    grpc_handshake_manager_shutdown(exec_ctx, state->handshake_mgr);
+  } else {
+    // Beware that the call to grpc_create_chttp2_transport() has to happen
+    // before grpc_tcp_server_destroy(). This is fine here, but similar code
+    // asynchronously doing a handshake instead of calling
+    // grpc_tcp_server_start() (as in server_secure_chttp2.c) needs to add
+    // synchronization to avoid this case.
+    grpc_transport *transport =
+        grpc_create_chttp2_transport(exec_ctx, args, endpoint, 0);
+    grpc_server_setup_transport(exec_ctx, state->server, transport,
+                                state->accepting_pollset,
+                                grpc_server_get_channel_args(state->server));
+    grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0);
+  }
+  // Clean up.
+  grpc_channel_args_destroy(args);
+  grpc_handshake_manager_destroy(exec_ctx, state->handshake_mgr);
+  gpr_free(state);
+}
+
+static void on_accept(grpc_exec_ctx *exec_ctx, void *server, grpc_endpoint *tcp,
+                      grpc_pollset *accepting_pollset,
+                      grpc_tcp_server_acceptor *acceptor) {
+  server_connect_state *state = gpr_malloc(sizeof(server_connect_state));
+  state->server = server;
+  state->accepting_pollset = accepting_pollset;
+  state->acceptor = acceptor;
+  state->handshake_mgr = grpc_handshake_manager_create();
+  // TODO(roth): We should really get this timeout value from channel
+  // args instead of hard-coding it.
+  const gpr_timespec deadline = gpr_time_add(
+      gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(120, GPR_TIMESPAN));
+  grpc_handshake_manager_do_handshake(
+      exec_ctx, state->handshake_mgr, tcp, grpc_server_get_channel_args(server),
+      deadline, acceptor, on_handshake_done, state);
 }
 
 /* Server callback: start listening on our ports */
 static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp,
                   grpc_pollset **pollsets, size_t pollset_count) {
   grpc_tcp_server *tcp = tcpp;
-  grpc_tcp_server_start(exec_ctx, tcp, pollsets, pollset_count, new_transport,
+  grpc_tcp_server_start(exec_ctx, tcp, pollsets, pollset_count, on_accept,
                         server);
 }
 
@@ -74,6 +113,7 @@ static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp,
 static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp,
                     grpc_closure *destroy_done) {
   grpc_tcp_server *tcp = tcpp;
+  grpc_tcp_server_shutdown_listeners(exec_ctx, tcp);
   grpc_tcp_server_unref(exec_ctx, tcp);
   grpc_exec_ctx_sched(exec_ctx, destroy_done, GRPC_ERROR_NONE, NULL);
 }

src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c

@@ -42,6 +42,7 @@
 #include <grpc/support/useful.h>
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/channel/http_server_filter.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/resolve_address.h"
@@ -58,7 +59,7 @@ typedef struct server_secure_state {
   grpc_tcp_server *tcp;
   grpc_server_security_connector *sc;
   grpc_server_credentials *creds;
-  int is_shutdown;
+  bool is_shutdown;
   gpr_mu mu;
   gpr_refcount refcount;
   grpc_closure destroy_closure;
@@ -68,6 +69,12 @@ typedef struct server_secure_state {
 typedef struct server_secure_connect {
   server_secure_state *state;
   grpc_pollset *accepting_pollset;
+  grpc_tcp_server_acceptor *acceptor;
+  grpc_handshake_manager *handshake_mgr;
+  // TODO(roth): Remove the following two fields when we eliminate
+  // grpc_server_security_connector_do_handshake().
+  gpr_timespec deadline;
+  grpc_channel_args *args;
 } server_secure_connect;
 
 static void state_ref(server_secure_state *state) { gpr_ref(&state->refcount); }
@@ -89,21 +96,18 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
                                      grpc_endpoint *secure_endpoint,
                                      grpc_auth_context *auth_context) {
   server_secure_connect *state = statep;
-  grpc_transport *transport;
   if (status == GRPC_SECURITY_OK) {
     if (secure_endpoint) {
       gpr_mu_lock(&state->state->mu);
       if (!state->state->is_shutdown) {
-        transport = grpc_create_chttp2_transport(
+        grpc_transport *transport = grpc_create_chttp2_transport(
             exec_ctx, grpc_server_get_channel_args(state->state->server),
             secure_endpoint, 0);
-        grpc_channel_args *args_copy;
         grpc_arg args_to_add[2];
         args_to_add[0] = grpc_server_credentials_to_arg(state->state->creds);
         args_to_add[1] = grpc_auth_context_to_arg(auth_context);
-        args_copy = grpc_channel_args_copy_and_add(
+        grpc_channel_args *args_copy = grpc_channel_args_copy_and_add(
-            grpc_server_get_channel_args(state->state->server), args_to_add,
+            state->args, args_to_add, GPR_ARRAY_SIZE(args_to_add));
-            GPR_ARRAY_SIZE(args_to_add));
         grpc_server_setup_transport(exec_ctx, state->state->server, transport,
                                     state->accepting_pollset, args_copy);
         grpc_channel_args_destroy(args_copy);
@@ -118,10 +122,38 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
   } else {
     gpr_log(GPR_ERROR, "Secure transport failed with error %d", status);
   }
+  grpc_channel_args_destroy(state->args);
   state_unref(state->state);
   gpr_free(state);
 }
 
+static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
+                              grpc_channel_args *args, void *user_data,
+                              grpc_error *error) {
+  server_secure_connect *state = user_data;
+  if (error != GRPC_ERROR_NONE) {
+    const char *error_str = grpc_error_string(error);
+    gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str);
+    grpc_error_free_string(error_str);
+    GRPC_ERROR_UNREF(error);
+    grpc_handshake_manager_shutdown(exec_ctx, state->handshake_mgr);
+    grpc_handshake_manager_destroy(exec_ctx, state->handshake_mgr);
+    grpc_channel_args_destroy(args);
+    state_unref(state->state);
+    gpr_free(state);
+    return;
+  }
+  grpc_handshake_manager_destroy(exec_ctx, state->handshake_mgr);
+  state->handshake_mgr = NULL;
+  // TODO(roth, jboeuf): Convert security connector handshaking to use new
+  // handshake API, and then move the code from on_secure_handshake_done()
+  // into this function.
+  state->args = args;
+  grpc_server_security_connector_do_handshake(
+      exec_ctx, state->state->sc, state->acceptor, endpoint, state->deadline,
+      on_secure_handshake_done, state);
+}
+
 static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
                       grpc_pollset *accepting_pollset,
                       grpc_tcp_server_acceptor *acceptor) {
@@ -129,11 +161,16 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
   state->state = statep;
   state_ref(state->state);
   state->accepting_pollset = accepting_pollset;
-  grpc_server_security_connector_do_handshake(
+  state->acceptor = acceptor;
-      exec_ctx, state->state->sc, acceptor, tcp,
+  state->handshake_mgr = grpc_handshake_manager_create();
-      gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
+  // TODO(roth): We should really get this timeout value from channel
-                   gpr_time_from_seconds(120, GPR_TIMESPAN)),
+  // args instead of hard-coding it.
-      on_secure_handshake_done, state);
+  state->deadline = gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
+                                 gpr_time_from_seconds(120, GPR_TIMESPAN));
+  grpc_handshake_manager_do_handshake(
+      exec_ctx, state->handshake_mgr, tcp,
+      grpc_server_get_channel_args(state->state->server), state->deadline,
+      acceptor, on_handshake_done, state);
 }
 
 /* Server callback: start listening on our ports */
@@ -162,10 +199,11 @@ static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep,
   server_secure_state *state = statep;
   grpc_tcp_server *tcp;
   gpr_mu_lock(&state->mu);
-  state->is_shutdown = 1;
+  state->is_shutdown = true;
   state->destroy_callback = callback;
   tcp = state->tcp;
   gpr_mu_unlock(&state->mu);
+  grpc_tcp_server_shutdown_listeners(exec_ctx, tcp);
   grpc_tcp_server_unref(exec_ctx, tcp);
 }
 
@@ -226,7 +264,7 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
   state->tcp = tcp;
   state->sc = sc;
   state->creds = grpc_server_credentials_ref(creds);
-  state->is_shutdown = 0;
+  state->is_shutdown = false;
   gpr_mu_init(&state->mu);
   gpr_ref_init(&state->refcount, 1);
 

src/core/ext/transport/chttp2/transport/chttp2_plugin.c

@@ -36,11 +36,14 @@
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/transport/metadata.h"
 
+extern int grpc_http_write_state_trace;
+
 void grpc_chttp2_plugin_init(void) {
   grpc_chttp2_base64_encode_and_huffman_compress =
       grpc_chttp2_base64_encode_and_huffman_compress_impl;
   grpc_register_tracer("http", &grpc_http_trace);
   grpc_register_tracer("flowctl", &grpc_flowctl_trace);
+  grpc_register_tracer("http_write_state", &grpc_http_write_state_trace);
 }
 
 void grpc_chttp2_plugin_shutdown(void) {}

src/core/ext/transport/chttp2/transport/chttp2_transport.c

@@ -48,6 +48,7 @@
 #include "src/core/ext/transport/chttp2/transport/status_conversion.h"
 #include "src/core/ext/transport/chttp2/transport/timeout_encoding.h"
 #include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/workqueue.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/static_metadata.h"
@@ -60,9 +61,9 @@
 #define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024)
 
 #define MAX_CLIENT_STREAM_ID 0x7fffffffu
-
 int grpc_http_trace = 0;
 int grpc_flowctl_trace = 0;
+int grpc_http_write_state_trace = 0;
 
 #define TRANSPORT_FROM_WRITING(tw)                                        \
   ((grpc_chttp2_transport *)((char *)(tw)-offsetof(grpc_chttp2_transport, \
@@ -88,10 +89,16 @@ static const grpc_transport_vtable vtable;
 static void writing_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error);
 static void reading_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error);
 static void parsing_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error);
+static void initiate_writing(grpc_exec_ctx *exec_ctx, void *t,
+                             grpc_error *error);
+
+static void start_writing(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t);
+static void end_waiting_for_write(grpc_exec_ctx *exec_ctx,
+                                  grpc_chttp2_transport *t, grpc_error *error);
 
 /** Set a transport level setting, and push it to our peer */
-static void push_setting(grpc_chttp2_transport *t, grpc_chttp2_setting_id id,
+static void push_setting(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
-                         uint32_t value);
+                         grpc_chttp2_setting_id id, uint32_t value);
 
 /** Start disconnection chain */
 static void drop_connection(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
@@ -137,7 +144,7 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx,
                            grpc_chttp2_transport_global *transport_global);
 
 static void incoming_byte_stream_update_flow_control(
-    grpc_chttp2_transport_global *transport_global,
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global *stream_global, size_t max_size_hint,
     size_t have_already);
 static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx,
@@ -201,6 +208,7 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx,
   gpr_free(t);
 }
 
+/*#define REFCOUNTING_DEBUG 1*/
 #ifdef REFCOUNTING_DEBUG
 #define REF_TRANSPORT(t, r) ref_transport(t, r, __FILE__, __LINE__)
 #define UNREF_TRANSPORT(cl, t, r) unref_transport(cl, t, r, __FILE__, __LINE__)
@@ -231,7 +239,7 @@ static void ref_transport(grpc_chttp2_transport *t) { gpr_ref(&t->refs); }
 
 static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
                            const grpc_channel_args *channel_args,
-                           grpc_endpoint *ep, uint8_t is_client) {
+                           grpc_endpoint *ep, bool is_client) {
   size_t i;
   int j;
 
@@ -273,6 +281,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   grpc_closure_init(&t->writing_action, writing_action, t);
   grpc_closure_init(&t->reading_action, reading_action, t);
   grpc_closure_init(&t->parsing_action, parsing_action, t);
+  grpc_closure_init(&t->initiate_writing, initiate_writing, t);
 
   gpr_slice_buffer_init(&t->parsing.qbuf);
   grpc_chttp2_goaway_parser_init(&t->parsing.goaway_parser);
@@ -286,6 +295,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     gpr_slice_buffer_add(
         &t->global.qbuf,
         gpr_slice_from_copied_string(GRPC_CHTTP2_CLIENT_CONNECT_STRING));
+    grpc_chttp2_initiate_write(exec_ctx, &t->global, false, "initial_write");
   }
   /* 8 is a random stab in the dark as to a good initial size: it's small enough
      that it shouldn't waste memory for infrequently used connections, yet
@@ -311,11 +321,12 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
 
   /* configure http2 the way we like it */
   if (is_client) {
-    push_setting(t, GRPC_CHTTP2_SETTINGS_ENABLE_PUSH, 0);
+    push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_ENABLE_PUSH, 0);
-    push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, 0);
+    push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, 0);
   }
-  push_setting(t, GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, DEFAULT_WINDOW);
+  push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE,
-  push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE,
+               DEFAULT_WINDOW);
+  push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE,
                DEFAULT_MAX_HEADER_LIST_SIZE);
 
   if (channel_args) {
@@ -329,7 +340,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
           gpr_log(GPR_ERROR, "%s: must be an integer",
                   GRPC_ARG_MAX_CONCURRENT_STREAMS);
         } else {
-          push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS,
+          push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS,
                        (uint32_t)channel_args->args[i].value.integer);
         }
       } else if (0 == strcmp(channel_args->args[i].key,
@@ -368,7 +379,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
           gpr_log(GPR_ERROR, "%s: must be non-negative",
                   GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER);
         } else {
-          push_setting(t, GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE,
+          push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE,
                        (uint32_t)channel_args->args[i].value.integer);
         }
       } else if (0 == strcmp(channel_args->args[i].key,
@@ -393,7 +404,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
           gpr_log(GPR_ERROR, "%s: must be non-negative",
                   GRPC_ARG_MAX_METADATA_SIZE);
         } else {
-          push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE,
+          push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE,
                        (uint32_t)channel_args->args[i].value.integer);
         }
       }
@@ -444,6 +455,9 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx,
                                    grpc_chttp2_transport *t,
                                    grpc_error *error) {
   if (!t->closed) {
+    if (grpc_http_write_state_trace) {
+      gpr_log(GPR_DEBUG, "W:%p close transport", t);
+    }
     t->closed = 1;
     connectivity_state_set(exec_ctx, &t->global, GRPC_CHANNEL_SHUTDOWN,
                            GRPC_ERROR_REF(error), "close_transport");
@@ -590,7 +604,8 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_incoming_metadata_buffer_destroy(
       &s->global.received_trailing_metadata);
   gpr_slice_buffer_destroy(&s->writing.flow_controlled_buffer);
-  GRPC_ERROR_UNREF(s->global.removal_error);
+  GRPC_ERROR_UNREF(s->global.read_closed_error);
+  GRPC_ERROR_UNREF(s->global.write_closed_error);
 
   UNREF_TRANSPORT(exec_ctx, t, "stream");
 
@@ -634,6 +649,36 @@ grpc_chttp2_stream_parsing *grpc_chttp2_parsing_accept_stream(
  * LOCK MANAGEMENT
  */
 
+static const char *write_state_name(grpc_chttp2_write_state state) {
+  switch (state) {
+    case GRPC_CHTTP2_WRITING_INACTIVE:
+      return "INACTIVE";
+    case GRPC_CHTTP2_WRITE_REQUESTED_NO_POLLER:
+      return "REQUESTED[p=0]";
+    case GRPC_CHTTP2_WRITE_REQUESTED_WITH_POLLER:
+      return "REQUESTED[p=1]";
+    case GRPC_CHTTP2_WRITE_SCHEDULED:
+      return "SCHEDULED";
+    case GRPC_CHTTP2_WRITING:
+      return "WRITING";
+    case GRPC_CHTTP2_WRITING_STALE_WITH_POLLER:
+      return "WRITING[p=1]";
+    case GRPC_CHTTP2_WRITING_STALE_NO_POLLER:
+      return "WRITING[p=0]";
+  }
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
+}
+
+static void set_write_state(grpc_chttp2_transport *t,
+                            grpc_chttp2_write_state state, const char *reason) {
+  if (grpc_http_write_state_trace) {
+    gpr_log(GPR_DEBUG, "W:%p %s -> %s because %s", t,
+            write_state_name(t->executor.write_state), write_state_name(state),
+            reason);
+  }
+  t->executor.write_state = state;
+}
+
 static void finish_global_actions(grpc_exec_ctx *exec_ctx,
                                   grpc_chttp2_transport *t) {
   grpc_chttp2_executor_action_header *hdr;
@@ -642,13 +687,6 @@ static void finish_global_actions(grpc_exec_ctx *exec_ctx,
   GPR_TIMER_BEGIN("finish_global_actions", 0);
 
   for (;;) {
-    if (!t->executor.writing_active && !t->closed &&
-        grpc_chttp2_unlocking_check_writes(exec_ctx, &t->global, &t->writing)) {
-      t->executor.writing_active = 1;
-      REF_TRANSPORT(t, "writing");
-      prevent_endpoint_shutdown(t);
-      grpc_exec_ctx_sched(exec_ctx, &t->writing_action, GRPC_ERROR_NONE, NULL);
-    }
     check_read_ops(exec_ctx, &t->global);
 
     gpr_mu_lock(&t->executor.mu);
@@ -669,8 +707,28 @@ static void finish_global_actions(grpc_exec_ctx *exec_ctx,
       continue;
     } else {
       t->executor.global_active = false;
+      switch (t->executor.write_state) {
+        case GRPC_CHTTP2_WRITE_REQUESTED_WITH_POLLER:
+          set_write_state(t, GRPC_CHTTP2_WRITE_SCHEDULED, "unlocking");
+          REF_TRANSPORT(t, "initiate_writing");
+          gpr_mu_unlock(&t->executor.mu);
+          grpc_exec_ctx_sched(
+              exec_ctx, &t->initiate_writing, GRPC_ERROR_NONE,
+              t->ep != NULL ? grpc_endpoint_get_workqueue(t->ep) : NULL);
+          break;
+        case GRPC_CHTTP2_WRITE_REQUESTED_NO_POLLER:
+          start_writing(exec_ctx, t);
+          gpr_mu_unlock(&t->executor.mu);
+          break;
+        case GRPC_CHTTP2_WRITING_INACTIVE:
+        case GRPC_CHTTP2_WRITING:
+        case GRPC_CHTTP2_WRITING_STALE_WITH_POLLER:
+        case GRPC_CHTTP2_WRITING_STALE_NO_POLLER:
+        case GRPC_CHTTP2_WRITE_SCHEDULED:
+          gpr_mu_unlock(&t->executor.mu);
+          break;
+      }
     }
-    gpr_mu_unlock(&t->executor.mu);
     break;
   }
 
@@ -741,16 +799,118 @@ void grpc_chttp2_run_with_global_lock(grpc_exec_ctx *exec_ctx,
  * OUTPUT PROCESSING
  */
 
-void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global,
+void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx,
-                                 grpc_chttp2_stream_global *stream_global) {
+                                grpc_chttp2_transport_global *transport_global,
+                                bool covered_by_poller, const char *reason) {
+  /* Perform state checks, and transition to a scheduled state if appropriate.
+     Each time we finish the global lock execution, we check if we need to
+     write. If we do:
+      - (if there is a poller surrounding the write) schedule
+        initiate_writing, which locks and calls initiate_writing_locked to...
+      - call start_writing, which verifies (under the global lock) that there
+        are things that need to be written by calling
+        grpc_chttp2_unlocking_check_writes, and if so schedules writing_action
+        against the current exec_ctx, to be executed OUTSIDE of the global lock
+      - eventually writing_action results in grpc_chttp2_terminate_writing being
+        called, which re-takes the global lock, updates state, checks if we need
+        to do *another* write immediately, and if so loops back to
+        start_writing.
+
+      Current problems:
+       - too much lock entry/exiting
+       - the writing thread can become stuck indefinitely (punt through the
+         workqueue periodically to fix) */
+
+  grpc_chttp2_transport *t = TRANSPORT_FROM_GLOBAL(transport_global);
+  switch (t->executor.write_state) {
+    case GRPC_CHTTP2_WRITING_INACTIVE:
+      set_write_state(t, covered_by_poller
+                             ? GRPC_CHTTP2_WRITE_REQUESTED_WITH_POLLER
+                             : GRPC_CHTTP2_WRITE_REQUESTED_NO_POLLER,
+                      reason);
+      break;
+    case GRPC_CHTTP2_WRITE_REQUESTED_WITH_POLLER:
+      /* nothing to do: write already requested */
+      break;
+    case GRPC_CHTTP2_WRITE_REQUESTED_NO_POLLER:
+      if (covered_by_poller) {
+        /* upgrade to note poller is available to cover the write */
+        set_write_state(t, GRPC_CHTTP2_WRITE_REQUESTED_WITH_POLLER, reason);
+      }
+      break;
+    case GRPC_CHTTP2_WRITE_SCHEDULED:
+      /* nothing to do: write already scheduled */
+      break;
+    case GRPC_CHTTP2_WRITING:
+      set_write_state(t,
+                      covered_by_poller ? GRPC_CHTTP2_WRITING_STALE_WITH_POLLER
+                                        : GRPC_CHTTP2_WRITING_STALE_NO_POLLER,
+                      reason);
+      break;
+    case GRPC_CHTTP2_WRITING_STALE_WITH_POLLER:
+      /* nothing to do: write already requested */
+      break;
+    case GRPC_CHTTP2_WRITING_STALE_NO_POLLER:
+      if (covered_by_poller) {
+        /* upgrade to note poller is available to cover the write */
+        set_write_state(t, GRPC_CHTTP2_WRITING_STALE_WITH_POLLER, reason);
+      }
+      break;
+  }
+}
+
+static void start_writing(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) {
+  GPR_ASSERT(t->executor.write_state == GRPC_CHTTP2_WRITE_SCHEDULED ||
+             t->executor.write_state == GRPC_CHTTP2_WRITE_REQUESTED_NO_POLLER);
+  if (!t->closed &&
+      grpc_chttp2_unlocking_check_writes(exec_ctx, &t->global, &t->writing)) {
+    set_write_state(t, GRPC_CHTTP2_WRITING, "start_writing");
+    REF_TRANSPORT(t, "writing");
+    prevent_endpoint_shutdown(t);
+    grpc_exec_ctx_sched(exec_ctx, &t->writing_action, GRPC_ERROR_NONE, NULL);
+  } else {
+    if (t->closed) {
+      set_write_state(t, GRPC_CHTTP2_WRITING_INACTIVE,
+                      "start_writing:transport_closed");
+    } else {
+      set_write_state(t, GRPC_CHTTP2_WRITING_INACTIVE,
+                      "start_writing:nothing_to_write");
+    }
+    end_waiting_for_write(exec_ctx, t, GRPC_ERROR_CREATE("Nothing to write"));
+    if (t->ep && !t->endpoint_reading) {
+      destroy_endpoint(exec_ctx, t);
+    }
+  }
+}
+
+static void initiate_writing_locked(grpc_exec_ctx *exec_ctx,
+                                    grpc_chttp2_transport *t,
+                                    grpc_chttp2_stream *s_unused,
+                                    void *arg_ignored) {
+  start_writing(exec_ctx, t);
+  UNREF_TRANSPORT(exec_ctx, t, "initiate_writing");
+}
+
+static void initiate_writing(grpc_exec_ctx *exec_ctx, void *arg,
+                             grpc_error *error) {
+  grpc_chttp2_run_with_global_lock(exec_ctx, arg, NULL, initiate_writing_locked,
+                                   NULL, 0);
+}
+
+void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx,
+                                 grpc_chttp2_transport_global *transport_global,
+                                 grpc_chttp2_stream_global *stream_global,
+                                 bool covered_by_poller, const char *reason) {
   if (!TRANSPORT_FROM_GLOBAL(transport_global)->closed &&
       grpc_chttp2_list_add_writable_stream(transport_global, stream_global)) {
     GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing");
+    grpc_chttp2_initiate_write(exec_ctx, transport_global, covered_by_poller,
+                               reason);
   }
 }
 
-static void push_setting(grpc_chttp2_transport *t, grpc_chttp2_setting_id id,
+static void push_setting(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
-                         uint32_t value) {
+                         grpc_chttp2_setting_id id, uint32_t value) {
   const grpc_chttp2_setting_parameters *sp =
       &grpc_chttp2_settings_parameters[id];
   uint32_t use_value = GPR_CLAMP(value, sp->min_value, sp->max_value);
@@ -761,9 +921,22 @@ static void push_setting(grpc_chttp2_transport *t, grpc_chttp2_setting_id id,
   if (use_value != t->global.settings[GRPC_LOCAL_SETTINGS][id]) {
     t->global.settings[GRPC_LOCAL_SETTINGS][id] = use_value;
     t->global.dirtied_local_settings = 1;
+    grpc_chttp2_initiate_write(exec_ctx, &t->global, false, "push_setting");
   }
 }
 
+static void end_waiting_for_write(grpc_exec_ctx *exec_ctx,
+                                  grpc_chttp2_transport *t, grpc_error *error) {
+  grpc_chttp2_stream_global *stream_global;
+  while (grpc_chttp2_list_pop_closed_waiting_for_writing(&t->global,
+                                                         &stream_global)) {
+    fail_pending_writes(exec_ctx, &t->global, stream_global,
+                        GRPC_ERROR_REF(error));
+    GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "finish_writes");
+  }
+  GRPC_ERROR_UNREF(error);
+}
+
 static void terminate_writing_with_lock(grpc_exec_ctx *exec_ctx,
                                         grpc_chttp2_transport *t,
                                         grpc_chttp2_stream *s_ignored,
@@ -778,24 +951,32 @@ static void terminate_writing_with_lock(grpc_exec_ctx *exec_ctx,
 
   grpc_chttp2_cleanup_writing(exec_ctx, &t->global, &t->writing);
 
-  grpc_chttp2_stream_global *stream_global;
+  end_waiting_for_write(exec_ctx, t, error);
-  while (grpc_chttp2_list_pop_closed_waiting_for_writing(&t->global,
+
-                                                         &stream_global)) {
+  switch (t->executor.write_state) {
-    fail_pending_writes(exec_ctx, &t->global, stream_global,
+    case GRPC_CHTTP2_WRITING_INACTIVE:
-                        GRPC_ERROR_REF(error));
+    case GRPC_CHTTP2_WRITE_REQUESTED_WITH_POLLER:
-    GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "finish_writes");
+    case GRPC_CHTTP2_WRITE_REQUESTED_NO_POLLER:
+    case GRPC_CHTTP2_WRITE_SCHEDULED:
+      GPR_UNREACHABLE_CODE(break);
+    case GRPC_CHTTP2_WRITING:
+      set_write_state(t, GRPC_CHTTP2_WRITING_INACTIVE, "terminate_writing");
+      break;
+    case GRPC_CHTTP2_WRITING_STALE_WITH_POLLER:
+      set_write_state(t, GRPC_CHTTP2_WRITE_REQUESTED_WITH_POLLER,
+                      "terminate_writing");
+      break;
+    case GRPC_CHTTP2_WRITING_STALE_NO_POLLER:
+      set_write_state(t, GRPC_CHTTP2_WRITE_REQUESTED_NO_POLLER,
+                      "terminate_writing");
+      break;
   }
 
-  /* leave the writing flag up on shutdown to prevent further writes in
-     unlock()
-     from starting */
-  t->executor.writing_active = 0;
   if (t->ep && !t->endpoint_reading) {
     destroy_endpoint(exec_ctx, t);
   }
 
   UNREF_TRANSPORT(exec_ctx, t, "writing");
-  GRPC_ERROR_UNREF(error);
 }
 
 void grpc_chttp2_terminate_writing(grpc_exec_ctx *exec_ctx,
@@ -878,7 +1059,8 @@ static void maybe_start_some_streams(
         stream_global->id, STREAM_FROM_GLOBAL(stream_global));
     stream_global->in_stream_map = true;
     transport_global->concurrent_stream_count++;
-    grpc_chttp2_become_writable(transport_global, stream_global);
+    grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global, true,
+                                "new_stream");
   }
   /* cancel out streams that will never be started */
   while (transport_global->next_stream_id >= MAX_CLIENT_STREAM_ID &&
@@ -1018,9 +1200,11 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
           maybe_start_some_streams(exec_ctx, transport_global);
         } else {
           GPR_ASSERT(stream_global->id != 0);
-          grpc_chttp2_become_writable(transport_global, stream_global);
+          grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global,
+                                      true, "op.send_initial_metadata");
         }
       } else {
+        stream_global->send_trailing_metadata = NULL;
         grpc_chttp2_complete_closure_step(
             exec_ctx, transport_global, stream_global,
             &stream_global->send_initial_metadata_finished,
@@ -1042,7 +1226,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
     } else {
       stream_global->send_message = op->send_message;
       if (stream_global->id != 0) {
-        grpc_chttp2_become_writable(transport_global, stream_global);
+        grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global,
+                                    true, "op.send_message");
       }
     }
   }
@@ -1075,6 +1260,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
         grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
       }
       if (stream_global->write_closed) {
+        stream_global->send_trailing_metadata = NULL;
         grpc_chttp2_complete_closure_step(
             exec_ctx, transport_global, stream_global,
             &stream_global->send_trailing_metadata_finished,
@@ -1085,7 +1271,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
       } else if (stream_global->id != 0) {
         /* TODO(ctiller): check if there's flow control for any outstanding
            bytes before going writable */
-        grpc_chttp2_become_writable(transport_global, stream_global);
+        grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global,
+                                    true, "op.send_trailing_metadata");
       }
     }
   }
@@ -1106,8 +1293,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx,
         (stream_global->incoming_frames.head == NULL ||
          stream_global->incoming_frames.head->is_tail)) {
       incoming_byte_stream_update_flow_control(
-          transport_global, stream_global, transport_global->stream_lookahead,
+          exec_ctx, transport_global, stream_global,
-          0);
+          transport_global->stream_lookahead, 0);
     }
     grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
   }
@@ -1135,7 +1322,8 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
                                    sizeof(*op));
 }
 
-static void send_ping_locked(grpc_chttp2_transport *t, grpc_closure *on_recv) {
+static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
+                             grpc_closure *on_recv) {
   grpc_chttp2_outstanding_ping *p = gpr_malloc(sizeof(*p));
   p->next = &t->global.pings;
   p->prev = p->next->prev;
@@ -1150,6 +1338,7 @@ static void send_ping_locked(grpc_chttp2_transport *t, grpc_closure *on_recv) {
   p->id[7] = (uint8_t)(t->global.ping_counter & 0xff);
   p->on_recv = on_recv;
   gpr_slice_buffer_add(&t->global.qbuf, grpc_chttp2_ping_create(0, p->id));
+  grpc_chttp2_initiate_write(exec_ctx, &t->global, true, "send_ping");
 }
 
 static void ack_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
@@ -1209,6 +1398,7 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx,
     close_transport = grpc_chttp2_has_streams(t)
                           ? GRPC_ERROR_NONE
                           : GRPC_ERROR_CREATE("GOAWAY sent");
+    grpc_chttp2_initiate_write(exec_ctx, &t->global, false, "goaway_sent");
   }
 
   if (op->set_accept_stream) {
@@ -1226,7 +1416,7 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx,
   }
 
   if (op->send_ping) {
-    send_ping_locked(t, op->send_ping);
+    send_ping_locked(exec_ctx, t, op->send_ping);
   }
 
   if (close_transport != GRPC_ERROR_NONE) {
@@ -1414,6 +1604,8 @@ static void cancel_from_api(grpc_exec_ctx *exec_ctx,
           &transport_global->qbuf,
           grpc_chttp2_rst_stream_create(stream_global->id, (uint32_t)http_error,
                                         &stream_global->stats.outgoing));
+      grpc_chttp2_initiate_write(exec_ctx, transport_global, false,
+                                 "rst_stream");
     }
 
     const char *msg =
@@ -1473,10 +1665,39 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx,
   }
 }
 
+static void add_error(grpc_error *error, grpc_error **refs, size_t *nrefs) {
+  if (error == GRPC_ERROR_NONE) return;
+  for (size_t i = 0; i < *nrefs; i++) {
+    if (error == refs[i]) {
+      return;
+    }
+  }
+  refs[*nrefs] = error;
+  ++*nrefs;
+}
+
+static grpc_error *removal_error(grpc_error *extra_error,
+                                 grpc_chttp2_stream_global *stream_global) {
+  grpc_error *refs[3];
+  size_t nrefs = 0;
+  add_error(stream_global->read_closed_error, refs, &nrefs);
+  add_error(stream_global->write_closed_error, refs, &nrefs);
+  add_error(extra_error, refs, &nrefs);
+  grpc_error *error = GRPC_ERROR_NONE;
+  if (nrefs > 0) {
+    error = GRPC_ERROR_CREATE_REFERENCING("Failed due to stream removal", refs,
+                                          nrefs);
+  }
+  GRPC_ERROR_UNREF(extra_error);
+  return error;
+}
+
 static void fail_pending_writes(grpc_exec_ctx *exec_ctx,
                                 grpc_chttp2_transport_global *transport_global,
                                 grpc_chttp2_stream_global *stream_global,
                                 grpc_error *error) {
+  error = removal_error(error, stream_global);
+  stream_global->send_message = NULL;
   grpc_chttp2_complete_closure_step(
       exec_ctx, transport_global, stream_global,
       &stream_global->send_initial_metadata_finished, GRPC_ERROR_REF(error));
@@ -1499,14 +1720,17 @@ void grpc_chttp2_mark_stream_closed(
   }
   grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
   if (close_reads && !stream_global->read_closed) {
+    stream_global->read_closed_error = GRPC_ERROR_REF(error);
     stream_global->read_closed = true;
     stream_global->published_initial_metadata = true;
     stream_global->published_trailing_metadata = true;
     decrement_active_streams_locked(exec_ctx, transport_global, stream_global);
   }
   if (close_writes && !stream_global->write_closed) {
+    stream_global->write_closed_error = GRPC_ERROR_REF(error);
     stream_global->write_closed = true;
-    if (TRANSPORT_FROM_GLOBAL(transport_global)->executor.writing_active) {
+    if (TRANSPORT_FROM_GLOBAL(transport_global)->executor.write_state !=
+        GRPC_CHTTP2_WRITING_INACTIVE) {
       GRPC_CHTTP2_STREAM_REF(stream_global, "finish_writes");
       grpc_chttp2_list_add_closed_waiting_for_writing(transport_global,
                                                       stream_global);
@@ -1516,7 +1740,6 @@ void grpc_chttp2_mark_stream_closed(
     }
   }
   if (stream_global->read_closed && stream_global->write_closed) {
-    stream_global->removal_error = GRPC_ERROR_REF(error);
     if (stream_global->id != 0 &&
         TRANSPORT_FROM_GLOBAL(transport_global)->executor.parsing_active) {
       grpc_chttp2_list_add_closed_waiting_for_parsing(transport_global,
@@ -1524,7 +1747,8 @@ void grpc_chttp2_mark_stream_closed(
     } else {
       if (stream_global->id != 0) {
         remove_stream(exec_ctx, TRANSPORT_FROM_GLOBAL(transport_global),
-                      stream_global->id, GRPC_ERROR_REF(error));
+                      stream_global->id,
+                      removal_error(GRPC_ERROR_REF(error), stream_global));
       }
       GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2");
     }
@@ -1649,6 +1873,8 @@ static void close_from_api(grpc_exec_ctx *exec_ctx,
 
   grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, 1,
                                  1, error);
+  grpc_chttp2_initiate_write(exec_ctx, transport_global, false,
+                             "close_from_api");
 }
 
 typedef struct {
@@ -1678,8 +1904,14 @@ static void drop_connection(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
 }
 
 /** update window from a settings change */
+typedef struct {
+  grpc_chttp2_transport *t;
+  grpc_exec_ctx *exec_ctx;
+} update_global_window_args;
+
 static void update_global_window(void *args, uint32_t id, void *stream) {
-  grpc_chttp2_transport *t = args;
+  update_global_window_args *a = args;
+  grpc_chttp2_transport *t = a->t;
   grpc_chttp2_stream *s = stream;
   grpc_chttp2_transport_global *transport_global = &t->global;
   grpc_chttp2_stream_global *stream_global = &s->global;
@@ -1693,7 +1925,8 @@ static void update_global_window(void *args, uint32_t id, void *stream) {
   is_zero = stream_global->outgoing_window <= 0;
 
   if (was_zero && !is_zero) {
-    grpc_chttp2_become_writable(transport_global, stream_global);
+    grpc_chttp2_become_writable(a->exec_ctx, transport_global, stream_global,
+                                true, "update_global_window");
   }
 }
 
@@ -1772,6 +2005,7 @@ static grpc_error *try_http_parsing(grpc_exec_ctx *exec_ctx,
 static void parsing_action(grpc_exec_ctx *exec_ctx, void *arg,
                            grpc_error *error) {
   grpc_chttp2_transport *t = arg;
+  grpc_error *err = GRPC_ERROR_NONE;
   GPR_TIMER_BEGIN("reading_action.parse", 0);
   size_t i = 0;
   grpc_error *errors[3] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE,
@@ -1780,15 +2014,13 @@ static void parsing_action(grpc_exec_ctx *exec_ctx, void *arg,
     errors[1] = grpc_chttp2_perform_read(exec_ctx, &t->parsing,
                                          t->read_buffer.slices[i]);
   };
-  if (i != t->read_buffer.count) {
+  if (errors[1] == GRPC_ERROR_NONE) {
+    err = GRPC_ERROR_REF(error);
+  } else {
     errors[2] = try_http_parsing(exec_ctx, t);
+    err = GRPC_ERROR_CREATE_REFERENCING("Failed parsing HTTP/2", errors,
+                                        GPR_ARRAY_SIZE(errors));
   }
-  grpc_error *err =
-      errors[0] == GRPC_ERROR_NONE && errors[1] == GRPC_ERROR_NONE &&
-              errors[2] == GRPC_ERROR_NONE
-          ? GRPC_ERROR_NONE
-          : GRPC_ERROR_CREATE_REFERENCING("Failed parsing HTTP/2", errors,
-                                          GPR_ARRAY_SIZE(errors));
   for (i = 0; i < GPR_ARRAY_SIZE(errors); i++) {
     GRPC_ERROR_UNREF(errors[i]);
   }
@@ -1802,14 +2034,19 @@ static void post_parse_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   grpc_chttp2_transport_global *transport_global = &t->global;
   grpc_chttp2_transport_parsing *transport_parsing = &t->parsing;
   /* copy parsing qbuf to global qbuf */
-  gpr_slice_buffer_move_into(&t->parsing.qbuf, &t->global.qbuf);
+  if (t->parsing.qbuf.count > 0) {
+    gpr_slice_buffer_move_into(&t->parsing.qbuf, &t->global.qbuf);
+    grpc_chttp2_initiate_write(exec_ctx, transport_global, false,
+                               "parsing_qbuf");
+  }
   /* merge stream lists */
   grpc_chttp2_stream_map_move_into(&t->new_stream_map, &t->parsing_stream_map);
   transport_global->concurrent_stream_count =
       (uint32_t)grpc_chttp2_stream_map_size(&t->parsing_stream_map);
   if (transport_parsing->initial_window_update != 0) {
+    update_global_window_args args = {t, exec_ctx};
     grpc_chttp2_stream_map_for_each(&t->parsing_stream_map,
-                                    update_global_window, t);
+                                    update_global_window, &args);
     transport_parsing->initial_window_update = 0;
   }
   /* handle higher level things */
@@ -1832,7 +2069,7 @@ static void post_parse_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     GPR_ASSERT(stream_global->write_closed);
     GPR_ASSERT(stream_global->read_closed);
     remove_stream(exec_ctx, t, stream_global->id,
-                  GRPC_ERROR_REF(stream_global->removal_error));
+                  removal_error(GRPC_ERROR_NONE, stream_global));
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2");
   }
 
@@ -1855,11 +2092,12 @@ static void post_reading_action_locked(grpc_exec_ctx *exec_ctx,
     }
     drop_connection(exec_ctx, t, GRPC_ERROR_REF(error));
     t->endpoint_reading = 0;
-    if (!t->executor.writing_active && t->ep) {
+    if (grpc_http_write_state_trace) {
-      grpc_endpoint_destroy(exec_ctx, t->ep);
+      gpr_log(GPR_DEBUG, "R:%p -> 0 ws=%s", t,
-      t->ep = NULL;
+              write_state_name(t->executor.write_state));
-      /* safe as we still have a ref for read */
+    }
-      UNREF_TRANSPORT(exec_ctx, t, "disconnect");
+    if (t->executor.write_state == GRPC_CHTTP2_WRITING_INACTIVE && t->ep) {
+      destroy_endpoint(exec_ctx, t);
     }
   } else if (!t->closed) {
     keep_reading = true;
@@ -1943,7 +2181,7 @@ static void incoming_byte_stream_unref(grpc_exec_ctx *exec_ctx,
 }
 
 static void incoming_byte_stream_update_flow_control(
-    grpc_chttp2_transport_global *transport_global,
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global *stream_global, size_t max_size_hint,
     size_t have_already) {
   uint32_t max_recv_bytes;
@@ -1978,7 +2216,8 @@ static void incoming_byte_stream_update_flow_control(
                                    add_max_recv_bytes);
     grpc_chttp2_list_add_unannounced_incoming_window_available(transport_global,
                                                                stream_global);
-    grpc_chttp2_become_writable(transport_global, stream_global);
+    grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global,
+                                false, "read_incoming_stream");
   }
 }
 
@@ -2000,8 +2239,9 @@ static void incoming_byte_stream_next_locked(grpc_exec_ctx *exec_ctx,
   grpc_chttp2_stream_global *stream_global = &bs->stream->global;
 
   if (bs->is_tail) {
-    incoming_byte_stream_update_flow_control(
+    incoming_byte_stream_update_flow_control(exec_ctx, transport_global,
-        transport_global, stream_global, arg->max_size_hint, bs->slices.length);
+                                             stream_global, arg->max_size_hint,
+                                             bs->slices.length);
   }
   if (bs->slices.count > 0) {
     *arg->slice = gpr_slice_buffer_take_first(&bs->slices);
@@ -2185,7 +2425,7 @@ static char *format_flowctl_context_var(const char *context, const char *var,
   if (context == NULL) {
     *scope = NULL;
     gpr_asprintf(&buf, "%s(%" PRId64 ")", var, val);
-    result = gpr_leftpad(buf, ' ', 40);
+    result = gpr_leftpad(buf, ' ', 60);
     gpr_free(buf);
     return result;
   }
@@ -2198,7 +2438,7 @@ static char *format_flowctl_context_var(const char *context, const char *var,
     gpr_free(tmp);
   }
   gpr_asprintf(&buf, "%s.%s(%" PRId64 ")", underscore_pos + 1, var, val);
-  result = gpr_leftpad(buf, ' ', 40);
+  result = gpr_leftpad(buf, ' ', 60);
   gpr_free(buf);
   return result;
 }
@@ -2231,7 +2471,7 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase,
 
   tmp_phase = gpr_leftpad(phase, ' ', 8);
   tmp_scope1 = gpr_leftpad(scope1, ' ', 11);
-  gpr_asprintf(&prefix, "FLOW %s: %s %s ", phase, clisvr, scope1);
+  gpr_asprintf(&prefix, "FLOW %s: %s %s ", tmp_phase, clisvr, scope1);
   gpr_free(tmp_phase);
   gpr_free(tmp_scope1);
 

src/core/ext/transport/chttp2/transport/internal.h

@@ -305,6 +305,22 @@ typedef struct grpc_chttp2_executor_action_header {
   void *arg;
 } grpc_chttp2_executor_action_header;
 
+typedef enum {
+  /** no writing activity */
+  GRPC_CHTTP2_WRITING_INACTIVE,
+  /** write has been requested, but not scheduled yet */
+  GRPC_CHTTP2_WRITE_REQUESTED_WITH_POLLER,
+  GRPC_CHTTP2_WRITE_REQUESTED_NO_POLLER,
+  /** write has been requested and scheduled against the workqueue */
+  GRPC_CHTTP2_WRITE_SCHEDULED,
+  /** write has been initiated after being reaped from the workqueue */
+  GRPC_CHTTP2_WRITING,
+  /** write has been initiated, AND another write needs to be started once it's
+      done */
+  GRPC_CHTTP2_WRITING_STALE_WITH_POLLER,
+  GRPC_CHTTP2_WRITING_STALE_NO_POLLER,
+} grpc_chttp2_write_state;
+
 struct grpc_chttp2_transport {
   grpc_transport base; /* must be first */
   gpr_refcount refs;
@@ -319,10 +335,10 @@ struct grpc_chttp2_transport {
 
     /** is a thread currently in the global lock */
     bool global_active;
-    /** is a thread currently writing */
-    bool writing_active;
     /** is a thread currently parsing */
     bool parsing_active;
+    /** write execution state of the transport */
+    grpc_chttp2_write_state write_state;
 
     grpc_chttp2_executor_action_header *pending_actions_head;
     grpc_chttp2_executor_action_header *pending_actions_tail;
@@ -342,7 +358,8 @@ struct grpc_chttp2_transport {
   /** global state for reading/writing */
   grpc_chttp2_transport_global global;
   /** state only accessible by the chain of execution that
-      set writing_active=1 */
+      set writing_state >= GRPC_WRITING, and only by the writing closure
+      chain. */
   grpc_chttp2_transport_writing writing;
   /** state only accessible by the chain of execution that
       set parsing_active=1 */
@@ -363,6 +380,8 @@ struct grpc_chttp2_transport {
   grpc_closure reading_action;
   /** closure to actually do parsing */
   grpc_closure parsing_action;
+  /** closure to initiate writing */
+  grpc_closure initiate_writing;
 
   /** incoming read bytes */
   gpr_slice_buffer read_buffer;
@@ -436,8 +455,10 @@ typedef struct {
   bool seen_error;
   bool exceeded_metadata_size;
 
-  /** the error that resulted in this stream being removed */
+  /** the error that resulted in this stream being read-closed */
-  grpc_error *removal_error;
+  grpc_error *read_closed_error;
+  /** the error that resulted in this stream being write-closed */
+  grpc_error *write_closed_error;
 
   bool published_initial_metadata;
   bool published_trailing_metadata;
@@ -514,15 +535,20 @@ struct grpc_chttp2_stream {
 };
 
 /** Transport writing call flow:
-    chttp2_transport.c calls grpc_chttp2_unlocking_check_writes to see if writes
+    grpc_chttp2_initiate_write() is called anywhere that we know bytes need to
-   are required;
+    go out on the wire.
-    if they are, chttp2_transport.c calls grpc_chttp2_perform_writes to do the
+    If no other write has been started, a task is enqueued onto our workqueue.
-   writes.
+    When that task executes, it obtains the global lock, and gathers the data
-    Once writes have been completed (meaning another write could potentially be
+    to write.
-   started),
+    The global lock is dropped and we do the syscall to write.
-    grpc_chttp2_terminate_writing is called. This will call
+    After writing, a follow-up check is made to see if another round of writing
-   grpc_chttp2_cleanup_writing, at which
+    should be performed.
-    point the write phase is complete. */
+
+    The actual call chain is documented in the implementation of this function.
+    */
+void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx,
+                                grpc_chttp2_transport_global *transport_global,
+                                bool covered_by_poller, const char *reason);
 
 /** Someone is unlocking the transport mutex: check to see if writes
     are required, and schedule them if so */
@@ -610,9 +636,8 @@ int grpc_chttp2_list_pop_check_read_ops(
 void grpc_chttp2_list_add_writing_stalled_by_transport(
     grpc_chttp2_transport_writing *transport_writing,
     grpc_chttp2_stream_writing *stream_writing);
-void grpc_chttp2_list_flush_writing_stalled_by_transport(
+bool grpc_chttp2_list_flush_writing_stalled_by_transport(
-    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing,
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing);
-    bool is_window_available);
 
 void grpc_chttp2_list_add_stalled_by_transport(
     grpc_chttp2_transport_writing *transport_writing,
@@ -822,7 +847,9 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx,
 
 /** add a ref to the stream and add it to the writable list;
     ref will be dropped in writing.c */
-void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global,
+void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx,
-                                 grpc_chttp2_stream_global *stream_global);
+                                 grpc_chttp2_transport_global *transport_global,
+                                 grpc_chttp2_stream_global *stream_global,
+                                 bool covered_by_poller, const char *reason);
 
 #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INTERNAL_H */

src/core/ext/transport/chttp2/transport/parsing.c

@@ -154,10 +154,8 @@ void grpc_chttp2_publish_reads(
                                   transport_parsing, outgoing_window);
   is_zero = transport_global->outgoing_window <= 0;
   if (was_zero && !is_zero) {
-    while (grpc_chttp2_list_pop_stalled_by_transport(transport_global,
+    grpc_chttp2_initiate_write(exec_ctx, transport_global, false,
-                                                     &stream_global)) {
+                               "new_global_flow_control");
-      grpc_chttp2_become_writable(transport_global, stream_global);
-    }
   }
 
   if (transport_parsing->incoming_window <
@@ -168,6 +166,8 @@ void grpc_chttp2_publish_reads(
                                       announce_incoming_window, announce_bytes);
     GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", transport_parsing,
                                       incoming_window, announce_bytes);
+    grpc_chttp2_initiate_write(exec_ctx, transport_global, false,
+                               "global incoming window");
   }
 
   /* for each stream that saw an update, fixup global state */
@@ -190,7 +190,8 @@ void grpc_chttp2_publish_reads(
                                  outgoing_window);
     is_zero = stream_global->outgoing_window <= 0;
     if (was_zero && !is_zero) {
-      grpc_chttp2_become_writable(transport_global, stream_global);
+      grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global,
+                                  false, "stream.read_flow_control");
     }
 
     stream_global->max_recv_bytes -= (uint32_t)GPR_MIN(

src/core/ext/transport/chttp2/transport/stream_lists.c

@@ -329,6 +329,7 @@ void grpc_chttp2_list_add_writing_stalled_by_transport(
     grpc_chttp2_transport_writing *transport_writing,
     grpc_chttp2_stream_writing *stream_writing) {
   grpc_chttp2_stream *stream = STREAM_FROM_WRITING(stream_writing);
+  gpr_log(GPR_DEBUG, "writing stalled %d", stream->global.id);
   if (!stream->included[GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT]) {
     GRPC_CHTTP2_STREAM_REF(&stream->global, "chttp2_writing_stalled");
   }
@@ -336,27 +337,28 @@ void grpc_chttp2_list_add_writing_stalled_by_transport(
                   GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT);
 }
 
-void grpc_chttp2_list_flush_writing_stalled_by_transport(
+bool grpc_chttp2_list_flush_writing_stalled_by_transport(
-    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing,
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing) {
-    bool is_window_available) {
   grpc_chttp2_stream *stream;
+  bool out = false;
   grpc_chttp2_transport *transport = TRANSPORT_FROM_WRITING(transport_writing);
   while (stream_list_pop(transport, &stream,
                          GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT)) {
-    if (is_window_available) {
+    gpr_log(GPR_DEBUG, "move %d from writing stalled to just stalled",
-      grpc_chttp2_become_writable(&transport->global, &stream->global);
+            stream->global.id);
-    } else {
+    grpc_chttp2_list_add_stalled_by_transport(transport_writing,
-      grpc_chttp2_list_add_stalled_by_transport(transport_writing,
+                                              &stream->writing);
-                                                &stream->writing);
-    }
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &stream->global,
                              "chttp2_writing_stalled");
+    out = true;
   }
+  return out;
 }
 
 void grpc_chttp2_list_add_stalled_by_transport(
     grpc_chttp2_transport_writing *transport_writing,
     grpc_chttp2_stream_writing *stream_writing) {
+  gpr_log(GPR_DEBUG, "stalled %d", stream_writing->id);
   stream_list_add(TRANSPORT_FROM_WRITING(transport_writing),
                   STREAM_FROM_WRITING(stream_writing),
                   GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);

src/core/ext/transport/chttp2/transport/writing.c

@@ -75,9 +75,13 @@ int grpc_chttp2_unlocking_check_writes(
 
   GRPC_CHTTP2_FLOW_MOVE_TRANSPORT("write", transport_writing, outgoing_window,
                                   transport_global, outgoing_window);
-  bool is_window_available = transport_writing->outgoing_window > 0;
+  if (transport_writing->outgoing_window > 0) {
-  grpc_chttp2_list_flush_writing_stalled_by_transport(
+    while (grpc_chttp2_list_pop_stalled_by_transport(transport_global,
-      exec_ctx, transport_writing, is_window_available);
+                                                     &stream_global)) {
+      grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global,
+                                  false, "transport.read_flow_control");
+    }
+  }
 
   /* for each grpc_chttp2_stream that's become writable, frame it's data
      (according to available window sizes) and add to the output buffer */
@@ -331,6 +335,12 @@ void grpc_chttp2_cleanup_writing(
   grpc_chttp2_stream_writing *stream_writing;
   grpc_chttp2_stream_global *stream_global;
 
+  if (grpc_chttp2_list_flush_writing_stalled_by_transport(exec_ctx,
+                                                          transport_writing)) {
+    grpc_chttp2_initiate_write(exec_ctx, transport_global, false,
+                               "resume_stalled_stream");
+  }
+
   while (grpc_chttp2_list_pop_written_stream(
       transport_global, transport_writing, &stream_global, &stream_writing)) {
     if (stream_writing->sent_initial_metadata) {

src/core/lib/channel/channel_args.h

@@ -37,6 +37,8 @@
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
 
+// Channel args are intentionally immutable, to avoid the need for locking.
+
 /** Copy the arguments in \a src into a new instance */
 grpc_channel_args *grpc_channel_args_copy(const grpc_channel_args *src);
 

src/core/lib/channel/handshaker.c

@@ -0,0 +1,197 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <string.h>
+
+#include <grpc/impl/codegen/alloc.h>
+#include <grpc/impl/codegen/log.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/handshaker.h"
+
+//
+// grpc_handshaker
+//
+
+void grpc_handshaker_init(const struct grpc_handshaker_vtable* vtable,
+                          grpc_handshaker* handshaker) {
+  handshaker->vtable = vtable;
+}
+
+void grpc_handshaker_destroy(grpc_exec_ctx* exec_ctx,
+                             grpc_handshaker* handshaker) {
+  handshaker->vtable->destroy(exec_ctx, handshaker);
+}
+
+void grpc_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
+                              grpc_handshaker* handshaker) {
+  handshaker->vtable->shutdown(exec_ctx, handshaker);
+}
+
+void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx,
+                                  grpc_handshaker* handshaker,
+                                  grpc_endpoint* endpoint,
+                                  grpc_channel_args* args,
+                                  gpr_timespec deadline,
+                                  grpc_tcp_server_acceptor* acceptor,
+                                  grpc_handshaker_done_cb cb, void* user_data) {
+  handshaker->vtable->do_handshake(exec_ctx, handshaker, endpoint, args,
+                                   deadline, acceptor, cb, user_data);
+}
+
+//
+// grpc_handshake_manager
+//
+
+// State used while chaining handshakers.
+struct grpc_handshaker_state {
+  // The index of the handshaker to invoke next.
+  size_t index;
+  // The deadline for all handshakers.
+  gpr_timespec deadline;
+  // The acceptor to call the handshakers with.
+  grpc_tcp_server_acceptor* acceptor;
+  // The final callback and user_data to invoke after the last handshaker.
+  grpc_handshaker_done_cb final_cb;
+  void* final_user_data;
+};
+
+struct grpc_handshake_manager {
+  // An array of handshakers added via grpc_handshake_manager_add().
+  size_t count;
+  grpc_handshaker** handshakers;
+  // State used while chaining handshakers.
+  struct grpc_handshaker_state* state;
+};
+
+grpc_handshake_manager* grpc_handshake_manager_create() {
+  grpc_handshake_manager* mgr = gpr_malloc(sizeof(grpc_handshake_manager));
+  memset(mgr, 0, sizeof(*mgr));
+  return mgr;
+}
+
+static bool is_power_of_2(size_t n) { return (n & (n - 1)) == 0; }
+
+void grpc_handshake_manager_add(grpc_handshake_manager* mgr,
+                                grpc_handshaker* handshaker) {
+  // To avoid allocating memory for each handshaker we add, we double
+  // the number of elements every time we need more.
+  size_t realloc_count = 0;
+  if (mgr->count == 0) {
+    realloc_count = 2;
+  } else if (mgr->count >= 2 && is_power_of_2(mgr->count)) {
+    realloc_count = mgr->count * 2;
+  }
+  if (realloc_count > 0) {
+    mgr->handshakers =
+        gpr_realloc(mgr->handshakers, realloc_count * sizeof(grpc_handshaker*));
+  }
+  mgr->handshakers[mgr->count++] = handshaker;
+}
+
+void grpc_handshake_manager_destroy(grpc_exec_ctx* exec_ctx,
+                                    grpc_handshake_manager* mgr) {
+  for (size_t i = 0; i < mgr->count; ++i) {
+    grpc_handshaker_destroy(exec_ctx, mgr->handshakers[i]);
+  }
+  gpr_free(mgr->handshakers);
+  gpr_free(mgr);
+}
+
+void grpc_handshake_manager_shutdown(grpc_exec_ctx* exec_ctx,
+                                     grpc_handshake_manager* mgr) {
+  for (size_t i = 0; i < mgr->count; ++i) {
+    grpc_handshaker_shutdown(exec_ctx, mgr->handshakers[i]);
+  }
+  if (mgr->state != NULL) {
+    gpr_free(mgr->state);
+    mgr->state = NULL;
+  }
+}
+
+// A function used as the handshaker-done callback when chaining
+// handshakers together.
+static void call_next_handshaker(grpc_exec_ctx* exec_ctx,
+                                 grpc_endpoint* endpoint,
+                                 grpc_channel_args* args, void* user_data,
+                                 grpc_error* error) {
+  grpc_handshake_manager* mgr = user_data;
+  GPR_ASSERT(mgr->state != NULL);
+  GPR_ASSERT(mgr->state->index < mgr->count);
+  // If we got an error, skip all remaining handshakers and invoke the
+  // caller-supplied callback immediately.
+  if (error != GRPC_ERROR_NONE) {
+    mgr->state->final_cb(exec_ctx, endpoint, args, mgr->state->final_user_data,
+                         error);
+    return;
+  }
+  grpc_handshaker_done_cb cb = call_next_handshaker;
+  // If this is the last handshaker, use the caller-supplied callback
+  // and user_data instead of chaining back to this function again.
+  if (mgr->state->index == mgr->count - 1) {
+    cb = mgr->state->final_cb;
+    user_data = mgr->state->final_user_data;
+  }
+  // Invoke handshaker.
+  grpc_handshaker_do_handshake(exec_ctx, mgr->handshakers[mgr->state->index],
+                               endpoint, args, mgr->state->deadline,
+                               mgr->state->acceptor, cb, user_data);
+  ++mgr->state->index;
+  // If this is the last handshaker, clean up state.
+  if (mgr->state->index == mgr->count) {
+    gpr_free(mgr->state);
+    mgr->state = NULL;
+  }
+}
+
+void grpc_handshake_manager_do_handshake(
+    grpc_exec_ctx* exec_ctx, grpc_handshake_manager* mgr,
+    grpc_endpoint* endpoint, const grpc_channel_args* args,
+    gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor,
+    grpc_handshaker_done_cb cb, void* user_data) {
+  grpc_channel_args* args_copy = grpc_channel_args_copy(args);
+  if (mgr->count == 0) {
+    // No handshakers registered, so we just immediately call the done
+    // callback with the passed-in endpoint.
+    cb(exec_ctx, endpoint, args_copy, user_data, GRPC_ERROR_NONE);
+  } else {
+    GPR_ASSERT(mgr->state == NULL);
+    mgr->state = gpr_malloc(sizeof(struct grpc_handshaker_state));
+    memset(mgr->state, 0, sizeof(*mgr->state));
+    mgr->state->deadline = deadline;
+    mgr->state->acceptor = acceptor;
+    mgr->state->final_cb = cb;
+    mgr->state->final_user_data = user_data;
+    call_next_handshaker(exec_ctx, endpoint, args_copy, mgr, GRPC_ERROR_NONE);
+  }
+}

src/core/lib/channel/handshaker.h

@@ -0,0 +1,145 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_CHANNEL_HANDSHAKER_H
+#define GRPC_CORE_LIB_CHANNEL_HANDSHAKER_H
+
+#include <grpc/impl/codegen/grpc_types.h>
+#include <grpc/impl/codegen/time.h>
+
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/tcp_server.h"
+
+/// Handshakers are used to perform initial handshakes on a connection
+/// before the client sends the initial request.  Some examples of what
+/// a handshaker can be used for includes support for HTTP CONNECT on
+/// the client side and various types of security initialization.
+///
+/// In general, handshakers should be used via a handshake manager.
+
+///
+/// grpc_handshaker
+///
+
+typedef struct grpc_handshaker grpc_handshaker;
+
+/// Callback type invoked when a handshaker is done.
+/// Takes ownership of \a args.
+typedef void (*grpc_handshaker_done_cb)(grpc_exec_ctx* exec_ctx,
+                                        grpc_endpoint* endpoint,
+                                        grpc_channel_args* args,
+                                        void* user_data, grpc_error* error);
+
+struct grpc_handshaker_vtable {
+  /// Destroys the handshaker.
+  void (*destroy)(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker);
+
+  /// Shuts down the handshaker (e.g., to clean up when the operation is
+  /// aborted in the middle).
+  void (*shutdown)(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker);
+
+  /// Performs handshaking.  When finished, calls \a cb with \a user_data.
+  /// Takes ownership of \a args.
+  /// \a acceptor will be NULL for client-side handshakers.
+  void (*do_handshake)(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker,
+                       grpc_endpoint* endpoint, grpc_channel_args* args,
+                       gpr_timespec deadline,
+                       grpc_tcp_server_acceptor* acceptor,
+                       grpc_handshaker_done_cb cb, void* user_data);
+};
+
+/// Base struct.  To subclass, make this the first member of the
+/// implementation struct.
+struct grpc_handshaker {
+  const struct grpc_handshaker_vtable* vtable;
+};
+
+/// Called by concrete implementations to initialize the base struct.
+void grpc_handshaker_init(const struct grpc_handshaker_vtable* vtable,
+                          grpc_handshaker* handshaker);
+
+/// Convenient wrappers for invoking methods via the vtable.
+/// These probably do not need to be called from anywhere but
+/// grpc_handshake_manager.
+void grpc_handshaker_destroy(grpc_exec_ctx* exec_ctx,
+                             grpc_handshaker* handshaker);
+void grpc_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
+                              grpc_handshaker* handshaker);
+void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx,
+                                  grpc_handshaker* handshaker,
+                                  grpc_endpoint* endpoint,
+                                  grpc_channel_args* args,
+                                  gpr_timespec deadline,
+                                  grpc_tcp_server_acceptor* acceptor,
+                                  grpc_handshaker_done_cb cb, void* user_data);
+
+///
+/// grpc_handshake_manager
+///
+
+typedef struct grpc_handshake_manager grpc_handshake_manager;
+
+/// Creates a new handshake manager.  Caller takes ownership.
+grpc_handshake_manager* grpc_handshake_manager_create();
+
+/// Adds a handshaker to the handshake manager.
+/// Takes ownership of \a handshaker.
+void grpc_handshake_manager_add(grpc_handshake_manager* mgr,
+                                grpc_handshaker* handshaker);
+
+/// Destroys the handshake manager.
+void grpc_handshake_manager_destroy(grpc_exec_ctx* exec_ctx,
+                                    grpc_handshake_manager* mgr);
+
+/// Shuts down the handshake manager (e.g., to clean up when the operation is
+/// aborted in the middle).
+/// The caller must still call grpc_handshake_manager_destroy() after
+/// calling this function.
+void grpc_handshake_manager_shutdown(grpc_exec_ctx* exec_ctx,
+                                     grpc_handshake_manager* mgr);
+
+/// Invokes handshakers in the order they were added.
+/// Does NOT take ownership of \a args.  Instead, makes a copy before
+/// invoking the first handshaker.
+/// \a acceptor will be NULL for client-side handshakers.
+/// Invokes \a cb with \a user_data after either a handshaker fails or
+/// all handshakers have completed successfully.
+void grpc_handshake_manager_do_handshake(
+    grpc_exec_ctx* exec_ctx, grpc_handshake_manager* mgr,
+    grpc_endpoint* endpoint, const grpc_channel_args* args,
+    gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor,
+    grpc_handshaker_done_cb cb, void* user_data);
+
+#endif /* GRPC_CORE_LIB_CHANNEL_HANDSHAKER_H */

src/core/lib/iomgr/endpoint.c

@@ -65,3 +65,7 @@ void grpc_endpoint_destroy(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep) {
 char* grpc_endpoint_get_peer(grpc_endpoint* ep) {
   return ep->vtable->get_peer(ep);
 }
+
+grpc_workqueue* grpc_endpoint_get_workqueue(grpc_endpoint* ep) {
+  return ep->vtable->get_workqueue(ep);
+}

src/core/lib/iomgr/endpoint.h

@@ -51,6 +51,7 @@ struct grpc_endpoint_vtable {
                gpr_slice_buffer *slices, grpc_closure *cb);
   void (*write)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                 gpr_slice_buffer *slices, grpc_closure *cb);
+  grpc_workqueue *(*get_workqueue)(grpc_endpoint *ep);
   void (*add_to_pollset)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                          grpc_pollset *pollset);
   void (*add_to_pollset_set)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
@@ -69,6 +70,9 @@ void grpc_endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
 char *grpc_endpoint_get_peer(grpc_endpoint *ep);
 
+/* Retrieve a reference to the workqueue associated with this endpoint */
+grpc_workqueue *grpc_endpoint_get_workqueue(grpc_endpoint *ep);
+
 /* Write slices out to the socket.
 
    If the connection is ready for more data after the end of the call, it

src/core/lib/iomgr/ev_epoll_linux.c

@@ -57,6 +57,7 @@
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/iomgr/workqueue.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/block_annotate.h"
 
@@ -113,9 +114,7 @@ struct grpc_fd {
   grpc_closure *read_closure;
   grpc_closure *write_closure;
 
-  /* The polling island to which this fd belongs to and the mutex protecting the
+  /* The polling island to which this fd belongs to (protected by mu) */
-     the field */
-  gpr_mu pi_mu;
   struct polling_island *polling_island;
 
   struct grpc_fd *freelist_next;
@@ -152,16 +151,17 @@ static void fd_global_shutdown(void);
  * Polling island Declarations
  */
 
-// #define GRPC_PI_REF_COUNT_DEBUG
+//#define GRPC_PI_REF_COUNT_DEBUG
 #ifdef GRPC_PI_REF_COUNT_DEBUG
 
 #define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__)
-#define PI_UNREF(p, r) pi_unref_dbg((p), (r), __FILE__, __LINE__)
+#define PI_UNREF(exec_ctx, p, r) \
+  pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__)
 
 #else /* defined(GRPC_PI_REF_COUNT_DEBUG) */
 
 #define PI_ADD_REF(p, r) pi_add_ref((p))
-#define PI_UNREF(p, r) pi_unref((p))
+#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p))
 
 #endif /* !defined(GPRC_PI_REF_COUNT_DEBUG) */
 
@@ -172,7 +172,7 @@ typedef struct polling_island {
      Once the ref count becomes zero, this structure is destroyed which means
      we should ensure that there is never a scenario where a PI_ADD_REF() is
      racing with a PI_UNREF() that just made the ref_count zero. */
-  gpr_refcount ref_count;
+  gpr_atm ref_count;
 
   /* Pointer to the polling_island this merged into.
    * merged_to value is only set once in polling_island's lifetime (and that too
@@ -184,6 +184,9 @@ typedef struct polling_island {
    * (except mu and ref_count) are invalid and must be ignored. */
   gpr_atm merged_to;
 
+  /* The workqueue associated with this polling island */
+  grpc_workqueue *workqueue;
+
   /* The fd of the underlying epoll set */
   int epoll_fd;
 
@@ -191,11 +194,6 @@ typedef struct polling_island {
   size_t fd_cnt;
   size_t fd_capacity;
   grpc_fd **fds;
-
-  /* Polling islands that are no longer needed are kept in a freelist so that
-     they can be reused. This field points to the next polling island in the
-     free list */
-  struct polling_island *next_free;
 } polling_island;
 
 /*******************************************************************************
@@ -253,13 +251,14 @@ struct grpc_pollset_set {
  * Common helpers
  */
 
-static void append_error(grpc_error **composite, grpc_error *error,
+static bool append_error(grpc_error **composite, grpc_error *error,
                          const char *desc) {
-  if (error == GRPC_ERROR_NONE) return;
+  if (error == GRPC_ERROR_NONE) return true;
   if (*composite == GRPC_ERROR_NONE) {
     *composite = GRPC_ERROR_CREATE(desc);
   }
   *composite = grpc_error_add_child(*composite, error);
+  return false;
 }
 
 /*******************************************************************************
@@ -275,11 +274,8 @@ static void append_error(grpc_error **composite, grpc_error *error,
    threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */
 static grpc_wakeup_fd polling_island_wakeup_fd;
 
-/* Polling island freelist */
+/* Forward declaration */
-static gpr_mu g_pi_freelist_mu;
+static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi);
-static polling_island *g_pi_freelist = NULL;
-
-static void polling_island_delete(); /* Forward declaration */
 
 #ifdef GRPC_TSAN
 /* Currently TSAN may incorrectly flag data races between epoll_ctl and
@@ -293,28 +289,35 @@ gpr_atm g_epoll_sync;
 #endif /* defined(GRPC_TSAN) */
 
 #ifdef GRPC_PI_REF_COUNT_DEBUG
-void pi_add_ref(polling_island *pi);
+static void pi_add_ref(polling_island *pi);
-void pi_unref(polling_island *pi);
+static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi);
 
-void pi_add_ref_dbg(polling_island *pi, char *reason, char *file, int line) {
+static void pi_add_ref_dbg(polling_island *pi, char *reason, char *file,
-  long old_cnt = gpr_atm_acq_load(&(pi->ref_count.count));
+                           int line) {
+  long old_cnt = gpr_atm_acq_load(&pi->ref_count);
   pi_add_ref(pi);
   gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)",
           (void *)pi, old_cnt, old_cnt + 1, reason, file, line);
 }
 
-void pi_unref_dbg(polling_island *pi, char *reason, char *file, int line) {
+static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi,
-  long old_cnt = gpr_atm_acq_load(&(pi->ref_count.count));
+                         char *reason, char *file, int line) {
-  pi_unref(pi);
+  long old_cnt = gpr_atm_acq_load(&pi->ref_count);
+  pi_unref(exec_ctx, pi);
   gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)",
           (void *)pi, old_cnt, (old_cnt - 1), reason, file, line);
 }
 #endif
 
-void pi_add_ref(polling_island *pi) { gpr_ref(&pi->ref_count); }
+static void pi_add_ref(polling_island *pi) {
+  gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1);
+}
 
-void pi_unref(polling_island *pi) {
+static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) {
-  /* If ref count went to zero, delete the polling island.
+  /* If ref count went to one, we're back to just the workqueue owning a ref.
+     Unref the workqueue to break the loop.
+
+     If ref count went to zero, delete the polling island.
      Note that this deletion not be done under a lock. Once the ref count goes
      to zero, we are guaranteed that no one else holds a reference to the
      polling island (and that there is no racing pi_add_ref() call either).
@@ -322,12 +325,20 @@ void pi_unref(polling_island *pi) {
      Also, if we are deleting the polling island and the merged_to field is
      non-empty, we should remove a ref to the merged_to polling island
    */
-  if (gpr_unref(&pi->ref_count)) {
+  switch (gpr_atm_full_fetch_add(&pi->ref_count, -1)) {
-    polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to);
+    case 2: /* last external ref: the only one now owned is by the workqueue */
-    polling_island_delete(pi);
+      GRPC_WORKQUEUE_UNREF(exec_ctx, pi->workqueue, "polling_island");
-    if (next != NULL) {
+      break;
-      PI_UNREF(next, "pi_delete"); /* Recursive call */
+    case 1: {
+      polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to);
+      polling_island_delete(exec_ctx, pi);
+      if (next != NULL) {
+        PI_UNREF(exec_ctx, next, "pi_delete"); /* Recursive call */
+      }
+      break;
     }
+    case 0:
+      GPR_UNREACHABLE_CODE(return );
   }
 }
 
@@ -462,69 +473,68 @@ static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd,
 }
 
 /* Might return NULL in case of an error */
-static polling_island *polling_island_create(grpc_fd *initial_fd,
+static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx,
+                                             grpc_fd *initial_fd,
                                              grpc_error **error) {
   polling_island *pi = NULL;
-  char *err_msg;
   const char *err_desc = "polling_island_create";
 
-  /* Try to get one from the polling island freelist */
+  *error = GRPC_ERROR_NONE;
-  gpr_mu_lock(&g_pi_freelist_mu);
-  if (g_pi_freelist != NULL) {
-    pi = g_pi_freelist;
-    g_pi_freelist = g_pi_freelist->next_free;
-    pi->next_free = NULL;
-  }
-  gpr_mu_unlock(&g_pi_freelist_mu);
 
-  /* Create new polling island if we could not get one from the free list */
+  pi = gpr_malloc(sizeof(*pi));
-  if (pi == NULL) {
+  gpr_mu_init(&pi->mu);
-    pi = gpr_malloc(sizeof(*pi));
+  pi->fd_cnt = 0;
-    gpr_mu_init(&pi->mu);
+  pi->fd_capacity = 0;
-    pi->fd_cnt = 0;
+  pi->fds = NULL;
-    pi->fd_capacity = 0;
+  pi->epoll_fd = -1;
-    pi->fds = NULL;
+  pi->workqueue = NULL;
-  }
 
-  gpr_ref_init(&pi->ref_count, 0);
+  gpr_atm_rel_store(&pi->ref_count, 0);
   gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL);
 
   pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
 
   if (pi->epoll_fd < 0) {
-    gpr_asprintf(&err_msg, "epoll_create1 failed with error %d (%s)", errno,
+    append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc);
-                 strerror(errno));
+    goto done;
-    append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc);
+  }
-    gpr_free(err_msg);
-  } else {
-    polling_island_add_wakeup_fd_locked(pi, &grpc_global_wakeup_fd, error);
-    pi->next_free = NULL;
 
-    if (initial_fd != NULL) {
+  polling_island_add_wakeup_fd_locked(pi, &grpc_global_wakeup_fd, error);
-      /* Lock the polling island here just in case we got this structure from
+
-         the freelist and the polling island lock was not released yet (by the
+  if (initial_fd != NULL) {
-         code that adds the polling island to the freelist) */
+    polling_island_add_fds_locked(pi, &initial_fd, 1, true, error);
-      gpr_mu_lock(&pi->mu);
+  }
-      polling_island_add_fds_locked(pi, &initial_fd, 1, true, error);
+
-      gpr_mu_unlock(&pi->mu);
+  if (append_error(error, grpc_workqueue_create(exec_ctx, &pi->workqueue),
-    }
+                   err_desc) &&
+      *error == GRPC_ERROR_NONE) {
+    polling_island_add_fds_locked(pi, &pi->workqueue->wakeup_read_fd, 1, true,
+                                  error);
+    GPR_ASSERT(pi->workqueue->wakeup_read_fd->polling_island == NULL);
+    pi->workqueue->wakeup_read_fd->polling_island = pi;
+    PI_ADD_REF(pi, "fd");
   }
 
+done:
+  if (*error != GRPC_ERROR_NONE) {
+    if (pi->workqueue != NULL) {
+      GRPC_WORKQUEUE_UNREF(exec_ctx, pi->workqueue, "polling_island");
+    }
+    polling_island_delete(exec_ctx, pi);
+    pi = NULL;
+  }
   return pi;
 }
 
-static void polling_island_delete(polling_island *pi) {
+static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) {
   GPR_ASSERT(pi->fd_cnt == 0);
 
-  gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL);
+  if (pi->epoll_fd >= 0) {
-
+    close(pi->epoll_fd);
-  close(pi->epoll_fd);
+  }
-  pi->epoll_fd = -1;
+  gpr_mu_destroy(&pi->mu);
-
+  gpr_free(pi->fds);
-  gpr_mu_lock(&g_pi_freelist_mu);
+  gpr_free(pi);
-  pi->next_free = g_pi_freelist;
-  g_pi_freelist = pi;
-  gpr_mu_unlock(&g_pi_freelist_mu);
 }
 
 /* Attempts to gets the last polling island in the linked list (liked by the
@@ -704,9 +714,6 @@ static polling_island *polling_island_merge(polling_island *p,
 static grpc_error *polling_island_global_init() {
   grpc_error *error = GRPC_ERROR_NONE;
 
-  gpr_mu_init(&g_pi_freelist_mu);
-  g_pi_freelist = NULL;
-
   error = grpc_wakeup_fd_init(&polling_island_wakeup_fd);
   if (error == GRPC_ERROR_NONE) {
     error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd);
@@ -716,18 +723,6 @@ static grpc_error *polling_island_global_init() {
 }
 
 static void polling_island_global_shutdown() {
-  polling_island *next;
-  gpr_mu_lock(&g_pi_freelist_mu);
-  gpr_mu_unlock(&g_pi_freelist_mu);
-  while (g_pi_freelist != NULL) {
-    next = g_pi_freelist->next_free;
-    gpr_mu_destroy(&g_pi_freelist->mu);
-    gpr_free(g_pi_freelist->fds);
-    gpr_free(g_pi_freelist);
-    g_pi_freelist = next;
-  }
-  gpr_mu_destroy(&g_pi_freelist_mu);
-
   grpc_wakeup_fd_destroy(&polling_island_wakeup_fd);
 }
 
@@ -845,7 +840,6 @@ static grpc_fd *fd_create(int fd, const char *name) {
   if (new_fd == NULL) {
     new_fd = gpr_malloc(sizeof(grpc_fd));
     gpr_mu_init(&new_fd->mu);
-    gpr_mu_init(&new_fd->pi_mu);
   }
 
   /* Note: It is not really needed to get the new_fd->mu lock here. If this is a
@@ -896,6 +890,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       const char *reason) {
   bool is_fd_closed = false;
   grpc_error *error = GRPC_ERROR_NONE;
+  polling_island *unref_pi = NULL;
 
   gpr_mu_lock(&fd->mu);
   fd->on_done_closure = on_done;
@@ -923,21 +918,26 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
      - Unlock the latest polling island
      - Set fd->polling_island to NULL (but remove the ref on the polling island
        before doing this.) */
-  gpr_mu_lock(&fd->pi_mu);
   if (fd->polling_island != NULL) {
     polling_island *pi_latest = polling_island_lock(fd->polling_island);
     polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error);
     gpr_mu_unlock(&pi_latest->mu);
 
-    PI_UNREF(fd->polling_island, "fd_orphan");
+    unref_pi = fd->polling_island;
     fd->polling_island = NULL;
   }
-  gpr_mu_unlock(&fd->pi_mu);
 
   grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, error, NULL);
 
   gpr_mu_unlock(&fd->mu);
   UNREF_BY(fd, 2, reason); /* Drop the reference */
+  if (unref_pi != NULL) {
+    /* Unref stale polling island here, outside the fd lock above.
+       The polling island owns a workqueue which owns an fd, and unreffing
+       inside the lock can cause an eventual lock loop that makes TSAN very
+       unhappy. */
+    PI_UNREF(exec_ctx, unref_pi, "fd_orphan");
+  }
   GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error));
 }
 
@@ -1037,6 +1037,17 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   gpr_mu_unlock(&fd->mu);
 }
 
+static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) {
+  gpr_mu_lock(&fd->mu);
+  grpc_workqueue *workqueue = NULL;
+  if (fd->polling_island != NULL) {
+    workqueue =
+        GRPC_WORKQUEUE_REF(fd->polling_island->workqueue, "get_workqueue");
+  }
+  gpr_mu_unlock(&fd->mu);
+  return workqueue;
+}
+
 /*******************************************************************************
  * Pollset Definitions
  */
@@ -1227,9 +1238,10 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   gpr_mu_unlock(&fd->mu);
 }
 
-static void pollset_release_polling_island(grpc_pollset *ps, char *reason) {
+static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx,
+                                           grpc_pollset *ps, char *reason) {
   if (ps->polling_island != NULL) {
-    PI_UNREF(ps->polling_island, reason);
+    PI_UNREF(exec_ctx, ps->polling_island, reason);
   }
   ps->polling_island = NULL;
 }
@@ -1242,7 +1254,7 @@ static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx,
   pollset->finish_shutdown_called = true;
 
   /* Release the ref and set pollset->polling_island to NULL */
-  pollset_release_polling_island(pollset, "ps_shutdown");
+  pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown");
   grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL);
 }
 
@@ -1281,7 +1293,7 @@ static void pollset_reset(grpc_pollset *pollset) {
   pollset->finish_shutdown_called = false;
   pollset->kicked_without_pollers = false;
   pollset->shutdown_done = NULL;
-  pollset_release_polling_island(pollset, "ps_reset");
+  GPR_ASSERT(pollset->polling_island == NULL);
 }
 
 #define GRPC_EPOLL_MAX_EVENTS 1000
@@ -1309,7 +1321,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
      this function (i.e pollset_work_and_unlock()) is called */
 
   if (pollset->polling_island == NULL) {
-    pollset->polling_island = polling_island_create(NULL, error);
+    pollset->polling_island = polling_island_create(exec_ctx, NULL, error);
     if (pollset->polling_island == NULL) {
       GPR_TIMER_END("pollset_work_and_unlock", 0);
       return; /* Fatal error. We cannot continue */
@@ -1329,7 +1341,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
     /* Always do PI_ADD_REF before PI_UNREF because PI_UNREF may cause the
        polling island to be deleted */
     PI_ADD_REF(pi, "ps");
-    PI_UNREF(pollset->polling_island, "ps");
+    PI_UNREF(exec_ctx, pollset->polling_island, "ps");
     pollset->polling_island = pi;
   }
 
@@ -1400,7 +1412,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
      that we got before releasing the polling island lock). This is because
      pollset->polling_island pointer might get udpated in other parts of the
      code when there is an island merge while we are doing epoll_wait() above */
-  PI_UNREF(pi, "ps_work");
+  PI_UNREF(exec_ctx, pi, "ps_work");
 
   GPR_TIMER_END("pollset_work_and_unlock", 0);
 }
@@ -1517,10 +1529,11 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   grpc_error *error = GRPC_ERROR_NONE;
 
   gpr_mu_lock(&pollset->mu);
-  gpr_mu_lock(&fd->pi_mu);
+  gpr_mu_lock(&fd->mu);
 
   polling_island *pi_new = NULL;
 
+retry:
   /* 1) If fd->polling_island and pollset->polling_island are both non-NULL and
    *    equal, do nothing.
    * 2) If fd->polling_island and pollset->polling_island are both NULL, create
@@ -1535,15 +1548,44 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
    *    polling_island fields in both fd and pollset to point to the merged
    *    polling island.
    */
+
+  if (fd->orphaned) {
+    gpr_mu_unlock(&fd->mu);
+    gpr_mu_unlock(&pollset->mu);
+    /* early out */
+    return;
+  }
+
   if (fd->polling_island == pollset->polling_island) {
     pi_new = fd->polling_island;
     if (pi_new == NULL) {
-      pi_new = polling_island_create(fd, &error);
+      /* Unlock before creating a new polling island: the polling island will
-
+         create a workqueue which creates a file descriptor, and holding an fd
-      GRPC_POLLING_TRACE(
+         lock here can eventually cause a loop to appear to TSAN (making it
-          "pollset_add_fd: Created new polling island. pi_new: %p (fd: %d, "
+         unhappy). We don't think it's a real loop (there's an epoch point where
-          "pollset: %p)",
+         that loop possibility disappears), but the advantages of keeping TSAN
-          (void *)pi_new, fd->fd, (void *)pollset);
+         happy outweigh any performance advantage we might have by keeping the
+         lock held. */
+      gpr_mu_unlock(&fd->mu);
+      pi_new = polling_island_create(exec_ctx, fd, &error);
+      gpr_mu_lock(&fd->mu);
+      /* Need to reverify any assumptions made between the initial lock and
+         getting to this branch: if they've changed, we need to throw away our
+         work and figure things out again. */
+      if (fd->polling_island != NULL) {
+        GRPC_POLLING_TRACE(
+            "pollset_add_fd: Raced creating new polling island. pi_new: %p "
+            "(fd: %d, pollset: %p)",
+            (void *)pi_new, fd->fd, (void *)pollset);
+        PI_ADD_REF(pi_new, "dance_of_destruction");
+        PI_UNREF(exec_ctx, pi_new, "dance_of_destruction");
+        goto retry;
+      } else {
+        GRPC_POLLING_TRACE(
+            "pollset_add_fd: Created new polling island. pi_new: %p (fd: %d, "
+            "pollset: %p)",
+            (void *)pi_new, fd->fd, (void *)pollset);
+      }
     }
   } else if (fd->polling_island == NULL) {
     pi_new = polling_island_lock(pollset->polling_island);
@@ -1579,7 +1621,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (fd->polling_island != pi_new) {
     PI_ADD_REF(pi_new, "fd");
     if (fd->polling_island != NULL) {
-      PI_UNREF(fd->polling_island, "fd");
+      PI_UNREF(exec_ctx, fd->polling_island, "fd");
     }
     fd->polling_island = pi_new;
   }
@@ -1587,13 +1629,15 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (pollset->polling_island != pi_new) {
     PI_ADD_REF(pi_new, "ps");
     if (pollset->polling_island != NULL) {
-      PI_UNREF(pollset->polling_island, "ps");
+      PI_UNREF(exec_ctx, pollset->polling_island, "ps");
     }
     pollset->polling_island = pi_new;
   }
 
-  gpr_mu_unlock(&fd->pi_mu);
+  gpr_mu_unlock(&fd->mu);
   gpr_mu_unlock(&pollset->mu);
+
+  GRPC_LOG_IF_ERROR("pollset_add_fd", error);
 }
 
 /*******************************************************************************
@@ -1744,9 +1788,9 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
 void *grpc_fd_get_polling_island(grpc_fd *fd) {
   polling_island *pi;
 
-  gpr_mu_lock(&fd->pi_mu);
+  gpr_mu_lock(&fd->mu);
   pi = fd->polling_island;
-  gpr_mu_unlock(&fd->pi_mu);
+  gpr_mu_unlock(&fd->mu);
 
   return pi;
 }
@@ -1794,6 +1838,7 @@ static const grpc_event_engine_vtable vtable = {
     .fd_notify_on_read = fd_notify_on_read,
     .fd_notify_on_write = fd_notify_on_write,
     .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
+    .fd_get_workqueue = fd_get_workqueue,
 
     .pollset_init = pollset_init,
     .pollset_shutdown = pollset_shutdown,

src/core/lib/iomgr/ev_poll_and_epoll_posix.c

@@ -725,6 +725,8 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
   GRPC_FD_UNREF(fd, "poll");
 }
 
+static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { return NULL; }
+
 /*******************************************************************************
  * pollset_posix.c
  */
@@ -2006,6 +2008,7 @@ static const grpc_event_engine_vtable vtable = {
     .fd_notify_on_read = fd_notify_on_read,
     .fd_notify_on_write = fd_notify_on_write,
     .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
+    .fd_get_workqueue = fd_get_workqueue,
 
     .pollset_init = pollset_init,
     .pollset_shutdown = pollset_shutdown,

src/core/lib/iomgr/ev_poll_posix.c

@@ -617,6 +617,8 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
   GRPC_FD_UNREF(fd, "poll");
 }
 
+static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { return NULL; }
+
 /*******************************************************************************
  * pollset_posix.c
  */
@@ -842,6 +844,11 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   *worker_hdl = &worker;
   grpc_error *error = GRPC_ERROR_NONE;
 
+  /* Avoid malloc for small number of elements. */
+  enum { inline_elements = 96 };
+  struct pollfd pollfd_space[inline_elements];
+  struct grpc_fd_watcher watcher_space[inline_elements];
+
   /* pollset->mu already held */
   int added_worker = 0;
   int locked = 1;
@@ -897,15 +904,23 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       int r;
       size_t i, fd_count;
       nfds_t pfd_count;
-      /* TODO(ctiller): inline some elements to avoid an allocation */
       grpc_fd_watcher *watchers;
       struct pollfd *pfds;
 
       timeout = poll_deadline_to_millis_timeout(deadline, now);
-      /* TODO(ctiller): perform just one malloc here if we exceed the inline
+
-       * case */
+      if (pollset->fd_count + 2 <= inline_elements) {
-      pfds = gpr_malloc(sizeof(*pfds) * (pollset->fd_count + 2));
+        pfds = pollfd_space;
-      watchers = gpr_malloc(sizeof(*watchers) * (pollset->fd_count + 2));
+        watchers = watcher_space;
+      } else {
+        /* Allocate one buffer to hold both pfds and watchers arrays */
+        const size_t pfd_size = sizeof(*pfds) * (pollset->fd_count + 2);
+        const size_t watch_size = sizeof(*watchers) * (pollset->fd_count + 2);
+        void *buf = gpr_malloc(pfd_size + watch_size);
+        pfds = buf;
+        watchers = (void *)((char *)buf + pfd_size);
+      }
+
       fd_count = 0;
       pfd_count = 2;
       pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd);
@@ -972,8 +987,11 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
         }
       }
 
-      gpr_free(pfds);
+      if (pfds != pollfd_space) {
-      gpr_free(watchers);
+        /* pfds and watchers are in the same memory block pointed to by pfds */
+        gpr_free(pfds);
+      }
+
       GPR_TIMER_END("maybe_work_and_unlock", 0);
       locked = 0;
     } else {
@@ -1234,6 +1252,7 @@ static const grpc_event_engine_vtable vtable = {
     .fd_notify_on_read = fd_notify_on_read,
     .fd_notify_on_write = fd_notify_on_write,
     .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
+    .fd_get_workqueue = fd_get_workqueue,
 
     .pollset_init = pollset_init,
     .pollset_shutdown = pollset_shutdown,

src/core/lib/iomgr/ev_posix.c

@@ -148,6 +148,10 @@ grpc_fd *grpc_fd_create(int fd, const char *name) {
   return g_event_engine->fd_create(fd, name);
 }
 
+grpc_workqueue *grpc_fd_get_workqueue(grpc_fd *fd) {
+  return g_event_engine->fd_get_workqueue(fd);
+}
+
 int grpc_fd_wrapped_fd(grpc_fd *fd) {
   return g_event_engine->fd_wrapped_fd(fd);
 }

src/core/lib/iomgr/ev_posix.h

@@ -56,6 +56,7 @@ typedef struct grpc_event_engine_vtable {
   void (*fd_notify_on_write)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
   bool (*fd_is_shutdown)(grpc_fd *fd);
+  grpc_workqueue *(*fd_get_workqueue)(grpc_fd *fd);
   grpc_pollset *(*fd_get_read_notifier_pollset)(grpc_exec_ctx *exec_ctx,
                                                 grpc_fd *fd);
 
@@ -107,6 +108,9 @@ const char *grpc_get_poll_strategy_name();
    This takes ownership of closing fd. */
 grpc_fd *grpc_fd_create(int fd, const char *name);
 
+/* Get a workqueue that's associated with this fd */
+grpc_workqueue *grpc_fd_get_workqueue(grpc_fd *fd);
+
 /* Return the wrapped fd, or -1 if it has been released or closed. */
 int grpc_fd_wrapped_fd(grpc_fd *fd);
 

src/core/lib/iomgr/exec_ctx.c

@@ -37,6 +37,7 @@
 #include <grpc/support/sync.h>
 #include <grpc/support/thd.h>
 
+#include "src/core/lib/iomgr/workqueue.h"
 #include "src/core/lib/profiling/timers.h"
 
 bool grpc_exec_ctx_ready_to_finish(grpc_exec_ctx *exec_ctx) {
@@ -85,14 +86,17 @@ void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) {
 void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
                          grpc_error *error,
                          grpc_workqueue *offload_target_or_null) {
-  GPR_ASSERT(offload_target_or_null == NULL);
+  if (offload_target_or_null == NULL) {
-  grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
+    grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
+  } else {
+    grpc_workqueue_enqueue(exec_ctx, offload_target_or_null, closure, error);
+    GRPC_WORKQUEUE_UNREF(exec_ctx, offload_target_or_null, "exec_ctx_sched");
+  }
 }
 
 void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx,
                                 grpc_closure_list *list,
                                 grpc_workqueue *offload_target_or_null) {
-  GPR_ASSERT(offload_target_or_null == NULL);
   grpc_closure_list_move(list, &exec_ctx->closure_list);
 }
 

src/core/lib/iomgr/exec_ctx.h

@@ -93,7 +93,11 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx);
 /** Finish any pending work for a grpc_exec_ctx. Must be called before
  *  the instance is destroyed, or work may be lost. */
 void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx);
-/** Add a closure to be executed at the next flush/finish point */
+/** Add a closure to be executed in the future.
+    If \a offload_target_or_null is NULL, the closure will be executed at the
+    next exec_ctx.{finish,flush} point.
+    If \a offload_target_or_null is non-NULL, the closure will be scheduled
+    against the workqueue, and a reference to the workqueue will be consumed. */
 void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
                          grpc_error *error,
                          grpc_workqueue *offload_target_or_null);

src/core/lib/iomgr/iomgr.c

@@ -45,6 +45,7 @@
 
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/network_status_tracker.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
@@ -62,6 +63,7 @@ void grpc_iomgr_init(void) {
   grpc_timer_list_init(gpr_now(GPR_CLOCK_MONOTONIC));
   g_root_object.next = g_root_object.prev = &g_root_object;
   g_root_object.name = "root";
+  grpc_network_status_init();
   grpc_iomgr_platform_init();
 }
 
@@ -140,6 +142,7 @@ void grpc_iomgr_shutdown(void) {
 
   grpc_iomgr_platform_shutdown();
   grpc_exec_ctx_global_shutdown();
+  grpc_network_status_shutdown();
   gpr_mu_destroy(&g_mu);
   gpr_cv_destroy(&g_rcv);
 }

src/core/lib/iomgr/network_status_tracker.c

@@ -42,9 +42,8 @@ typedef struct endpoint_ll_node {
 
 static endpoint_ll_node *head = NULL;
 static gpr_mu g_endpoint_mutex;
-static gpr_once g_once_init = GPR_ONCE_INIT;
 
-static void destroy_network_status_monitor(void) {
+void grpc_network_status_shutdown(void) {
   if (head != NULL) {
     gpr_log(GPR_ERROR,
             "Memory leaked as all network endpoints were not shut down");
@@ -52,14 +51,21 @@ static void destroy_network_status_monitor(void) {
   gpr_mu_destroy(&g_endpoint_mutex);
 }
 
-static void initialize_network_status_monitor(void) {
+void grpc_network_status_init(void) {
   gpr_mu_init(&g_endpoint_mutex);
-  atexit(destroy_network_status_monitor);
   // TODO(makarandd): Install callback with OS to monitor network status.
 }
 
+void grpc_destroy_network_status_monitor() {
+  for (endpoint_ll_node *curr = head; curr != NULL;) {
+    endpoint_ll_node *next = curr->next;
+    gpr_free(curr);
+    curr = next;
+  }
+  gpr_mu_destroy(&g_endpoint_mutex);
+}
+
 void grpc_network_status_register_endpoint(grpc_endpoint *ep) {
-  gpr_once_init(&g_once_init, initialize_network_status_monitor);
   gpr_mu_lock(&g_endpoint_mutex);
   if (head == NULL) {
     head = (endpoint_ll_node *)gpr_malloc(sizeof(endpoint_ll_node));

src/core/lib/iomgr/network_status_tracker.h

@@ -35,7 +35,11 @@
 #define GRPC_CORE_LIB_IOMGR_NETWORK_STATUS_TRACKER_H
 #include "src/core/lib/iomgr/endpoint.h"
 
+void grpc_network_status_init(void);
+void grpc_network_status_shutdown(void);
+
 void grpc_network_status_register_endpoint(grpc_endpoint *ep);
 void grpc_network_status_unregister_endpoint(grpc_endpoint *ep);
 void grpc_network_status_shutdown_all_endpoints();
+
 #endif /* GRPC_CORE_LIB_IOMGR_NETWORK_STATUS_TRACKER_H */

src/core/lib/iomgr/tcp_posix.c

@@ -284,7 +284,7 @@ static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 }
 
 /* returns true if done, false if pending; if returning true, *error is set */
-#define MAX_WRITE_IOVEC 16
+#define MAX_WRITE_IOVEC 1024
 static bool tcp_flush(grpc_tcp *tcp, grpc_error **error) {
   struct msghdr msg;
   struct iovec iov[MAX_WRITE_IOVEC];
@@ -450,9 +450,19 @@ static char *tcp_get_peer(grpc_endpoint *ep) {
   return gpr_strdup(tcp->peer_string);
 }
 
-static const grpc_endpoint_vtable vtable = {
+static grpc_workqueue *tcp_get_workqueue(grpc_endpoint *ep) {
-    tcp_read,     tcp_write,   tcp_add_to_pollset, tcp_add_to_pollset_set,
+  grpc_tcp *tcp = (grpc_tcp *)ep;
-    tcp_shutdown, tcp_destroy, tcp_get_peer};
+  return grpc_fd_get_workqueue(tcp->em_fd);
+}
+
+static const grpc_endpoint_vtable vtable = {tcp_read,
+                                            tcp_write,
+                                            tcp_get_workqueue,
+                                            tcp_add_to_pollset,
+                                            tcp_add_to_pollset_set,
+                                            tcp_shutdown,
+                                            tcp_destroy,
+                                            tcp_get_peer};
 
 grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size,
                                const char *peer_string) {

src/core/lib/iomgr/tcp_server.h

@@ -105,4 +105,8 @@ void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s,
    a call (exec_ctx!=NULL) to shutdown_complete. */
 void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s);
 
+/* Shutdown the fds of listeners. */
+void grpc_tcp_server_shutdown_listeners(grpc_exec_ctx *exec_ctx,
+                                        grpc_tcp_server *s);
+
 #endif /* GRPC_CORE_LIB_IOMGR_TCP_SERVER_H */

src/core/lib/iomgr/tcp_server_posix.c

@@ -491,7 +491,8 @@ static grpc_error *clone_port(grpc_tcp_listener *listener, unsigned count) {
   }
 
   for (unsigned i = 0; i < count; i++) {
-    int fd, port;
+    int fd = -1;
+    int port = -1;
     grpc_dualstack_mode dsmode;
     err = grpc_create_dualstack_socket(&listener->addr.sockaddr, SOCK_STREAM, 0,
                                        &dsmode, &fd);
@@ -740,4 +741,17 @@ void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   }
 }
 
+void grpc_tcp_server_shutdown_listeners(grpc_exec_ctx *exec_ctx,
+                                        grpc_tcp_server *s) {
+  gpr_mu_lock(&s->mu);
+  /* shutdown all fd's */
+  if (s->active_ports) {
+    grpc_tcp_listener *sp;
+    for (sp = s->head; sp; sp = sp->next) {
+      grpc_fd_shutdown(exec_ctx, sp->emfd);
+    }
+  }
+  gpr_mu_unlock(&s->mu);
+}
+
 #endif

src/core/lib/iomgr/tcp_server_windows.c

@@ -540,4 +540,7 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
   gpr_mu_unlock(&s->mu);
 }
 
+void grpc_tcp_server_shutdown_listeners(grpc_exec_ctx *exec_ctx,
+                                        grpc_tcp_server *s) {}
+
 #endif /* GPR_WINSOCK_SOCKET */

src/core/lib/iomgr/tcp_windows.c

@@ -389,9 +389,16 @@ static char *win_get_peer(grpc_endpoint *ep) {
   return gpr_strdup(tcp->peer_string);
 }
 
-static grpc_endpoint_vtable vtable = {
+static grpc_workqueue *win_get_workqueue(grpc_endpoint *ep) { return NULL; }
-    win_read,     win_write,   win_add_to_pollset, win_add_to_pollset_set,
+
-    win_shutdown, win_destroy, win_get_peer};
+static grpc_endpoint_vtable vtable = {win_read,
+                                      win_write,
+                                      win_get_workqueue,
+                                      win_add_to_pollset,
+                                      win_add_to_pollset_set,
+                                      win_shutdown,
+                                      win_destroy,
+                                      win_get_peer};
 
 grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) {
   grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp));