From e73e9e1a98600dfbf405c9bcbbfdcd4bd05d78f4 Mon Sep 17 00:00:00 2001 From: Christian Svensson Date: Thu, 10 Mar 2016 17:04:11 +0100 Subject: [PATCH 001/155] Allow directly specifiying connection path Make insecure_channel / secure_channel accept port=None and only use the 'host' argument. This enables using UNIX sockets without mangling the path of the socket. Signed-off-by: Christian Svensson --- src/python/grpcio/grpc/beta/implementations.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio/grpc/beta/implementations.py b/src/python/grpcio/grpc/beta/implementations.py index a0ca330d2c0..6f1ca2bb5ed 100644 --- a/src/python/grpcio/grpc/beta/implementations.py +++ b/src/python/grpcio/grpc/beta/implementations.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -187,12 +187,13 @@ def insecure_channel(host, port): Args: host: The name of the remote host to which to connect. port: The port of the remote host to which to connect. + If None only the 'host' part will be used. Returns: A Channel to the remote host through which RPCs may be conducted. """ intermediary_low_channel = _intermediary_low.Channel( - '%s:%d' % (host, port), None) + '%s:%d' % (host, port) if port else host, None) return Channel(intermediary_low_channel._internal, intermediary_low_channel) # pylint: disable=protected-access @@ -202,13 +203,15 @@ def secure_channel(host, port, channel_credentials): Args: host: The name of the remote host to which to connect. port: The port of the remote host to which to connect. + If None only the 'host' part will be used. channel_credentials: A ChannelCredentials. Returns: A secure Channel to the remote host through which RPCs may be conducted. """ intermediary_low_channel = _intermediary_low.Channel( - '%s:%d' % (host, port), channel_credentials._low_credentials) + '%s:%d' % (host, port) if port else host, + channel_credentials._low_credentials) return Channel(intermediary_low_channel._internal, intermediary_low_channel) # pylint: disable=protected-access From 7f0793ad623ae1dd2b175306a0e7eb060d53511b Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 25 Apr 2016 12:35:58 -0700 Subject: [PATCH 002/155] Updated load balancer proto to v1 --- BUILD | 12 +- Makefile | 18 +- binding.gyp | 2 +- build.yaml | 6 +- config.m4 | 4 +- gRPC.podspec | 6 +- grpc.gemspec | 4 +- package.xml | 4 +- .../ext/lb_policy/grpclb/load_balancer_api.c | 14 +- .../ext/lb_policy/grpclb/load_balancer_api.h | 10 +- .../proto/grpc/lb/v0/load_balancer.pb.h | 182 ------------------ .../grpc/lb/{v0 => v1}/load_balancer.pb.c | 59 +++--- .../proto/grpc/lb/v1/load_balancer.pb.h | 178 +++++++++++++++++ src/proto/grpc/lb/v0/load_balancer.options | 6 - src/proto/grpc/lb/v1/load_balancer.options | 6 + .../grpc/lb/{v0 => v1}/load_balancer.proto | 11 +- src/python/grpcio/grpc_core_dependencies.py | 2 +- test/cpp/grpclb/grpclb_api_test.cc | 11 +- tools/codegen/core/gen_nano_proto.sh | 6 +- tools/distrib/check_include_guards.py | 2 +- tools/distrib/check_nanopb_output.sh | 6 +- tools/doxygen/Doxyfile.core.internal | 4 +- tools/run_tests/sources_and_headers.json | 10 +- vsprojects/vcxproj/grpc/grpc.vcxproj | 4 +- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 12 +- .../grpc_unsecure/grpc_unsecure.vcxproj | 4 +- .../grpc_unsecure.vcxproj.filters | 12 +- .../grpclb_api_test/grpclb_api_test.vcxproj | 8 +- .../grpclb_api_test.vcxproj.filters | 8 +- 29 files changed, 299 insertions(+), 312 deletions(-) delete mode 100644 src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h rename src/core/ext/lb_policy/grpclb/proto/grpc/lb/{v0 => v1}/load_balancer.pb.c (55%) create mode 100644 src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h delete mode 100644 src/proto/grpc/lb/v0/load_balancer.options create mode 100644 src/proto/grpc/lb/v1/load_balancer.options rename src/proto/grpc/lb/{v0 => v1}/load_balancer.proto (94%) diff --git a/BUILD b/BUILD index f8e7661ad46..ff5220bc8ed 100644 --- a/BUILD +++ b/BUILD @@ -284,7 +284,7 @@ cc_library( "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", "src/core/ext/census/aggregation.h", "src/core/ext/census/census_interface.h", "src/core/ext/census/census_rpc_stats.h", @@ -438,7 +438,7 @@ cc_library( "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c", "src/core/ext/transport/chttp2/client/insecure/channel_create.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", - "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", "src/core/ext/lb_policy/pick_first/pick_first.c", "src/core/ext/lb_policy/round_robin/round_robin.c", "src/core/ext/resolver/dns/native/dns_resolver.c", @@ -615,7 +615,7 @@ cc_library( "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", "src/core/ext/census/aggregation.h", "src/core/ext/census/census_interface.h", "src/core/ext/census/census_rpc_stats.h", @@ -751,7 +751,7 @@ cc_library( "src/core/ext/resolver/dns/native/dns_resolver.c", "src/core/ext/resolver/sockaddr/sockaddr_resolver.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", - "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", "src/core/ext/lb_policy/pick_first/pick_first.c", "src/core/ext/lb_policy/round_robin/round_robin.c", "src/core/ext/census/context.c", @@ -1450,7 +1450,7 @@ objc_library( "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c", "src/core/ext/transport/chttp2/client/insecure/channel_create.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", - "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", "src/core/ext/lb_policy/pick_first/pick_first.c", "src/core/ext/lb_policy/round_robin/round_robin.c", "src/core/ext/resolver/dns/native/dns_resolver.c", @@ -1620,7 +1620,7 @@ objc_library( "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h", "src/core/ext/census/aggregation.h", "src/core/ext/census/census_interface.h", "src/core/ext/census/census_rpc_stats.h", diff --git a/Makefile b/Makefile index 12022c2dd95..027a24664a8 100644 --- a/Makefile +++ b/Makefile @@ -1862,15 +1862,15 @@ $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc: $(Q) echo "$(GRPCXX_UNSECURE_PC_FILE)" | tr , '\n' >$@ ifeq ($(NO_PROTOC),true) -$(GENDIR)/src/proto/grpc/lb/v0/load_balancer.pb.cc: protoc_dep_error -$(GENDIR)/src/proto/grpc/lb/v0/load_balancer.grpc.pb.cc: protoc_dep_error +$(GENDIR)/src/proto/grpc/lb/v1/load_balancer.pb.cc: protoc_dep_error +$(GENDIR)/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc: protoc_dep_error else -$(GENDIR)/src/proto/grpc/lb/v0/load_balancer.pb.cc: src/proto/grpc/lb/v0/load_balancer.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) +$(GENDIR)/src/proto/grpc/lb/v1/load_balancer.pb.cc: src/proto/grpc/lb/v1/load_balancer.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(E) "[PROTOC] Generating protobuf CC file from $<" $(Q) mkdir -p `dirname $@` $(Q) $(PROTOC) --cpp_out=$(GENDIR) $< -$(GENDIR)/src/proto/grpc/lb/v0/load_balancer.grpc.pb.cc: src/proto/grpc/lb/v0/load_balancer.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) +$(GENDIR)/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc: src/proto/grpc/lb/v1/load_balancer.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(E) "[GRPC] Generating gRPC's protobuf service CC file from $<" $(Q) mkdir -p `dirname $@` $(Q) $(PROTOC) --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $< @@ -2597,7 +2597,7 @@ LIBGRPC_SRC = \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ + src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_decode.c \ third_party/nanopb/pb_encode.c \ @@ -2917,7 +2917,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/resolver/dns/native/dns_resolver.c \ src/core/ext/resolver/sockaddr/sockaddr_resolver.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ + src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_decode.c \ third_party/nanopb/pb_encode.c \ @@ -10602,7 +10602,7 @@ endif GRPCLB_API_TEST_SRC = \ - $(GENDIR)/src/proto/grpc/lb/v0/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v0/load_balancer.grpc.pb.cc \ + $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc \ test/cpp/grpclb/grpclb_api_test.cc \ GRPCLB_API_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPCLB_API_TEST_SRC)))) @@ -10634,7 +10634,7 @@ endif endif -$(OBJDIR)/$(CONFIG)/src/proto/grpc/lb/v0/load_balancer.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a +$(OBJDIR)/$(CONFIG)/src/proto/grpc/lb/v1/load_balancer.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(OBJDIR)/$(CONFIG)/test/cpp/grpclb/grpclb_api_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a @@ -10645,7 +10645,7 @@ ifneq ($(NO_DEPS),true) -include $(GRPCLB_API_TEST_OBJS:.o=.dep) endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/grpclb/grpclb_api_test.o: $(GENDIR)/src/proto/grpc/lb/v0/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v0/load_balancer.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/grpclb/grpclb_api_test.o: $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc HYBRID_END2END_TEST_SRC = \ diff --git a/binding.gyp b/binding.gyp index 058743edbf8..0fecc66c460 100644 --- a/binding.gyp +++ b/binding.gyp @@ -708,7 +708,7 @@ 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', - 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', + 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', 'third_party/nanopb/pb_decode.c', 'third_party/nanopb/pb_encode.c', diff --git a/build.yaml b/build.yaml index 26308e8db5d..d5cfb3c2d61 100644 --- a/build.yaml +++ b/build.yaml @@ -359,10 +359,10 @@ filegroups: - name: grpc_lb_policy_grpclb headers: - src/core/ext/lb_policy/grpclb/load_balancer_api.h - - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h + - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h src: - src/core/ext/lb_policy/grpclb/load_balancer_api.c - - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c + - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c uses: - grpc_base - grpc_client_config @@ -2601,7 +2601,7 @@ targets: build: test language: c++ src: - - src/proto/grpc/lb/v0/load_balancer.proto + - src/proto/grpc/lb/v1/load_balancer.proto - test/cpp/grpclb/grpclb_api_test.cc deps: - grpc++_test_util diff --git a/config.m4 b/config.m4 index 2d930a648e1..c60297213d2 100644 --- a/config.m4 +++ b/config.m4 @@ -227,7 +227,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ - src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ + src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_decode.c \ third_party/nanopb/pb_encode.c \ @@ -553,7 +553,7 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/census) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/client_config) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/lb_policy/grpclb) - PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/lb_policy/pick_first) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/lb_policy/round_robin) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/resolver/dns/native) diff --git a/gRPC.podspec b/gRPC.podspec index d66e03354bb..6f631c1c558 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -286,7 +286,7 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', - 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', + 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', 'third_party/nanopb/pb.h', 'third_party/nanopb/pb_common.h', 'third_party/nanopb/pb_decode.h', @@ -472,7 +472,7 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', - 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', + 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', 'third_party/nanopb/pb_decode.c', 'third_party/nanopb/pb_encode.c', @@ -628,7 +628,7 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', - 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', + 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', 'third_party/nanopb/pb.h', 'third_party/nanopb/pb_common.h', 'third_party/nanopb/pb_decode.h', diff --git a/grpc.gemspec b/grpc.gemspec index f83b7d35872..1512da6b569 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -298,7 +298,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/client_config/subchannel_index.h ) s.files += %w( src/core/ext/client_config/uri_parser.h ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.h ) - s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h ) + s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h ) s.files += %w( third_party/nanopb/pb.h ) s.files += %w( third_party/nanopb/pb_common.h ) s.files += %w( third_party/nanopb/pb_decode.h ) @@ -456,7 +456,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/transport/chttp2/server/insecure/server_chttp2.c ) s.files += %w( src/core/ext/transport/chttp2/client/insecure/channel_create.c ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.c ) - s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c ) + s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c ) s.files += %w( third_party/nanopb/pb_common.c ) s.files += %w( third_party/nanopb/pb_decode.c ) s.files += %w( third_party/nanopb/pb_encode.c ) diff --git a/package.xml b/package.xml index d192ebde2cc..295901aaffe 100644 --- a/package.xml +++ b/package.xml @@ -301,7 +301,7 @@ - + @@ -459,7 +459,7 @@ - + diff --git a/src/core/ext/lb_policy/grpclb/load_balancer_api.c b/src/core/ext/lb_policy/grpclb/load_balancer_api.c index 459d6d9954d..59b89997dd9 100644 --- a/src/core/ext/lb_policy/grpclb/load_balancer_api.c +++ b/src/core/ext/lb_policy/grpclb/load_balancer_api.c @@ -50,7 +50,7 @@ static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field, decode_serverlist_arg *dec_arg = *arg; if (dec_arg->first_pass != 0) { /* first pass */ grpc_grpclb_server server; - if (!pb_decode(stream, grpc_lb_v0_Server_fields, &server)) { + if (!pb_decode(stream, grpc_lb_v1_Server_fields, &server)) { return false; } dec_arg->num_servers++; @@ -61,7 +61,7 @@ static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field, dec_arg->servers = gpr_malloc(sizeof(grpc_grpclb_server *) * dec_arg->num_servers); } - if (!pb_decode(stream, grpc_lb_v0_Server_fields, server)) { + if (!pb_decode(stream, grpc_lb_v1_Server_fields, server)) { return false; } dec_arg->servers[dec_arg->i++] = server; @@ -87,13 +87,13 @@ gpr_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request) { pb_ostream_t outputstream; gpr_slice slice; memset(&sizestream, 0, sizeof(pb_ostream_t)); - pb_encode(&sizestream, grpc_lb_v0_LoadBalanceRequest_fields, request); + pb_encode(&sizestream, grpc_lb_v1_LoadBalanceRequest_fields, request); encoded_length = sizestream.bytes_written; slice = gpr_slice_malloc(encoded_length); outputstream = pb_ostream_from_buffer(GPR_SLICE_START_PTR(slice), encoded_length); - GPR_ASSERT(pb_encode(&outputstream, grpc_lb_v0_LoadBalanceRequest_fields, + GPR_ASSERT(pb_encode(&outputstream, grpc_lb_v1_LoadBalanceRequest_fields, request) != 0); return slice; } @@ -109,7 +109,7 @@ grpc_grpclb_response *grpc_grpclb_response_parse(gpr_slice encoded_response) { GPR_SLICE_LENGTH(encoded_response)); grpc_grpclb_response *res = gpr_malloc(sizeof(grpc_grpclb_response)); memset(res, 0, sizeof(*res)); - status = pb_decode(&stream, grpc_lb_v0_LoadBalanceResponse_fields, res); + status = pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, res); if (!status) { grpc_grpclb_response_destroy(res); return NULL; @@ -132,7 +132,7 @@ grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist( res->server_list.servers.funcs.decode = decode_serverlist; res->server_list.servers.arg = &arg; arg.first_pass = 1; - status = pb_decode(&stream, grpc_lb_v0_LoadBalanceResponse_fields, res); + status = pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, res); if (!status) { grpc_grpclb_response_destroy(res); return NULL; @@ -140,7 +140,7 @@ grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist( arg.first_pass = 0; status = - pb_decode(&stream_at_start, grpc_lb_v0_LoadBalanceResponse_fields, res); + pb_decode(&stream_at_start, grpc_lb_v1_LoadBalanceResponse_fields, res); if (!status) { grpc_grpclb_response_destroy(res); return NULL; diff --git a/src/core/ext/lb_policy/grpclb/load_balancer_api.h b/src/core/ext/lb_policy/grpclb/load_balancer_api.h index 968f7d278ae..71b5616d0c8 100644 --- a/src/core/ext/lb_policy/grpclb/load_balancer_api.h +++ b/src/core/ext/lb_policy/grpclb/load_balancer_api.h @@ -37,7 +37,7 @@ #include #include "src/core/ext/client_config/lb_policy_factory.h" -#include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h" +#include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" #ifdef __cplusplus extern "C" { @@ -45,10 +45,10 @@ extern "C" { #define GRPC_GRPCLB_SERVICE_NAME_MAX_LENGTH 128 -typedef grpc_lb_v0_LoadBalanceRequest grpc_grpclb_request; -typedef grpc_lb_v0_LoadBalanceResponse grpc_grpclb_response; -typedef grpc_lb_v0_Server grpc_grpclb_server; -typedef grpc_lb_v0_Duration grpc_grpclb_duration; +typedef grpc_lb_v1_LoadBalanceRequest grpc_grpclb_request; +typedef grpc_lb_v1_LoadBalanceResponse grpc_grpclb_response; +typedef grpc_lb_v1_Server grpc_grpclb_server; +typedef grpc_lb_v1_Duration grpc_grpclb_duration; typedef struct grpc_grpclb_serverlist { grpc_grpclb_server **servers; size_t num_servers; diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h deleted file mode 100644 index 3599f881bb1..00000000000 --- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h +++ /dev/null @@ -1,182 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ -/* Automatically generated nanopb header */ -/* Generated by nanopb-0.3.5-dev */ - -#ifndef PB_LOAD_BALANCER_PB_H_INCLUDED -#define PB_LOAD_BALANCER_PB_H_INCLUDED -#include "third_party/nanopb/pb.h" -#if PB_PROTO_HEADER_VERSION != 30 -#error Regenerate this file with the current version of nanopb generator. -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/* Struct definitions */ -typedef struct _grpc_lb_v0_ClientStats { - bool has_total_requests; - int64_t total_requests; - bool has_client_rpc_errors; - int64_t client_rpc_errors; - bool has_dropped_requests; - int64_t dropped_requests; -} grpc_lb_v0_ClientStats; - -typedef struct _grpc_lb_v0_Duration { - bool has_seconds; - int64_t seconds; - bool has_nanos; - int32_t nanos; -} grpc_lb_v0_Duration; - -typedef struct _grpc_lb_v0_InitialLoadBalanceRequest { - bool has_name; - char name[128]; -} grpc_lb_v0_InitialLoadBalanceRequest; - -typedef PB_BYTES_ARRAY_T(64) grpc_lb_v0_Server_load_balance_token_t; -typedef struct _grpc_lb_v0_Server { - bool has_ip_address; - char ip_address[46]; - bool has_port; - int32_t port; - bool has_load_balance_token; - grpc_lb_v0_Server_load_balance_token_t load_balance_token; - bool has_drop_request; - bool drop_request; -} grpc_lb_v0_Server; - -typedef struct _grpc_lb_v0_InitialLoadBalanceResponse { - bool has_client_config; - char client_config[64]; - bool has_load_balancer_delegate; - char load_balancer_delegate[64]; - bool has_client_stats_report_interval; - grpc_lb_v0_Duration client_stats_report_interval; -} grpc_lb_v0_InitialLoadBalanceResponse; - -typedef struct _grpc_lb_v0_LoadBalanceRequest { - bool has_initial_request; - grpc_lb_v0_InitialLoadBalanceRequest initial_request; - bool has_client_stats; - grpc_lb_v0_ClientStats client_stats; -} grpc_lb_v0_LoadBalanceRequest; - -typedef struct _grpc_lb_v0_ServerList { - pb_callback_t servers; - bool has_expiration_interval; - grpc_lb_v0_Duration expiration_interval; -} grpc_lb_v0_ServerList; - -typedef struct _grpc_lb_v0_LoadBalanceResponse { - bool has_initial_response; - grpc_lb_v0_InitialLoadBalanceResponse initial_response; - bool has_server_list; - grpc_lb_v0_ServerList server_list; -} grpc_lb_v0_LoadBalanceResponse; - -/* Default values for struct fields */ - -/* Initializer values for message structs */ -#define grpc_lb_v0_Duration_init_default {false, 0, false, 0} -#define grpc_lb_v0_LoadBalanceRequest_init_default {false, grpc_lb_v0_InitialLoadBalanceRequest_init_default, false, grpc_lb_v0_ClientStats_init_default} -#define grpc_lb_v0_InitialLoadBalanceRequest_init_default {false, ""} -#define grpc_lb_v0_ClientStats_init_default {false, 0, false, 0, false, 0} -#define grpc_lb_v0_LoadBalanceResponse_init_default {false, grpc_lb_v0_InitialLoadBalanceResponse_init_default, false, grpc_lb_v0_ServerList_init_default} -#define grpc_lb_v0_InitialLoadBalanceResponse_init_default {false, "", false, "", false, grpc_lb_v0_Duration_init_default} -#define grpc_lb_v0_ServerList_init_default {{{NULL}, NULL}, false, grpc_lb_v0_Duration_init_default} -#define grpc_lb_v0_Server_init_default {false, "", false, 0, false, {0, {0}}, false, 0} -#define grpc_lb_v0_Duration_init_zero {false, 0, false, 0} -#define grpc_lb_v0_LoadBalanceRequest_init_zero {false, grpc_lb_v0_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v0_ClientStats_init_zero} -#define grpc_lb_v0_InitialLoadBalanceRequest_init_zero {false, ""} -#define grpc_lb_v0_ClientStats_init_zero {false, 0, false, 0, false, 0} -#define grpc_lb_v0_LoadBalanceResponse_init_zero {false, grpc_lb_v0_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v0_ServerList_init_zero} -#define grpc_lb_v0_InitialLoadBalanceResponse_init_zero {false, "", false, "", false, grpc_lb_v0_Duration_init_zero} -#define grpc_lb_v0_ServerList_init_zero {{{NULL}, NULL}, false, grpc_lb_v0_Duration_init_zero} -#define grpc_lb_v0_Server_init_zero {false, "", false, 0, false, {0, {0}}, false, 0} - -/* Field tags (for use in manual encoding/decoding) */ -#define grpc_lb_v0_ClientStats_total_requests_tag 1 -#define grpc_lb_v0_ClientStats_client_rpc_errors_tag 2 -#define grpc_lb_v0_ClientStats_dropped_requests_tag 3 -#define grpc_lb_v0_Duration_seconds_tag 1 -#define grpc_lb_v0_Duration_nanos_tag 2 -#define grpc_lb_v0_InitialLoadBalanceRequest_name_tag 1 -#define grpc_lb_v0_Server_ip_address_tag 1 -#define grpc_lb_v0_Server_port_tag 2 -#define grpc_lb_v0_Server_load_balance_token_tag 3 -#define grpc_lb_v0_Server_drop_request_tag 4 -#define grpc_lb_v0_InitialLoadBalanceResponse_client_config_tag 1 -#define grpc_lb_v0_InitialLoadBalanceResponse_load_balancer_delegate_tag 2 -#define grpc_lb_v0_InitialLoadBalanceResponse_client_stats_report_interval_tag 3 -#define grpc_lb_v0_LoadBalanceRequest_initial_request_tag 1 -#define grpc_lb_v0_LoadBalanceRequest_client_stats_tag 2 -#define grpc_lb_v0_ServerList_servers_tag 1 -#define grpc_lb_v0_ServerList_expiration_interval_tag 3 -#define grpc_lb_v0_LoadBalanceResponse_initial_response_tag 1 -#define grpc_lb_v0_LoadBalanceResponse_server_list_tag 2 - -/* Struct field encoding specification for nanopb */ -extern const pb_field_t grpc_lb_v0_Duration_fields[3]; -extern const pb_field_t grpc_lb_v0_LoadBalanceRequest_fields[3]; -extern const pb_field_t grpc_lb_v0_InitialLoadBalanceRequest_fields[2]; -extern const pb_field_t grpc_lb_v0_ClientStats_fields[4]; -extern const pb_field_t grpc_lb_v0_LoadBalanceResponse_fields[3]; -extern const pb_field_t grpc_lb_v0_InitialLoadBalanceResponse_fields[4]; -extern const pb_field_t grpc_lb_v0_ServerList_fields[3]; -extern const pb_field_t grpc_lb_v0_Server_fields[5]; - -/* Maximum encoded size of messages (where known) */ -#define grpc_lb_v0_Duration_size 22 -#define grpc_lb_v0_LoadBalanceRequest_size 169 -#define grpc_lb_v0_InitialLoadBalanceRequest_size 131 -#define grpc_lb_v0_ClientStats_size 33 -#define grpc_lb_v0_LoadBalanceResponse_size (165 + grpc_lb_v0_ServerList_size) -#define grpc_lb_v0_InitialLoadBalanceResponse_size 156 -#define grpc_lb_v0_Server_size 127 - -/* Message IDs (where set with "msgid" option) */ -#ifdef PB_MSGID - -#define LOAD_BALANCER_MESSAGES \ - - -#endif - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c similarity index 55% rename from src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c rename to src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c index 97196731810..52e11c40bb0 100644 --- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c +++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c @@ -33,7 +33,7 @@ /* Automatically generated nanopb constant definitions */ /* Generated by nanopb-0.3.5-dev */ -#include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h" +#include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" #if PB_PROTO_HEADER_VERSION != 30 #error Regenerate this file with the current version of nanopb generator. @@ -41,54 +41,53 @@ -const pb_field_t grpc_lb_v0_Duration_fields[3] = { - PB_FIELD( 1, INT64 , OPTIONAL, STATIC , FIRST, grpc_lb_v0_Duration, seconds, seconds, 0), - PB_FIELD( 2, INT32 , OPTIONAL, STATIC , OTHER, grpc_lb_v0_Duration, nanos, seconds, 0), +const pb_field_t grpc_lb_v1_Duration_fields[3] = { + PB_FIELD( 1, INT64 , OPTIONAL, STATIC , FIRST, grpc_lb_v1_Duration, seconds, seconds, 0), + PB_FIELD( 2, INT32 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Duration, nanos, seconds, 0), PB_LAST_FIELD }; -const pb_field_t grpc_lb_v0_LoadBalanceRequest_fields[3] = { - PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, grpc_lb_v0_LoadBalanceRequest, initial_request, initial_request, &grpc_lb_v0_InitialLoadBalanceRequest_fields), - PB_FIELD( 2, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v0_LoadBalanceRequest, client_stats, initial_request, &grpc_lb_v0_ClientStats_fields), +const pb_field_t grpc_lb_v1_LoadBalanceRequest_fields[3] = { + PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, grpc_lb_v1_LoadBalanceRequest, initial_request, initial_request, &grpc_lb_v1_InitialLoadBalanceRequest_fields), + PB_FIELD( 2, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v1_LoadBalanceRequest, client_stats, initial_request, &grpc_lb_v1_ClientStats_fields), PB_LAST_FIELD }; -const pb_field_t grpc_lb_v0_InitialLoadBalanceRequest_fields[2] = { - PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v0_InitialLoadBalanceRequest, name, name, 0), +const pb_field_t grpc_lb_v1_InitialLoadBalanceRequest_fields[2] = { + PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v1_InitialLoadBalanceRequest, name, name, 0), PB_LAST_FIELD }; -const pb_field_t grpc_lb_v0_ClientStats_fields[4] = { - PB_FIELD( 1, INT64 , OPTIONAL, STATIC , FIRST, grpc_lb_v0_ClientStats, total_requests, total_requests, 0), - PB_FIELD( 2, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v0_ClientStats, client_rpc_errors, total_requests, 0), - PB_FIELD( 3, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v0_ClientStats, dropped_requests, client_rpc_errors, 0), +const pb_field_t grpc_lb_v1_ClientStats_fields[4] = { + PB_FIELD( 1, INT64 , OPTIONAL, STATIC , FIRST, grpc_lb_v1_ClientStats, total_requests, total_requests, 0), + PB_FIELD( 2, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, client_rpc_errors, total_requests, 0), + PB_FIELD( 3, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, dropped_requests, client_rpc_errors, 0), PB_LAST_FIELD }; -const pb_field_t grpc_lb_v0_LoadBalanceResponse_fields[3] = { - PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, grpc_lb_v0_LoadBalanceResponse, initial_response, initial_response, &grpc_lb_v0_InitialLoadBalanceResponse_fields), - PB_FIELD( 2, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v0_LoadBalanceResponse, server_list, initial_response, &grpc_lb_v0_ServerList_fields), +const pb_field_t grpc_lb_v1_LoadBalanceResponse_fields[3] = { + PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, grpc_lb_v1_LoadBalanceResponse, initial_response, initial_response, &grpc_lb_v1_InitialLoadBalanceResponse_fields), + PB_FIELD( 2, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v1_LoadBalanceResponse, server_list, initial_response, &grpc_lb_v1_ServerList_fields), PB_LAST_FIELD }; -const pb_field_t grpc_lb_v0_InitialLoadBalanceResponse_fields[4] = { - PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v0_InitialLoadBalanceResponse, client_config, client_config, 0), - PB_FIELD( 2, STRING , OPTIONAL, STATIC , OTHER, grpc_lb_v0_InitialLoadBalanceResponse, load_balancer_delegate, client_config, 0), - PB_FIELD( 3, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v0_Duration_fields), +const pb_field_t grpc_lb_v1_InitialLoadBalanceResponse_fields[3] = { + PB_FIELD( 2, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v1_InitialLoadBalanceResponse, load_balancer_delegate, load_balancer_delegate, 0), + PB_FIELD( 3, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v1_Duration_fields), PB_LAST_FIELD }; -const pb_field_t grpc_lb_v0_ServerList_fields[3] = { - PB_FIELD( 1, MESSAGE , REPEATED, CALLBACK, FIRST, grpc_lb_v0_ServerList, servers, servers, &grpc_lb_v0_Server_fields), - PB_FIELD( 3, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v0_ServerList, expiration_interval, servers, &grpc_lb_v0_Duration_fields), +const pb_field_t grpc_lb_v1_ServerList_fields[3] = { + PB_FIELD( 1, MESSAGE , REPEATED, CALLBACK, FIRST, grpc_lb_v1_ServerList, servers, servers, &grpc_lb_v1_Server_fields), + PB_FIELD( 3, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ServerList, expiration_interval, servers, &grpc_lb_v1_Duration_fields), PB_LAST_FIELD }; -const pb_field_t grpc_lb_v0_Server_fields[5] = { - PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v0_Server, ip_address, ip_address, 0), - PB_FIELD( 2, INT32 , OPTIONAL, STATIC , OTHER, grpc_lb_v0_Server, port, ip_address, 0), - PB_FIELD( 3, BYTES , OPTIONAL, STATIC , OTHER, grpc_lb_v0_Server, load_balance_token, port, 0), - PB_FIELD( 4, BOOL , OPTIONAL, STATIC , OTHER, grpc_lb_v0_Server, drop_request, load_balance_token, 0), +const pb_field_t grpc_lb_v1_Server_fields[5] = { + PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v1_Server, ip_address, ip_address, 0), + PB_FIELD( 2, INT32 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, port, ip_address, 0), + PB_FIELD( 3, STRING , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, load_balance_token, port, 0), + PB_FIELD( 4, BOOL , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, drop_request, load_balance_token, 0), PB_LAST_FIELD }; @@ -102,7 +101,7 @@ const pb_field_t grpc_lb_v0_Server_fields[5] = { * numbers or field sizes that are larger than what can fit in 8 or 16 bit * field descriptors. */ -PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v0_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server) +PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v1_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server) #endif #if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) @@ -113,7 +112,7 @@ PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) * numbers or field sizes that are larger than what can fit in the default * 8 bit descriptors. */ -PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v0_ServerList, servers) < 256 && pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server) +PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v1_ServerList, servers) < 256 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server) #endif diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h new file mode 100644 index 00000000000..d5dc39ab94f --- /dev/null +++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h @@ -0,0 +1,178 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +/* Automatically generated nanopb header */ +/* Generated by nanopb-0.3.5-dev */ + +#ifndef PB_LOAD_BALANCER_PB_H_INCLUDED +#define PB_LOAD_BALANCER_PB_H_INCLUDED +#include "third_party/nanopb/pb.h" +#if PB_PROTO_HEADER_VERSION != 30 +#error Regenerate this file with the current version of nanopb generator. +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Struct definitions */ +typedef struct _grpc_lb_v1_ClientStats { + bool has_total_requests; + int64_t total_requests; + bool has_client_rpc_errors; + int64_t client_rpc_errors; + bool has_dropped_requests; + int64_t dropped_requests; +} grpc_lb_v1_ClientStats; + +typedef struct _grpc_lb_v1_Duration { + bool has_seconds; + int64_t seconds; + bool has_nanos; + int32_t nanos; +} grpc_lb_v1_Duration; + +typedef struct _grpc_lb_v1_InitialLoadBalanceRequest { + bool has_name; + char name[128]; +} grpc_lb_v1_InitialLoadBalanceRequest; + +typedef struct _grpc_lb_v1_Server { + bool has_ip_address; + char ip_address[46]; + bool has_port; + int32_t port; + bool has_load_balance_token; + char load_balance_token[64]; + bool has_drop_request; + bool drop_request; +} grpc_lb_v1_Server; + +typedef struct _grpc_lb_v1_InitialLoadBalanceResponse { + bool has_load_balancer_delegate; + char load_balancer_delegate[64]; + bool has_client_stats_report_interval; + grpc_lb_v1_Duration client_stats_report_interval; +} grpc_lb_v1_InitialLoadBalanceResponse; + +typedef struct _grpc_lb_v1_LoadBalanceRequest { + bool has_initial_request; + grpc_lb_v1_InitialLoadBalanceRequest initial_request; + bool has_client_stats; + grpc_lb_v1_ClientStats client_stats; +} grpc_lb_v1_LoadBalanceRequest; + +typedef struct _grpc_lb_v1_ServerList { + pb_callback_t servers; + bool has_expiration_interval; + grpc_lb_v1_Duration expiration_interval; +} grpc_lb_v1_ServerList; + +typedef struct _grpc_lb_v1_LoadBalanceResponse { + bool has_initial_response; + grpc_lb_v1_InitialLoadBalanceResponse initial_response; + bool has_server_list; + grpc_lb_v1_ServerList server_list; +} grpc_lb_v1_LoadBalanceResponse; + +/* Default values for struct fields */ + +/* Initializer values for message structs */ +#define grpc_lb_v1_Duration_init_default {false, 0, false, 0} +#define grpc_lb_v1_LoadBalanceRequest_init_default {false, grpc_lb_v1_InitialLoadBalanceRequest_init_default, false, grpc_lb_v1_ClientStats_init_default} +#define grpc_lb_v1_InitialLoadBalanceRequest_init_default {false, ""} +#define grpc_lb_v1_ClientStats_init_default {false, 0, false, 0, false, 0} +#define grpc_lb_v1_LoadBalanceResponse_init_default {false, grpc_lb_v1_InitialLoadBalanceResponse_init_default, false, grpc_lb_v1_ServerList_init_default} +#define grpc_lb_v1_InitialLoadBalanceResponse_init_default {false, "", false, grpc_lb_v1_Duration_init_default} +#define grpc_lb_v1_ServerList_init_default {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_default} +#define grpc_lb_v1_Server_init_default {false, "", false, 0, false, "", false, 0} +#define grpc_lb_v1_Duration_init_zero {false, 0, false, 0} +#define grpc_lb_v1_LoadBalanceRequest_init_zero {false, grpc_lb_v1_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v1_ClientStats_init_zero} +#define grpc_lb_v1_InitialLoadBalanceRequest_init_zero {false, ""} +#define grpc_lb_v1_ClientStats_init_zero {false, 0, false, 0, false, 0} +#define grpc_lb_v1_LoadBalanceResponse_init_zero {false, grpc_lb_v1_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v1_ServerList_init_zero} +#define grpc_lb_v1_InitialLoadBalanceResponse_init_zero {false, "", false, grpc_lb_v1_Duration_init_zero} +#define grpc_lb_v1_ServerList_init_zero {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_zero} +#define grpc_lb_v1_Server_init_zero {false, "", false, 0, false, "", false, 0} + +/* Field tags (for use in manual encoding/decoding) */ +#define grpc_lb_v1_ClientStats_total_requests_tag 1 +#define grpc_lb_v1_ClientStats_client_rpc_errors_tag 2 +#define grpc_lb_v1_ClientStats_dropped_requests_tag 3 +#define grpc_lb_v1_Duration_seconds_tag 1 +#define grpc_lb_v1_Duration_nanos_tag 2 +#define grpc_lb_v1_InitialLoadBalanceRequest_name_tag 1 +#define grpc_lb_v1_Server_ip_address_tag 1 +#define grpc_lb_v1_Server_port_tag 2 +#define grpc_lb_v1_Server_load_balance_token_tag 3 +#define grpc_lb_v1_Server_drop_request_tag 4 +#define grpc_lb_v1_InitialLoadBalanceResponse_load_balancer_delegate_tag 2 +#define grpc_lb_v1_InitialLoadBalanceResponse_client_stats_report_interval_tag 3 +#define grpc_lb_v1_LoadBalanceRequest_initial_request_tag 1 +#define grpc_lb_v1_LoadBalanceRequest_client_stats_tag 2 +#define grpc_lb_v1_ServerList_servers_tag 1 +#define grpc_lb_v1_ServerList_expiration_interval_tag 3 +#define grpc_lb_v1_LoadBalanceResponse_initial_response_tag 1 +#define grpc_lb_v1_LoadBalanceResponse_server_list_tag 2 + +/* Struct field encoding specification for nanopb */ +extern const pb_field_t grpc_lb_v1_Duration_fields[3]; +extern const pb_field_t grpc_lb_v1_LoadBalanceRequest_fields[3]; +extern const pb_field_t grpc_lb_v1_InitialLoadBalanceRequest_fields[2]; +extern const pb_field_t grpc_lb_v1_ClientStats_fields[4]; +extern const pb_field_t grpc_lb_v1_LoadBalanceResponse_fields[3]; +extern const pb_field_t grpc_lb_v1_InitialLoadBalanceResponse_fields[3]; +extern const pb_field_t grpc_lb_v1_ServerList_fields[3]; +extern const pb_field_t grpc_lb_v1_Server_fields[5]; + +/* Maximum encoded size of messages (where known) */ +#define grpc_lb_v1_Duration_size 22 +#define grpc_lb_v1_LoadBalanceRequest_size 169 +#define grpc_lb_v1_InitialLoadBalanceRequest_size 131 +#define grpc_lb_v1_ClientStats_size 33 +#define grpc_lb_v1_LoadBalanceResponse_size (98 + grpc_lb_v1_ServerList_size) +#define grpc_lb_v1_InitialLoadBalanceResponse_size 90 +#define grpc_lb_v1_Server_size 127 + +/* Message IDs (where set with "msgid" option) */ +#ifdef PB_MSGID + +#define LOAD_BALANCER_MESSAGES \ + + +#endif + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/src/proto/grpc/lb/v0/load_balancer.options b/src/proto/grpc/lb/v0/load_balancer.options deleted file mode 100644 index 6d4528f838a..00000000000 --- a/src/proto/grpc/lb/v0/load_balancer.options +++ /dev/null @@ -1,6 +0,0 @@ -grpc.lb.v0.InitialLoadBalanceRequest.name max_size:128 -grpc.lb.v0.InitialLoadBalanceResponse.client_config max_size:64 -grpc.lb.v0.InitialLoadBalanceResponse.load_balancer_delegate max_size:64 -grpc.lb.v0.Server.ip_address max_size:46 -grpc.lb.v0.Server.load_balance_token max_size:64 -load_balancer.proto no_unions:true diff --git a/src/proto/grpc/lb/v1/load_balancer.options b/src/proto/grpc/lb/v1/load_balancer.options new file mode 100644 index 00000000000..d90366996e7 --- /dev/null +++ b/src/proto/grpc/lb/v1/load_balancer.options @@ -0,0 +1,6 @@ +grpc.lb.v1.InitialLoadBalanceRequest.name max_size:128 +grpc.lb.v1.InitialLoadBalanceResponse.client_config max_size:64 +grpc.lb.v1.InitialLoadBalanceResponse.load_balancer_delegate max_size:64 +grpc.lb.v1.Server.ip_address max_size:46 +grpc.lb.v1.Server.load_balance_token max_size:64 +load_balancer.proto no_unions:true diff --git a/src/proto/grpc/lb/v0/load_balancer.proto b/src/proto/grpc/lb/v1/load_balancer.proto similarity index 94% rename from src/proto/grpc/lb/v0/load_balancer.proto rename to src/proto/grpc/lb/v1/load_balancer.proto index e88a4f8c4a7..1bcad0b1d4b 100644 --- a/src/proto/grpc/lb/v0/load_balancer.proto +++ b/src/proto/grpc/lb/v1/load_balancer.proto @@ -29,7 +29,7 @@ syntax = "proto3"; -package grpc.lb.v0; +package grpc.lb.v1; message Duration { @@ -94,9 +94,8 @@ message LoadBalanceResponse { message InitialLoadBalanceResponse { oneof initial_response_type { - // Contains gRPC config options like RPC deadline or flow control. - // TODO(yetianx): Change to ClientConfig after it is defined. - string client_config = 1; + // TODO(zhangkun83): ClientConfig not yet defined + //ClientConfig client_config = 1; // This is an application layer redirect that indicates the client should // use the specified server for load balancing. When this field is set in @@ -134,9 +133,7 @@ message Server { // An opaque token that is passed from the client to the server in metadata. // The server may expect this token to indicate that the request from the // client was load balanced. - // TODO(yetianx): Not used right now, and will be used after implementing - // load report. - bytes load_balance_token = 3; + string load_balance_token = 3; // Indicates whether this particular request should be dropped by the client // when this server is chosen from the list. diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index c5a0a398b43..e1fe5fc1f67 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -221,7 +221,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', - 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', + 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', 'third_party/nanopb/pb_decode.c', 'third_party/nanopb/pb_encode.c', diff --git a/test/cpp/grpclb/grpclb_api_test.cc b/test/cpp/grpclb/grpclb_api_test.cc index 92f93c869c7..bf77878e0a6 100644 --- a/test/cpp/grpclb/grpclb_api_test.cc +++ b/test/cpp/grpclb/grpclb_api_test.cc @@ -35,13 +35,13 @@ #include #include "src/core/ext/lb_policy/grpclb/load_balancer_api.h" -#include "src/proto/grpc/lb/v0/load_balancer.pb.h" // C++ version +#include "src/proto/grpc/lb/v1/load_balancer.pb.h" // C++ version namespace grpc { namespace { -using grpc::lb::v0::LoadBalanceRequest; -using grpc::lb::v0::LoadBalanceResponse; +using grpc::lb::v1::LoadBalanceRequest; +using grpc::lb::v1::LoadBalanceResponse; class GrpclbTest : public ::testing::Test {}; @@ -60,9 +60,7 @@ TEST_F(GrpclbTest, CreateRequest) { TEST_F(GrpclbTest, ParseResponse) { LoadBalanceResponse response; - const std::string client_config_str = "I'm a client config"; auto* initial_response = response.mutable_initial_response(); - initial_response->set_client_config(client_config_str); auto* client_stats_report_interval = initial_response->mutable_client_stats_report_interval(); client_stats_report_interval->set_seconds(123); @@ -73,10 +71,7 @@ TEST_F(GrpclbTest, ParseResponse) { gpr_slice_from_copied_string(encoded_response.c_str()); grpc_grpclb_response* c_response = grpc_grpclb_response_parse(encoded_slice); EXPECT_TRUE(c_response->has_initial_response); - EXPECT_TRUE(c_response->initial_response.has_client_config); EXPECT_FALSE(c_response->initial_response.has_load_balancer_delegate); - EXPECT_TRUE(strcmp(c_response->initial_response.client_config, - client_config_str.c_str()) == 0); EXPECT_EQ(c_response->initial_response.client_stats_report_interval.seconds, 123); EXPECT_EQ(c_response->initial_response.client_stats_report_interval.nanos, diff --git a/tools/codegen/core/gen_nano_proto.sh b/tools/codegen/core/gen_nano_proto.sh index e2d2f672e93..b216a20379e 100755 --- a/tools/codegen/core/gen_nano_proto.sh +++ b/tools/codegen/core/gen_nano_proto.sh @@ -29,11 +29,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# # Example usage: # tools/codegen/core/gen_nano_proto.sh \ -# src/proto/grpc/lb/v0/load_balancer.proto -# $PWD/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0 +# src/proto/grpc/lb/v1/load_balancer.proto \ +# $PWD/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1 \ +# src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1 # # Exit statuses: # 1: Incorrect number of arguments diff --git a/tools/distrib/check_include_guards.py b/tools/distrib/check_include_guards.py index 897a899e7ed..b1fdb942cda 100755 --- a/tools/distrib/check_include_guards.py +++ b/tools/distrib/check_include_guards.py @@ -169,7 +169,7 @@ argp.add_argument('--precommit', args = argp.parse_args() KNOWN_BAD = set([ - 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', + 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', ]) diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 92cb8ecbb49..c0707051a60 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -58,15 +58,15 @@ popd # # Checks for load_balancer.proto # -readonly LOAD_BALANCER_GRPC_OUTPUT_PATH='src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0' +readonly LOAD_BALANCER_GRPC_OUTPUT_PATH='src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1' # nanopb-compile the proto to a temp location ./tools/codegen/core/gen_nano_proto.sh \ - src/proto/grpc/lb/v0/load_balancer.proto \ + src/proto/grpc/lb/v1/load_balancer.proto \ "$NANOPB_TMP_OUTPUT" \ "$LOAD_BALANCER_GRPC_OUTPUT_PATH" # compare outputs to checked compiled code -if ! diff -r $NANOPB_TMP_OUTPUT src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0; then +if ! diff -r $NANOPB_TMP_OUTPUT src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1; then echo "Outputs differ: $NANOPB_TMP_OUTPUT vs $LOAD_BALANCER_GRPC_OUTPUT_PATH" exit 2 fi diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 3a774a70d6b..ebf573d3b20 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -913,7 +913,7 @@ src/core/ext/client_config/subchannel_call_holder.h \ src/core/ext/client_config/subchannel_index.h \ src/core/ext/client_config/uri_parser.h \ src/core/ext/lb_policy/grpclb/load_balancer_api.h \ -src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h \ +src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h \ third_party/nanopb/pb.h \ third_party/nanopb/pb_common.h \ third_party/nanopb/pb_decode.h \ @@ -1071,7 +1071,7 @@ src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ -src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ +src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_decode.c \ third_party/nanopb/pb_encode.c \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 38e68f3b1a1..2599df23039 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -2201,8 +2201,8 @@ "grpc_test_util" ], "headers": [ - "src/proto/grpc/lb/v0/load_balancer.grpc.pb.h", - "src/proto/grpc/lb/v0/load_balancer.pb.h" + "src/proto/grpc/lb/v1/load_balancer.grpc.pb.h", + "src/proto/grpc/lb/v1/load_balancer.pb.h" ], "language": "c++", "name": "grpclb_api_test", @@ -5860,15 +5860,15 @@ ], "headers": [ "src/core/ext/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h" + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" ], "language": "c", "name": "grpc_lb_policy_grpclb", "src": [ "src/core/ext/lb_policy/grpclb/load_balancer_api.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", - "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h" + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", + "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" ], "third_party": false, "type": "filegroup" diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 4eec05a3b1f..26109c71020 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -422,7 +422,7 @@ - + @@ -729,7 +729,7 @@ - + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 17c88c4805e..8d7072b04a5 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -442,8 +442,8 @@ src\core\ext\lb_policy\grpclb - - src\core\ext\lb_policy\grpclb\proto\grpc\lb\v0 + + src\core\ext\lb_policy\grpclb\proto\grpc\lb\v1 third_party\nanopb @@ -959,8 +959,8 @@ src\core\ext\lb_policy\grpclb - - src\core\ext\lb_policy\grpclb\proto\grpc\lb\v0 + + src\core\ext\lb_policy\grpclb\proto\grpc\lb\v1 third_party\nanopb @@ -1037,8 +1037,8 @@ {adf7e553-94ef-14fd-e845-03104f00a06f} - - {0406d191-8817-38c3-a562-e3541201f424} + + {bc357e2d-8ddd-a688-88a3-255228fc0818} {b63ded00-b24f-708e-333f-ce199e421875} diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 26050dcf74e..31ab8f4ba4c 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -397,7 +397,7 @@ - + @@ -668,7 +668,7 @@ - + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index a4acf513bc1..869d92c7382 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -388,8 +388,8 @@ src\core\ext\lb_policy\grpclb - - src\core\ext\lb_policy\grpclb\proto\grpc\lb\v0 + + src\core\ext\lb_policy\grpclb\proto\grpc\lb\v1 third_party\nanopb @@ -851,8 +851,8 @@ src\core\ext\lb_policy\grpclb - - src\core\ext\lb_policy\grpclb\proto\grpc\lb\v0 + + src\core\ext\lb_policy\grpclb\proto\grpc\lb\v1 third_party\nanopb @@ -929,8 +929,8 @@ {21858d9d-30b5-8847-5882-6b47df0fa293} - - {1795a20b-3e7c-e27d-eae1-96582fa9a958} + + {e9256e96-ea3d-c1fd-6426-9d53d9f08f66} {e27f9ecf-97bb-1a2e-3135-a41f732dcf55} diff --git a/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj b/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj index 1509ece9f96..91b11a1f0f4 100644 --- a/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj +++ b/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj @@ -160,13 +160,13 @@ - + - + - + - + diff --git a/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj.filters b/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj.filters index 6c57b8c162a..50f0a3eac25 100644 --- a/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj.filters +++ b/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj.filters @@ -1,8 +1,8 @@ - - src\proto\grpc\lb\v0 + + src\proto\grpc\lb\v1 test\cpp\grpclb @@ -22,8 +22,8 @@ {2981699e-c196-c599-bc17-c177770f89ee} - - {3d04774a-1c2f-e100-435e-08af5d539250} + + {6cce8ddf-d9a9-1d71-0810-d1e6f8685d76} {64736e1d-eb77-664f-34ab-6cf41263d3d8} From ac2e88f9567c7e90f615738fb759537385dcc858 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 27 Apr 2016 14:13:27 -0700 Subject: [PATCH 003/155] Initial attempt at adding support for max metadata size. Currently, the code does not seem to be properly causing the RPC to fail when the max size is exceeded. --- include/grpc/impl/codegen/grpc_types.h | 2 ++ .../chttp2/transport/chttp2_transport.c | 15 ++++++++++ .../chttp2/transport/incoming_metadata.c | 1 + .../chttp2/transport/incoming_metadata.h | 1 + .../ext/transport/chttp2/transport/internal.h | 3 ++ .../ext/transport/chttp2/transport/parsing.c | 22 +++++++++++--- src/core/lib/transport/metadata.h | 2 ++ test/core/end2end/cq_verifier.c | 8 +++++ test/core/end2end/cq_verifier.h | 1 + test/core/end2end/tests/large_metadata.c | 29 ++++++++++++++----- 10 files changed, 73 insertions(+), 11 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 4c7373006b2..b5203b6e4d2 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -152,6 +152,8 @@ typedef struct { channel). If this parameter is specified and the underlying is not an SSL channel, it will just be ignored. */ #define GRPC_SSL_TARGET_NAME_OVERRIDE_ARG "grpc.ssl_target_name_override" +/* Maximum metadata size */ +#define GRPC_ARG_MAX_METADATA_SIZE "grpc.max_metadata_size" /** Result of a grpc call. If the caller satisfies the prerequisites of a particular operation, the grpc_call_error returned will be GRPC_CALL_OK. diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 01507f5ca6f..c24950a1890 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -56,6 +56,8 @@ #define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024) #define MAX_WINDOW 0x7fffffffu +#define DEFAULT_MAX_METADATA_SIZE 16 * 1024 + #define MAX_CLIENT_STREAM_ID 0x7fffffffu int grpc_http_trace = 0; @@ -250,6 +252,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->global.ping_counter = 1; t->global.pings.next = t->global.pings.prev = &t->global.pings; t->parsing.is_client = is_client; + t->parsing.max_metadata_size = DEFAULT_MAX_METADATA_SIZE; t->parsing.deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; t->writing.is_client = is_client; @@ -372,6 +375,18 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &t->writing.hpack_compressor, (uint32_t)channel_args->args[i].value.integer); } + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_MAX_METADATA_SIZE)) { + if (channel_args->args[i].type != GRPC_ARG_INTEGER) { + gpr_log(GPR_ERROR, "%s: must be an integer", + GRPC_ARG_MAX_METADATA_SIZE); + } else if (channel_args->args[i].value.integer < 0) { + gpr_log(GPR_ERROR, "%s: must be non-negative", + GRPC_ARG_MAX_METADATA_SIZE); + } else { + t->parsing.max_metadata_size = + (uint32_t)channel_args->args[i].value.integer; + } } } } diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.c b/src/core/ext/transport/chttp2/transport/incoming_metadata.c index db21744f0c0..3e463a79957 100644 --- a/src/core/ext/transport/chttp2/transport/incoming_metadata.c +++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.c @@ -65,6 +65,7 @@ void grpc_chttp2_incoming_metadata_buffer_add( gpr_realloc(buffer->elems, sizeof(*buffer->elems) * buffer->capacity); } buffer->elems[buffer->count++].md = elem; + buffer->size += GRPC_MDELEM_LENGTH(elem); } void grpc_chttp2_incoming_metadata_buffer_set_deadline( diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.h b/src/core/ext/transport/chttp2/transport/incoming_metadata.h index 17ecf8e1811..7db5db8de05 100644 --- a/src/core/ext/transport/chttp2/transport/incoming_metadata.h +++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.h @@ -42,6 +42,7 @@ typedef struct { size_t capacity; gpr_timespec deadline; int published; + size_t size; /* total size of metadata */ } grpc_chttp2_incoming_metadata_buffer; /** assumes everything initially zeroed */ diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 98cd38abd43..d547a6e9c11 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -229,6 +229,9 @@ struct grpc_chttp2_transport_parsing { /** is this transport a client? (boolean) */ uint8_t is_client; + /** max metadata size */ + uint32_t max_metadata_size; + /** were settings updated? */ uint8_t settings_updated; /** was a settings ack received? */ diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index e827a43f7a4..0cf4d87f3c0 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -624,8 +624,15 @@ static void on_initial_header(void *tp, grpc_mdelem *md) { gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), *cached_timeout)); GRPC_MDELEM_UNREF(md); } else { - grpc_chttp2_incoming_metadata_buffer_add( - &stream_parsing->metadata_buffer[0], md); + const size_t new_size = stream_parsing->metadata_buffer[0].size + + GRPC_MDELEM_LENGTH(md); + if (new_size > transport_parsing->max_metadata_size) { + stream_parsing->seen_error = 1; + GRPC_MDELEM_UNREF(md); + } else { + grpc_chttp2_incoming_metadata_buffer_add( + &stream_parsing->metadata_buffer[0], md); + } } grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); @@ -652,8 +659,15 @@ static void on_trailing_header(void *tp, grpc_mdelem *md) { stream_parsing->seen_error = 1; } - grpc_chttp2_incoming_metadata_buffer_add(&stream_parsing->metadata_buffer[1], - md); + const size_t new_size = stream_parsing->metadata_buffer[1].size + + GRPC_MDELEM_LENGTH(md); + if (new_size > transport_parsing->max_metadata_size) { + stream_parsing->seen_error = 1; + GRPC_MDELEM_UNREF(md); + } else { + grpc_chttp2_incoming_metadata_buffer_add( + &stream_parsing->metadata_buffer[1], md); + } grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 713d9e6782c..277c2579338 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -146,6 +146,8 @@ void grpc_mdelem_unref(grpc_mdelem *md); const char *grpc_mdstr_as_c_string(grpc_mdstr *s); #define GRPC_MDSTR_LENGTH(s) (GPR_SLICE_LENGTH(s->slice)) +#define GRPC_MDELEM_LENGTH(e) (GRPC_MDSTR_LENGTH(e->key) + \ + GRPC_MDSTR_LENGTH(e->value)) int grpc_mdstr_is_legal_header(grpc_mdstr *s); int grpc_mdstr_is_legal_nonbin_header(grpc_mdstr *s); diff --git a/test/core/end2end/cq_verifier.c b/test/core/end2end/cq_verifier.c index 77afe588d79..5f1a33242ee 100644 --- a/test/core/end2end/cq_verifier.c +++ b/test/core/end2end/cq_verifier.c @@ -107,6 +107,14 @@ int contains_metadata(grpc_metadata_array *array, const char *key, return has_metadata(array->metadata, array->count, key, value); } +int contains_metadata_key(grpc_metadata_array *array, const char *key) { + for (size_t i = 0; i < array->count; ++i) { + if (strcmp(array->metadata[i].key, key) == 0) + return 1; + } + return 0; +} + static gpr_slice merge_slices(gpr_slice *slices, size_t nslices) { size_t i; size_t len = 0; diff --git a/test/core/end2end/cq_verifier.h b/test/core/end2end/cq_verifier.h index b3e07c45a58..a54065950d2 100644 --- a/test/core/end2end/cq_verifier.h +++ b/test/core/end2end/cq_verifier.h @@ -62,5 +62,6 @@ void cq_expect_completion(cq_verifier *v, void *tag, int success); int byte_buffer_eq_string(grpc_byte_buffer *byte_buffer, const char *string); int contains_metadata(grpc_metadata_array *array, const char *key, const char *value); +int contains_metadata_key(grpc_metadata_array *array, const char *key); #endif /* GRPC_TEST_CORE_END2END_CQ_VERIFIER_H */ diff --git a/test/core/end2end/tests/large_metadata.c b/test/core/end2end/tests/large_metadata.c index 0e5d6b4fe01..f09b55a2c0c 100644 --- a/test/core/end2end/tests/large_metadata.c +++ b/test/core/end2end/tests/large_metadata.c @@ -98,7 +98,8 @@ static void end_test(grpc_end2end_test_fixture *f) { } /* Request with a large amount of metadata.*/ -static void test_request_with_large_metadata(grpc_end2end_test_config config) { +static void test_request_with_large_metadata(grpc_end2end_test_config config, + int allow_large_metadata) { grpc_call *c; grpc_call *s; gpr_slice request_payload_slice = gpr_slice_from_copied_string("hello world"); @@ -106,8 +107,16 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { grpc_raw_byte_buffer_create(&request_payload_slice, 1); gpr_timespec deadline = five_seconds_time(); grpc_metadata meta; - grpc_end2end_test_fixture f = - begin_test(config, "test_request_with_large_metadata", NULL, NULL); + const char *test_name = allow_large_metadata + ? "test_request_with_large_metadata_allowed" + : "test_request_with_large_metadata_not_allowed"; + const size_t large_size = 64 * 1024; + grpc_arg arg = { GRPC_ARG_INTEGER, GRPC_ARG_MAX_METADATA_SIZE, + { .integer=(int)large_size + 1024 } }; + grpc_channel_args args = { 1, &arg }; + grpc_channel_args* use_args = allow_large_metadata ? &args : NULL; + grpc_end2end_test_fixture f = begin_test( + config, test_name, use_args, use_args); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -121,7 +130,6 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { char *details = NULL; size_t details_capacity = 0; int was_cancelled = 2; - const size_t large_size = 64 * 1024; c = grpc_channel_create_call(f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, "/foo", "foo.test.google.fr", deadline, NULL); @@ -214,13 +222,19 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { cq_expect_completion(cqv, tag(1), 1); cq_verify(cqv); +// FIXME: why is this assert passing with allow_large_metadata=false? GPR_ASSERT(status == GRPC_STATUS_OK); GPR_ASSERT(0 == strcmp(details, "xyz")); GPR_ASSERT(0 == strcmp(call_details.method, "/foo")); GPR_ASSERT(0 == strcmp(call_details.host, "foo.test.google.fr")); GPR_ASSERT(was_cancelled == 0); - GPR_ASSERT(byte_buffer_eq_string(request_payload_recv, "hello world")); - GPR_ASSERT(contains_metadata(&request_metadata_recv, "key", meta.value)); + if (allow_large_metadata) { + GPR_ASSERT(byte_buffer_eq_string(request_payload_recv, "hello world")); + GPR_ASSERT(contains_metadata(&request_metadata_recv, "key", meta.value)); + } else { + GPR_ASSERT(request_payload_recv == NULL); + GPR_ASSERT(!contains_metadata_key(&request_metadata_recv, "key")); + } gpr_free(details); grpc_metadata_array_destroy(&initial_metadata_recv); @@ -243,7 +257,8 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { } void large_metadata(grpc_end2end_test_config config) { - test_request_with_large_metadata(config); + test_request_with_large_metadata(config, 1); + test_request_with_large_metadata(config, 0); } void large_metadata_pre_init(void) {} From 1b2db6333d4783e18b8d72db7893a758c4fcd2b9 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 27 Apr 2016 15:06:54 -0700 Subject: [PATCH 004/155] Moved protos out of core_codegen interface --- BUILD | 5 + Makefile | 4 + build.yaml | 1 + gRPC.podspec | 1 + grpc.gemspec | 1 + .../impl/codegen/core_codegen_interface.h | 32 +-- include/grpc++/impl/codegen/proto_utils.h | 161 +++++++++++++- include/grpc/byte_buffer_reader.h | 21 +- .../grpc/impl/codegen/byte_buffer_reader.h | 57 +++++ package.xml | 1 + src/compiler/csharp_generator.cc | 4 + src/core/lib/transport/metadata.c | 29 ++- src/cpp/common/core_codegen.cc | 204 ++++-------------- src/cpp/common/core_codegen.h | 26 ++- src/csharp/Grpc.Examples/MathGrpc.cs | 6 + src/csharp/Grpc.HealthCheck/HealthGrpc.cs | 6 + .../Grpc.IntegrationTesting/MetricsGrpc.cs | 6 + .../Grpc.IntegrationTesting/ServicesGrpc.cs | 12 ++ .../Grpc.IntegrationTesting/TestGrpc.cs | 18 ++ .../core/surface/public_headers_must_be_c89.c | 1 + tools/doxygen/Doxyfile.c++ | 1 + tools/doxygen/Doxyfile.c++.internal | 1 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + tools/run_tests/sources_and_headers.json | 2 + vsprojects/vcxproj/grpc++/grpc++.vcxproj | 1 + .../vcxproj/grpc++/grpc++.vcxproj.filters | 3 + .../grpc++_unsecure/grpc++_unsecure.vcxproj | 1 + .../grpc++_unsecure.vcxproj.filters | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj | 1 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 3 + .../grpc_unsecure/grpc_unsecure.vcxproj | 1 + .../grpc_unsecure.vcxproj.filters | 3 + .../codegen_test_full.vcxproj | 1 + .../codegen_test_full.vcxproj.filters | 3 + .../codegen_test_minimal.vcxproj | 1 + .../codegen_test_minimal.vcxproj.filters | 3 + 37 files changed, 410 insertions(+), 216 deletions(-) create mode 100644 include/grpc/impl/codegen/byte_buffer_reader.h diff --git a/BUILD b/BUILD index b69c9404113..aa06678ffd1 100644 --- a/BUILD +++ b/BUILD @@ -461,6 +461,7 @@ cc_library( "include/grpc/grpc.h", "include/grpc/status.h", "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", "include/grpc/impl/codegen/grpc_types.h", @@ -772,6 +773,7 @@ cc_library( "include/grpc/grpc.h", "include/grpc/status.h", "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", "include/grpc/impl/codegen/grpc_types.h", @@ -944,6 +946,7 @@ cc_library( "include/grpc++/impl/codegen/sync_stream.h", "include/grpc++/impl/codegen/time.h", "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", "include/grpc/impl/codegen/grpc_types.h", @@ -1089,6 +1092,7 @@ cc_library( "include/grpc++/impl/codegen/sync_stream.h", "include/grpc++/impl/codegen/time.h", "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", "include/grpc/impl/codegen/grpc_types.h", @@ -1476,6 +1480,7 @@ objc_library( "include/grpc/grpc.h", "include/grpc/status.h", "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", "include/grpc/impl/codegen/grpc_types.h", diff --git a/Makefile b/Makefile index 3b7dbd9638a..89cfd4136ab 100644 --- a/Makefile +++ b/Makefile @@ -2629,6 +2629,7 @@ PUBLIC_HEADERS_C += \ include/grpc/grpc.h \ include/grpc/status.h \ include/grpc/impl/codegen/byte_buffer.h \ + include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ include/grpc/impl/codegen/grpc_types.h \ @@ -2949,6 +2950,7 @@ PUBLIC_HEADERS_C += \ include/grpc/grpc.h \ include/grpc/status.h \ include/grpc/impl/codegen/byte_buffer.h \ + include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ include/grpc/impl/codegen/grpc_types.h \ @@ -3235,6 +3237,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ include/grpc/impl/codegen/byte_buffer.h \ + include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ include/grpc/impl/codegen/grpc_types.h \ @@ -3538,6 +3541,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ include/grpc/impl/codegen/byte_buffer.h \ + include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ include/grpc/impl/codegen/grpc_types.h \ diff --git a/build.yaml b/build.yaml index 6c00b42a1e7..daa8c6aa95a 100644 --- a/build.yaml +++ b/build.yaml @@ -349,6 +349,7 @@ filegroups: - name: grpc_codegen public_headers: - include/grpc/impl/codegen/byte_buffer.h + - include/grpc/impl/codegen/byte_buffer_reader.h - include/grpc/impl/codegen/compression_types.h - include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/grpc_types.h diff --git a/gRPC.podspec b/gRPC.podspec index d66e03354bb..ace137f104b 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -303,6 +303,7 @@ Pod::Spec.new do |s| 'include/grpc/grpc.h', 'include/grpc/status.h', 'include/grpc/impl/codegen/byte_buffer.h', + 'include/grpc/impl/codegen/byte_buffer_reader.h', 'include/grpc/impl/codegen/compression_types.h', 'include/grpc/impl/codegen/connectivity_state.h', 'include/grpc/impl/codegen/grpc_types.h', diff --git a/grpc.gemspec b/grpc.gemspec index a9f0f681df0..2f72e02152c 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -147,6 +147,7 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/grpc.h ) s.files += %w( include/grpc/status.h ) s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) + s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h ) s.files += %w( include/grpc/impl/codegen/compression_types.h ) s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) s.files += %w( include/grpc/impl/codegen/grpc_types.h ) diff --git a/include/grpc++/impl/codegen/core_codegen_interface.h b/include/grpc++/impl/codegen/core_codegen_interface.h index 16424bab358..aa9013c4cec 100644 --- a/include/grpc++/impl/codegen/core_codegen_interface.h +++ b/include/grpc++/impl/codegen/core_codegen_interface.h @@ -49,18 +49,6 @@ namespace grpc { /// \warning This interface should be considered internal and private. class CoreCodegenInterface { public: - // Serialize the msg into a buffer created inside the function. The caller - // should destroy the returned buffer when done with it. If serialization - // fails, - // false is returned and buffer is left unchanged. - virtual Status SerializeProto(const grpc::protobuf::Message& msg, - grpc_byte_buffer** buffer) = 0; - - // The caller keeps ownership of buffer and msg. - virtual Status DeserializeProto(grpc_byte_buffer* buffer, - grpc::protobuf::Message* msg, - int max_message_size) = 0; - /// Upon a failed assertion, log the error. virtual void assert_fail(const char* failed_assertion) = 0; @@ -76,9 +64,29 @@ class CoreCodegenInterface { virtual void gpr_free(void* p) = 0; virtual void grpc_byte_buffer_destroy(grpc_byte_buffer* bb) = 0; + + virtual void grpc_byte_buffer_reader_init(grpc_byte_buffer_reader* reader, + grpc_byte_buffer* buffer) = 0; + virtual void grpc_byte_buffer_reader_destroy( + grpc_byte_buffer_reader* reader) = 0; + virtual int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader* reader, + gpr_slice* slice) = 0; + + virtual grpc_byte_buffer* grpc_raw_byte_buffer_create(gpr_slice* slice, + size_t nslices) = 0; + + virtual gpr_slice gpr_slice_malloc(size_t length) = 0; + virtual void gpr_slice_unref(gpr_slice slice) = 0; + virtual gpr_slice gpr_slice_split_tail(gpr_slice* s, size_t split) = 0; + virtual void gpr_slice_buffer_add(gpr_slice_buffer* sb, gpr_slice slice) = 0; + virtual void gpr_slice_buffer_pop(gpr_slice_buffer* sb) = 0; + virtual void grpc_metadata_array_init(grpc_metadata_array* array) = 0; virtual void grpc_metadata_array_destroy(grpc_metadata_array* array) = 0; + virtual const Status& ok() = 0; + virtual const Status& cancelled() = 0; + virtual gpr_timespec gpr_inf_future(gpr_clock_type type) = 0; }; diff --git a/include/grpc++/impl/codegen/proto_utils.h b/include/grpc++/impl/codegen/proto_utils.h index 2aaa3c3b300..d044ddc642d 100644 --- a/include/grpc++/impl/codegen/proto_utils.h +++ b/include/grpc++/impl/codegen/proto_utils.h @@ -41,26 +41,179 @@ #include #include #include +#include #include +#include namespace grpc { extern CoreCodegenInterface* g_core_codegen_interface; +namespace { + +const int kGrpcBufferWriterMaxBufferLength = 8192; + +class GrpcBufferWriter GRPC_FINAL + : public ::grpc::protobuf::io::ZeroCopyOutputStream { + public: + explicit GrpcBufferWriter(grpc_byte_buffer** bp, int block_size) + : block_size_(block_size), byte_count_(0), have_backup_(false) { + *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(NULL, 0); + slice_buffer_ = &(*bp)->data.raw.slice_buffer; + } + + ~GrpcBufferWriter() GRPC_OVERRIDE { + if (have_backup_) { + g_core_codegen_interface->gpr_slice_unref(backup_slice_); + } + } + + bool Next(void** data, int* size) GRPC_OVERRIDE { + if (have_backup_) { + slice_ = backup_slice_; + have_backup_ = false; + } else { + slice_ = g_core_codegen_interface->gpr_slice_malloc(block_size_); + } + *data = GPR_SLICE_START_PTR(slice_); + // On win x64, int is only 32bit + GPR_CODEGEN_ASSERT(GPR_SLICE_LENGTH(slice_) <= INT_MAX); + byte_count_ += * size = (int)GPR_SLICE_LENGTH(slice_); + g_core_codegen_interface->gpr_slice_buffer_add(slice_buffer_, slice_); + return true; + } + + void BackUp(int count) GRPC_OVERRIDE { + g_core_codegen_interface->gpr_slice_buffer_pop(slice_buffer_); + if (count == block_size_) { + backup_slice_ = slice_; + } else { + backup_slice_ = g_core_codegen_interface->gpr_slice_split_tail( + &slice_, GPR_SLICE_LENGTH(slice_) - count); + g_core_codegen_interface->gpr_slice_buffer_add(slice_buffer_, slice_); + } + have_backup_ = true; + byte_count_ -= count; + } + + grpc::protobuf::int64 ByteCount() const GRPC_OVERRIDE { return byte_count_; } + + private: + const int block_size_; + int64_t byte_count_; + gpr_slice_buffer* slice_buffer_; + bool have_backup_; + gpr_slice backup_slice_; + gpr_slice slice_; +}; + +class GrpcBufferReader GRPC_FINAL + : public ::grpc::protobuf::io::ZeroCopyInputStream { + public: + explicit GrpcBufferReader(grpc_byte_buffer* buffer) + : byte_count_(0), backup_count_(0) { + g_core_codegen_interface->grpc_byte_buffer_reader_init(&reader_, buffer); + } + ~GrpcBufferReader() GRPC_OVERRIDE { + g_core_codegen_interface->grpc_byte_buffer_reader_destroy(&reader_); + } + + bool Next(const void** data, int* size) GRPC_OVERRIDE { + if (backup_count_ > 0) { + *data = GPR_SLICE_START_PTR(slice_) + GPR_SLICE_LENGTH(slice_) - + backup_count_; + GPR_CODEGEN_ASSERT(backup_count_ <= INT_MAX); + *size = (int)backup_count_; + backup_count_ = 0; + return true; + } + if (!g_core_codegen_interface->grpc_byte_buffer_reader_next(&reader_, + &slice_)) { + return false; + } + g_core_codegen_interface->gpr_slice_unref(slice_); + *data = GPR_SLICE_START_PTR(slice_); + // On win x64, int is only 32bit + GPR_CODEGEN_ASSERT(GPR_SLICE_LENGTH(slice_) <= INT_MAX); + byte_count_ += * size = (int)GPR_SLICE_LENGTH(slice_); + return true; + } + + void BackUp(int count) GRPC_OVERRIDE { backup_count_ = count; } + + bool Skip(int count) GRPC_OVERRIDE { + const void* data; + int size; + while (Next(&data, &size)) { + if (size >= count) { + BackUp(size - count); + return true; + } + // size < count; + count -= size; + } + // error or we have too large count; + return false; + } + + grpc::protobuf::int64 ByteCount() const GRPC_OVERRIDE { + return byte_count_ - backup_count_; + } + + private: + int64_t byte_count_; + int64_t backup_count_; + grpc_byte_buffer_reader reader_; + gpr_slice slice_; +}; +} // namespace + template class SerializationTraits::value>::type> { public: static Status Serialize(const grpc::protobuf::Message& msg, - grpc_byte_buffer** buffer, bool* own_buffer) { + grpc_byte_buffer** bp, bool* own_buffer) { *own_buffer = true; - return g_core_codegen_interface->SerializeProto(msg, buffer); + int byte_size = msg.ByteSize(); + if (byte_size <= kGrpcBufferWriterMaxBufferLength) { + gpr_slice slice = g_core_codegen_interface->gpr_slice_malloc(byte_size); + GPR_CODEGEN_ASSERT( + GPR_SLICE_END_PTR(slice) == + msg.SerializeWithCachedSizesToArray(GPR_SLICE_START_PTR(slice))); + *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(&slice, 1); + g_core_codegen_interface->gpr_slice_unref(slice); + return g_core_codegen_interface->ok(); + } else { + GrpcBufferWriter writer(bp, kGrpcBufferWriterMaxBufferLength); + return msg.SerializeToZeroCopyStream(&writer) + ? g_core_codegen_interface->ok() + : Status(StatusCode::INTERNAL, "Failed to serialize message"); + } } + static Status Deserialize(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg, int max_message_size) { - return g_core_codegen_interface->DeserializeProto(buffer, msg, - max_message_size); + if (buffer == nullptr) { + return Status(StatusCode::INTERNAL, "No payload"); + } + Status result = g_core_codegen_interface->ok(); + { + GrpcBufferReader reader(buffer); + ::grpc::protobuf::io::CodedInputStream decoder(&reader); + if (max_message_size > 0) { + decoder.SetTotalBytesLimit(max_message_size, max_message_size); + } + if (!msg->ParseFromCodedStream(&decoder)) { + result = Status(StatusCode::INTERNAL, msg->InitializationErrorString()); + } + if (!decoder.ConsumedEntireMessage()) { + result = Status(StatusCode::INTERNAL, "Did not read entire message"); + } + } + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer); + return result; } }; diff --git a/include/grpc/byte_buffer_reader.h b/include/grpc/byte_buffer_reader.h index 9a1c6178ab6..e95bf2f80dc 100644 --- a/include/grpc/byte_buffer_reader.h +++ b/include/grpc/byte_buffer_reader.h @@ -34,25 +34,6 @@ #ifndef GRPC_BYTE_BUFFER_READER_H #define GRPC_BYTE_BUFFER_READER_H -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -struct grpc_byte_buffer_reader { - grpc_byte_buffer *buffer_in; - grpc_byte_buffer *buffer_out; - /* Different current objects correspond to different types of byte buffers */ - union { - /* Index into a slice buffer's array of slices */ - unsigned index; - } current; -}; - -#ifdef __cplusplus -} -#endif +#include #endif /* GRPC_BYTE_BUFFER_READER_H */ diff --git a/include/grpc/impl/codegen/byte_buffer_reader.h b/include/grpc/impl/codegen/byte_buffer_reader.h new file mode 100644 index 00000000000..10c382924ec --- /dev/null +++ b/include/grpc/impl/codegen/byte_buffer_reader.h @@ -0,0 +1,57 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_IMPL_CODEGEN_BYTE_BUFFER_READER_H +#define GRPC_IMPL_CODEGEN_BYTE_BUFFER_READER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct grpc_byte_buffer_reader { + grpc_byte_buffer *buffer_in; + grpc_byte_buffer *buffer_out; + /* Different current objects correspond to different types of byte buffers */ + union { + /* Index into a slice buffer's array of slices */ + unsigned index; + } current; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_IMPL_CODEGEN_BYTE_BUFFER_READER_H */ diff --git a/package.xml b/package.xml index 3ae810df5eb..b9b8f0297f7 100644 --- a/package.xml +++ b/package.xml @@ -154,6 +154,7 @@ + diff --git a/src/compiler/csharp_generator.cc b/src/compiler/csharp_generator.cc index 69e2738d53d..4def6c5e31f 100644 --- a/src/compiler/csharp_generator.cc +++ b/src/compiler/csharp_generator.cc @@ -350,10 +350,12 @@ void GenerateServerClass(Printer* out, const ServiceDescriptor *service) { void GenerateClientStub(Printer* out, const ServiceDescriptor *service) { out->Print("// client stub\n"); + out->Print("#pragma warning disable 0618\n"); out->Print( "public class $name$ : ClientBase<$name$>, $interface$\n", "name", GetClientClassName(service), "interface", GetClientInterfaceName(service)); + out->Print("#pragma warning restore 0618\n"); out->Print("{\n"); out->Indent(); @@ -480,10 +482,12 @@ void GenerateBindServiceMethod(Printer* out, const ServiceDescriptor *service, bool use_server_class) { out->Print( "// creates service definition that can be registered with a server\n"); + out->Print("#pragma warning disable 0618\n"); out->Print( "public static ServerServiceDefinition BindService($interface$ serviceImpl)\n", "interface", use_server_class ? GetServerClassName(service) : GetServerInterfaceName(service)); + out->Print("#pragma warning restore 0618\n"); out->Print("{\n"); out->Indent(); diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c index 779efbb97d5..5847ec9053d 100644 --- a/src/core/lib/transport/metadata.c +++ b/src/core/lib/transport/metadata.c @@ -386,10 +386,18 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) { for (s = shard->strs[idx]; s; s = s->bucket_next) { if (s->hash == hash && GPR_SLICE_LENGTH(s->slice) == length && 0 == memcmp(buf, GPR_SLICE_START_PTR(s->slice), length)) { - GRPC_MDSTR_REF((grpc_mdstr *)s); - gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_mdstr_from_buffer", 0); - return (grpc_mdstr *)s; + if (gpr_atm_full_fetch_add(&s->refcnt, 1) == 0) { + /* If we get here, we've added a ref to something that was about to + * die - drop it immediately. + * The *only* possible path here (given the shard mutex) should be to + * drop from one ref back to zero - assert that with a CAS */ + GPR_ASSERT(gpr_atm_rel_cas(&s->refcnt, 1, 0)); + /* and treat this as if we were never here... sshhh */ + } else { + gpr_mu_unlock(&shard->mu); + GPR_TIMER_END("grpc_mdstr_from_buffer", 0); + return (grpc_mdstr *)s; + } } } @@ -397,7 +405,7 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) { if (length + 1 < GPR_SLICE_INLINED_SIZE) { /* string data goes directly into the slice */ s = gpr_malloc(sizeof(internal_string)); - gpr_atm_rel_store(&s->refcnt, 2); + gpr_atm_rel_store(&s->refcnt, 1); s->slice.refcount = NULL; memcpy(s->slice.data.inlined.bytes, buf, length); s->slice.data.inlined.bytes[length] = 0; @@ -406,7 +414,7 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) { /* string data goes after the internal_string header, and we +1 for null terminator */ s = gpr_malloc(sizeof(internal_string) + length + 1); - gpr_atm_rel_store(&s->refcnt, 2); + gpr_atm_rel_store(&s->refcnt, 1); s->refcount.ref = slice_ref; s->refcount.unref = slice_unref; s->slice.refcount = &s->refcount; @@ -675,20 +683,19 @@ const char *grpc_mdstr_as_c_string(grpc_mdstr *s) { grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *gs DEBUG_ARGS) { internal_string *s = (internal_string *)gs; if (is_mdstr_static(gs)) return gs; - GPR_ASSERT(gpr_atm_full_fetch_add(&s->refcnt, 1) != 0); + GPR_ASSERT(gpr_atm_full_fetch_add(&s->refcnt, 1) > 0); return gs; } void grpc_mdstr_unref(grpc_mdstr *gs DEBUG_ARGS) { internal_string *s = (internal_string *)gs; if (is_mdstr_static(gs)) return; - if (2 == gpr_atm_full_fetch_add(&s->refcnt, -1)) { + if (1 == gpr_atm_full_fetch_add(&s->refcnt, -1)) { strtab_shard *shard = &g_strtab_shard[SHARD_IDX(s->hash, LOG2_STRTAB_SHARD_COUNT)]; gpr_mu_lock(&shard->mu); - if (1 == gpr_atm_no_barrier_load(&s->refcnt)) { - internal_destroy_string(shard, s); - } + GPR_ASSERT(0 == gpr_atm_no_barrier_load(&s->refcnt)); + internal_destroy_string(shard, s); gpr_mu_unlock(&shard->mu); } } diff --git a/src/cpp/common/core_codegen.cc b/src/cpp/common/core_codegen.cc index 33a8f755e69..8e8d42eb294 100644 --- a/src/cpp/common/core_codegen.cc +++ b/src/cpp/common/core_codegen.cc @@ -48,124 +48,6 @@ #include "src/core/lib/profiling/timers.h" -namespace { - -const int kGrpcBufferWriterMaxBufferLength = 8192; - -class GrpcBufferWriter GRPC_FINAL - : public ::grpc::protobuf::io::ZeroCopyOutputStream { - public: - explicit GrpcBufferWriter(grpc_byte_buffer** bp, int block_size) - : block_size_(block_size), byte_count_(0), have_backup_(false) { - *bp = grpc_raw_byte_buffer_create(NULL, 0); - slice_buffer_ = &(*bp)->data.raw.slice_buffer; - } - - ~GrpcBufferWriter() GRPC_OVERRIDE { - if (have_backup_) { - gpr_slice_unref(backup_slice_); - } - } - - bool Next(void** data, int* size) GRPC_OVERRIDE { - if (have_backup_) { - slice_ = backup_slice_; - have_backup_ = false; - } else { - slice_ = gpr_slice_malloc(block_size_); - } - *data = GPR_SLICE_START_PTR(slice_); - // On win x64, int is only 32bit - GPR_ASSERT(GPR_SLICE_LENGTH(slice_) <= INT_MAX); - byte_count_ += * size = (int)GPR_SLICE_LENGTH(slice_); - gpr_slice_buffer_add(slice_buffer_, slice_); - return true; - } - - void BackUp(int count) GRPC_OVERRIDE { - gpr_slice_buffer_pop(slice_buffer_); - if (count == block_size_) { - backup_slice_ = slice_; - } else { - backup_slice_ = - gpr_slice_split_tail(&slice_, GPR_SLICE_LENGTH(slice_) - count); - gpr_slice_buffer_add(slice_buffer_, slice_); - } - have_backup_ = true; - byte_count_ -= count; - } - - grpc::protobuf::int64 ByteCount() const GRPC_OVERRIDE { return byte_count_; } - - private: - const int block_size_; - int64_t byte_count_; - gpr_slice_buffer* slice_buffer_; - bool have_backup_; - gpr_slice backup_slice_; - gpr_slice slice_; -}; - -class GrpcBufferReader GRPC_FINAL - : public ::grpc::protobuf::io::ZeroCopyInputStream { - public: - explicit GrpcBufferReader(grpc_byte_buffer* buffer) - : byte_count_(0), backup_count_(0) { - grpc_byte_buffer_reader_init(&reader_, buffer); - } - ~GrpcBufferReader() GRPC_OVERRIDE { - grpc_byte_buffer_reader_destroy(&reader_); - } - - bool Next(const void** data, int* size) GRPC_OVERRIDE { - if (backup_count_ > 0) { - *data = GPR_SLICE_START_PTR(slice_) + GPR_SLICE_LENGTH(slice_) - - backup_count_; - GPR_ASSERT(backup_count_ <= INT_MAX); - *size = (int)backup_count_; - backup_count_ = 0; - return true; - } - if (!grpc_byte_buffer_reader_next(&reader_, &slice_)) { - return false; - } - gpr_slice_unref(slice_); - *data = GPR_SLICE_START_PTR(slice_); - // On win x64, int is only 32bit - GPR_ASSERT(GPR_SLICE_LENGTH(slice_) <= INT_MAX); - byte_count_ += * size = (int)GPR_SLICE_LENGTH(slice_); - return true; - } - - void BackUp(int count) GRPC_OVERRIDE { backup_count_ = count; } - - bool Skip(int count) GRPC_OVERRIDE { - const void* data; - int size; - while (Next(&data, &size)) { - if (size >= count) { - BackUp(size - count); - return true; - } - // size < count; - count -= size; - } - // error or we have too large count; - return false; - } - - grpc::protobuf::int64 ByteCount() const GRPC_OVERRIDE { - return byte_count_ - backup_count_; - } - - private: - int64_t byte_count_; - int64_t backup_count_; - grpc_byte_buffer_reader reader_; - gpr_slice slice_; -}; -} // namespace - namespace grpc { grpc_completion_queue* CoreCodegen::grpc_completion_queue_create( @@ -192,6 +74,44 @@ void CoreCodegen::grpc_byte_buffer_destroy(grpc_byte_buffer* bb) { ::grpc_byte_buffer_destroy(bb); } +void CoreCodegen::grpc_byte_buffer_reader_init(grpc_byte_buffer_reader* reader, + grpc_byte_buffer* buffer) { + ::grpc_byte_buffer_reader_init(reader, buffer); +} + +void CoreCodegen::grpc_byte_buffer_reader_destroy( + grpc_byte_buffer_reader* reader) { + ::grpc_byte_buffer_reader_destroy(reader); +} + +int CoreCodegen::grpc_byte_buffer_reader_next(grpc_byte_buffer_reader* reader, + gpr_slice* slice) { + return ::grpc_byte_buffer_reader_next(reader, slice); +} + +grpc_byte_buffer* CoreCodegen::grpc_raw_byte_buffer_create(gpr_slice* slice, + size_t nslices) { + return ::grpc_raw_byte_buffer_create(slice, nslices); +} + +gpr_slice CoreCodegen::gpr_slice_malloc(size_t length) { + return ::gpr_slice_malloc(length); +} + +void CoreCodegen::gpr_slice_unref(gpr_slice slice) { ::gpr_slice_unref(slice); } + +gpr_slice CoreCodegen::gpr_slice_split_tail(gpr_slice* s, size_t split) { + return ::gpr_slice_split_tail(s, split); +} + +void CoreCodegen::gpr_slice_buffer_add(gpr_slice_buffer* sb, gpr_slice slice) { + ::gpr_slice_buffer_add(sb, slice); +} + +void CoreCodegen::gpr_slice_buffer_pop(gpr_slice_buffer* sb) { + ::gpr_slice_buffer_pop(sb); +} + void CoreCodegen::grpc_metadata_array_init(grpc_metadata_array* array) { ::grpc_metadata_array_init(array); } @@ -200,6 +120,10 @@ void CoreCodegen::grpc_metadata_array_destroy(grpc_metadata_array* array) { ::grpc_metadata_array_destroy(array); } +const Status& CoreCodegen::ok() { return grpc::Status::OK; } + +const Status& CoreCodegen::cancelled() { return grpc::Status::CANCELLED; } + gpr_timespec CoreCodegen::gpr_inf_future(gpr_clock_type type) { return ::gpr_inf_future(type); } @@ -209,48 +133,4 @@ void CoreCodegen::assert_fail(const char* failed_assertion) { abort(); } -Status CoreCodegen::SerializeProto(const grpc::protobuf::Message& msg, - grpc_byte_buffer** bp) { - GPR_TIMER_SCOPE("SerializeProto", 0); - int byte_size = msg.ByteSize(); - if (byte_size <= kGrpcBufferWriterMaxBufferLength) { - gpr_slice slice = gpr_slice_malloc(byte_size); - GPR_ASSERT(GPR_SLICE_END_PTR(slice) == - msg.SerializeWithCachedSizesToArray(GPR_SLICE_START_PTR(slice))); - *bp = grpc_raw_byte_buffer_create(&slice, 1); - gpr_slice_unref(slice); - return Status::OK; - } else { - GrpcBufferWriter writer(bp, kGrpcBufferWriterMaxBufferLength); - return msg.SerializeToZeroCopyStream(&writer) - ? Status::OK - : Status(StatusCode::INTERNAL, "Failed to serialize message"); - } -} - -Status CoreCodegen::DeserializeProto(grpc_byte_buffer* buffer, - grpc::protobuf::Message* msg, - int max_message_size) { - GPR_TIMER_SCOPE("DeserializeProto", 0); - if (buffer == nullptr) { - return Status(StatusCode::INTERNAL, "No payload"); - } - Status result = Status::OK; - { - GrpcBufferReader reader(buffer); - ::grpc::protobuf::io::CodedInputStream decoder(&reader); - if (max_message_size > 0) { - decoder.SetTotalBytesLimit(max_message_size, max_message_size); - } - if (!msg->ParseFromCodedStream(&decoder)) { - result = Status(StatusCode::INTERNAL, msg->InitializationErrorString()); - } - if (!decoder.ConsumedEntireMessage()) { - result = Status(StatusCode::INTERNAL, "Did not read entire message"); - } - } - grpc_byte_buffer_destroy(buffer); - return result; -} - } // namespace grpc diff --git a/src/cpp/common/core_codegen.h b/src/cpp/common/core_codegen.h index e15cb4c34aa..656b11e7e72 100644 --- a/src/cpp/common/core_codegen.h +++ b/src/cpp/common/core_codegen.h @@ -42,13 +42,6 @@ namespace grpc { /// Implementation of the core codegen interface. class CoreCodegen : public CoreCodegenInterface { private: - Status SerializeProto(const grpc::protobuf::Message& msg, - grpc_byte_buffer** bp) override; - - Status DeserializeProto(grpc_byte_buffer* buffer, - grpc::protobuf::Message* msg, - int max_message_size) override; - grpc_completion_queue* grpc_completion_queue_create(void* reserved) override; void grpc_completion_queue_destroy(grpc_completion_queue* cq) override; grpc_event grpc_completion_queue_pluck(grpc_completion_queue* cq, void* tag, @@ -60,11 +53,30 @@ class CoreCodegen : public CoreCodegenInterface { void grpc_byte_buffer_destroy(grpc_byte_buffer* bb) override; + void grpc_byte_buffer_reader_init(grpc_byte_buffer_reader* reader, + grpc_byte_buffer* buffer) override; + void grpc_byte_buffer_reader_destroy( + grpc_byte_buffer_reader* reader) override; + int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader* reader, + gpr_slice* slice) override; + + grpc_byte_buffer* grpc_raw_byte_buffer_create(gpr_slice* slice, + size_t nslices) override; + + gpr_slice gpr_slice_malloc(size_t length) override; + void gpr_slice_unref(gpr_slice slice) override; + gpr_slice gpr_slice_split_tail(gpr_slice* s, size_t split) override; + void gpr_slice_buffer_add(gpr_slice_buffer* sb, gpr_slice slice) override; + void gpr_slice_buffer_pop(gpr_slice_buffer* sb) override; + void grpc_metadata_array_init(grpc_metadata_array* array) override; void grpc_metadata_array_destroy(grpc_metadata_array* array) override; gpr_timespec gpr_inf_future(gpr_clock_type type) override; + virtual const Status& ok() override; + virtual const Status& cancelled() override; + void assert_fail(const char* failed_assertion) override; }; diff --git a/src/csharp/Grpc.Examples/MathGrpc.cs b/src/csharp/Grpc.Examples/MathGrpc.cs index f3bb0d1cdcb..1a6482df90e 100644 --- a/src/csharp/Grpc.Examples/MathGrpc.cs +++ b/src/csharp/Grpc.Examples/MathGrpc.cs @@ -103,7 +103,9 @@ namespace Math { } // client stub + #pragma warning disable 0618 public class MathClient : ClientBase, IMathClient + #pragma warning restore 0618 { public MathClient(Channel channel) : base(channel) { @@ -167,7 +169,9 @@ namespace Math { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(IMath serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_Div, serviceImpl.Div) @@ -177,7 +181,9 @@ namespace Math { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(MathBase serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_Div, serviceImpl.Div) diff --git a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs index 72e11cca3a1..e7f779753d7 100644 --- a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs +++ b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs @@ -56,7 +56,9 @@ namespace Grpc.Health.V1 { } // client stub + #pragma warning disable 0618 public class HealthClient : ClientBase, IHealthClient + #pragma warning restore 0618 { public HealthClient(Channel channel) : base(channel) { @@ -96,14 +98,18 @@ namespace Grpc.Health.V1 { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(IHealth serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_Check, serviceImpl.Check).Build(); } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(HealthBase serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_Check, serviceImpl.Check).Build(); diff --git a/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs b/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs index cc01ae91a14..11c1572c194 100644 --- a/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs @@ -72,7 +72,9 @@ namespace Grpc.Testing { } // client stub + #pragma warning disable 0618 public class MetricsServiceClient : ClientBase, IMetricsServiceClient + #pragma warning restore 0618 { public MetricsServiceClient(Channel channel) : base(channel) { @@ -120,7 +122,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(IMetricsService serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_GetAllGauges, serviceImpl.GetAllGauges) @@ -128,7 +132,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(MetricsServiceBase serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_GetAllGauges, serviceImpl.GetAllGauges) diff --git a/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs b/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs index 46b16cf202d..18cf0672e30 100644 --- a/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs @@ -71,7 +71,9 @@ namespace Grpc.Testing { } // client stub + #pragma warning disable 0618 public class BenchmarkServiceClient : ClientBase, IBenchmarkServiceClient + #pragma warning restore 0618 { public BenchmarkServiceClient(Channel channel) : base(channel) { @@ -119,7 +121,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(IBenchmarkService serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_UnaryCall, serviceImpl.UnaryCall) @@ -127,7 +131,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(BenchmarkServiceBase serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_UnaryCall, serviceImpl.UnaryCall) @@ -241,7 +247,9 @@ namespace Grpc.Testing { } // client stub + #pragma warning disable 0618 public class WorkerServiceClient : ClientBase, IWorkerServiceClient + #pragma warning restore 0618 { public WorkerServiceClient(Channel channel) : base(channel) { @@ -313,7 +321,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(IWorkerService serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_RunServer, serviceImpl.RunServer) @@ -323,7 +333,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(WorkerServiceBase serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_RunServer, serviceImpl.RunServer) diff --git a/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs b/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs index 31746cbe715..3b915f6df12 100644 --- a/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs @@ -138,7 +138,9 @@ namespace Grpc.Testing { } // client stub + #pragma warning disable 0618 public class TestServiceClient : ClientBase, ITestServiceClient + #pragma warning restore 0618 { public TestServiceClient(Channel channel) : base(channel) { @@ -226,7 +228,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(ITestService serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_EmptyCall, serviceImpl.EmptyCall) @@ -238,7 +242,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(TestServiceBase serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_EmptyCall, serviceImpl.EmptyCall) @@ -303,7 +309,9 @@ namespace Grpc.Testing { } // client stub + #pragma warning disable 0618 public class UnimplementedServiceClient : ClientBase, IUnimplementedServiceClient + #pragma warning restore 0618 { public UnimplementedServiceClient(Channel channel) : base(channel) { @@ -343,14 +351,18 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(IUnimplementedService serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_UnimplementedCall, serviceImpl.UnimplementedCall).Build(); } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(UnimplementedServiceBase serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_UnimplementedCall, serviceImpl.UnimplementedCall).Build(); @@ -429,7 +441,9 @@ namespace Grpc.Testing { } // client stub + #pragma warning disable 0618 public class ReconnectServiceClient : ClientBase, IReconnectServiceClient + #pragma warning restore 0618 { public ReconnectServiceClient(Channel channel) : base(channel) { @@ -485,7 +499,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(IReconnectService serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_Start, serviceImpl.Start) @@ -493,7 +509,9 @@ namespace Grpc.Testing { } // creates service definition that can be registered with a server + #pragma warning disable 0618 public static ServerServiceDefinition BindService(ReconnectServiceBase serviceImpl) + #pragma warning restore 0618 { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_Start, serviceImpl.Start) diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 0eede6c23bd..3eeb55d033d 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 7dc04960475..664ca03d978 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -833,6 +833,7 @@ include/grpc++/impl/codegen/sync_no_cxx11.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ include/grpc/impl/codegen/byte_buffer.h \ +include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ include/grpc/impl/codegen/grpc_types.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 312fd17cb22..5188ef1e8da 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -833,6 +833,7 @@ include/grpc++/impl/codegen/sync_no_cxx11.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ include/grpc/impl/codegen/byte_buffer.h \ +include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ include/grpc/impl/codegen/grpc_types.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 034d9c6e6f4..42bdb2f0433 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -766,6 +766,7 @@ include/grpc/compression.h \ include/grpc/grpc.h \ include/grpc/status.h \ include/grpc/impl/codegen/byte_buffer.h \ +include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ include/grpc/impl/codegen/grpc_types.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 3a774a70d6b..7a99598e055 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -766,6 +766,7 @@ include/grpc/compression.h \ include/grpc/grpc.h \ include/grpc/status.h \ include/grpc/impl/codegen/byte_buffer.h \ +include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ include/grpc/impl/codegen/grpc_types.h \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index e94387cb4d7..cfe204840d5 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -5883,6 +5883,7 @@ ], "headers": [ "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", "include/grpc/impl/codegen/grpc_types.h", @@ -5893,6 +5894,7 @@ "name": "grpc_codegen", "src": [ "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", "include/grpc/impl/codegen/grpc_types.h", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 29cab37d521..0ec53acf655 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -331,6 +331,7 @@ + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index 15e2807fd41..491aeaeb678 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -315,6 +315,9 @@ include\grpc\impl\codegen + + include\grpc\impl\codegen + include\grpc\impl\codegen diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index fcda361ef19..96bee4101c8 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -331,6 +331,7 @@ + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 1dc95f985ad..fe9eed781c7 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -300,6 +300,9 @@ include\grpc\impl\codegen + + include\grpc\impl\codegen + include\grpc\impl\codegen diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 4eec05a3b1f..03f4eaa5bec 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -273,6 +273,7 @@ + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 17c88c4805e..4617e3de0d7 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -516,6 +516,9 @@ include\grpc\impl\codegen + + include\grpc\impl\codegen + include\grpc\impl\codegen diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 26050dcf74e..0eb6535c6bb 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -264,6 +264,7 @@ + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index a4acf513bc1..f544fe6158a 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -456,6 +456,9 @@ include\grpc\impl\codegen + + include\grpc\impl\codegen + include\grpc\impl\codegen diff --git a/vsprojects/vcxproj/test/codegen_test_full/codegen_test_full.vcxproj b/vsprojects/vcxproj/test/codegen_test_full/codegen_test_full.vcxproj index cd0b40c8730..34e939cf845 100644 --- a/vsprojects/vcxproj/test/codegen_test_full/codegen_test_full.vcxproj +++ b/vsprojects/vcxproj/test/codegen_test_full/codegen_test_full.vcxproj @@ -191,6 +191,7 @@ + diff --git a/vsprojects/vcxproj/test/codegen_test_full/codegen_test_full.vcxproj.filters b/vsprojects/vcxproj/test/codegen_test_full/codegen_test_full.vcxproj.filters index 029b8ef7741..d66236580c2 100644 --- a/vsprojects/vcxproj/test/codegen_test_full/codegen_test_full.vcxproj.filters +++ b/vsprojects/vcxproj/test/codegen_test_full/codegen_test_full.vcxproj.filters @@ -120,6 +120,9 @@ include\grpc\impl\codegen + + include\grpc\impl\codegen + include\grpc\impl\codegen diff --git a/vsprojects/vcxproj/test/codegen_test_minimal/codegen_test_minimal.vcxproj b/vsprojects/vcxproj/test/codegen_test_minimal/codegen_test_minimal.vcxproj index 6d138fae1cb..890d77df22c 100644 --- a/vsprojects/vcxproj/test/codegen_test_minimal/codegen_test_minimal.vcxproj +++ b/vsprojects/vcxproj/test/codegen_test_minimal/codegen_test_minimal.vcxproj @@ -191,6 +191,7 @@ + diff --git a/vsprojects/vcxproj/test/codegen_test_minimal/codegen_test_minimal.vcxproj.filters b/vsprojects/vcxproj/test/codegen_test_minimal/codegen_test_minimal.vcxproj.filters index dc3f0b2d040..4e0ba656fc7 100644 --- a/vsprojects/vcxproj/test/codegen_test_minimal/codegen_test_minimal.vcxproj.filters +++ b/vsprojects/vcxproj/test/codegen_test_minimal/codegen_test_minimal.vcxproj.filters @@ -120,6 +120,9 @@ include\grpc\impl\codegen + + include\grpc\impl\codegen + include\grpc\impl\codegen From c74f62bcfcabd8b7468bcca16dda28a29b377877 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 28 Apr 2016 00:56:31 -0700 Subject: [PATCH 005/155] Don't try to read files that have been moved/renamed. --- tools/distrib/check_include_guards.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/distrib/check_include_guards.py b/tools/distrib/check_include_guards.py index 897a899e7ed..6c160c64b6f 100755 --- a/tools/distrib/check_include_guards.py +++ b/tools/distrib/check_include_guards.py @@ -31,6 +31,7 @@ import argparse import os +import os.path import re import sys import subprocess @@ -187,6 +188,8 @@ filename_list = [] try: filename_list = subprocess.check_output(FILE_LIST_COMMAND, shell=True).splitlines() + # Filter out non-existent files (ie, file removed or renamed) + filename_list = (f for f in filename_list if os.path.isfile(f)) except subprocess.CalledProcessError: sys.exit(0) From 0c6070f68d5dbac069fd19e1ddc394c2ea3c4775 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 28 Apr 2016 11:26:34 -0700 Subject: [PATCH 006/155] Mark stream as cancelled if we exceed the metadata size limit. Also take this opportunity to convert the seen_error field to a bool. --- .../chttp2/transport/chttp2_transport.c | 31 ++++++++++++++----- .../ext/transport/chttp2/transport/internal.h | 6 ++-- .../ext/transport/chttp2/transport/parsing.c | 14 ++++++--- test/core/end2end/tests/large_metadata.c | 8 ++--- 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index c24950a1890..b73ec2a7e9d 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -858,7 +858,7 @@ static void perform_stream_op_locked( add_closure_barrier(on_complete); stream_global->send_initial_metadata = op->send_initial_metadata; if (contains_non_ok_status(transport_global, op->send_initial_metadata)) { - stream_global->seen_error = 1; + stream_global->seen_error = true; grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } if (!stream_global->write_closed) { @@ -899,7 +899,7 @@ static void perform_stream_op_locked( add_closure_barrier(on_complete); stream_global->send_trailing_metadata = op->send_trailing_metadata; if (contains_non_ok_status(transport_global, op->send_trailing_metadata)) { - stream_global->seen_error = 1; + stream_global->seen_error = true; grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } if (stream_global->write_closed) { @@ -1076,6 +1076,16 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx, grpc_chttp2_list_pop_check_read_ops(transport_global, &stream_global)) { if (stream_global->recv_initial_metadata_ready != NULL && stream_global->published_initial_metadata) { + if (stream_global->seen_error) { + while ((bs = grpc_chttp2_incoming_frame_queue_pop( + &stream_global->incoming_frames)) != NULL) { + grpc_byte_stream_destroy(exec_ctx, bs); + } + if (stream_global->exceeded_metadata_size) { + cancel_from_api(exec_ctx, transport_global, stream_global, + GRPC_STATUS_RESOURCE_EXHAUSTED); + } + } grpc_chttp2_incoming_metadata_buffer_publish( &stream_global->received_initial_metadata, stream_global->recv_initial_metadata); @@ -1105,10 +1115,15 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx, } if (stream_global->recv_trailing_metadata_finished != NULL && stream_global->read_closed && stream_global->write_closed) { - while (stream_global->seen_error && - (bs = grpc_chttp2_incoming_frame_queue_pop( - &stream_global->incoming_frames)) != NULL) { - grpc_byte_stream_destroy(exec_ctx, bs); + if (stream_global->seen_error) { + while ((bs = grpc_chttp2_incoming_frame_queue_pop( + &stream_global->incoming_frames)) != NULL) { + grpc_byte_stream_destroy(exec_ctx, bs); + } + if (stream_global->exceeded_metadata_size) { + cancel_from_api(exec_ctx, transport_global, stream_global, + GRPC_STATUS_RESOURCE_EXHAUSTED); + } } if (stream_global->incoming_frames.head == NULL) { grpc_chttp2_incoming_metadata_buffer_publish( @@ -1175,7 +1190,7 @@ static void cancel_from_api(grpc_exec_ctx *exec_ctx, NULL); } if (status != GRPC_STATUS_OK && !stream_global->seen_error) { - stream_global->seen_error = 1; + stream_global->seen_error = true; grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, 1, @@ -1187,7 +1202,7 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream_global *stream_global, grpc_status_code status, gpr_slice *slice) { if (status != GRPC_STATUS_OK) { - stream_global->seen_error = 1; + stream_global->seen_error = true; grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } /* stream_global->recv_trailing_metadata_finished gives us a diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index d547a6e9c11..be38ffda1ff 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -410,7 +410,8 @@ typedef struct { uint8_t in_stream_map; /** has this stream seen an error? if 1, then pending incoming frames can be thrown away */ - uint8_t seen_error; + bool seen_error; + bool exceeded_metadata_size; uint8_t published_initial_metadata; uint8_t published_trailing_metadata; @@ -457,7 +458,8 @@ struct grpc_chttp2_stream_parsing { /** which metadata did we get (on this parse) */ uint8_t got_metadata_on_parse[2]; /** should we raise the seen_error flag in transport_global */ - uint8_t seen_error; + bool seen_error; + bool exceeded_metadata_size; /** window available for peer to send to us */ int64_t incoming_window; /** parsing state for data frames */ diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 0cf4d87f3c0..11cbb80ca88 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -167,7 +167,9 @@ void grpc_chttp2_publish_reads( while (grpc_chttp2_list_pop_parsing_seen_stream( transport_global, transport_parsing, &stream_global, &stream_parsing)) { if (stream_parsing->seen_error) { - stream_global->seen_error = 1; + stream_global->seen_error = true; + stream_global->exceeded_metadata_size = + stream_parsing->exceeded_metadata_size; grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } @@ -603,7 +605,7 @@ static void on_initial_header(void *tp, grpc_mdelem *md) { if (md->key == GRPC_MDSTR_GRPC_STATUS && md != GRPC_MDELEM_GRPC_STATUS_0) { /* TODO(ctiller): check for a status like " 0" */ - stream_parsing->seen_error = 1; + stream_parsing->seen_error = true; } if (md->key == GRPC_MDSTR_GRPC_TIMEOUT) { @@ -627,7 +629,8 @@ static void on_initial_header(void *tp, grpc_mdelem *md) { const size_t new_size = stream_parsing->metadata_buffer[0].size + GRPC_MDELEM_LENGTH(md); if (new_size > transport_parsing->max_metadata_size) { - stream_parsing->seen_error = 1; + stream_parsing->seen_error = true; + stream_parsing->exceeded_metadata_size = true; GRPC_MDELEM_UNREF(md); } else { grpc_chttp2_incoming_metadata_buffer_add( @@ -656,13 +659,14 @@ static void on_trailing_header(void *tp, grpc_mdelem *md) { if (md->key == GRPC_MDSTR_GRPC_STATUS && md != GRPC_MDELEM_GRPC_STATUS_0) { /* TODO(ctiller): check for a status like " 0" */ - stream_parsing->seen_error = 1; + stream_parsing->seen_error = true; } const size_t new_size = stream_parsing->metadata_buffer[1].size + GRPC_MDELEM_LENGTH(md); if (new_size > transport_parsing->max_metadata_size) { - stream_parsing->seen_error = 1; + stream_parsing->seen_error = true; + stream_parsing->exceeded_metadata_size = true; GRPC_MDELEM_UNREF(md); } else { grpc_chttp2_incoming_metadata_buffer_add( diff --git a/test/core/end2end/tests/large_metadata.c b/test/core/end2end/tests/large_metadata.c index f09b55a2c0c..2aa6381e9e2 100644 --- a/test/core/end2end/tests/large_metadata.c +++ b/test/core/end2end/tests/large_metadata.c @@ -199,7 +199,7 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config, error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), NULL); GPR_ASSERT(GRPC_CALL_OK == error); - cq_expect_completion(cqv, tag(102), 1); + cq_expect_completion(cqv, tag(102), allow_large_metadata); cq_verify(cqv); op = ops; @@ -222,13 +222,13 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config, cq_expect_completion(cqv, tag(1), 1); cq_verify(cqv); -// FIXME: why is this assert passing with allow_large_metadata=false? - GPR_ASSERT(status == GRPC_STATUS_OK); - GPR_ASSERT(0 == strcmp(details, "xyz")); + GPR_ASSERT(status == (allow_large_metadata ? GRPC_STATUS_OK + : GRPC_STATUS_RESOURCE_EXHAUSTED)); GPR_ASSERT(0 == strcmp(call_details.method, "/foo")); GPR_ASSERT(0 == strcmp(call_details.host, "foo.test.google.fr")); GPR_ASSERT(was_cancelled == 0); if (allow_large_metadata) { + GPR_ASSERT(0 == strcmp(details, "xyz")); GPR_ASSERT(byte_buffer_eq_string(request_payload_recv, "hello world")); GPR_ASSERT(contains_metadata(&request_metadata_recv, "key", meta.value)); } else { From c2de452309c91b934382c2ea77eff5d9e53caad1 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 29 Apr 2016 10:25:27 -0700 Subject: [PATCH 007/155] Fix header-size computation to comply with the HTTP/2 RFC. --- src/core/lib/transport/metadata.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 277c2579338..77c32c72dee 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -146,8 +146,10 @@ void grpc_mdelem_unref(grpc_mdelem *md); const char *grpc_mdstr_as_c_string(grpc_mdstr *s); #define GRPC_MDSTR_LENGTH(s) (GPR_SLICE_LENGTH(s->slice)) + +/* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */ #define GRPC_MDELEM_LENGTH(e) (GRPC_MDSTR_LENGTH(e->key) + \ - GRPC_MDSTR_LENGTH(e->value)) + GRPC_MDSTR_LENGTH(e->value) + 32) int grpc_mdstr_is_legal_header(grpc_mdstr *s); int grpc_mdstr_is_legal_nonbin_header(grpc_mdstr *s); From 00598719bcf40a95505f2deeb53906c5944373fd Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 29 Apr 2016 13:38:50 -0700 Subject: [PATCH 008/155] Use HTTP/2 MAX_HEADER_LIST_SIZE setting instead of adding a new member in the grpc_chttp2_transport_parsing struct. --- .../transport/chttp2/transport/chttp2_transport.c | 11 ++++------- .../transport/chttp2/transport/frame_settings.c | 3 ++- src/core/ext/transport/chttp2/transport/internal.h | 3 --- src/core/ext/transport/chttp2/transport/parsing.c | 14 ++++++++++++-- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index b73ec2a7e9d..63147865257 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -56,8 +56,6 @@ #define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024) #define MAX_WINDOW 0x7fffffffu -#define DEFAULT_MAX_METADATA_SIZE 16 * 1024 - #define MAX_CLIENT_STREAM_ID 0x7fffffffu int grpc_http_trace = 0; @@ -67,8 +65,8 @@ int grpc_flowctl_trace = 0; ((grpc_chttp2_transport *)((char *)(tw)-offsetof(grpc_chttp2_transport, \ writing))) -#define TRANSPORT_FROM_PARSING(tw) \ - ((grpc_chttp2_transport *)((char *)(tw)-offsetof(grpc_chttp2_transport, \ +#define TRANSPORT_FROM_PARSING(tp) \ + ((grpc_chttp2_transport *)((char *)(tp)-offsetof(grpc_chttp2_transport, \ parsing))) #define TRANSPORT_FROM_GLOBAL(tg) \ @@ -252,7 +250,6 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->global.ping_counter = 1; t->global.pings.next = t->global.pings.prev = &t->global.pings; t->parsing.is_client = is_client; - t->parsing.max_metadata_size = DEFAULT_MAX_METADATA_SIZE; t->parsing.deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; t->writing.is_client = is_client; @@ -384,8 +381,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, gpr_log(GPR_ERROR, "%s: must be non-negative", GRPC_ARG_MAX_METADATA_SIZE); } else { - t->parsing.max_metadata_size = - (uint32_t)channel_args->args[i].value.integer; + push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, + (uint32_t)channel_args->args[i].value.integer); } } } diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c index a3c1e15f35e..7fa66247e4d 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.c +++ b/src/core/ext/transport/chttp2/transport/frame_settings.c @@ -44,6 +44,7 @@ #include "src/core/ext/transport/chttp2/transport/http2_errors.h" #include "src/core/lib/debug/trace.h" +#define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024) #define MAX_MAX_HEADER_LIST_SIZE (1024 * 1024 * 1024) /* HTTP/2 mandated initial connection settings */ @@ -62,7 +63,7 @@ const grpc_chttp2_setting_parameters GRPC_CHTTP2_FLOW_CONTROL_ERROR}, {"MAX_FRAME_SIZE", 16384, 16384, 16777215, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_CHTTP2_PROTOCOL_ERROR}, - {"MAX_HEADER_LIST_SIZE", MAX_MAX_HEADER_LIST_SIZE, 0, + {"MAX_HEADER_LIST_SIZE", DEFAULT_MAX_HEADER_LIST_SIZE, 0, MAX_MAX_HEADER_LIST_SIZE, GRPC_CHTTP2_CLAMP_INVALID_VALUE, GRPC_CHTTP2_PROTOCOL_ERROR}, }; diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index be38ffda1ff..2884b3be9b2 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -229,9 +229,6 @@ struct grpc_chttp2_transport_parsing { /** is this transport a client? (boolean) */ uint8_t is_client; - /** max metadata size */ - uint32_t max_metadata_size; - /** were settings updated? */ uint8_t settings_updated; /** was a settings ack received? */ diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 11cbb80ca88..f1018733379 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -45,6 +45,10 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/transport/static_metadata.h" +#define TRANSPORT_FROM_PARSING(tp) \ + ((grpc_chttp2_transport *)((char *)(tp)-offsetof(grpc_chttp2_transport, \ + parsing))) + static int init_frame_parser(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); static int init_header_frame_parser( @@ -628,7 +632,10 @@ static void on_initial_header(void *tp, grpc_mdelem *md) { } else { const size_t new_size = stream_parsing->metadata_buffer[0].size + GRPC_MDELEM_LENGTH(md); - if (new_size > transport_parsing->max_metadata_size) { + grpc_chttp2_transport_global *transport_global = + &TRANSPORT_FROM_PARSING(transport_parsing)->global; + if (new_size > transport_global->settings + [GRPC_LOCAL_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]) { stream_parsing->seen_error = true; stream_parsing->exceeded_metadata_size = true; GRPC_MDELEM_UNREF(md); @@ -664,7 +671,10 @@ static void on_trailing_header(void *tp, grpc_mdelem *md) { const size_t new_size = stream_parsing->metadata_buffer[1].size + GRPC_MDELEM_LENGTH(md); - if (new_size > transport_parsing->max_metadata_size) { + grpc_chttp2_transport_global *transport_global = + &TRANSPORT_FROM_PARSING(transport_parsing)->global; + if (new_size > transport_global->settings + [GRPC_LOCAL_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]) { stream_parsing->seen_error = true; stream_parsing->exceeded_metadata_size = true; GRPC_MDELEM_UNREF(md); From eadcb89b7ac37207a945a520b99eb24de1513bc8 Mon Sep 17 00:00:00 2001 From: "David G. Quintas" Date: Fri, 29 Apr 2016 14:54:31 -0700 Subject: [PATCH 009/155] Fixed name of test config for h2_census --- test/core/end2end/fixtures/h2_census.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/end2end/fixtures/h2_census.c b/test/core/end2end/fixtures/h2_census.c index ff2f028f096..e46b39e476f 100644 --- a/test/core/end2end/fixtures/h2_census.c +++ b/test/core/end2end/fixtures/h2_census.c @@ -111,7 +111,7 @@ void chttp2_tear_down_fullstack(grpc_end2end_test_fixture *f) { /* All test configurations */ static grpc_end2end_test_config configs[] = { - {"chttp2/fullstack", FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION, + {"chttp2/fullstack+census", FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION, chttp2_create_fixture_fullstack, chttp2_init_client_fullstack, chttp2_init_server_fullstack, chttp2_tear_down_fullstack}, }; From 9fab4386695eb5575b943b0d1fec8141aec72ea1 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 29 Apr 2016 15:05:00 -0700 Subject: [PATCH 010/155] Made Node tests support multiple versions and set the Node default version to 4 everywhere possible --- templates/tools/dockerfile/node_deps.include | 4 ++++ .../interoptest/grpc_interop_node/Dockerfile | 5 ++++- .../interoptest/grpc_interop_node/build_interop.sh | 2 -- .../stress_test/grpc_interop_stress_node/Dockerfile | 5 ++++- .../grpc_interop_stress_node/build_interop_stress.sh | 2 -- .../dockerfile/test/multilang_jessie_x64/Dockerfile | 5 ++++- tools/dockerfile/test/node_jessie_x64/Dockerfile | 5 ++++- tools/run_tests/run_tests.py | 12 +++++++++--- 8 files changed, 29 insertions(+), 11 deletions(-) diff --git a/templates/tools/dockerfile/node_deps.include b/templates/tools/dockerfile/node_deps.include index 7d37d671467..7855fbfee36 100644 --- a/templates/tools/dockerfile/node_deps.include +++ b/templates/tools/dockerfile/node_deps.include @@ -4,4 +4,8 @@ # Install nvm RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +# Install all versions of node that we want to test RUN /bin/bash -l -c "nvm install 0.12 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 4" \ No newline at end of file diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile index 64314f88642..2a8d35a5dc1 100644 --- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile @@ -69,8 +69,11 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Install nvm RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +# Install all versions of node that we want to test RUN /bin/bash -l -c "nvm install 0.12 && npm config set cache /tmp/npm-cache" - +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 4" # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc RUN ln -s /usr/bin/ccache /usr/local/bin/g++ diff --git a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh index b99fd444eea..976f55d9ab5 100755 --- a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh @@ -38,8 +38,6 @@ git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -nvm use 0.12 -nvm alias default 0.12 # prevent the need to run 'nvm use' in every shell # build Node interop client & server npm install -g node-gyp diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile index f70add4e31a..4fd7cc29a3c 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile @@ -69,8 +69,11 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Install nvm RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +# Install all versions of node that we want to test RUN /bin/bash -l -c "nvm install 0.12 && npm config set cache /tmp/npm-cache" - +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 4" # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean RUN pip install --upgrade google-api-python-client diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh index b99fd444eea..976f55d9ab5 100755 --- a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh +++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh @@ -38,8 +38,6 @@ git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -nvm use 0.12 -nvm alias default 0.12 # prevent the need to run 'nvm use' in every shell # build Node interop client & server npm install -g node-gyp diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index 71ebf2bf715..5c3f77405ea 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -90,8 +90,11 @@ RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev c # Install nvm RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +# Install all versions of node that we want to test RUN /bin/bash -l -c "nvm install 0.12 && npm config set cache /tmp/npm-cache" - +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 4" #================= # PHP dependencies diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 64314f88642..2a8d35a5dc1 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -69,8 +69,11 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Install nvm RUN touch .profile RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +# Install all versions of node that we want to test RUN /bin/bash -l -c "nvm install 0.12 && npm config set cache /tmp/npm-cache" - +RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" +RUN /bin/bash -l -c "nvm alias default 4" # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc RUN ln -s /usr/bin/ccache /usr/local/bin/g++ diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index dea481ef90d..d50d5aac95e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -272,12 +272,17 @@ class NodeLanguage(object): def __init__(self): self.platform = platform_string() - self.node_version = '0.12' def configure(self, config, args): self.config = config self.args = args - _check_compiler(self.args.compiler, ['default']) + _check_compiler(self.args.compiler, ['default', 'node0.12', + 'node4', 'node5']) + if self.args.compiler == 'default': + self.node_version = '4' + else: + # Take off the word "node" + self.node_version = self.args.compiler[4:] def test_specs(self): if self.platform == 'windows': @@ -802,7 +807,8 @@ argp.add_argument('--compiler', 'gcc4.4', 'gcc4.9', 'gcc5.3', 'clang3.4', 'clang3.6', 'vs2010', 'vs2013', 'vs2015', - 'python2.7', 'python3.4'], + 'python2.7', 'python3.4', + 'node0.12', 'node4', 'node5'], default='default', help='Selects compiler to use. Allowed values depend on the platform and language.') argp.add_argument('--build_only', From fad045004d5576a3f2d32f6d27b0fc285ff589d0 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Fri, 29 Apr 2016 16:43:46 -0700 Subject: [PATCH 011/155] debug logging fix --- src/core/ext/lb_policy/round_robin/round_robin.c | 6 ++++-- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 3 ++- src/core/ext/transport/chttp2/transport/hpack_encoder.c | 6 +++++- src/core/ext/transport/chttp2/transport/hpack_table.c | 4 +++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index 3f6051b8925..dcdc0c62857 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -306,8 +306,10 @@ static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) { size_t i; p->started_picking = 1; - gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%d", p, - p->num_subchannels); + if (grpc_lb_round_robin_trace) { + gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%d", p, + p->num_subchannels); + } for (i = 0; i < p->num_subchannels; i++) { subchannel_data *sd = p->subchannels[i]; diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index fcf2abfe66e..02ad0c0370a 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -776,7 +776,8 @@ void grpc_chttp2_add_incoming_goaway( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, uint32_t goaway_error, gpr_slice goaway_text) { char *msg = gpr_dump_slice(goaway_text, GPR_DUMP_HEX | GPR_DUMP_ASCII); - gpr_log(GPR_DEBUG, "got goaway [%d]: %s", goaway_error, msg); + GRPC_CHTTP2_IF_TRACING(gpr_log( + GPR_DEBUG, "got goaway [%d]: %s", goaway_error, msg)); gpr_free(msg); gpr_slice_unref(goaway_text); transport_global->seen_goaway = 1; diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 555027c8660..ebeee37f0d9 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -63,6 +63,8 @@ /* don't consider adding anything bigger than this to the hpack table */ #define MAX_DECODER_SPACE_USAGE 512 +extern int grpc_http_trace; + typedef struct { int is_first_frame; /* number of bytes in 'output' when we started the frame - used to calculate @@ -532,7 +534,9 @@ void grpc_chttp2_hpack_compressor_set_max_table_size( } } c->advertise_table_size_change = 1; - gpr_log(GPR_DEBUG, "set max table size from encoder to %d", max_table_size); + if (grpc_http_trace) { + gpr_log(GPR_DEBUG, "set max table size from encoder to %d", max_table_size); + } } void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor *c, diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.c b/src/core/ext/transport/chttp2/transport/hpack_table.c index 4d64506de29..295f31c44f2 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.c +++ b/src/core/ext/transport/chttp2/transport/hpack_table.c @@ -253,7 +253,9 @@ void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl *tbl, if (tbl->max_bytes == max_bytes) { return; } - gpr_log(GPR_DEBUG, "Update hpack parser max size to %d", max_bytes); + if (grpc_http_trace) { + gpr_log(GPR_DEBUG, "Update hpack parser max size to %d", max_bytes); + } while (tbl->mem_used > max_bytes) { evict1(tbl); } From 088aa27adfd0852a36f7e6eedf77cad8eb896ef2 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Fri, 29 Apr 2016 18:15:43 -0700 Subject: [PATCH 012/155] format fixes --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 02ad0c0370a..9ca551f0c5f 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -776,8 +776,8 @@ void grpc_chttp2_add_incoming_goaway( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, uint32_t goaway_error, gpr_slice goaway_text) { char *msg = gpr_dump_slice(goaway_text, GPR_DUMP_HEX | GPR_DUMP_ASCII); - GRPC_CHTTP2_IF_TRACING(gpr_log( - GPR_DEBUG, "got goaway [%d]: %s", goaway_error, msg)); + GRPC_CHTTP2_IF_TRACING( + gpr_log(GPR_DEBUG, "got goaway [%d]: %s", goaway_error, msg)); gpr_free(msg); gpr_slice_unref(goaway_text); transport_global->seen_goaway = 1; From 7a5f019bd3a086f4024724be936d6cd87b963daa Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 29 Apr 2016 19:34:25 -0700 Subject: [PATCH 013/155] Removed leftover function declaration from channel args --- src/core/lib/channel/channel_args.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h index 0a51780a14e..23c7b7b897b 100644 --- a/src/core/lib/channel/channel_args.h +++ b/src/core/lib/channel/channel_args.h @@ -56,10 +56,6 @@ grpc_channel_args *grpc_channel_args_merge(const grpc_channel_args *a, /** Destroy arguments created by \a grpc_channel_args_copy */ void grpc_channel_args_destroy(grpc_channel_args *a); -/** Reads census_enabled settings from channel args. Returns 1 if census_enabled - * is specified in channel args, otherwise returns 0. */ -int grpc_channel_args_is_census_enabled(const grpc_channel_args *a); - /** Returns the compression algorithm set in \a a. */ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( const grpc_channel_args *a); From a49b13bc332a38dfcf83162003cbaa0a1384143b Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 19 Apr 2016 10:31:25 -0700 Subject: [PATCH 014/155] cronet wrapper code --- include/grpc/grpc_security.h | 5 + .../client/secure/cronet_channel_create.c | 72 +++ .../cronet/transport/cronet_c_for_grpc.h | 202 ++++++ .../cronet/transport/cronet_transport.c | 586 ++++++++++++++++++ 4 files changed, 865 insertions(+) create mode 100644 src/core/ext/transport/cronet/client/secure/cronet_channel_create.c create mode 100644 src/core/ext/transport/cronet/transport/cronet_c_for_grpc.h create mode 100644 src/core/ext/transport/cronet/transport/cronet_transport.c diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index 79199cc5d68..d753aaf96c1 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -299,6 +299,11 @@ GRPCAPI grpc_channel *grpc_secure_channel_create( grpc_channel_credentials *creds, const char *target, const grpc_channel_args *args, void *reserved); +GRPCAPI grpc_channel *grpc_custom_secure_channel_create( + void *engine, const char *target, + const grpc_channel_args *args, void *reserved); + + /* --- grpc_server_credentials object. --- A server credentials object represents a way to authenticate a server. */ diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c new file mode 100644 index 00000000000..23189809f72 --- /dev/null +++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c @@ -0,0 +1,72 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include +#include + +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/transport/transport_impl.h" + +#ifdef COMPILE_WITH_CRONET +// Cronet transport object +struct grpc_cronet_transport { + grpc_transport base; /* must be first element in this structure */ + void *engine; + char *host; +}; + +typedef struct grpc_cronet_transport grpc_cronet_transport; + +extern grpc_transport_vtable cronet_vtable; + +GRPCAPI grpc_channel *grpc_custom_secure_channel_create( + void *engine, const char *target, + const grpc_channel_args *args, void *reserved) { + grpc_cronet_transport *ct = gpr_malloc(sizeof(grpc_cronet_transport)); + ct->base.vtable = &cronet_vtable; + ct->engine = engine; + ct->host = gpr_malloc(strlen(target) + 1); + strcpy(ct->host, target); + gpr_log( + GPR_DEBUG, "grpc_create_cronet_transport: cronet_engine = %p, target=%s", + engine, ct->host); + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + return grpc_channel_create(&exec_ctx, target, args, + GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); +} +#endif // COMPILE_WITH_CRONET diff --git a/src/core/ext/transport/cronet/transport/cronet_c_for_grpc.h b/src/core/ext/transport/cronet/transport/cronet_c_for_grpc.h new file mode 100644 index 00000000000..15a511aebd0 --- /dev/null +++ b/src/core/ext/transport/cronet/transport/cronet_c_for_grpc.h @@ -0,0 +1,202 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ +#define COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Cronet Engine API. */ + +/* Opaque object representing Cronet Engine. Created and configured outside + * of this API to facilitate sharing with other components */ +typedef struct cronet_engine { void* obj; } cronet_engine; + +void cronet_engine_add_quic_hint(cronet_engine* engine, + const char* host, + int port, + int alternate_port); + +/* Cronet Bidirectional Stream API */ + +/* Opaque object representing Cronet Bidirectional Stream. */ +typedef struct cronet_bidirectional_stream { + void* obj; + void* annotation; +} cronet_bidirectional_stream; + +/* A single request or response header element. */ +typedef struct cronet_bidirectional_stream_header { + const char* key; + const char* value; +} cronet_bidirectional_stream_header; + +/* Array of request or response headers or trailers. */ +typedef struct cronet_bidirectional_stream_header_array { + size_t count; + size_t capacity; + cronet_bidirectional_stream_header* headers; +} cronet_bidirectional_stream_header_array; + +/* Set of callbacks used to receive callbacks from bidirectional stream. */ +typedef struct cronet_bidirectional_stream_callback { + /* Invoked when request headers are sent. Indicates that stream has initiated + * the request. Consumer may call cronet_bidirectional_stream_write() to start + * writing data. + */ + void (*on_request_headers_sent)(cronet_bidirectional_stream* stream); + + /* Invoked when initial response headers are received. + * Consumer must call cronet_bidirectional_stream_read() to start reading. + * Consumer may call cronet_bidirectional_stream_write() to start writing or + * close the stream. Contents of |headers| is valid for duration of the call. + */ + void (*on_response_headers_received)( + cronet_bidirectional_stream* stream, + const cronet_bidirectional_stream_header_array* headers, + const char* negotiated_protocol); + + /* Invoked when data is read into the buffer passed to + * cronet_bidirectional_stream_read(). Only part of the buffer may be + * populated. To continue reading, call cronet_bidirectional_stream_read(). + * It may be invoked after on_response_trailers_received()}, if there was + * pending read data before trailers were received. + * + * If count is 0, it means the remote side has signaled that it will send no + * more data; future calls to cronet_bidirectional_stream_read() will result + * in the on_data_read() callback or on_succeded() callback if + * cronet_bidirectional_stream_write() was invoked with end_of_stream set to + * true. + */ + void (*on_read_completed)(cronet_bidirectional_stream* stream, + char* data, + int count); + + /** + * Invoked when all data passed to cronet_bidirectional_stream_write() is + * sent. + * To continue writing, call cronet_bidirectional_stream_write(). + */ + void (*on_write_completed)(cronet_bidirectional_stream* stream, + const char* data); + + /* Invoked when trailers are received before closing the stream. Only invoked + * when server sends trailers, which it may not. May be invoked while there is + * read data remaining in local buffer. Contents of |trailers| is valid for + * duration of the call. + */ + void (*on_response_trailers_received)( + cronet_bidirectional_stream* stream, + const cronet_bidirectional_stream_header_array* trailers); + + /** + * Invoked when there is no data to be read or written and the stream is + * closed successfully remotely and locally. Once invoked, no further callback + * methods will be invoked. + */ + void (*on_succeded)(cronet_bidirectional_stream* stream); + + /** + * Invoked if the stream failed for any reason after + * cronet_bidirectional_stream_start(). HTTP/2 error codes are + * mapped to chrome net error codes. Once invoked, no further callback methods + * will be invoked. + */ + void (*on_failed)(cronet_bidirectional_stream* stream, int net_error); + + /** + * Invoked if the stream was canceled via + * cronet_bidirectional_stream_cancel(). Once invoked, no further callback + * methods will be invoked. + */ + void (*on_canceled)(cronet_bidirectional_stream* stream); +} cronet_bidirectional_stream_callback; + +/* Create a new stream object that uses |engine| and |callback|. All stream + * tasks are performed asynchronously on the |engine| network thread. |callback| + * methods are invoked synchronously on the |engine| network thread, but must + * not run tasks on the current thread to prevent blocking networking operations + * and causing exceptions during shutdown. The |annotation| is stored in + * bidirectional stream for arbitrary use by application. + * + * Returned |cronet_bidirectional_stream*| is owned by the caller, and must be + * destroyed using |cronet_bidirectional_stream_destroy|. + * + * Both |calback| and |engine| must remain valid until stream is destroyed. + */ +cronet_bidirectional_stream* cronet_bidirectional_stream_create( + cronet_engine* engine, + void* annotation, + cronet_bidirectional_stream_callback* callback); + +/* TBD: The following methods return int. Should it be a custom type? */ + +/* Destroy stream object. Destroy could be called from any thread, including + * network thread, but is posted, so |stream| is valid until calling task is + * complete. + */ +int cronet_bidirectional_stream_destroy(cronet_bidirectional_stream* stream); + +/* Start the stream by sending request to |url| using |method| and |headers|. If + * |end_of_stream| is true, then no data is expected to be written. + */ +int cronet_bidirectional_stream_start( + cronet_bidirectional_stream* stream, + const char* url, + int priority, + const char* method, + const cronet_bidirectional_stream_header_array* headers, + bool end_of_stream); + +/* Read response data into |buffer| of |capacity| length. Must only be called at + * most once in response to each invocation of the + * on_response_headers_received() and on_read_completed() methods of the + * cronet_bidirectional_stream_callback. + * Each call will result in an invocation of one of the callback's + * on_read_completed method if data is read, its on_succeeded() method if + * the stream is closed, or its on_failed() method if there's an error. + */ +int cronet_bidirectional_stream_read(cronet_bidirectional_stream* stream, + char* buffer, + int capacity); + +/* Read response data into |buffer| of |capacity| length. Must only be called at + * most once in response to each invocation of the + * on_response_headers_received() and on_read_completed() methods of the + * cronet_bidirectional_stream_callback. + * Each call will result in an invocation of one of the callback's + * on_read_completed method if data is read, its on_succeeded() method if + * the stream is closed, or its on_failed() method if there's an error. + */ +int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream, + const char* buffer, + int count, + bool end_of_stream); + +/* Cancels the stream. Can be called at any time after + * cronet_bidirectional_stream_start(). The on_canceled() method of + * cronet_bidirectional_stream_callback will be invoked when cancelation + * is complete and no further callback methods will be invoked. If the + * stream has completed or has not started, calling + * cronet_bidirectional_stream_cancel() has no effect and on_canceled() will not + * be invoked. At most one callback method may be invoked after + * cronet_bidirectional_stream_cancel() has completed. + */ +int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream); + +/* Returns true if the |stream| was successfully started and is now done + * (succeeded, canceled, or failed). + * Returns false if the |stream| stream is not yet started or is in progress. + */ +bool cronet_bidirectional_stream_is_done(cronet_bidirectional_stream* stream); + +#ifdef __cplusplus +} +#endif + +#endif // COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c new file mode 100644 index 00000000000..3b9b1b08d81 --- /dev/null +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -0,0 +1,586 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/support/string.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/ext/transport/chttp2/transport/incoming_metadata.h" +#include "src/core/lib/transport/metadata_batch.h" +#include "src/core/lib/transport/transport_impl.h" +#include "src/core/ext/transport/cronet/transport/cronet_c_for_grpc.h" + +#ifdef COMPILE_WITH_CRONET + +#define GRPC_HEADER_SIZE_IN_BYTES 5 +#define MAX_HDRS 100 + +#define GRPC_CRONET_TRACE(...) \ + { \ + if (grpc_cronet_trace) gpr_log(__VA_ARGS__); \ + } +#define CRONET_READ(...) \ + { \ + GRPC_CRONET_TRACE(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); \ + cronet_bidirectional_stream_read(__VA_ARGS__); \ + } +#define SET_RECV_STATE(STATE) \ + { \ + GRPC_CRONET_TRACE(GPR_DEBUG, "next_state = %s", recv_state_name[STATE]); \ + cronet_recv_state = STATE; \ + } + +// Global flag that gets set with GRPC_TRACE env variable +int grpc_cronet_trace = 1; + +// Cronet transport object +struct grpc_cronet_transport { + grpc_transport base; /* must be first element in this structure */ + cronet_engine *engine; + const char *host; +}; + +typedef struct grpc_cronet_transport grpc_cronet_transport; + +enum send_state { + CRONET_SEND_IDLE = 0, + CRONET_REQ_STARTED, + CRONET_SEND_HEADER, + CRONET_WRITE, + CRONET_WRITE_COMPLETED, +}; + +enum recv_state { + CRONET_RECV_IDLE = 0, + CRONET_RECV_READ_LENGTH, + CRONET_RECV_READ_DATA, + CRONET_RECV_CLOSED, +}; + +const char *recv_state_name[] = {"CRONET_RECV_IDLE", "CRONET_RECV_READ_LENGTH", + "CRONET_RECV_READ_DATA,", + "CRONET_RECV_CLOSED"}; + +// Enum that identifies calling function. +enum e_caller { + PERFORM_STREAM_OP, + ON_READ_COMPLETE, + ON_RESPONSE_HEADERS_RECEIVED, + ON_RESPONSE_TRAILERS_RECEIVED +}; + +enum callback_id { + CB_SEND_INITIAL_METADATA = 0, + CB_SEND_MESSAGE, + CB_SEND_TRAILING_METADATA, + CB_RECV_MESSAGE, + CB_RECV_INITIAL_METADATA, + CB_RECV_TRAILING_METADATA, + CB_NUM_CALLBACKS +}; + +struct stream_obj { + // we store received bytes here as they trickle in. + gpr_slice_buffer write_slicebuffer; + cronet_bidirectional_stream *cbs; + gpr_slice slice; + gpr_slice_buffer read_slicebuffer; + struct grpc_slice_buffer_stream sbs; + char *read_buffer; + uint32_t remaining_read_bytes; + uint32_t total_read_bytes; + + char *write_buffer; + size_t write_buffer_size; + + // + char *url; + char *host; + + bool response_headers_received; + bool read_requested; + bool response_trailers_received; + bool read_closed; + + // Recv message stuff + grpc_byte_buffer **recv_message; + // Initial metadata stuff + grpc_metadata_batch *recv_initial_metadata; + // Trailing metadata stuff + grpc_metadata_batch *recv_trailing_metadata; + grpc_chttp2_incoming_metadata_buffer imb; + + // This mutex protects receive state machine execution + gpr_mu recv_mu; + // we can queue up up to 2 callbacks for each OP + grpc_closure *callback_list[CB_NUM_CALLBACKS][2]; + + // storage for header + cronet_bidirectional_stream_header headers[MAX_HDRS]; + uint32_t num_headers; + cronet_bidirectional_stream_header_array header_array; +}; + +typedef struct stream_obj stream_obj; + +void next_send_step(stream_obj *s); +void next_recv_step(stream_obj *s, enum e_caller caller); + +enum send_state cronet_send_state; +enum recv_state cronet_recv_state; + +static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_pollset *pollset) {} + +void enqueue_callbacks(grpc_closure *callback_list[]) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + if (callback_list[0]) { + // GRPC_CRONET_TRACE(GPR_DEBUG, "enqueuing callback = %p", + // callback_list[0]); + grpc_exec_ctx_enqueue(&exec_ctx, callback_list[0], true, NULL); + callback_list[0] = NULL; + } + if (callback_list[1]) { + // GRPC_CRONET_TRACE(GPR_DEBUG, "enqueuing callback = %p", + // callback_list[1]); + grpc_exec_ctx_enqueue(&exec_ctx, callback_list[1], true, NULL); + callback_list[1] = NULL; + } + grpc_exec_ctx_finish(&exec_ctx); +} + +void on_canceled(cronet_bidirectional_stream *stream) { + GRPC_CRONET_TRACE(GPR_DEBUG, "on_canceled %p", stream); +} +void on_failed(cronet_bidirectional_stream *stream, int net_error) { + GRPC_CRONET_TRACE(GPR_DEBUG, "on_failed %p, error = %d", stream, net_error); +} +void on_succeded(cronet_bidirectional_stream *stream) { + GRPC_CRONET_TRACE(GPR_DEBUG, "on_succeeded %p", stream); +} +void on_response_trailers_received( + cronet_bidirectional_stream *stream, + const cronet_bidirectional_stream_header_array *trailers) { + GRPC_CRONET_TRACE(GPR_DEBUG, "R: on_response_trailers_received"); + stream_obj *s = (stream_obj *)stream->annotation; + + memset(&s->imb, 0, sizeof(s->imb)); + grpc_chttp2_incoming_metadata_buffer_init(&s->imb); + int i = 0; + for (i = 0; i < trailers->count; i++) { + grpc_chttp2_incoming_metadata_buffer_add( + &s->imb, grpc_mdelem_from_metadata_strings( + grpc_mdstr_from_string(trailers->headers[i].key), + grpc_mdstr_from_string(trailers->headers[i].value))); + } + s->response_trailers_received = true; + next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); +} +void on_write_completed(cronet_bidirectional_stream *stream, + const char *data) { + GRPC_CRONET_TRACE(GPR_DEBUG, "W: on_write_completed"); + stream_obj *s = (stream_obj *)stream->annotation; + enqueue_callbacks(s->callback_list[CB_SEND_MESSAGE]); + cronet_send_state = CRONET_WRITE_COMPLETED; + next_send_step(s); +} + +void process_recv_message(stream_obj *s, const uint8_t *recv_data) { + gpr_slice read_data_slice = gpr_slice_malloc(s->total_read_bytes); + uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); + memcpy(dst_p, recv_data, s->total_read_bytes); + gpr_slice_buffer_add(&s->read_slicebuffer, read_data_slice); + grpc_slice_buffer_stream_init(&s->sbs, &s->read_slicebuffer, 0); + *s->recv_message = (grpc_byte_buffer *)&s->sbs; +} + +int parse_grpc_header(const uint8_t *data) { + const uint8_t *p = data + 1; + uint32_t length = 0; + length |= ((uint8_t)*p++) << 24; + length |= ((uint8_t)*p++) << 16; + length |= ((uint8_t)*p++) << 8; + length |= ((uint8_t)*p++); + return length; +} + +void on_read_completed(cronet_bidirectional_stream *stream, char *data, + int count) { + stream_obj *s = (stream_obj *)stream->annotation; + GRPC_CRONET_TRACE(GPR_DEBUG, + "R: on_read_completed count=%d, total=%d, remaining=%d", + count, s->total_read_bytes, s->remaining_read_bytes); + if (count > 0) { + GPR_ASSERT(s->recv_message); + s->remaining_read_bytes -= count; + next_recv_step(s, ON_READ_COMPLETE); + } else { + s->read_closed = true; + next_recv_step(s, ON_READ_COMPLETE); + } +} + +void on_response_headers_received( + cronet_bidirectional_stream *stream, + const cronet_bidirectional_stream_header_array *headers, + const char *negotiated_protocol) { + GRPC_CRONET_TRACE(GPR_DEBUG, "R: on_response_headers_received"); + stream_obj *s = (stream_obj *)stream->annotation; + enqueue_callbacks(s->callback_list[CB_RECV_INITIAL_METADATA]); + s->response_headers_received = true; + next_recv_step(s, ON_RESPONSE_HEADERS_RECEIVED); +} + +void on_request_headers_sent(cronet_bidirectional_stream *stream) { + GRPC_CRONET_TRACE(GPR_DEBUG, "W: on_request_headers_sent"); + stream_obj *s = (stream_obj *)stream->annotation; + enqueue_callbacks(s->callback_list[CB_SEND_INITIAL_METADATA]); + cronet_send_state = CRONET_SEND_HEADER; + next_send_step(s); +} + +// Callback function pointers (invoked by cronet in response to events) +cronet_bidirectional_stream_callback callbacks = {on_request_headers_sent, + on_response_headers_received, + on_read_completed, + on_write_completed, + on_response_trailers_received, + on_succeded, + on_failed, + on_canceled}; + + +void invoke_closing_callback(stream_obj *s) { + grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, + s->recv_trailing_metadata); + if (s->callback_list[CB_RECV_TRAILING_METADATA]) { + enqueue_callbacks(s->callback_list[CB_RECV_TRAILING_METADATA]); + } +} + +// This is invoked from perform_stream_op, and all on_xxxx callbacks. +void next_recv_step(stream_obj *s, enum e_caller caller) { + gpr_mu_lock(&s->recv_mu); + switch (cronet_recv_state) { + case CRONET_RECV_IDLE: + GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE"); + if (caller == PERFORM_STREAM_OP || + caller == ON_RESPONSE_HEADERS_RECEIVED) { + if (s->read_closed && s->response_trailers_received) { + invoke_closing_callback(s); + SET_RECV_STATE(CRONET_RECV_CLOSED); + } else if (s->response_headers_received == true && + s->read_requested == true) { + SET_RECV_STATE(CRONET_RECV_READ_LENGTH); + s->total_read_bytes = s->remaining_read_bytes = + GRPC_HEADER_SIZE_IN_BYTES; + GPR_ASSERT(s->read_buffer); + CRONET_READ(s->cbs, s->read_buffer, s->remaining_read_bytes); + } + } + break; + case CRONET_RECV_READ_LENGTH: + GRPC_CRONET_TRACE(GPR_DEBUG, + "cronet_recv_state = CRONET_RECV_READ_LENGTH"); + if (caller == ON_READ_COMPLETE) { + if (s->read_closed) { + invoke_closing_callback(s); + enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); + SET_RECV_STATE(CRONET_RECV_CLOSED); + } else { + GPR_ASSERT(s->remaining_read_bytes == 0); + SET_RECV_STATE(CRONET_RECV_READ_DATA); + s->total_read_bytes = s->remaining_read_bytes = + parse_grpc_header(s->read_buffer); + s->read_buffer = gpr_realloc(s->read_buffer, s->remaining_read_bytes); + GPR_ASSERT(s->read_buffer); + CRONET_READ(s->cbs, (char *)s->read_buffer, s->remaining_read_bytes); + } + } + break; + case CRONET_RECV_READ_DATA: + GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_DATA"); + if (caller == ON_READ_COMPLETE) { + if (s->remaining_read_bytes > 0) { + int offset = s->total_read_bytes - s->remaining_read_bytes; + GPR_ASSERT(s->read_buffer); + CRONET_READ(s->cbs, (char *)s->read_buffer + offset, + s->remaining_read_bytes); + } else { + gpr_slice_buffer_init(&s->read_slicebuffer); + uint8_t *p = s->read_buffer; + process_recv_message(s, p); + SET_RECV_STATE(CRONET_RECV_IDLE); + enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); + } + } + break; + case CRONET_RECV_CLOSED: + break; + default: + GPR_ASSERT(0); // Should not reach here + break; + } + gpr_mu_unlock(&s->recv_mu); +} + + +// This function takes the data from s->write_slicebuffer and assembles into +// a contiguous byte stream with 5 byte gRPC header prepended. +void create_grpc_frame(stream_obj *s) { + gpr_slice slice = gpr_slice_buffer_take_first(&s->write_slicebuffer); + uint8_t *raw_data = GPR_SLICE_START_PTR(slice); + size_t length = GPR_SLICE_LENGTH(slice); + s->write_buffer_size = length + GRPC_HEADER_SIZE_IN_BYTES; + s->write_buffer = gpr_realloc(s->write_buffer, s->write_buffer_size); + uint8_t *p = s->write_buffer; + // Append 5 byte header + *p++ = 0; + *p++ = (uint8_t)(length >> 24); + *p++ = (uint8_t)(length >> 16); + *p++ = (uint8_t)(length >> 8); + *p++ = (uint8_t)(length); + // append actual data + memcpy(p, raw_data, length); +} + +void do_write(stream_obj *s) { + gpr_slice_buffer *sb = &s->write_slicebuffer; + GPR_ASSERT(sb->count <= 1); + if (sb->count > 0) { + create_grpc_frame(s); + GRPC_CRONET_TRACE(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); + cronet_bidirectional_stream_write(s->cbs, s->write_buffer, + (int)s->write_buffer_size, + false); + } +} + +// +void next_send_step(stream_obj *s) { + switch(cronet_send_state) { + case CRONET_SEND_IDLE: + GPR_ASSERT(s->cbs); // cronet_bidirectional_stream is not initialized yet. + cronet_send_state = CRONET_REQ_STARTED; + GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", + s->url); + cronet_bidirectional_stream_start(s->cbs, s->url, 0, "POST", + &s->header_array, false); + break; + case CRONET_SEND_HEADER: + do_write(s); + cronet_send_state = CRONET_WRITE; + break; + case CRONET_WRITE_COMPLETED: + do_write(s); + break; + default: + GPR_ASSERT(0); + break; + } +} + +void create_url(const char *path, const char *host, stream_obj *s) { + const char prefix[] = "https://"; + s->url = gpr_malloc(strlen(prefix) + strlen(host) + strlen(path) + 1); + strcpy(s->url, prefix); + strcat(s->url, host); + strcat(s->url, path); +} + +static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, + const char *host, + stream_obj *s) { + grpc_linked_mdelem *curr = head; + while (s->num_headers < MAX_HDRS) { + grpc_mdelem *mdelem = curr->md; + curr = curr->next; + const char *key = grpc_mdstr_as_c_string(mdelem->key); + const char *value = grpc_mdstr_as_c_string(mdelem->value); + if (strcmp(key, ":scheme") == 0 || strcmp(key, ":method") == 0 || + strcmp(key, ":authority") == 0) { + // Cronet populates these fields on its own. + continue; + } + if (strcmp(key, ":path") == 0) { + // Create URL by appending :path value to the hostname + create_url(value, host, s); + GRPC_CRONET_TRACE(GPR_DEBUG, "extracted URL = %s", s->url); + continue; + } + s->headers[s->num_headers].key = key; + s->headers[s->num_headers].value = value; + s->num_headers++; + if (curr == NULL) { + break; + } + } +} + +static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_transport_stream_op *op) { + grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; + GPR_ASSERT(ct->engine); + stream_obj *s = (stream_obj *)gs; + if (op->recv_trailing_metadata) { + GRPC_CRONET_TRACE( + GPR_DEBUG, "perform_stream_op - recv_trailing_metadata: on_complete=%p", + op->on_complete); + s->recv_trailing_metadata = op->recv_trailing_metadata; + GPR_ASSERT(!s->callback_list[CB_RECV_TRAILING_METADATA][0]); + s->callback_list[CB_RECV_TRAILING_METADATA][0] = op->on_complete; + } + if (op->recv_message) { + GRPC_CRONET_TRACE(GPR_DEBUG, + "perform_stream_op - recv_message: on_complete=%p", + op->on_complete); + s->recv_message = op->recv_message; + GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][0]); + GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][1]); + s->callback_list[CB_RECV_MESSAGE][0] = op->recv_message_ready; + s->callback_list[CB_RECV_MESSAGE][1] = op->on_complete; + s->read_requested = true; + next_recv_step(s, PERFORM_STREAM_OP); + } + if (op->recv_initial_metadata) { + GRPC_CRONET_TRACE(GPR_DEBUG, + "perform_stream_op - recv_initial_metadata:=%p", + op->on_complete); + s->recv_initial_metadata = op->recv_initial_metadata; + GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][0]); + GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][1]); + s->callback_list[CB_RECV_INITIAL_METADATA][0] = + op->recv_initial_metadata_ready; + s->callback_list[CB_RECV_INITIAL_METADATA][1] = op->on_complete; + } + if (op->send_initial_metadata) { + GRPC_CRONET_TRACE( + GPR_DEBUG, "perform_stream_op - send_initial_metadata: on_complete=%p", + op->on_complete); + s->num_headers = 0; + convert_metadata_to_cronet_headers(op->send_initial_metadata->list.head, + ct->host, s); + s->header_array.count = s->num_headers; + s->header_array.capacity = s->num_headers; + s->header_array.headers = s->headers; + GPR_ASSERT(!s->callback_list[CB_SEND_INITIAL_METADATA][0]); + s->callback_list[CB_SEND_INITIAL_METADATA][0] = op->on_complete; + } + if (op->send_message) { + GRPC_CRONET_TRACE(GPR_DEBUG, + "perform_stream_op - send_message: on_complete=%p", + op->on_complete); + grpc_byte_stream_next(exec_ctx, op->send_message, &s->slice, + op->send_message->length, NULL); + gpr_slice_buffer_add(&s->write_slicebuffer, s->slice); + if (s->cbs == NULL) { + GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_bidirectional_stream_create"); + s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); + GPR_ASSERT(s->cbs); + s->read_closed = false; + s->response_trailers_received = false; + s->response_headers_received = false; + cronet_send_state = CRONET_SEND_IDLE; + cronet_recv_state = CRONET_RECV_IDLE; + } + GPR_ASSERT(!s->callback_list[CB_SEND_MESSAGE][0]); + s->callback_list[CB_SEND_MESSAGE][0] = op->on_complete; + next_send_step(s); + } + if (op->send_trailing_metadata) { + GRPC_CRONET_TRACE( + GPR_DEBUG, "perform_stream_op - send_trailing_metadata: on_complete=%p", + op->on_complete); + GPR_ASSERT(!s->callback_list[CB_SEND_TRAILING_METADATA][0]); + s->callback_list[CB_SEND_TRAILING_METADATA][0] = op->on_complete; + if (s->cbs) { + // Send an "empty" write to the far end to signal that we're done. + // This will induce the server to send down trailers. + GRPC_CRONET_TRACE(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); + cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); + } else { + // We never created a stream. This was probably an empty request. + invoke_closing_callback(s); + } + } +} + +static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_stream_refcount *refcount, + const void *server_data) { + stream_obj *s = (stream_obj *)gs; + memset(s->callback_list, 0, sizeof(s->callback_list)); + s->cbs = NULL; + gpr_mu_init(&s->recv_mu); + s->read_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); + s->write_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); + gpr_slice_buffer_init(&s->write_slicebuffer); + GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_transport - init_stream"); + return 0; +} + +static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs) { + GRPC_CRONET_TRACE(GPR_DEBUG, "Destroy stream"); + stream_obj *s = (stream_obj *)gs; + s->cbs = NULL; + gpr_free(s->read_buffer); + gpr_free(s->write_buffer); + gpr_mu_destroy(&s->recv_mu); +} + +static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { + grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; + gpr_free(ct->host); + GRPC_CRONET_TRACE(GPR_DEBUG, "Destroy transport"); +} + +const grpc_transport_vtable cronet_vtable = {sizeof(stream_obj), + "cronet_http", + init_stream, + set_pollset_do_nothing, + perform_stream_op, + destroy_stream, + destroy_transport, + NULL, + NULL}; +#endif // COMPILE_WITH_CRONET From df665073044eac553b2844450cb68a6eafbc0d01 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 19 Apr 2016 14:23:56 -0700 Subject: [PATCH 015/155] moved cronet_c_for_grpc.h in third_party --- src/core/ext/transport/cronet/transport/cronet_transport.c | 2 +- .../objective_c/Cronet}/cronet_c_for_grpc.h | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {src/core/ext/transport/cronet/transport => third_party/objective_c/Cronet}/cronet_c_for_grpc.h (100%) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 3b9b1b08d81..671bee638ef 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -47,7 +47,7 @@ #include "src/core/ext/transport/chttp2/transport/incoming_metadata.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport_impl.h" -#include "src/core/ext/transport/cronet/transport/cronet_c_for_grpc.h" +#include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" #ifdef COMPILE_WITH_CRONET diff --git a/src/core/ext/transport/cronet/transport/cronet_c_for_grpc.h b/third_party/objective_c/Cronet/cronet_c_for_grpc.h similarity index 100% rename from src/core/ext/transport/cronet/transport/cronet_c_for_grpc.h rename to third_party/objective_c/Cronet/cronet_c_for_grpc.h From 77044830ff927c7c55a342030e2e2a9d73d0d723 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Mon, 25 Apr 2016 09:27:49 -0700 Subject: [PATCH 016/155] fixed clang formating --- include/grpc/grpc_security.h | 5 ++--- .../client/secure/cronet_channel_create.c | 12 ++++++------ .../cronet/transport/cronet_transport.c | 19 ++++++++----------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index d753aaf96c1..150ad05a272 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -300,9 +300,8 @@ GRPCAPI grpc_channel *grpc_secure_channel_create( const grpc_channel_args *args, void *reserved); GRPCAPI grpc_channel *grpc_custom_secure_channel_create( - void *engine, const char *target, - const grpc_channel_args *args, void *reserved); - + void *engine, const char *target, const grpc_channel_args *args, + void *reserved); /* --- grpc_server_credentials object. --- diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c index 23189809f72..914c5670864 100644 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c +++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c @@ -54,19 +54,19 @@ typedef struct grpc_cronet_transport grpc_cronet_transport; extern grpc_transport_vtable cronet_vtable; GRPCAPI grpc_channel *grpc_custom_secure_channel_create( - void *engine, const char *target, - const grpc_channel_args *args, void *reserved) { + void *engine, const char *target, const grpc_channel_args *args, + void *reserved) { grpc_cronet_transport *ct = gpr_malloc(sizeof(grpc_cronet_transport)); ct->base.vtable = &cronet_vtable; ct->engine = engine; ct->host = gpr_malloc(strlen(target) + 1); strcpy(ct->host, target); - gpr_log( - GPR_DEBUG, "grpc_create_cronet_transport: cronet_engine = %p, target=%s", - engine, ct->host); + gpr_log(GPR_DEBUG, + "grpc_create_cronet_transport: cronet_engine = %p, target=%s", engine, + ct->host); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; return grpc_channel_create(&exec_ctx, target, args, - GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); + GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); } #endif // COMPILE_WITH_CRONET diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 671bee638ef..5c27a4ea00d 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -41,10 +41,10 @@ #include #include +#include "src/core/ext/transport/chttp2/transport/incoming_metadata.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/channel.h" -#include "src/core/ext/transport/chttp2/transport/incoming_metadata.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport_impl.h" #include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" @@ -215,8 +215,7 @@ void on_response_trailers_received( s->response_trailers_received = true; next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); } -void on_write_completed(cronet_bidirectional_stream *stream, - const char *data) { +void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { GRPC_CRONET_TRACE(GPR_DEBUG, "W: on_write_completed"); stream_obj *s = (stream_obj *)stream->annotation; enqueue_callbacks(s->callback_list[CB_SEND_MESSAGE]); @@ -288,7 +287,6 @@ cronet_bidirectional_stream_callback callbacks = {on_request_headers_sent, on_failed, on_canceled}; - void invoke_closing_callback(stream_obj *s) { grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, s->recv_trailing_metadata); @@ -363,7 +361,6 @@ void next_recv_step(stream_obj *s, enum e_caller caller) { gpr_mu_unlock(&s->recv_mu); } - // This function takes the data from s->write_slicebuffer and assembles into // a contiguous byte stream with 5 byte gRPC header prepended. void create_grpc_frame(stream_obj *s) { @@ -390,22 +387,22 @@ void do_write(stream_obj *s) { create_grpc_frame(s); GRPC_CRONET_TRACE(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); cronet_bidirectional_stream_write(s->cbs, s->write_buffer, - (int)s->write_buffer_size, - false); + (int)s->write_buffer_size, false); } } -// +// void next_send_step(stream_obj *s) { - switch(cronet_send_state) { + switch (cronet_send_state) { case CRONET_SEND_IDLE: - GPR_ASSERT(s->cbs); // cronet_bidirectional_stream is not initialized yet. + GPR_ASSERT( + s->cbs); // cronet_bidirectional_stream is not initialized yet. cronet_send_state = CRONET_REQ_STARTED; GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", s->url); cronet_bidirectional_stream_start(s->cbs, s->url, 0, "POST", &s->header_array, false); - break; + break; case CRONET_SEND_HEADER: do_write(s); cronet_send_state = CRONET_WRITE; From be5186a79dee5ca96254b6d38fa2401522987f63 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 27 Apr 2016 13:47:10 -0700 Subject: [PATCH 017/155] created new grpc_cronet.h. Addressed feedback from jcanizales@ and ctiller@ --- include/grpc/grpc_cronet.h | 51 +++ .../client/secure/cronet_channel_create.c | 22 +- .../cronet/transport/cronet_transport.c | 315 ++++++++++-------- 3 files changed, 245 insertions(+), 143 deletions(-) create mode 100644 include/grpc/grpc_cronet.h diff --git a/include/grpc/grpc_cronet.h b/include/grpc/grpc_cronet.h new file mode 100644 index 00000000000..295e0f55e80 --- /dev/null +++ b/include/grpc/grpc_cronet.h @@ -0,0 +1,51 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_GRPC_CRONET_H +#define GRPC_GRPC_CRONET_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( + void *engine, const char *target, const grpc_channel_args *args, + void *reserved); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_GRPC_CRONET_H */ diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c index 914c5670864..96baa3984b5 100644 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c +++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c @@ -32,6 +32,9 @@ */ #include + +#ifdef GRPC_COMPILE_WITH_CRONET + #include #include @@ -41,23 +44,20 @@ #include "src/core/lib/surface/channel.h" #include "src/core/lib/transport/transport_impl.h" -#ifdef COMPILE_WITH_CRONET // Cronet transport object -struct grpc_cronet_transport { - grpc_transport base; /* must be first element in this structure */ +typedef struct cronet_transport { + grpc_transport base; // must be first element in this structure void *engine; char *host; -}; - -typedef struct grpc_cronet_transport grpc_cronet_transport; +} cronet_transport; -extern grpc_transport_vtable cronet_vtable; +extern grpc_transport_vtable grpc_cronet_vtable; -GRPCAPI grpc_channel *grpc_custom_secure_channel_create( +GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( void *engine, const char *target, const grpc_channel_args *args, void *reserved) { - grpc_cronet_transport *ct = gpr_malloc(sizeof(grpc_cronet_transport)); - ct->base.vtable = &cronet_vtable; + cronet_transport *ct = gpr_malloc(sizeof(cronet_transport)); + ct->base.vtable = &grpc_cronet_vtable; ct->engine = engine; ct->host = gpr_malloc(strlen(target) + 1); strcpy(ct->host, target); @@ -69,4 +69,4 @@ GRPCAPI grpc_channel *grpc_custom_secure_channel_create( return grpc_channel_create(&exec_ctx, target, args, GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); } -#endif // COMPILE_WITH_CRONET +#endif // GRPC_COMPILE_WITH_CRONET diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 5c27a4ea00d..c2bb9e0393a 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -49,25 +49,9 @@ #include "src/core/lib/transport/transport_impl.h" #include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" -#ifdef COMPILE_WITH_CRONET +#ifdef GRPC_COMPILE_WITH_CRONET #define GRPC_HEADER_SIZE_IN_BYTES 5 -#define MAX_HDRS 100 - -#define GRPC_CRONET_TRACE(...) \ - { \ - if (grpc_cronet_trace) gpr_log(__VA_ARGS__); \ - } -#define CRONET_READ(...) \ - { \ - GRPC_CRONET_TRACE(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); \ - cronet_bidirectional_stream_read(__VA_ARGS__); \ - } -#define SET_RECV_STATE(STATE) \ - { \ - GRPC_CRONET_TRACE(GPR_DEBUG, "next_state = %s", recv_state_name[STATE]); \ - cronet_recv_state = STATE; \ - } // Global flag that gets set with GRPC_TRACE env variable int grpc_cronet_trace = 1; @@ -76,7 +60,7 @@ int grpc_cronet_trace = 1; struct grpc_cronet_transport { grpc_transport base; /* must be first element in this structure */ cronet_engine *engine; - const char *host; + char *host; }; typedef struct grpc_cronet_transport grpc_cronet_transport; @@ -96,7 +80,8 @@ enum recv_state { CRONET_RECV_CLOSED, }; -const char *recv_state_name[] = {"CRONET_RECV_IDLE", "CRONET_RECV_READ_LENGTH", +static const char *recv_state_name[] = {"CRONET_RECV_IDLE", + "CRONET_RECV_READ_LENGTH", "CRONET_RECV_READ_DATA,", "CRONET_RECV_CLOSED"}; @@ -120,21 +105,20 @@ enum callback_id { struct stream_obj { // we store received bytes here as they trickle in. - gpr_slice_buffer write_slicebuffer; + gpr_slice_buffer write_slice_buffer; cronet_bidirectional_stream *cbs; gpr_slice slice; - gpr_slice_buffer read_slicebuffer; + gpr_slice_buffer read_slice_buffer; struct grpc_slice_buffer_stream sbs; char *read_buffer; - uint32_t remaining_read_bytes; - uint32_t total_read_bytes; + int remaining_read_bytes; + int total_read_bytes; char *write_buffer; size_t write_buffer_size; - // + // Hold the URL char *url; - char *host; bool response_headers_received; bool read_requested; @@ -155,57 +139,65 @@ struct stream_obj { grpc_closure *callback_list[CB_NUM_CALLBACKS][2]; // storage for header - cronet_bidirectional_stream_header headers[MAX_HDRS]; + cronet_bidirectional_stream_header *headers; uint32_t num_headers; cronet_bidirectional_stream_header_array header_array; + // state tracking + enum recv_state cronet_recv_state; + enum send_state cronet_send_state; }; typedef struct stream_obj stream_obj; -void next_send_step(stream_obj *s); -void next_recv_step(stream_obj *s, enum e_caller caller); +static void next_send_step(stream_obj *s); +static void next_recv_step(stream_obj *s, enum e_caller caller); -enum send_state cronet_send_state; -enum recv_state cronet_recv_state; static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_pollset *pollset) {} -void enqueue_callbacks(grpc_closure *callback_list[]) { +static void enqueue_callbacks(grpc_closure *callback_list[]) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; if (callback_list[0]) { - // GRPC_CRONET_TRACE(GPR_DEBUG, "enqueuing callback = %p", - // callback_list[0]); grpc_exec_ctx_enqueue(&exec_ctx, callback_list[0], true, NULL); callback_list[0] = NULL; } if (callback_list[1]) { - // GRPC_CRONET_TRACE(GPR_DEBUG, "enqueuing callback = %p", - // callback_list[1]); grpc_exec_ctx_enqueue(&exec_ctx, callback_list[1], true, NULL); callback_list[1] = NULL; } grpc_exec_ctx_finish(&exec_ctx); } -void on_canceled(cronet_bidirectional_stream *stream) { - GRPC_CRONET_TRACE(GPR_DEBUG, "on_canceled %p", stream); +static void on_canceled(cronet_bidirectional_stream *stream) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "on_canceled %p", stream); + } } -void on_failed(cronet_bidirectional_stream *stream, int net_error) { - GRPC_CRONET_TRACE(GPR_DEBUG, "on_failed %p, error = %d", stream, net_error); + +static void on_failed(cronet_bidirectional_stream *stream, int net_error) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "on_failed %p, error = %d", stream, net_error); + } } -void on_succeded(cronet_bidirectional_stream *stream) { - GRPC_CRONET_TRACE(GPR_DEBUG, "on_succeeded %p", stream); + +static void on_succeeded(cronet_bidirectional_stream *stream) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "on_succeeded %p", stream); + } } -void on_response_trailers_received( + +static void on_response_trailers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *trailers) { - GRPC_CRONET_TRACE(GPR_DEBUG, "R: on_response_trailers_received"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "R: on_response_trailers_received"); + } stream_obj *s = (stream_obj *)stream->annotation; memset(&s->imb, 0, sizeof(s->imb)); grpc_chttp2_incoming_metadata_buffer_init(&s->imb); - int i = 0; + unsigned int i = 0; for (i = 0; i < trailers->count; i++) { grpc_chttp2_incoming_metadata_buffer_add( &s->imb, grpc_mdelem_from_metadata_strings( @@ -215,26 +207,29 @@ void on_response_trailers_received( s->response_trailers_received = true; next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); } -void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { - GRPC_CRONET_TRACE(GPR_DEBUG, "W: on_write_completed"); + +static void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "W: on_write_completed"); + } stream_obj *s = (stream_obj *)stream->annotation; enqueue_callbacks(s->callback_list[CB_SEND_MESSAGE]); - cronet_send_state = CRONET_WRITE_COMPLETED; + s->cronet_send_state = CRONET_WRITE_COMPLETED; next_send_step(s); } -void process_recv_message(stream_obj *s, const uint8_t *recv_data) { - gpr_slice read_data_slice = gpr_slice_malloc(s->total_read_bytes); +static void process_recv_message(stream_obj *s, const uint8_t *recv_data) { + gpr_slice read_data_slice = gpr_slice_malloc((uint32_t) s->total_read_bytes); uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); memcpy(dst_p, recv_data, s->total_read_bytes); - gpr_slice_buffer_add(&s->read_slicebuffer, read_data_slice); - grpc_slice_buffer_stream_init(&s->sbs, &s->read_slicebuffer, 0); + gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice); + grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0); *s->recv_message = (grpc_byte_buffer *)&s->sbs; } -int parse_grpc_header(const uint8_t *data) { +static int parse_grpc_header(const uint8_t *data) { const uint8_t *p = data + 1; - uint32_t length = 0; + int length = 0; length |= ((uint8_t)*p++) << 24; length |= ((uint8_t)*p++) << 16; length |= ((uint8_t)*p++) << 8; @@ -242,12 +237,13 @@ int parse_grpc_header(const uint8_t *data) { return length; } -void on_read_completed(cronet_bidirectional_stream *stream, char *data, +static void on_read_completed(cronet_bidirectional_stream *stream, char *data, int count) { stream_obj *s = (stream_obj *)stream->annotation; - GRPC_CRONET_TRACE(GPR_DEBUG, - "R: on_read_completed count=%d, total=%d, remaining=%d", + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "R: on_read_completed count=%d, total=%d, remaining=%d", count, s->total_read_bytes, s->remaining_read_bytes); + } if (count > 0) { GPR_ASSERT(s->recv_message); s->remaining_read_bytes -= count; @@ -258,36 +254,40 @@ void on_read_completed(cronet_bidirectional_stream *stream, char *data, } } -void on_response_headers_received( +static void on_response_headers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *headers, const char *negotiated_protocol) { - GRPC_CRONET_TRACE(GPR_DEBUG, "R: on_response_headers_received"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "R: on_response_headers_received"); + } stream_obj *s = (stream_obj *)stream->annotation; enqueue_callbacks(s->callback_list[CB_RECV_INITIAL_METADATA]); s->response_headers_received = true; next_recv_step(s, ON_RESPONSE_HEADERS_RECEIVED); } -void on_request_headers_sent(cronet_bidirectional_stream *stream) { - GRPC_CRONET_TRACE(GPR_DEBUG, "W: on_request_headers_sent"); +static void on_request_headers_sent(cronet_bidirectional_stream *stream) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "W: on_request_headers_sent"); + } stream_obj *s = (stream_obj *)stream->annotation; enqueue_callbacks(s->callback_list[CB_SEND_INITIAL_METADATA]); - cronet_send_state = CRONET_SEND_HEADER; + s->cronet_send_state = CRONET_SEND_HEADER; next_send_step(s); } // Callback function pointers (invoked by cronet in response to events) -cronet_bidirectional_stream_callback callbacks = {on_request_headers_sent, +static cronet_bidirectional_stream_callback callbacks = {on_request_headers_sent, on_response_headers_received, on_read_completed, on_write_completed, on_response_trailers_received, - on_succeded, + on_succeeded, on_failed, on_canceled}; -void invoke_closing_callback(stream_obj *s) { +static void invoke_closing_callback(stream_obj *s) { grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, s->recv_trailing_metadata); if (s->callback_list[CB_RECV_TRAILING_METADATA]) { @@ -295,59 +295,75 @@ void invoke_closing_callback(stream_obj *s) { } } +static void set_recv_state(stream_obj *s, enum recv_state state) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "next_state = %s", recv_state_name[state]); + } + s->cronet_recv_state = state; +} + + // This is invoked from perform_stream_op, and all on_xxxx callbacks. -void next_recv_step(stream_obj *s, enum e_caller caller) { +static void next_recv_step(stream_obj *s, enum e_caller caller) { gpr_mu_lock(&s->recv_mu); - switch (cronet_recv_state) { + switch (s->cronet_recv_state) { case CRONET_RECV_IDLE: - GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE"); + } if (caller == PERFORM_STREAM_OP || caller == ON_RESPONSE_HEADERS_RECEIVED) { if (s->read_closed && s->response_trailers_received) { invoke_closing_callback(s); - SET_RECV_STATE(CRONET_RECV_CLOSED); + set_recv_state(s, CRONET_RECV_CLOSED); } else if (s->response_headers_received == true && s->read_requested == true) { - SET_RECV_STATE(CRONET_RECV_READ_LENGTH); + set_recv_state(s, CRONET_RECV_READ_LENGTH); s->total_read_bytes = s->remaining_read_bytes = GRPC_HEADER_SIZE_IN_BYTES; GPR_ASSERT(s->read_buffer); - CRONET_READ(s->cbs, s->read_buffer, s->remaining_read_bytes); + if (grpc_cronet_trace) {gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");} + cronet_bidirectional_stream_read(s->cbs, s->read_buffer, s->remaining_read_bytes); } } break; case CRONET_RECV_READ_LENGTH: - GRPC_CRONET_TRACE(GPR_DEBUG, - "cronet_recv_state = CRONET_RECV_READ_LENGTH"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_LENGTH"); + } if (caller == ON_READ_COMPLETE) { if (s->read_closed) { invoke_closing_callback(s); enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); - SET_RECV_STATE(CRONET_RECV_CLOSED); + set_recv_state(s, CRONET_RECV_CLOSED); } else { GPR_ASSERT(s->remaining_read_bytes == 0); - SET_RECV_STATE(CRONET_RECV_READ_DATA); + set_recv_state(s, CRONET_RECV_READ_DATA); s->total_read_bytes = s->remaining_read_bytes = - parse_grpc_header(s->read_buffer); - s->read_buffer = gpr_realloc(s->read_buffer, s->remaining_read_bytes); + parse_grpc_header((const uint8_t *)s->read_buffer); + s->read_buffer = gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes); GPR_ASSERT(s->read_buffer); - CRONET_READ(s->cbs, (char *)s->read_buffer, s->remaining_read_bytes); + if (grpc_cronet_trace) {gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");} + cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer, s->remaining_read_bytes); } } break; case CRONET_RECV_READ_DATA: - GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_DATA"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_DATA"); + } if (caller == ON_READ_COMPLETE) { if (s->remaining_read_bytes > 0) { int offset = s->total_read_bytes - s->remaining_read_bytes; GPR_ASSERT(s->read_buffer); - CRONET_READ(s->cbs, (char *)s->read_buffer + offset, + if (grpc_cronet_trace) {gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");} + cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer + offset, s->remaining_read_bytes); } else { - gpr_slice_buffer_init(&s->read_slicebuffer); - uint8_t *p = s->read_buffer; + gpr_slice_buffer_init(&s->read_slice_buffer); + uint8_t *p = (uint8_t *)s->read_buffer; process_recv_message(s, p); - SET_RECV_STATE(CRONET_RECV_IDLE); + set_recv_state(s, CRONET_RECV_IDLE); enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); } } @@ -361,15 +377,15 @@ void next_recv_step(stream_obj *s, enum e_caller caller) { gpr_mu_unlock(&s->recv_mu); } -// This function takes the data from s->write_slicebuffer and assembles into +// This function takes the data from s->write_slice_buffer and assembles into // a contiguous byte stream with 5 byte gRPC header prepended. -void create_grpc_frame(stream_obj *s) { - gpr_slice slice = gpr_slice_buffer_take_first(&s->write_slicebuffer); +static void create_grpc_frame(stream_obj *s) { + gpr_slice slice = gpr_slice_buffer_take_first(&s->write_slice_buffer); uint8_t *raw_data = GPR_SLICE_START_PTR(slice); size_t length = GPR_SLICE_LENGTH(slice); s->write_buffer_size = length + GRPC_HEADER_SIZE_IN_BYTES; s->write_buffer = gpr_realloc(s->write_buffer, s->write_buffer_size); - uint8_t *p = s->write_buffer; + uint8_t *p = (uint8_t *)s->write_buffer; // Append 5 byte header *p++ = 0; *p++ = (uint8_t)(length >> 24); @@ -380,32 +396,37 @@ void create_grpc_frame(stream_obj *s) { memcpy(p, raw_data, length); } -void do_write(stream_obj *s) { - gpr_slice_buffer *sb = &s->write_slicebuffer; +static void do_write(stream_obj *s) { + gpr_slice_buffer *sb = &s->write_slice_buffer; GPR_ASSERT(sb->count <= 1); if (sb->count > 0) { create_grpc_frame(s); - GRPC_CRONET_TRACE(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); + } cronet_bidirectional_stream_write(s->cbs, s->write_buffer, (int)s->write_buffer_size, false); } } // -void next_send_step(stream_obj *s) { - switch (cronet_send_state) { +static void next_send_step(stream_obj *s) { + switch (s->cronet_send_state) { case CRONET_SEND_IDLE: GPR_ASSERT( s->cbs); // cronet_bidirectional_stream is not initialized yet. - cronet_send_state = CRONET_REQ_STARTED; - GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", - s->url); + s->cronet_send_state = CRONET_REQ_STARTED; + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", s->url); + } cronet_bidirectional_stream_start(s->cbs, s->url, 0, "POST", &s->header_array, false); + // we no longer need the memory that was allocated earlier. + gpr_free(s->header_array.headers); break; case CRONET_SEND_HEADER: do_write(s); - cronet_send_state = CRONET_WRITE; + s->cronet_send_state = CRONET_WRITE; break; case CRONET_WRITE_COMPLETED: do_write(s); @@ -416,19 +437,25 @@ void next_send_step(stream_obj *s) { } } -void create_url(const char *path, const char *host, stream_obj *s) { - const char prefix[] = "https://"; - s->url = gpr_malloc(strlen(prefix) + strlen(host) + strlen(path) + 1); - strcpy(s->url, prefix); - strcat(s->url, host); - strcat(s->url, path); -} - static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, const char *host, stream_obj *s) { grpc_linked_mdelem *curr = head; - while (s->num_headers < MAX_HDRS) { + // Walk the linked list and get number of header fields + uint32_t num_headers_available = 0; + while (curr != NULL) { + curr = curr->next; + num_headers_available++; + } + // Allocate enough memory + s->headers = (cronet_bidirectional_stream_header *) + gpr_malloc(sizeof(cronet_bidirectional_stream_header) * num_headers_available); + + // Walk the linked list again, this time copying the header fields. s->num_headers + // can be less than num_headers_available, as some headers are not used for cronet + curr = head; + s->num_headers = 0; + while (s->num_headers < num_headers_available) { grpc_mdelem *mdelem = curr->md; curr = curr->next; const char *key = grpc_mdstr_as_c_string(mdelem->key); @@ -440,8 +467,10 @@ static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, } if (strcmp(key, ":path") == 0) { // Create URL by appending :path value to the hostname - create_url(value, host, s); - GRPC_CRONET_TRACE(GPR_DEBUG, "extracted URL = %s", s->url); + gpr_asprintf(&s->url, "https://%s%s", host, value); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "extracted URL = %s", s->url); + } continue; } s->headers[s->num_headers].key = key; @@ -459,18 +488,21 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, GPR_ASSERT(ct->engine); stream_obj *s = (stream_obj *)gs; if (op->recv_trailing_metadata) { - GRPC_CRONET_TRACE( + if (grpc_cronet_trace) { + gpr_log( GPR_DEBUG, "perform_stream_op - recv_trailing_metadata: on_complete=%p", op->on_complete); + } s->recv_trailing_metadata = op->recv_trailing_metadata; GPR_ASSERT(!s->callback_list[CB_RECV_TRAILING_METADATA][0]); s->callback_list[CB_RECV_TRAILING_METADATA][0] = op->on_complete; } if (op->recv_message) { - GRPC_CRONET_TRACE(GPR_DEBUG, - "perform_stream_op - recv_message: on_complete=%p", - op->on_complete); - s->recv_message = op->recv_message; + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "perform_stream_op - recv_message: on_complete=%p", + op->on_complete); + } + s->recv_message = (grpc_byte_buffer **)op->recv_message; GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][0]); GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][1]); s->callback_list[CB_RECV_MESSAGE][0] = op->recv_message_ready; @@ -479,9 +511,10 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, next_recv_step(s, PERFORM_STREAM_OP); } if (op->recv_initial_metadata) { - GRPC_CRONET_TRACE(GPR_DEBUG, - "perform_stream_op - recv_initial_metadata:=%p", - op->on_complete); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "perform_stream_op - recv_initial_metadata:=%p", + op->on_complete); + } s->recv_initial_metadata = op->recv_initial_metadata; GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][0]); GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][1]); @@ -490,9 +523,11 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->callback_list[CB_RECV_INITIAL_METADATA][1] = op->on_complete; } if (op->send_initial_metadata) { - GRPC_CRONET_TRACE( + if (grpc_cronet_trace) { + gpr_log( GPR_DEBUG, "perform_stream_op - send_initial_metadata: on_complete=%p", op->on_complete); + } s->num_headers = 0; convert_metadata_to_cronet_headers(op->send_initial_metadata->list.head, ct->host, s); @@ -503,36 +538,45 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->callback_list[CB_SEND_INITIAL_METADATA][0] = op->on_complete; } if (op->send_message) { - GRPC_CRONET_TRACE(GPR_DEBUG, - "perform_stream_op - send_message: on_complete=%p", - op->on_complete); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "perform_stream_op - send_message: on_complete=%p", + op->on_complete); + } grpc_byte_stream_next(exec_ctx, op->send_message, &s->slice, op->send_message->length, NULL); - gpr_slice_buffer_add(&s->write_slicebuffer, s->slice); + // Check that compression flag is not ON. We don't support compression yet. + // TODO (makdharma): add compression support + GPR_ASSERT(op->send_message->flags == 0); + gpr_slice_buffer_add(&s->write_slice_buffer, s->slice); if (s->cbs == NULL) { - GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_bidirectional_stream_create"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); + } s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); GPR_ASSERT(s->cbs); s->read_closed = false; s->response_trailers_received = false; s->response_headers_received = false; - cronet_send_state = CRONET_SEND_IDLE; - cronet_recv_state = CRONET_RECV_IDLE; + s->cronet_send_state = CRONET_SEND_IDLE; + s->cronet_recv_state = CRONET_RECV_IDLE; } GPR_ASSERT(!s->callback_list[CB_SEND_MESSAGE][0]); s->callback_list[CB_SEND_MESSAGE][0] = op->on_complete; next_send_step(s); } if (op->send_trailing_metadata) { - GRPC_CRONET_TRACE( - GPR_DEBUG, "perform_stream_op - send_trailing_metadata: on_complete=%p", + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "perform_stream_op - send_trailing_metadata: on_complete=%p", op->on_complete); + } GPR_ASSERT(!s->callback_list[CB_SEND_TRAILING_METADATA][0]); s->callback_list[CB_SEND_TRAILING_METADATA][0] = op->on_complete; if (s->cbs) { // Send an "empty" write to the far end to signal that we're done. // This will induce the server to send down trailers. - GRPC_CRONET_TRACE(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); + } cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); } else { // We never created a stream. This was probably an empty request. @@ -550,34 +594,41 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, gpr_mu_init(&s->recv_mu); s->read_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); s->write_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); - gpr_slice_buffer_init(&s->write_slicebuffer); - GRPC_CRONET_TRACE(GPR_DEBUG, "cronet_transport - init_stream"); + gpr_slice_buffer_init(&s->write_slice_buffer); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "cronet_transport - init_stream"); + } return 0; } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs) { - GRPC_CRONET_TRACE(GPR_DEBUG, "Destroy stream"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "Destroy stream"); + } stream_obj *s = (stream_obj *)gs; s->cbs = NULL; gpr_free(s->read_buffer); gpr_free(s->write_buffer); + gpr_free(s->url); gpr_mu_destroy(&s->recv_mu); } static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; gpr_free(ct->host); - GRPC_CRONET_TRACE(GPR_DEBUG, "Destroy transport"); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "Destroy transport"); + } } -const grpc_transport_vtable cronet_vtable = {sizeof(stream_obj), +const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), "cronet_http", init_stream, set_pollset_do_nothing, perform_stream_op, + NULL, destroy_stream, destroy_transport, - NULL, NULL}; -#endif // COMPILE_WITH_CRONET +#endif // GRPC_COMPILE_WITH_CRONET From 808131932e8ef8b9ec239256407f6e2e0f270948 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 29 Apr 2016 13:36:32 -0700 Subject: [PATCH 018/155] reverting stuff --- include/grpc/grpc_security.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index 150ad05a272..79199cc5d68 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -299,10 +299,6 @@ GRPCAPI grpc_channel *grpc_secure_channel_create( grpc_channel_credentials *creds, const char *target, const grpc_channel_args *args, void *reserved); -GRPCAPI grpc_channel *grpc_custom_secure_channel_create( - void *engine, const char *target, const grpc_channel_args *args, - void *reserved); - /* --- grpc_server_credentials object. --- A server credentials object represents a way to authenticate a server. */ From 2389ad629514cdd58f0fda0042f04b89b4013d44 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 29 Apr 2016 14:49:59 -0700 Subject: [PATCH 019/155] Clang formatting fixes --- .../client/secure/cronet_channel_create.c | 2 +- .../cronet/transport/cronet_transport.c | 106 +++++++++--------- 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c index 96baa3984b5..a6cb1f70a70 100644 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c +++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c @@ -46,7 +46,7 @@ // Cronet transport object typedef struct cronet_transport { - grpc_transport base; // must be first element in this structure + grpc_transport base; // must be first element in this structure void *engine; char *host; } cronet_transport; diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index c2bb9e0393a..d337e84606b 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -80,10 +80,9 @@ enum recv_state { CRONET_RECV_CLOSED, }; -static const char *recv_state_name[] = {"CRONET_RECV_IDLE", - "CRONET_RECV_READ_LENGTH", - "CRONET_RECV_READ_DATA,", - "CRONET_RECV_CLOSED"}; +static const char *recv_state_name[] = { + "CRONET_RECV_IDLE", "CRONET_RECV_READ_LENGTH", "CRONET_RECV_READ_DATA,", + "CRONET_RECV_CLOSED"}; // Enum that identifies calling function. enum e_caller { @@ -152,7 +151,6 @@ typedef struct stream_obj stream_obj; static void next_send_step(stream_obj *s); static void next_recv_step(stream_obj *s, enum e_caller caller); - static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_pollset *pollset) {} @@ -208,7 +206,8 @@ static void on_response_trailers_received( next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); } -static void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { +static void on_write_completed(cronet_bidirectional_stream *stream, + const char *data) { if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "W: on_write_completed"); } @@ -219,7 +218,7 @@ static void on_write_completed(cronet_bidirectional_stream *stream, const char * } static void process_recv_message(stream_obj *s, const uint8_t *recv_data) { - gpr_slice read_data_slice = gpr_slice_malloc((uint32_t) s->total_read_bytes); + gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes); uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); memcpy(dst_p, recv_data, s->total_read_bytes); gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice); @@ -238,11 +237,11 @@ static int parse_grpc_header(const uint8_t *data) { } static void on_read_completed(cronet_bidirectional_stream *stream, char *data, - int count) { + int count) { stream_obj *s = (stream_obj *)stream->annotation; if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "R: on_read_completed count=%d, total=%d, remaining=%d", - count, s->total_read_bytes, s->remaining_read_bytes); + count, s->total_read_bytes, s->remaining_read_bytes); } if (count > 0) { GPR_ASSERT(s->recv_message); @@ -278,14 +277,15 @@ static void on_request_headers_sent(cronet_bidirectional_stream *stream) { } // Callback function pointers (invoked by cronet in response to events) -static cronet_bidirectional_stream_callback callbacks = {on_request_headers_sent, - on_response_headers_received, - on_read_completed, - on_write_completed, - on_response_trailers_received, - on_succeeded, - on_failed, - on_canceled}; +static cronet_bidirectional_stream_callback callbacks = { + on_request_headers_sent, + on_response_headers_received, + on_read_completed, + on_write_completed, + on_response_trailers_received, + on_succeeded, + on_failed, + on_canceled}; static void invoke_closing_callback(stream_obj *s) { grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, @@ -302,7 +302,6 @@ static void set_recv_state(stream_obj *s, enum recv_state state) { s->cronet_recv_state = state; } - // This is invoked from perform_stream_op, and all on_xxxx callbacks. static void next_recv_step(stream_obj *s, enum e_caller caller) { gpr_mu_lock(&s->recv_mu); @@ -322,8 +321,11 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { s->total_read_bytes = s->remaining_read_bytes = GRPC_HEADER_SIZE_IN_BYTES; GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) {gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");} - cronet_bidirectional_stream_read(s->cbs, s->read_buffer, s->remaining_read_bytes); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); + } + cronet_bidirectional_stream_read(s->cbs, s->read_buffer, + s->remaining_read_bytes); } } break; @@ -341,10 +343,14 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { set_recv_state(s, CRONET_RECV_READ_DATA); s->total_read_bytes = s->remaining_read_bytes = parse_grpc_header((const uint8_t *)s->read_buffer); - s->read_buffer = gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes); + s->read_buffer = + gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes); GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) {gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");} - cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer, s->remaining_read_bytes); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); + } + cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer, + s->remaining_read_bytes); } } break; @@ -356,9 +362,11 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { if (s->remaining_read_bytes > 0) { int offset = s->total_read_bytes - s->remaining_read_bytes; GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) {gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");} - cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer + offset, - s->remaining_read_bytes); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); + } + cronet_bidirectional_stream_read( + s->cbs, (char *)s->read_buffer + offset, s->remaining_read_bytes); } else { gpr_slice_buffer_init(&s->read_slice_buffer); uint8_t *p = (uint8_t *)s->read_buffer; @@ -448,11 +456,13 @@ static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, num_headers_available++; } // Allocate enough memory - s->headers = (cronet_bidirectional_stream_header *) - gpr_malloc(sizeof(cronet_bidirectional_stream_header) * num_headers_available); + s->headers = (cronet_bidirectional_stream_header *)gpr_malloc( + sizeof(cronet_bidirectional_stream_header) * num_headers_available); - // Walk the linked list again, this time copying the header fields. s->num_headers - // can be less than num_headers_available, as some headers are not used for cronet + // Walk the linked list again, this time copying the header fields. + // s->num_headers + // can be less than num_headers_available, as some headers are not used for + // cronet curr = head; s->num_headers = 0; while (s->num_headers < num_headers_available) { @@ -489,9 +499,9 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, stream_obj *s = (stream_obj *)gs; if (op->recv_trailing_metadata) { if (grpc_cronet_trace) { - gpr_log( - GPR_DEBUG, "perform_stream_op - recv_trailing_metadata: on_complete=%p", - op->on_complete); + gpr_log(GPR_DEBUG, + "perform_stream_op - recv_trailing_metadata: on_complete=%p", + op->on_complete); } s->recv_trailing_metadata = op->recv_trailing_metadata; GPR_ASSERT(!s->callback_list[CB_RECV_TRAILING_METADATA][0]); @@ -500,7 +510,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, if (op->recv_message) { if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "perform_stream_op - recv_message: on_complete=%p", - op->on_complete); + op->on_complete); } s->recv_message = (grpc_byte_buffer **)op->recv_message; GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][0]); @@ -513,7 +523,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, if (op->recv_initial_metadata) { if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "perform_stream_op - recv_initial_metadata:=%p", - op->on_complete); + op->on_complete); } s->recv_initial_metadata = op->recv_initial_metadata; GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][0]); @@ -524,9 +534,9 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, } if (op->send_initial_metadata) { if (grpc_cronet_trace) { - gpr_log( - GPR_DEBUG, "perform_stream_op - send_initial_metadata: on_complete=%p", - op->on_complete); + gpr_log(GPR_DEBUG, + "perform_stream_op - send_initial_metadata: on_complete=%p", + op->on_complete); } s->num_headers = 0; convert_metadata_to_cronet_headers(op->send_initial_metadata->list.head, @@ -540,7 +550,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, if (op->send_message) { if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "perform_stream_op - send_message: on_complete=%p", - op->on_complete); + op->on_complete); } grpc_byte_stream_next(exec_ctx, op->send_message, &s->slice, op->send_message->length, NULL); @@ -566,8 +576,9 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, } if (op->send_trailing_metadata) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - send_trailing_metadata: on_complete=%p", - op->on_complete); + gpr_log(GPR_DEBUG, + "perform_stream_op - send_trailing_metadata: on_complete=%p", + op->on_complete); } GPR_ASSERT(!s->callback_list[CB_SEND_TRAILING_METADATA][0]); s->callback_list[CB_SEND_TRAILING_METADATA][0] = op->on_complete; @@ -622,13 +633,8 @@ static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { } } -const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), - "cronet_http", - init_stream, - set_pollset_do_nothing, - perform_stream_op, - NULL, - destroy_stream, - destroy_transport, - NULL}; +const grpc_transport_vtable grpc_cronet_vtable = { + sizeof(stream_obj), "cronet_http", init_stream, + set_pollset_do_nothing, perform_stream_op, NULL, + destroy_stream, destroy_transport, NULL}; #endif // GRPC_COMPILE_WITH_CRONET From 63ee8ce43cb9bcd9832fb0a5303b8228a009c1c2 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Sun, 1 May 2016 15:12:18 -0700 Subject: [PATCH 020/155] Reran tools/buildgen/generate_projects.sh again. --- grpc.def | 1 + src/python/grpcio/grpc/_cython/imports.generated.c | 2 ++ src/python/grpcio/grpc/_cython/imports.generated.h | 4 ++++ src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 ++ src/ruby/ext/grpc/rb_grpc_imports.generated.h | 4 ++++ 5 files changed, 13 insertions(+) diff --git a/grpc.def b/grpc.def index 61948ed1b89..09a94a6cd0c 100644 --- a/grpc.def +++ b/grpc.def @@ -87,6 +87,7 @@ EXPORTS grpc_header_nonbin_value_is_legal grpc_is_binary_header grpc_call_error_to_string + grpc_cronet_secure_channel_create grpc_auth_property_iterator_next grpc_auth_context_property_iterator grpc_auth_context_peer_identity diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c index f0a40dbb35c..09551472b5f 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.c +++ b/src/python/grpcio/grpc/_cython/imports.generated.c @@ -125,6 +125,7 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; +grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -395,6 +396,7 @@ void pygrpc_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); + grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h index d5e810b7cf4..54c8aaad13e 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.h +++ b/src/python/grpcio/grpc/_cython/imports.generated.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -325,6 +326,9 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import +typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); +extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; +#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index bc43f9d36b5..cebbe8c40fe 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -125,6 +125,7 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; +grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -391,6 +392,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); + grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index b67361ca25b..d7ea6c574c8 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -325,6 +326,9 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import +typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); +extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; +#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import From 125590f9d5ebc53271846401d6087e47eced9897 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 2 May 2016 07:37:16 -0700 Subject: [PATCH 021/155] Add grpc_metadata_batch_size() function. --- src/core/lib/transport/metadata.h | 4 ++-- src/core/lib/transport/metadata_batch.c | 9 +++++++++ src/core/lib/transport/metadata_batch.h | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 77c32c72dee..4ecbbd1b1b8 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -148,8 +148,8 @@ const char *grpc_mdstr_as_c_string(grpc_mdstr *s); #define GRPC_MDSTR_LENGTH(s) (GPR_SLICE_LENGTH(s->slice)) /* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */ -#define GRPC_MDELEM_LENGTH(e) (GRPC_MDSTR_LENGTH(e->key) + \ - GRPC_MDSTR_LENGTH(e->value) + 32) +#define GRPC_MDELEM_LENGTH(e) (GRPC_MDSTR_LENGTH((e)->key) + \ + GRPC_MDSTR_LENGTH((e)->value) + 32) int grpc_mdstr_is_legal_header(grpc_mdstr *s); int grpc_mdstr_is_legal_nonbin_header(grpc_mdstr *s); diff --git a/src/core/lib/transport/metadata_batch.c b/src/core/lib/transport/metadata_batch.c index 4567221a48f..4e1cd8e2c1c 100644 --- a/src/core/lib/transport/metadata_batch.c +++ b/src/core/lib/transport/metadata_batch.c @@ -192,3 +192,12 @@ int grpc_metadata_batch_is_empty(grpc_metadata_batch *batch) { gpr_time_cmp(gpr_inf_future(batch->deadline.clock_type), batch->deadline) == 0; } + +size_t grpc_metadata_batch_size(grpc_metadata_batch *batch) { + size_t size = 0; + for (grpc_linked_mdelem* elem = batch->list.head; + elem != NULL; elem = elem->next) { + size += GRPC_MDELEM_LENGTH(elem->md); + } + return size; +} diff --git a/src/core/lib/transport/metadata_batch.h b/src/core/lib/transport/metadata_batch.h index b62668876e4..7af823f7ca4 100644 --- a/src/core/lib/transport/metadata_batch.h +++ b/src/core/lib/transport/metadata_batch.h @@ -66,6 +66,9 @@ void grpc_metadata_batch_destroy(grpc_metadata_batch *batch); void grpc_metadata_batch_clear(grpc_metadata_batch *batch); int grpc_metadata_batch_is_empty(grpc_metadata_batch *batch); +/* Returns the transport size of the batch. */ +size_t grpc_metadata_batch_size(grpc_metadata_batch *batch); + /** Moves the metadata information from \a src to \a dst. Upon return, \a src is * zeroed. */ void grpc_metadata_batch_move(grpc_metadata_batch *dst, From ebbbce3e6e6f08a9d35292296cbd066b0a4c4a67 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 2 May 2016 09:54:04 -0700 Subject: [PATCH 022/155] Changed writing code to honor the peer's header size limit setting. Changed large_metadata test to only cover the case where both the client and server support large metadata; I will cover the other cases in separate tests in a subsequent commit. --- .../chttp2/transport/chttp2_transport.c | 85 ++++++++++++------- test/core/end2end/tests/large_metadata.c | 35 +++----- 2 files changed, 70 insertions(+), 50 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 63147865257..1eaffc66920 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -854,24 +854,37 @@ static void perform_stream_op_locked( stream_global->send_initial_metadata_finished = add_closure_barrier(on_complete); stream_global->send_initial_metadata = op->send_initial_metadata; - if (contains_non_ok_status(transport_global, op->send_initial_metadata)) { - stream_global->seen_error = true; - grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); - } - if (!stream_global->write_closed) { - if (transport_global->is_client) { - GPR_ASSERT(stream_global->id == 0); - grpc_chttp2_list_add_waiting_for_concurrency(transport_global, - stream_global); - maybe_start_some_streams(exec_ctx, transport_global); + const size_t metadata_size = grpc_metadata_batch_size( + op->send_initial_metadata); + const size_t metadata_peer_limit = + transport_global->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; + if (metadata_size > metadata_peer_limit) { + gpr_log(GPR_DEBUG, + "initial metadata size exceeds peer limit (%lu vs. %lu)", + metadata_size, metadata_peer_limit); + cancel_from_api(exec_ctx, transport_global, stream_global, + GRPC_STATUS_RESOURCE_EXHAUSTED); + } else { + if (contains_non_ok_status(transport_global, op->send_initial_metadata)) { + stream_global->seen_error = true; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + if (!stream_global->write_closed) { + if (transport_global->is_client) { + GPR_ASSERT(stream_global->id == 0); + grpc_chttp2_list_add_waiting_for_concurrency(transport_global, + stream_global); + maybe_start_some_streams(exec_ctx, transport_global); + } else { + GPR_ASSERT(stream_global->id != 0); + grpc_chttp2_become_writable(transport_global, stream_global); + } } else { - GPR_ASSERT(stream_global->id != 0); - grpc_chttp2_become_writable(transport_global, stream_global); + grpc_chttp2_complete_closure_step( + exec_ctx, stream_global, + &stream_global->send_initial_metadata_finished, 0); } - } else { - grpc_chttp2_complete_closure_step( - exec_ctx, stream_global, - &stream_global->send_initial_metadata_finished, 0); } } @@ -895,19 +908,33 @@ static void perform_stream_op_locked( stream_global->send_trailing_metadata_finished = add_closure_barrier(on_complete); stream_global->send_trailing_metadata = op->send_trailing_metadata; - if (contains_non_ok_status(transport_global, op->send_trailing_metadata)) { - stream_global->seen_error = true; - grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); - } - if (stream_global->write_closed) { - grpc_chttp2_complete_closure_step( - exec_ctx, stream_global, - &stream_global->send_trailing_metadata_finished, - grpc_metadata_batch_is_empty(op->send_trailing_metadata)); - } else if (stream_global->id != 0) { - /* TODO(ctiller): check if there's flow control for any outstanding - bytes before going writable */ - grpc_chttp2_become_writable(transport_global, stream_global); + const size_t metadata_size = grpc_metadata_batch_size( + op->send_trailing_metadata); + const size_t metadata_peer_limit = + transport_global->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; + if (metadata_size > metadata_peer_limit) { + gpr_log(GPR_DEBUG, + "trailing metadata size exceeds peer limit (%lu vs. %lu)", + metadata_size, metadata_peer_limit); + cancel_from_api(exec_ctx, transport_global, stream_global, + GRPC_STATUS_RESOURCE_EXHAUSTED); + } else { + if (contains_non_ok_status(transport_global, + op->send_trailing_metadata)) { + stream_global->seen_error = true; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + if (stream_global->write_closed) { + grpc_chttp2_complete_closure_step( + exec_ctx, stream_global, + &stream_global->send_trailing_metadata_finished, + grpc_metadata_batch_is_empty(op->send_trailing_metadata)); + } else if (stream_global->id != 0) { + /* TODO(ctiller): check if there's flow control for any outstanding + bytes before going writable */ + grpc_chttp2_become_writable(transport_global, stream_global); + } } } diff --git a/test/core/end2end/tests/large_metadata.c b/test/core/end2end/tests/large_metadata.c index 2aa6381e9e2..b78d5b8292c 100644 --- a/test/core/end2end/tests/large_metadata.c +++ b/test/core/end2end/tests/large_metadata.c @@ -97,9 +97,8 @@ static void end_test(grpc_end2end_test_fixture *f) { grpc_completion_queue_destroy(f->cq); } -/* Request with a large amount of metadata.*/ -static void test_request_with_large_metadata(grpc_end2end_test_config config, - int allow_large_metadata) { +/* Request with a large amount of metadata. */ +static void test_request_with_large_metadata(grpc_end2end_test_config config) { grpc_call *c; grpc_call *s; gpr_slice request_payload_slice = gpr_slice_from_copied_string("hello world"); @@ -107,16 +106,12 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config, grpc_raw_byte_buffer_create(&request_payload_slice, 1); gpr_timespec deadline = five_seconds_time(); grpc_metadata meta; - const char *test_name = allow_large_metadata - ? "test_request_with_large_metadata_allowed" - : "test_request_with_large_metadata_not_allowed"; const size_t large_size = 64 * 1024; grpc_arg arg = { GRPC_ARG_INTEGER, GRPC_ARG_MAX_METADATA_SIZE, { .integer=(int)large_size + 1024 } }; grpc_channel_args args = { 1, &arg }; - grpc_channel_args* use_args = allow_large_metadata ? &args : NULL; grpc_end2end_test_fixture f = begin_test( - config, test_name, use_args, use_args); + config, "test_request_with_large_metadata", &args, &args); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -146,6 +141,7 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config, grpc_metadata_array_init(&request_metadata_recv); grpc_call_details_init(&call_details); + /* Client: send request. */ op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; op->data.send_initial_metadata.count = 1; @@ -182,9 +178,11 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config, grpc_server_request_call(f.server, &s, &call_details, &request_metadata_recv, f.cq, f.cq, tag(101)); GPR_ASSERT(GRPC_CALL_OK == error); + cq_expect_completion(cqv, tag(101), 1); cq_verify(cqv); + /* Server: send initial metadata and receive request. */ op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; op->data.send_initial_metadata.count = 0; @@ -199,9 +197,11 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config, error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), NULL); GPR_ASSERT(GRPC_CALL_OK == error); - cq_expect_completion(cqv, tag(102), allow_large_metadata); + cq_expect_completion(cqv, tag(102), 1); cq_verify(cqv); + /* Server: receive close and send status. This should trigger + completion of request on client. */ op = ops; op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; op->data.recv_close_on_server.cancelled = &was_cancelled; @@ -222,19 +222,13 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config, cq_expect_completion(cqv, tag(1), 1); cq_verify(cqv); - GPR_ASSERT(status == (allow_large_metadata ? GRPC_STATUS_OK - : GRPC_STATUS_RESOURCE_EXHAUSTED)); + GPR_ASSERT(status == GRPC_STATUS_OK); + GPR_ASSERT(0 == strcmp(details, "xyz")); GPR_ASSERT(0 == strcmp(call_details.method, "/foo")); GPR_ASSERT(0 == strcmp(call_details.host, "foo.test.google.fr")); GPR_ASSERT(was_cancelled == 0); - if (allow_large_metadata) { - GPR_ASSERT(0 == strcmp(details, "xyz")); - GPR_ASSERT(byte_buffer_eq_string(request_payload_recv, "hello world")); - GPR_ASSERT(contains_metadata(&request_metadata_recv, "key", meta.value)); - } else { - GPR_ASSERT(request_payload_recv == NULL); - GPR_ASSERT(!contains_metadata_key(&request_metadata_recv, "key")); - } + GPR_ASSERT(byte_buffer_eq_string(request_payload_recv, "hello world")); + GPR_ASSERT(contains_metadata(&request_metadata_recv, "key", meta.value)); gpr_free(details); grpc_metadata_array_destroy(&initial_metadata_recv); @@ -257,8 +251,7 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config, } void large_metadata(grpc_end2end_test_config config) { - test_request_with_large_metadata(config, 1); - test_request_with_large_metadata(config, 0); + test_request_with_large_metadata(config); } void large_metadata_pre_init(void) {} From 6a45e9340701f3107f5d39cc5f5c0366204faa23 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 2 May 2016 14:25:49 -0700 Subject: [PATCH 023/155] Made ruby plugin support empty package names --- src/compiler/ruby_generator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/ruby_generator.cc b/src/compiler/ruby_generator.cc index 5ac56ad289c..4ca06a4554d 100644 --- a/src/compiler/ruby_generator.cc +++ b/src/compiler/ruby_generator.cc @@ -98,7 +98,7 @@ void PrintService(const ServiceDescriptor *service, const grpc::string &package, out->Print("self.marshal_class_method = :encode\n"); out->Print("self.unmarshal_class_method = :decode\n"); std::map pkg_vars = - ListToDict({"service.name", service->name(), "pkg.name", package, }); + ListToDict({"service_full_namename", service->full_name()}); out->Print(pkg_vars, "self.service_name = '$pkg.name$.$service.name$'\n"); out->Print("\n"); for (int i = 0; i < service->method_count(); ++i) { From df8b62cea7c0d37cdf6918f1761137d95ab8a879 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 2 May 2016 14:34:24 -0700 Subject: [PATCH 024/155] Add coverage for secure async end2end testing --- test/cpp/end2end/async_end2end_test.cc | 225 +++++++++++++++---------- 1 file changed, 136 insertions(+), 89 deletions(-) diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 7e4d6046d60..8d58726f13d 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -51,6 +51,7 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" #include "test/cpp/util/string_ref_helper.h" +#include "test/cpp/util/test_credentials_provider.h" #ifdef GPR_POSIX_SOCKET #include "src/core/lib/iomgr/ev_posix.h" @@ -58,6 +59,7 @@ using grpc::testing::EchoRequest; using grpc::testing::EchoResponse; +using grpc::testing::kTlsCredentialsType; using std::chrono::system_clock; GPR_TLS_DECL(g_is_async_end2end_test); @@ -197,20 +199,28 @@ class Verifier { bool spin_; }; -class AsyncEnd2endTest : public ::testing::TestWithParam { +class TestScenario { + public: + TestScenario(bool non_block, const grpc::string& creds_type) + : disable_blocking(non_block), credentials_type(creds_type) {} + bool disable_blocking; + const grpc::string credentials_type; +}; + +class AsyncEnd2endTest : public ::testing::TestWithParam { protected: AsyncEnd2endTest() {} void SetUp() GRPC_OVERRIDE { - poll_overrider_.reset(new PollingOverrider(!GetParam())); + poll_overrider_.reset(new PollingOverrider(!GetParam().disable_blocking)); int port = grpc_pick_unused_port_or_die(); server_address_ << "localhost:" << port; // Setup server ServerBuilder builder; - builder.AddListeningPort(server_address_.str(), - grpc::InsecureServerCredentials()); + auto server_creds = GetServerCredentials(GetParam().credentials_type); + builder.AddListeningPort(server_address_.str(), server_creds); builder.RegisterService(&service_); cq_ = builder.AddCompletionQueue(); server_ = builder.BuildAndStart(); @@ -230,8 +240,11 @@ class AsyncEnd2endTest : public ::testing::TestWithParam { } void ResetStub() { + ChannelArguments args; + auto channel_creds = + GetChannelCredentials(GetParam().credentials_type, &args); std::shared_ptr channel = - CreateChannel(server_address_.str(), InsecureChannelCredentials()); + CreateCustomChannel(server_address_.str(), channel_creds, args); stub_ = grpc::testing::EchoTestService::NewStub(channel); } @@ -254,15 +267,15 @@ class AsyncEnd2endTest : public ::testing::TestWithParam { service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); response_reader->Finish(&recv_response, &recv_status, tag(4)); - Verifier(GetParam()).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); @@ -310,23 +323,25 @@ TEST_P(AsyncEnd2endTest, AsyncNextRpc) { std::chrono::system_clock::now()); std::chrono::system_clock::time_point time_limit( std::chrono::system_clock::now() + std::chrono::seconds(10)); - Verifier(GetParam()).Verify(cq_.get(), time_now); - Verifier(GetParam()).Verify(cq_.get(), time_now); + Verifier(GetParam().disable_blocking).Verify(cq_.get(), time_now); + Verifier(GetParam().disable_blocking).Verify(cq_.get(), time_now); service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get(), time_limit); + Verifier(GetParam().disable_blocking) + .Expect(2, true) + .Verify(cq_.get(), time_limit); EXPECT_EQ(send_request.message(), recv_request.message()); send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(3)); - Verifier(GetParam()) + Verifier(GetParam().disable_blocking) .Expect(3, true) .Verify(cq_.get(), std::chrono::system_clock::time_point::max()); response_reader->Finish(&recv_response, &recv_status, tag(4)); - Verifier(GetParam()) + Verifier(GetParam().disable_blocking) .Expect(4, true) .Verify(cq_.get(), std::chrono::system_clock::time_point::max()); @@ -354,34 +369,37 @@ TEST_P(AsyncEnd2endTest, SimpleClientStreaming) { service_.RequestRequestStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Expect(1, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(2, true) + .Expect(1, true) + .Verify(cq_.get()); cli_stream->Write(send_request, tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); srv_stream.Read(&recv_request, tag(4)); - Verifier(GetParam()).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); cli_stream->Write(send_request, tag(5)); - Verifier(GetParam()).Expect(5, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); srv_stream.Read(&recv_request, tag(6)); - Verifier(GetParam()).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); cli_stream->WritesDone(tag(7)); - Verifier(GetParam()).Expect(7, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(7, true).Verify(cq_.get()); srv_stream.Read(&recv_request, tag(8)); - Verifier(GetParam()).Expect(8, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(8, false).Verify(cq_.get()); send_response.set_message(recv_request.message()); srv_stream.Finish(send_response, Status::OK, tag(9)); - Verifier(GetParam()).Expect(9, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(9, true).Verify(cq_.get()); cli_stream->Finish(&recv_status, tag(10)); - Verifier(GetParam()).Expect(10, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(10, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); @@ -407,32 +425,35 @@ TEST_P(AsyncEnd2endTest, SimpleServerStreaming) { service_.RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(1, true).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(1, true) + .Expect(2, true) + .Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); send_response.set_message(recv_request.message()); srv_stream.Write(send_response, tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); cli_stream->Read(&recv_response, tag(4)); - Verifier(GetParam()).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); srv_stream.Write(send_response, tag(5)); - Verifier(GetParam()).Expect(5, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); cli_stream->Read(&recv_response, tag(6)); - Verifier(GetParam()).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); srv_stream.Finish(Status::OK, tag(7)); - Verifier(GetParam()).Expect(7, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(7, true).Verify(cq_.get()); cli_stream->Read(&recv_response, tag(8)); - Verifier(GetParam()).Expect(8, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(8, false).Verify(cq_.get()); cli_stream->Finish(&recv_status, tag(9)); - Verifier(GetParam()).Expect(9, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(9, true).Verify(cq_.get()); EXPECT_TRUE(recv_status.ok()); } @@ -457,34 +478,37 @@ TEST_P(AsyncEnd2endTest, SimpleBidiStreaming) { service_.RequestBidiStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(1, true).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(1, true) + .Expect(2, true) + .Verify(cq_.get()); cli_stream->Write(send_request, tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); srv_stream.Read(&recv_request, tag(4)); - Verifier(GetParam()).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); send_response.set_message(recv_request.message()); srv_stream.Write(send_response, tag(5)); - Verifier(GetParam()).Expect(5, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); cli_stream->Read(&recv_response, tag(6)); - Verifier(GetParam()).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); cli_stream->WritesDone(tag(7)); - Verifier(GetParam()).Expect(7, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(7, true).Verify(cq_.get()); srv_stream.Read(&recv_request, tag(8)); - Verifier(GetParam()).Expect(8, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(8, false).Verify(cq_.get()); srv_stream.Finish(Status::OK, tag(9)); - Verifier(GetParam()).Expect(9, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(9, true).Verify(cq_.get()); cli_stream->Finish(&recv_status, tag(10)); - Verifier(GetParam()).Expect(10, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(10, true).Verify(cq_.get()); EXPECT_TRUE(recv_status.ok()); } @@ -516,7 +540,7 @@ TEST_P(AsyncEnd2endTest, ClientInitialMetadataRpc) { service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); auto client_initial_metadata = srv_ctx.client_metadata(); EXPECT_EQ(meta1.second, @@ -530,10 +554,10 @@ TEST_P(AsyncEnd2endTest, ClientInitialMetadataRpc) { send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); response_reader->Finish(&recv_response, &recv_status, tag(4)); - Verifier(GetParam()).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); @@ -561,15 +585,15 @@ TEST_P(AsyncEnd2endTest, ServerInitialMetadataRpc) { service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); srv_ctx.AddInitialMetadata(meta1.first, meta1.second); srv_ctx.AddInitialMetadata(meta2.first, meta2.second); response_writer.SendInitialMetadata(tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); response_reader->ReadInitialMetadata(tag(4)); - Verifier(GetParam()).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); auto server_initial_metadata = cli_ctx.GetServerInitialMetadata(); EXPECT_EQ(meta1.second, ToString(server_initial_metadata.find(meta1.first)->second)); @@ -579,10 +603,10 @@ TEST_P(AsyncEnd2endTest, ServerInitialMetadataRpc) { send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(5)); - Verifier(GetParam()).Expect(5, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); response_reader->Finish(&recv_response, &recv_status, tag(6)); - Verifier(GetParam()).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); @@ -610,20 +634,20 @@ TEST_P(AsyncEnd2endTest, ServerTrailingMetadataRpc) { service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); response_writer.SendInitialMetadata(tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); send_response.set_message(recv_request.message()); srv_ctx.AddTrailingMetadata(meta1.first, meta1.second); srv_ctx.AddTrailingMetadata(meta2.first, meta2.second); response_writer.Finish(send_response, Status::OK, tag(4)); - Verifier(GetParam()).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); response_reader->Finish(&recv_response, &recv_status, tag(5)); - Verifier(GetParam()).Expect(5, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); auto server_trailing_metadata = cli_ctx.GetServerTrailingMetadata(); @@ -671,7 +695,7 @@ TEST_P(AsyncEnd2endTest, MetadataRpc) { service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); auto client_initial_metadata = srv_ctx.client_metadata(); EXPECT_EQ(meta1.second, @@ -683,9 +707,9 @@ TEST_P(AsyncEnd2endTest, MetadataRpc) { srv_ctx.AddInitialMetadata(meta3.first, meta3.second); srv_ctx.AddInitialMetadata(meta4.first, meta4.second); response_writer.SendInitialMetadata(tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); response_reader->ReadInitialMetadata(tag(4)); - Verifier(GetParam()).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); auto server_initial_metadata = cli_ctx.GetServerInitialMetadata(); EXPECT_EQ(meta3.second, ToString(server_initial_metadata.find(meta3.first)->second)); @@ -698,10 +722,10 @@ TEST_P(AsyncEnd2endTest, MetadataRpc) { srv_ctx.AddTrailingMetadata(meta6.first, meta6.second); response_writer.Finish(send_response, Status::OK, tag(5)); - Verifier(GetParam()).Expect(5, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); response_reader->Finish(&recv_response, &recv_status, tag(6)); - Verifier(GetParam()).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); auto server_trailing_metadata = cli_ctx.GetServerTrailingMetadata(); @@ -734,15 +758,15 @@ TEST_P(AsyncEnd2endTest, ServerCheckCancellation) { service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); cli_ctx.TryCancel(); - Verifier(GetParam()).Expect(5, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); EXPECT_TRUE(srv_ctx.IsCancelled()); response_reader->Finish(&recv_response, &recv_status, tag(4)); - Verifier(GetParam()).Expect(4, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, false).Verify(cq_.get()); EXPECT_EQ(StatusCode::CANCELLED, recv_status.error_code()); } @@ -769,25 +793,28 @@ TEST_P(AsyncEnd2endTest, ServerCheckDone) { service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); - Verifier(GetParam()).Expect(5, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); EXPECT_FALSE(srv_ctx.IsCancelled()); response_reader->Finish(&recv_response, &recv_status, tag(4)); - Verifier(GetParam()).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); } TEST_P(AsyncEnd2endTest, UnimplementedRpc) { + ChannelArguments args; + auto channel_creds = + GetChannelCredentials(GetParam().credentials_type, &args); std::shared_ptr channel = - CreateChannel(server_address_.str(), InsecureChannelCredentials()); + CreateCustomChannel(server_address_.str(), channel_creds, args); std::unique_ptr stub; stub = grpc::testing::UnimplementedService::NewStub(channel); EchoRequest send_request; @@ -800,7 +827,7 @@ TEST_P(AsyncEnd2endTest, UnimplementedRpc) { stub->AsyncUnimplemented(&cli_ctx, send_request, cq_.get())); response_reader->Finish(&recv_response, &recv_status, tag(4)); - Verifier(GetParam()).Expect(4, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(4, false).Verify(cq_.get()); EXPECT_EQ(StatusCode::UNIMPLEMENTED, recv_status.error_code()); EXPECT_EQ("", recv_status.error_message()); @@ -847,23 +874,25 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // Initiate the 'RequestStream' call on client std::unique_ptr> cli_stream( stub_->AsyncRequestStream(&cli_ctx, &recv_response, cq_.get(), tag(1))); - Verifier(GetParam()).Expect(1, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(1, true).Verify(cq_.get()); // On the server, request to be notified of 'RequestStream' calls // and receive the 'RequestStream' call just made by the client srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestRequestStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); // Client sends 3 messages (tags 3, 4 and 5) for (int tag_idx = 3; tag_idx <= 5; tag_idx++) { send_request.set_message("Ping " + std::to_string(tag_idx)); cli_stream->Write(send_request, tag(tag_idx)); - Verifier(GetParam()).Expect(tag_idx, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(tag_idx, true) + .Verify(cq_.get()); } cli_stream->WritesDone(tag(6)); - Verifier(GetParam()).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); bool expected_server_cq_result = true; bool ignore_cq_result = false; @@ -871,7 +900,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { srv_ctx.TryCancel(); - Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(11, true).Verify(cq_.get()); EXPECT_TRUE(srv_ctx.IsCancelled()); // Since cancellation is done before server reads any results, we know @@ -881,7 +910,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { std::thread* server_try_cancel_thd = NULL; - auto verif = Verifier(GetParam()); + auto verif = Verifier(GetParam().disable_blocking); if (server_try_cancel == CANCEL_DURING_PROCESSING) { server_try_cancel_thd = @@ -939,13 +968,13 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // Server sends the final message and cancelled status (but the RPC is // already cancelled at this point. So we expect the operation to fail) srv_stream.Finish(send_response, Status::CANCELLED, tag(9)); - Verifier(GetParam()).Expect(9, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(9, false).Verify(cq_.get()); // Client will see the cancellation cli_stream->Finish(&recv_status, tag(10)); // TODO(sreek): The expectation here should be true. This is a bug (github // issue #4972) - Verifier(GetParam()).Expect(10, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(10, false).Verify(cq_.get()); EXPECT_FALSE(recv_status.ok()); EXPECT_EQ(::grpc::StatusCode::CANCELLED, recv_status.error_code()); } @@ -979,13 +1008,13 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // Initiate the 'ResponseStream' call on the client std::unique_ptr> cli_stream( stub_->AsyncResponseStream(&cli_ctx, send_request, cq_.get(), tag(1))); - Verifier(GetParam()).Expect(1, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(1, true).Verify(cq_.get()); // On the server, request to be notified of 'ResponseStream' calls and // receive the call just made by the client srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); bool expected_cq_result = true; @@ -994,7 +1023,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { srv_ctx.TryCancel(); - Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(11, true).Verify(cq_.get()); EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point @@ -1004,7 +1033,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { std::thread* server_try_cancel_thd = NULL; - auto verif = Verifier(GetParam()); + auto verif = Verifier(GetParam().disable_blocking); if (server_try_cancel == CANCEL_DURING_PROCESSING) { server_try_cancel_thd = @@ -1064,7 +1093,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // Client attemts to read the three messages from the server for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { cli_stream->Read(&recv_response, tag(tag_idx)); - Verifier(GetParam()) + Verifier(GetParam().disable_blocking) .Expect(tag_idx, expected_cq_result) .Verify(cq_.get(), ignore_cq_result); } @@ -1075,11 +1104,11 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // Server finishes the stream (but the RPC is already cancelled) srv_stream.Finish(Status::CANCELLED, tag(9)); - Verifier(GetParam()).Expect(9, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(9, false).Verify(cq_.get()); // Client will see the cancellation cli_stream->Finish(&recv_status, tag(10)); - Verifier(GetParam()).Expect(10, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(10, true).Verify(cq_.get()); EXPECT_FALSE(recv_status.ok()); EXPECT_EQ(::grpc::StatusCode::CANCELLED, recv_status.error_code()); } @@ -1114,19 +1143,19 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // Initiate the call from the client side std::unique_ptr> cli_stream(stub_->AsyncBidiStream(&cli_ctx, cq_.get(), tag(1))); - Verifier(GetParam()).Expect(1, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(1, true).Verify(cq_.get()); // On the server, request to be notified of the 'BidiStream' call and // receive the call just made by the client srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestBidiStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); - Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(2, true).Verify(cq_.get()); // Client sends the first and the only message send_request.set_message("Ping"); cli_stream->Write(send_request, tag(3)); - Verifier(GetParam()).Expect(3, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); bool expected_cq_result = true; bool ignore_cq_result = false; @@ -1134,7 +1163,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { srv_ctx.TryCancel(); - Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(11, true).Verify(cq_.get()); EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point @@ -1144,7 +1173,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { std::thread* server_try_cancel_thd = NULL; - auto verif = Verifier(GetParam()); + auto verif = Verifier(GetParam().disable_blocking); if (server_try_cancel == CANCEL_DURING_PROCESSING) { server_try_cancel_thd = @@ -1244,10 +1273,10 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // know that cq results are supposed to return false on server. srv_stream.Finish(Status::CANCELLED, tag(9)); - Verifier(GetParam()).Expect(9, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(9, false).Verify(cq_.get()); cli_stream->Finish(&recv_status, tag(10)); - Verifier(GetParam()).Expect(10, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking).Expect(10, true).Verify(cq_.get()); EXPECT_FALSE(recv_status.ok()); EXPECT_EQ(grpc::StatusCode::CANCELLED, recv_status.error_code()); } @@ -1289,11 +1318,29 @@ TEST_P(AsyncEnd2endServerTryCancelTest, ServerBidiStreamingTryCancelAfter) { TestBidiStreamingServerCancel(CANCEL_AFTER_PROCESSING); } +std::vector CreateTestScenarios(bool test_disable_blocking, + bool test_secure) { + std::vector scenarios; + std::vector credentials_types; + if (test_secure) { + credentials_types = GetSecureCredentialsTypeList(); + } + credentials_types.push_back(kInsecureCredentialsType); + for (auto it = credentials_types.begin(); it != credentials_types.end(); + ++it) { + scenarios.push_back(TestScenario(false, *it)); + if (test_disable_blocking) { + scenarios.push_back(TestScenario(true, *it)); + } + } + return scenarios; +} + INSTANTIATE_TEST_CASE_P(AsyncEnd2end, AsyncEnd2endTest, - ::testing::Values(false, true)); + ::testing::ValuesIn(CreateTestScenarios(true, true))); INSTANTIATE_TEST_CASE_P(AsyncEnd2endServerTryCancel, AsyncEnd2endServerTryCancelTest, - ::testing::Values(false)); + ::testing::ValuesIn(CreateTestScenarios(false, false))); } // namespace } // namespace testing From 957fe8dc5de97d9b59de91522c8afab294abe9f4 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 2 May 2016 15:03:55 -0700 Subject: [PATCH 025/155] Updated a couple more scripts --- tools/gce/linux_performance_worker_init.sh | 3 +++ tools/run_tests/performance/run_worker_node.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 25ac3bcedea..df29581e69a 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -95,6 +95,9 @@ sudo pip install tox touch .profile curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash nvm install 0.12 && npm config set cache /tmp/npm-cache +nvm install 4 && npm config set cache /tmp/npm-cache +nvm install 5 && npm config set cache /tmp/npm-cache +nvm alias default 4 # C# dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) diff --git a/tools/run_tests/performance/run_worker_node.sh b/tools/run_tests/performance/run_worker_node.sh index 46b6ff01774..9a53a311f40 100755 --- a/tools/run_tests/performance/run_worker_node.sh +++ b/tools/run_tests/performance/run_worker_node.sh @@ -29,7 +29,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. source ~/.nvm/nvm.sh -nvm use 0.12 +nvm use 4 set -ex From dedae79cd46cb84187ca68854e191dbdb4cb9fae Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 2 May 2016 15:05:37 -0700 Subject: [PATCH 026/155] Finished ruby generator change --- src/compiler/ruby_generator.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/ruby_generator.cc b/src/compiler/ruby_generator.cc index 4ca06a4554d..936a186beb5 100644 --- a/src/compiler/ruby_generator.cc +++ b/src/compiler/ruby_generator.cc @@ -98,8 +98,8 @@ void PrintService(const ServiceDescriptor *service, const grpc::string &package, out->Print("self.marshal_class_method = :encode\n"); out->Print("self.unmarshal_class_method = :decode\n"); std::map pkg_vars = - ListToDict({"service_full_namename", service->full_name()}); - out->Print(pkg_vars, "self.service_name = '$pkg.name$.$service.name$'\n"); + ListToDict({"service_full_name", service->full_name()}); + out->Print(pkg_vars, "self.service_name = '$service_full_name$'\n"); out->Print("\n"); for (int i = 0; i < service->method_count(); ++i) { PrintMethod(service->method(i), package, out); From d7b1e704a829ccdc9e353e80faee634b1414265e Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 2 May 2016 15:10:21 -0700 Subject: [PATCH 027/155] Cover large message sent securely or insecurely over async --- test/cpp/end2end/async_end2end_test.cc | 80 +++++++++++++++++--------- 1 file changed, 54 insertions(+), 26 deletions(-) diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 8d58726f13d..4de181b9011 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -201,15 +201,24 @@ class Verifier { class TestScenario { public: - TestScenario(bool non_block, const grpc::string& creds_type) - : disable_blocking(non_block), credentials_type(creds_type) {} + TestScenario(bool non_block, const grpc::string& creds_type, + const grpc::string& content) + : disable_blocking(non_block), + credentials_type(creds_type), + message_content(content) {} + void Log() const { + gpr_log(GPR_INFO, + "Scenario: disable_blocking %d, credentials %s, message size %d", + disable_blocking, credentials_type.c_str(), message_content.size()); + } bool disable_blocking; const grpc::string credentials_type; + const grpc::string message_content; }; class AsyncEnd2endTest : public ::testing::TestWithParam { protected: - AsyncEnd2endTest() {} + AsyncEnd2endTest() { GetParam().Log(); } void SetUp() GRPC_OVERRIDE { poll_overrider_.reset(new PollingOverrider(!GetParam().disable_blocking)); @@ -260,7 +269,7 @@ class AsyncEnd2endTest : public ::testing::TestWithParam { ServerContext srv_ctx; grpc::ServerAsyncResponseWriter response_writer(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::unique_ptr> response_reader( stub_->AsyncEcho(&cli_ctx, send_request, cq_.get())); @@ -315,7 +324,7 @@ TEST_P(AsyncEnd2endTest, AsyncNextRpc) { ServerContext srv_ctx; grpc::ServerAsyncResponseWriter response_writer(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::unique_ptr> response_reader( stub_->AsyncEcho(&cli_ctx, send_request, cq_.get())); @@ -362,7 +371,7 @@ TEST_P(AsyncEnd2endTest, SimpleClientStreaming) { ServerContext srv_ctx; ServerAsyncReader srv_stream(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::unique_ptr> cli_stream( stub_->AsyncRequestStream(&cli_ctx, &recv_response, cq_.get(), tag(1))); @@ -418,7 +427,7 @@ TEST_P(AsyncEnd2endTest, SimpleServerStreaming) { ServerContext srv_ctx; ServerAsyncWriter srv_stream(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::unique_ptr> cli_stream( stub_->AsyncResponseStream(&cli_ctx, send_request, cq_.get(), tag(1))); @@ -471,7 +480,7 @@ TEST_P(AsyncEnd2endTest, SimpleBidiStreaming) { ServerContext srv_ctx; ServerAsyncReaderWriter srv_stream(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::unique_ptr> cli_stream(stub_->AsyncBidiStream(&cli_ctx, cq_.get(), tag(1))); @@ -527,7 +536,7 @@ TEST_P(AsyncEnd2endTest, ClientInitialMetadataRpc) { ServerContext srv_ctx; grpc::ServerAsyncResponseWriter response_writer(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::pair meta1("key1", "val1"); std::pair meta2("key2", "val2"); std::pair meta3("g.r.d-bin", "xyz"); @@ -576,7 +585,7 @@ TEST_P(AsyncEnd2endTest, ServerInitialMetadataRpc) { ServerContext srv_ctx; grpc::ServerAsyncResponseWriter response_writer(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::pair meta1("key1", "val1"); std::pair meta2("key2", "val2"); @@ -625,7 +634,7 @@ TEST_P(AsyncEnd2endTest, ServerTrailingMetadataRpc) { ServerContext srv_ctx; grpc::ServerAsyncResponseWriter response_writer(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::pair meta1("key1", "val1"); std::pair meta2("key2", "val2"); @@ -671,7 +680,7 @@ TEST_P(AsyncEnd2endTest, MetadataRpc) { ServerContext srv_ctx; grpc::ServerAsyncResponseWriter response_writer(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::pair meta1("key1", "val1"); std::pair meta2( "key2-bin", @@ -750,7 +759,7 @@ TEST_P(AsyncEnd2endTest, ServerCheckCancellation) { ServerContext srv_ctx; grpc::ServerAsyncResponseWriter response_writer(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::unique_ptr> response_reader( stub_->AsyncEcho(&cli_ctx, send_request, cq_.get())); @@ -785,7 +794,7 @@ TEST_P(AsyncEnd2endTest, ServerCheckDone) { ServerContext srv_ctx; grpc::ServerAsyncResponseWriter response_writer(&srv_ctx); - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::unique_ptr> response_reader( stub_->AsyncEcho(&cli_ctx, send_request, cq_.get())); @@ -822,7 +831,7 @@ TEST_P(AsyncEnd2endTest, UnimplementedRpc) { Status recv_status; ClientContext cli_ctx; - send_request.set_message("Hello"); + send_request.set_message(GetParam().message_content); std::unique_ptr> response_reader( stub->AsyncUnimplemented(&cli_ctx, send_request, cq_.get())); @@ -1319,28 +1328,47 @@ TEST_P(AsyncEnd2endServerTryCancelTest, ServerBidiStreamingTryCancelAfter) { } std::vector CreateTestScenarios(bool test_disable_blocking, - bool test_secure) { + bool test_secure, + int test_big_limit) { std::vector scenarios; std::vector credentials_types; - if (test_secure) { - credentials_types = GetSecureCredentialsTypeList(); - } + std::vector messages; + credentials_types.push_back(kInsecureCredentialsType); - for (auto it = credentials_types.begin(); it != credentials_types.end(); - ++it) { - scenarios.push_back(TestScenario(false, *it)); - if (test_disable_blocking) { - scenarios.push_back(TestScenario(true, *it)); + auto sec_list = GetSecureCredentialsTypeList(); + for (auto sec = sec_list.begin(); sec != sec_list.end(); sec++) { + credentials_types.push_back(*sec); + } + + messages.push_back("Hello"); + for (int sz = 1; sz < test_big_limit; sz *= 2) { + grpc::string big_msg; + for (int i = 0; i < sz * 1024; i++) { + char c = 'a' + (i % 26); + big_msg += c; + } + messages.push_back(big_msg); + } + + for (auto cred = credentials_types.begin(); cred != credentials_types.end(); + ++cred) { + for (auto msg = messages.begin(); msg != messages.end(); msg++) { + scenarios.push_back(TestScenario(false, *cred, *msg)); + if (test_disable_blocking) { + scenarios.push_back(TestScenario(true, *cred, *msg)); + } } } return scenarios; } INSTANTIATE_TEST_CASE_P(AsyncEnd2end, AsyncEnd2endTest, - ::testing::ValuesIn(CreateTestScenarios(true, true))); + ::testing::ValuesIn(CreateTestScenarios(true, true, + 1024))); INSTANTIATE_TEST_CASE_P(AsyncEnd2endServerTryCancel, AsyncEnd2endServerTryCancelTest, - ::testing::ValuesIn(CreateTestScenarios(false, false))); + ::testing::ValuesIn(CreateTestScenarios(false, false, + 0))); } // namespace } // namespace testing From 7edab7c5e32c74d996b077de6bb7cf5085846d62 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 29 Apr 2016 19:09:37 -0700 Subject: [PATCH 028/155] Ruby docker files and config --- .../Dockerfile.template | 41 ++++++++ .../grpc_interop_stress_ruby/Dockerfile | 99 +++++++++++++++++++ .../build_interop_stress.sh | 48 +++++++++ tools/gcp/stress_test/run_ruby.sh | 37 +++++++ tools/run_tests/stress_test/configs/ruby.json | 92 +++++++++++++++++ 5 files changed, 317 insertions(+) create mode 100644 templates/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile.template create mode 100644 tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile create mode 100755 tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh create mode 100755 tools/gcp/stress_test/run_ruby.sh create mode 100644 tools/run_tests/stress_test/configs/ruby.json diff --git a/templates/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile.template new file mode 100644 index 00000000000..8b933aaa324 --- /dev/null +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile.template @@ -0,0 +1,41 @@ +%YAML 1.2 +--- | + # Copyright 2015, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM debian:jessie + + <%include file="../../apt_get_basic.include"/> + <%include file="../../ccache_setup.include"/> + <%include file="../../cxx_deps.include"/> + <%include file="../../gcp_api_libraries.include"/> + <%include file="../../ruby_deps.include"/> + # Define the default command. + CMD ["bash"] + diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile new file mode 100644 index 00000000000..36b54ddafe3 --- /dev/null +++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/Dockerfile @@ -0,0 +1,99 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + + +#================== +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm use --default ruby-2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh new file mode 100755 index 00000000000..1b7567d87a6 --- /dev/null +++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds Ruby interop server and client in a base image. +set -e + +mkdir -p /var/local/git +git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc + +# Copy service account keys if available +cp -r /var/local/jenkins/service_account $HOME || true + +cd /var/local/git/grpc +rvm --default use ruby-2.1 + +# Build Ruby interop client and server +(cd src/ruby && gem update bundler && bundle && rake compile) + +# Build c++ metrics client to query the metrics from ruby stress client +make metrics_client -j + diff --git a/tools/gcp/stress_test/run_ruby.sh b/tools/gcp/stress_test/run_ruby.sh new file mode 100755 index 00000000000..80d0567447b --- /dev/null +++ b/tools/gcp/stress_test/run_ruby.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This is a wrapper script that was created to help run_server.py and +# run_client.py to launch 'node js' stress clients and stress servers +source /etc/profile.d/rvm.sh + +set -ex + +$@ diff --git a/tools/run_tests/stress_test/configs/ruby.json b/tools/run_tests/stress_test/configs/ruby.json new file mode 100644 index 00000000000..a323fa72b6b --- /dev/null +++ b/tools/run_tests/stress_test/configs/ruby.json @@ -0,0 +1,92 @@ +{ + "dockerImages": { + "grpc_stress_ruby" : { + "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "dockerFileDir": "grpc_interop_stress_ruby" + } + }, + + "clientTemplates": { + "baseTemplates": { + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", + "pollIntervalSecs": 60, + "clientArgs": { + "num_channels_per_server":5, + "num_stubs_per_channel":10, + "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", + "metrics_port": 8081 + }, + "metricsPort": 8081, + "metricsArgs": { + "metrics_server_address": "localhost:8081", + "total_only": "true" + } + } + }, + "templates": { + "ruby_client": { + "baseTemplate": "default", + "stressClientCmd": [ + "/var/local/git/grpc/tools/gcp/stress_test/run_ruby.sh", + "ruby", + "/var/local/git/grpc/src/ruby/stress/stress_client.rb" + ], + "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] + } + } + }, + + "serverTemplates": { + "baseTemplates":{ + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", + "serverPort": 8080, + "serverArgs": { + "port": 8080 + } + } + }, + "templates": { + "ruby_server": { + "baseTemplate": "default", + "stressServerCmd": [ + "/var/local/git/grpc/tools/gcp/stress_test/run_ruby.sh", + "ruby", + "/var/local/git/grpc/src/ruby/pb/test/server.rb" + ] + } + } + }, + + "testMatrix": { + "serverPodSpecs": { + "stress-server-ruby": { + "serverTemplate": "ruby_server", + "dockerImage": "grpc_stress_ruby", + "numInstances": 1 + } + }, + + "clientPodSpecs": { + "stress-client-ruby": { + "clientTemplate": "ruby_client", + "dockerImage": "grpc_stress_ruby", + "numInstances": 10, + "serverPodSpec": "stress-server-ruby" + } + } + }, + + "globalSettings": { + "buildDockerImages": true, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8001, + "datasetIdNamePrefix": "stress_test_ruby", + "summaryTableId": "summary", + "qpsTableId": "qps", + "podWarmupSecs": 60 + } +} + From 70f697ba1e6ad17422928a398807e84f8f469724 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Mon, 2 May 2016 22:29:21 -0700 Subject: [PATCH 029/155] reran generate_projects.sh after @nnoble's fix in PR6399. --- grpc.def | 1 - src/python/grpcio/grpc/_cython/imports.generated.c | 2 -- src/python/grpcio/grpc/_cython/imports.generated.h | 4 ---- src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 -- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 4 ---- 5 files changed, 13 deletions(-) diff --git a/grpc.def b/grpc.def index 09a94a6cd0c..61948ed1b89 100644 --- a/grpc.def +++ b/grpc.def @@ -87,7 +87,6 @@ EXPORTS grpc_header_nonbin_value_is_legal grpc_is_binary_header grpc_call_error_to_string - grpc_cronet_secure_channel_create grpc_auth_property_iterator_next grpc_auth_context_property_iterator grpc_auth_context_peer_identity diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c index 09551472b5f..f0a40dbb35c 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.c +++ b/src/python/grpcio/grpc/_cython/imports.generated.c @@ -125,7 +125,6 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; -grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -396,7 +395,6 @@ void pygrpc_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); - grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h index 54c8aaad13e..d5e810b7cf4 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.h +++ b/src/python/grpcio/grpc/_cython/imports.generated.h @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -326,9 +325,6 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import -typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); -extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; -#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index cebbe8c40fe..bc43f9d36b5 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -125,7 +125,6 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; -grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -392,7 +391,6 @@ void grpc_rb_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); - grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index d7ea6c574c8..b67361ca25b 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -326,9 +325,6 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import -typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); -extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; -#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import From 97eb4f6779ad232792c7f79738276d5ce13aa343 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 2 May 2016 13:08:12 -0700 Subject: [PATCH 030/155] add more tests --- .../Grpc.Core.Tests/Internal/AsyncCallTest.cs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs index a678e4dafe1..605072b3237 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs @@ -32,6 +32,7 @@ #endregion using System; +using System.Collections.Generic; using System.Runtime.InteropServices; using System.Threading.Tasks; @@ -118,6 +119,14 @@ namespace Grpc.Core.Internal.Tests AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.Internal); } + [Test] + public void ClientStreaming_StreamingReadNotAllowed() + { + asyncCall.ClientStreamingCallAsync(); + Assert.Throws(typeof(InvalidOperationException), + () => asyncCall.StartReadMessage((x,y) => {})); + } + [Test] public void ClientStreaming_NoRequest_Success() { @@ -142,6 +151,47 @@ namespace Grpc.Core.Internal.Tests AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.InvalidArgument); } + [Test] + public void ServerStreaming_StreamingSendNotAllowed() + { + asyncCall.StartServerStreamingCall("request1"); + Assert.Throws(typeof(InvalidOperationException), + () => asyncCall.StartSendMessage("abc", new WriteFlags(), (x,y) => {})); + } + + [Test] + public void ServerStreaming_NoResponse1_Success() + { + asyncCall.StartServerStreamingCall("request1"); + var responseStream = new ClientResponseStream(asyncCall); + var readTask = responseStream.MoveNext(); + + fakeCall.ReceivedResponseHeadersHandler(true, new Metadata()); + Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count); + + fakeCall.ReceivedMessageHandler(true, null); + fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata())); + + AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask); + } + + [Test] + public void ServerStreaming_NoResponse2_Success() + { + asyncCall.StartServerStreamingCall("request1"); + var responseStream = new ClientResponseStream(asyncCall); + var readTask = responseStream.MoveNext(); + + fakeCall.ReceivedResponseHeadersHandler(true, new Metadata()); + Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count); + + // try alternative order of completions + fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata())); + fakeCall.ReceivedMessageHandler(true, null); + + AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask); + } + ClientSideStatus CreateClientSideStatus(StatusCode statusCode) { return new ClientSideStatus(new Status(statusCode, ""), new Metadata()); @@ -163,6 +213,17 @@ namespace Grpc.Core.Internal.Tests Assert.AreEqual("response1", resultTask.Result); } + static void AssertStreamingResponseSuccess(AsyncCall asyncCall, FakeNativeCall fakeCall, Task moveNextTask) + { + Assert.IsTrue(moveNextTask.IsCompleted); + Assert.IsTrue(fakeCall.IsDisposed); + + Assert.IsFalse(moveNextTask.Result); + Assert.AreEqual(Status.DefaultSuccess, asyncCall.GetStatus()); + Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count); + Assert.AreEqual(0, asyncCall.GetTrailers().Count); + } + static void AssertUnaryResponseError(AsyncCall asyncCall, FakeNativeCall fakeCall, Task resultTask, StatusCode expectedStatusCode) { Assert.IsTrue(resultTask.IsCompleted); From 69274c2a0d316ccac66a5fa726255de0c8197834 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 2 May 2016 15:34:40 -0700 Subject: [PATCH 031/155] add more features --- .../Grpc.Core.Tests/Internal/AsyncCallTest.cs | 31 +++++++++++++++++-- .../Grpc.Core/Internal/AsyncCallBase.cs | 6 ++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs index 605072b3237..324b6825102 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs @@ -160,7 +160,7 @@ namespace Grpc.Core.Internal.Tests } [Test] - public void ServerStreaming_NoResponse1_Success() + public void ServerStreaming_NoResponse_Success1() { asyncCall.StartServerStreamingCall("request1"); var responseStream = new ClientResponseStream(asyncCall); @@ -176,7 +176,7 @@ namespace Grpc.Core.Internal.Tests } [Test] - public void ServerStreaming_NoResponse2_Success() + public void ServerStreaming_NoResponse_Success2() { asyncCall.StartServerStreamingCall("request1"); var responseStream = new ClientResponseStream(asyncCall); @@ -192,6 +192,22 @@ namespace Grpc.Core.Internal.Tests AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask); } + [Test] + public void ServerStreaming_NoResponse_ReadFailure() + { + asyncCall.StartServerStreamingCall("request1"); + var responseStream = new ClientResponseStream(asyncCall); + var readTask = responseStream.MoveNext(); + + fakeCall.ReceivedResponseHeadersHandler(true, new Metadata()); + Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count); + + fakeCall.ReceivedMessageHandler(false, null); // after a failed read, we rely on C core to deliver appropriate status code. + fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.Internal)); + + AssertStreamingResponseError(asyncCall, fakeCall, readTask, StatusCode.Internal); + } + ClientSideStatus CreateClientSideStatus(StatusCode statusCode) { return new ClientSideStatus(new Status(statusCode, ""), new Metadata()); @@ -236,6 +252,17 @@ namespace Grpc.Core.Internal.Tests Assert.AreEqual(0, asyncCall.GetTrailers().Count); } + static void AssertStreamingResponseError(AsyncCall asyncCall, FakeNativeCall fakeCall, Task moveNextTask, StatusCode expectedStatusCode) + { + Assert.IsTrue(moveNextTask.IsCompleted); + Assert.IsTrue(fakeCall.IsDisposed); + + var ex = Assert.ThrowsAsync(async () => await moveNextTask); + Assert.AreEqual(expectedStatusCode, asyncCall.GetStatus().StatusCode); + Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count); + Assert.AreEqual(0, asyncCall.GetTrailers().Count); + } + internal class FakeNativeCall : INativeCall { public UnaryResponseClientHandler UnaryResponseClientHandler diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs index ccd047f4695..877b997aba0 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs @@ -346,6 +346,10 @@ namespace Grpc.Core.Internal /// protected void HandleReadFinished(bool success, byte[] receivedMessage) { + // if success == false, received message will be null. It that case we will + // treat this completion as the last read an rely on C core to handle the failed + // read (e.g. deliver approriate statusCode on the clientside). + TRead msg = default(TRead); var deserializeException = (success && receivedMessage != null) ? TryDeserialize(receivedMessage, out msg) : null; @@ -370,8 +374,6 @@ namespace Grpc.Core.Internal ReleaseResourcesIfPossible(); } - // TODO: handle the case when success==false - if (deserializeException != null && !IsClient) { FireCompletion(origCompletionDelegate, default(TRead), new IOException("Failed to deserialize request message.", deserializeException)); From a83ad2aa65a04d16b484e7c4373b7db5575221a4 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 2 May 2016 17:13:00 -0700 Subject: [PATCH 032/155] add idempotency test --- .../Grpc.Core.Tests/ClientServerTest.cs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs index 6c13a4fa483..44be94c5ae6 100644 --- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs @@ -166,6 +166,37 @@ namespace Grpc.Core.Tests Assert.IsNotNull("xyz", call.GetTrailers()[0].Key); } + [Test] + public async Task ServerStreamingCall_EndOfStreamIsIdempotent() + { + helper.ServerStreamingHandler = new ServerStreamingServerMethod(async (request, responseStream, context) => + { + }); + + var call = Calls.AsyncServerStreamingCall(helper.CreateServerStreamingCall(), ""); + + Assert.IsFalse(await call.ResponseStream.MoveNext()); + Assert.IsFalse(await call.ResponseStream.MoveNext()); + } + + [Test] + public async Task ServerStreamingCall_ErrorCanBeAwaitedTwice() + { + helper.ServerStreamingHandler = new ServerStreamingServerMethod(async (request, responseStream, context) => + { + context.Status = new Status(StatusCode.InvalidArgument, ""); + }); + + var call = Calls.AsyncServerStreamingCall(helper.CreateServerStreamingCall(), ""); + + var ex = Assert.ThrowsAsync(async () => await call.ResponseStream.MoveNext()); + Assert.AreEqual(StatusCode.InvalidArgument, ex.Status.StatusCode); + + // attempting MoveNext again should result in throwing the same exception. + var ex2 = Assert.ThrowsAsync(async () => await call.ResponseStream.MoveNext()); + Assert.AreEqual(StatusCode.InvalidArgument, ex2.Status.StatusCode); + } + [Test] public async Task DuplexStreamingCall() { From 96f21a27cb7975d52aa92197536eef0ad8dca455 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 2 May 2016 17:17:18 -0700 Subject: [PATCH 033/155] make end-of-stream idempotent --- .../Grpc.Core.Tests/Internal/AsyncCallTest.cs | 39 +++++++++++++------ src/csharp/Grpc.Core/Internal/AsyncCall.cs | 5 +-- .../Grpc.Core/Internal/AsyncCallBase.cs | 39 ++++++++++++------- .../Grpc.Core/Internal/AsyncCallServer.cs | 5 +-- .../Internal/ClientResponseStream.cs | 4 +- .../Grpc.Core/Internal/ServerRequestStream.cs | 4 +- 6 files changed, 58 insertions(+), 38 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs index 324b6825102..96749cda14a 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs @@ -76,8 +76,8 @@ namespace Grpc.Core.Internal.Tests public void AsyncUnary_StreamingOperationsNotAllowed() { asyncCall.UnaryCallAsync("request1"); - Assert.Throws(typeof(InvalidOperationException), - () => asyncCall.StartReadMessage((x,y) => {})); + Assert.ThrowsAsync(typeof(InvalidOperationException), + async () => await asyncCall.ReadMessageAsync()); Assert.Throws(typeof(InvalidOperationException), () => asyncCall.StartSendMessage("abc", new WriteFlags(), (x,y) => {})); } @@ -123,8 +123,8 @@ namespace Grpc.Core.Internal.Tests public void ClientStreaming_StreamingReadNotAllowed() { asyncCall.ClientStreamingCallAsync(); - Assert.Throws(typeof(InvalidOperationException), - () => asyncCall.StartReadMessage((x,y) => {})); + Assert.ThrowsAsync(typeof(InvalidOperationException), + async () => await asyncCall.ReadMessageAsync()); } [Test] @@ -182,9 +182,6 @@ namespace Grpc.Core.Internal.Tests var responseStream = new ClientResponseStream(asyncCall); var readTask = responseStream.MoveNext(); - fakeCall.ReceivedResponseHeadersHandler(true, new Metadata()); - Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count); - // try alternative order of completions fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata())); fakeCall.ReceivedMessageHandler(true, null); @@ -199,15 +196,35 @@ namespace Grpc.Core.Internal.Tests var responseStream = new ClientResponseStream(asyncCall); var readTask = responseStream.MoveNext(); - fakeCall.ReceivedResponseHeadersHandler(true, new Metadata()); - Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count); - fakeCall.ReceivedMessageHandler(false, null); // after a failed read, we rely on C core to deliver appropriate status code. fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.Internal)); AssertStreamingResponseError(asyncCall, fakeCall, readTask, StatusCode.Internal); } + [Test] + public void ServerStreaming_MoreResponses_Success() + { + asyncCall.StartServerStreamingCall("request1"); + var responseStream = new ClientResponseStream(asyncCall); + + var readTask1 = responseStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, CreateResponsePayload()); + Assert.IsTrue(readTask1.Result); + Assert.AreEqual("response1", responseStream.Current); + + var readTask2 = responseStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, CreateResponsePayload()); + Assert.IsTrue(readTask2.Result); + Assert.AreEqual("response1", responseStream.Current); + + var readTask3 = responseStream.MoveNext(); + fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata())); + fakeCall.ReceivedMessageHandler(true, null); + + AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask3); + } + ClientSideStatus CreateClientSideStatus(StatusCode statusCode) { return new ClientSideStatus(new Status(statusCode, ""), new Metadata()); @@ -236,7 +253,6 @@ namespace Grpc.Core.Internal.Tests Assert.IsFalse(moveNextTask.Result); Assert.AreEqual(Status.DefaultSuccess, asyncCall.GetStatus()); - Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count); Assert.AreEqual(0, asyncCall.GetTrailers().Count); } @@ -259,7 +275,6 @@ namespace Grpc.Core.Internal.Tests var ex = Assert.ThrowsAsync(async () => await moveNextTask); Assert.AreEqual(expectedStatusCode, asyncCall.GetStatus().StatusCode); - Assert.AreEqual(0, asyncCall.ResponseHeadersAsync.Result.Count); Assert.AreEqual(0, asyncCall.GetTrailers().Count); } diff --git a/src/csharp/Grpc.Core/Internal/AsyncCall.cs b/src/csharp/Grpc.Core/Internal/AsyncCall.cs index 50ba617cdb1..f522174bd0f 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCall.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCall.cs @@ -241,11 +241,10 @@ namespace Grpc.Core.Internal /// /// Receives a streaming response. Only one pending read action is allowed at any given time. - /// completionDelegate is called when the operation finishes. /// - public void StartReadMessage(AsyncCompletionDelegate completionDelegate) + public Task ReadMessageAsync() { - StartReadMessageInternal(completionDelegate); + return ReadMessageInternalAsync(); } /// diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs index 877b997aba0..abacfabadb6 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs @@ -68,7 +68,7 @@ namespace Grpc.Core.Internal protected bool cancelRequested; protected AsyncCompletionDelegate sendCompletionDelegate; // Completion of a pending send or sendclose if not null. - protected AsyncCompletionDelegate readCompletionDelegate; // Completion of a pending send or sendclose if not null. + protected TaskCompletionSource streamingReadTcs; // Completion of a pending streaming read if not null. protected bool readingDone; // True if last read (i.e. read with null payload) was already received. protected bool halfcloseRequested; // True if send close have been initiated. @@ -150,15 +150,25 @@ namespace Grpc.Core.Internal /// Initiates reading a message. Only one read operation can be active at a time. /// completionDelegate is invoked upon completion. /// - protected void StartReadMessageInternal(AsyncCompletionDelegate completionDelegate) + protected Task ReadMessageInternalAsync() { lock (myLock) { - GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null"); CheckReadingAllowed(); + if (readingDone) + { + // the last read that returns null or throws an exception is idempotent + // and maintain its state. + GrpcPreconditions.CheckState(streamingReadTcs != null, "Call does not support streaming reads."); + return streamingReadTcs.Task; + } + + GrpcPreconditions.CheckState(streamingReadTcs == null, "Only one read can be pending at a time"); + GrpcPreconditions.CheckState(!disposed); call.StartReceiveMessage(HandleReadFinished); - readCompletionDelegate = completionDelegate; + streamingReadTcs = new TaskCompletionSource(); + return streamingReadTcs.Task; } } @@ -216,10 +226,6 @@ namespace Grpc.Core.Internal protected virtual void CheckReadingAllowed() { GrpcPreconditions.CheckState(started); - GrpcPreconditions.CheckState(!disposed); - - GrpcPreconditions.CheckState(!readingDone, "Stream has already been closed."); - GrpcPreconditions.CheckState(readCompletionDelegate == null, "Only one read can be pending at a time"); } protected void CheckNotCancelled() @@ -353,12 +359,10 @@ namespace Grpc.Core.Internal TRead msg = default(TRead); var deserializeException = (success && receivedMessage != null) ? TryDeserialize(receivedMessage, out msg) : null; - AsyncCompletionDelegate origCompletionDelegate = null; + TaskCompletionSource origTcs = null; lock (myLock) { - origCompletionDelegate = readCompletionDelegate; - readCompletionDelegate = null; - + origTcs = streamingReadTcs; if (receivedMessage == null) { // This was the last read. @@ -368,18 +372,25 @@ namespace Grpc.Core.Internal if (deserializeException != null && IsClient) { readingDone = true; + + // TODO(jtattermusch): it might be too late to set the status CancelWithStatus(DeserializeResponseFailureStatus); } + if (!readingDone) + { + streamingReadTcs = null; + } + ReleaseResourcesIfPossible(); } if (deserializeException != null && !IsClient) { - FireCompletion(origCompletionDelegate, default(TRead), new IOException("Failed to deserialize request message.", deserializeException)); + origTcs.SetException(new IOException("Failed to deserialize request message.", deserializeException)); return; } - FireCompletion(origCompletionDelegate, msg, null); + origTcs.SetResult(msg); } } } diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs index bea2b3660c3..cce480b2c4a 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs @@ -91,11 +91,10 @@ namespace Grpc.Core.Internal /// /// Receives a streaming request. Only one pending read action is allowed at any given time. - /// completionDelegate is called when the operation finishes. /// - public void StartReadMessage(AsyncCompletionDelegate completionDelegate) + public Task ReadMessageAsync() { - StartReadMessageInternal(completionDelegate); + return ReadMessageInternalAsync(); } /// diff --git a/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs b/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs index d6e34a0f044..ad9423ff58c 100644 --- a/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs +++ b/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs @@ -68,9 +68,7 @@ namespace Grpc.Core.Internal { throw new InvalidOperationException("Cancellation of individual reads is not supported."); } - var taskSource = new AsyncCompletionTaskSource(); - call.StartReadMessage(taskSource.CompletionDelegate); - var result = await taskSource.Task.ConfigureAwait(false); + var result = await call.ReadMessageAsync().ConfigureAwait(false); this.current = result; if (result == null) diff --git a/src/csharp/Grpc.Core/Internal/ServerRequestStream.cs b/src/csharp/Grpc.Core/Internal/ServerRequestStream.cs index e7be82c3185..d76030d1add 100644 --- a/src/csharp/Grpc.Core/Internal/ServerRequestStream.cs +++ b/src/csharp/Grpc.Core/Internal/ServerRequestStream.cs @@ -68,9 +68,7 @@ namespace Grpc.Core.Internal { throw new InvalidOperationException("Cancellation of individual reads is not supported."); } - var taskSource = new AsyncCompletionTaskSource(); - call.StartReadMessage(taskSource.CompletionDelegate); - var result = await taskSource.Task.ConfigureAwait(false); + var result = await call.ReadMessageAsync().ConfigureAwait(false); this.current = result; return result != null; } From a2966776f7f40e2095bba1f4d8b5922c239dfe9e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 2 May 2016 18:29:23 -0700 Subject: [PATCH 034/155] add more tests --- .../Grpc.Core.Tests/Internal/AsyncCallTest.cs | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs index 96749cda14a..ed2d22815b6 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs @@ -41,6 +41,9 @@ using NUnit.Framework; namespace Grpc.Core.Internal.Tests { + /// + /// Uses fake native call to test interaction of wrapping code with C core in different situations. + /// public class AsyncCallTest { Channel channel; @@ -151,6 +154,99 @@ namespace Grpc.Core.Internal.Tests AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.InvalidArgument); } + [Test] + public void ClientStreaming_MoreRequests_Success() + { + var resultTask = asyncCall.ClientStreamingCallAsync(); + var requestStream = new ClientRequestStream(asyncCall); + + var writeTask = requestStream.WriteAsync("request1"); + fakeCall.SendCompletionHandler(true); + writeTask.Wait(); + + var writeTask2 = requestStream.WriteAsync("request2"); + fakeCall.SendCompletionHandler(true); + writeTask2.Wait(); + + var completeTask = requestStream.CompleteAsync(); + fakeCall.SendCompletionHandler(true); + completeTask.Wait(); + + fakeCall.UnaryResponseClientHandler(true, + new ClientSideStatus(Status.DefaultSuccess, new Metadata()), + CreateResponsePayload(), + new Metadata()); + + AssertUnaryResponseSuccess(asyncCall, fakeCall, resultTask); + } + + [Test] + public void ClientStreaming_WriteFailure() + { + var resultTask = asyncCall.ClientStreamingCallAsync(); + var requestStream = new ClientRequestStream(asyncCall); + + var writeTask = requestStream.WriteAsync("request1"); + fakeCall.SendCompletionHandler(false); + Assert.ThrowsAsync(typeof(InvalidOperationException), async () => await writeTask); + + fakeCall.UnaryResponseClientHandler(true, + CreateClientSideStatus(StatusCode.Internal), + CreateResponsePayload(), + new Metadata()); + + AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.Internal); + } + + [Test] + public void ClientStreaming_WriteAfterReceivingStatusFails() + { + var resultTask = asyncCall.ClientStreamingCallAsync(); + var requestStream = new ClientRequestStream(asyncCall); + + fakeCall.UnaryResponseClientHandler(true, + new ClientSideStatus(Status.DefaultSuccess, new Metadata()), + CreateResponsePayload(), + new Metadata()); + + AssertUnaryResponseSuccess(asyncCall, fakeCall, resultTask); + Assert.Throws(typeof(InvalidOperationException), () => requestStream.WriteAsync("request1")); + } + + [Test] + public void ClientStreaming_CompleteAfterReceivingStatusSucceeds() + { + var resultTask = asyncCall.ClientStreamingCallAsync(); + var requestStream = new ClientRequestStream(asyncCall); + + fakeCall.UnaryResponseClientHandler(true, + new ClientSideStatus(Status.DefaultSuccess, new Metadata()), + CreateResponsePayload(), + new Metadata()); + + AssertUnaryResponseSuccess(asyncCall, fakeCall, resultTask); + Assert.DoesNotThrowAsync(async () => await requestStream.CompleteAsync()); + } + + [Test] + public void ClientStreaming_WriteAfterCancellationRequestFails() + { + var resultTask = asyncCall.ClientStreamingCallAsync(); + var requestStream = new ClientRequestStream(asyncCall); + + asyncCall.Cancel(); + Assert.IsTrue(fakeCall.IsCancelled); + + Assert.Throws(typeof(OperationCanceledException), () => requestStream.WriteAsync("request1")); + + fakeCall.UnaryResponseClientHandler(true, + CreateClientSideStatus(StatusCode.Cancelled), + CreateResponsePayload(), + new Metadata()); + + AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.Cancelled); + } + [Test] public void ServerStreaming_StreamingSendNotAllowed() { @@ -225,6 +321,116 @@ namespace Grpc.Core.Internal.Tests AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask3); } + [Test] + public void DuplexStreaming_NoRequestNoResponse_Success() + { + asyncCall.StartDuplexStreamingCall(); + var requestStream = new ClientRequestStream(asyncCall); + var responseStream = new ClientResponseStream(asyncCall); + + var writeTask1 = requestStream.CompleteAsync(); + fakeCall.SendCompletionHandler(true); + Assert.DoesNotThrowAsync(async () => await writeTask1); + + var readTask = responseStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata())); + + AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask); + } + + [Test] + public void DuplexStreaming_WriteAfterReceivingStatusFails() + { + asyncCall.StartDuplexStreamingCall(); + var requestStream = new ClientRequestStream(asyncCall); + var responseStream = new ClientResponseStream(asyncCall); + + var readTask = responseStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata())); + + AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask); + + Assert.ThrowsAsync(typeof(InvalidOperationException), async () => await requestStream.WriteAsync("request1")); + } + + [Test] + public void DuplexStreaming_CompleteAfterReceivingStatusFails() + { + asyncCall.StartDuplexStreamingCall(); + var requestStream = new ClientRequestStream(asyncCall); + var responseStream = new ClientResponseStream(asyncCall); + + var readTask = responseStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + fakeCall.ReceivedStatusOnClientHandler(true, new ClientSideStatus(Status.DefaultSuccess, new Metadata())); + + AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask); + + Assert.DoesNotThrowAsync(async () => await requestStream.CompleteAsync()); + } + + [Test] + public void DuplexStreaming_WriteAfterCancellationRequestFails() + { + asyncCall.StartDuplexStreamingCall(); + var requestStream = new ClientRequestStream(asyncCall); + var responseStream = new ClientResponseStream(asyncCall); + + asyncCall.Cancel(); + Assert.IsTrue(fakeCall.IsCancelled); + Assert.Throws(typeof(OperationCanceledException), () => requestStream.WriteAsync("request1")); + + var readTask = responseStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.Cancelled)); + + AssertStreamingResponseError(asyncCall, fakeCall, readTask, StatusCode.Cancelled); + } + + [Test] + public void DuplexStreaming_ReadAfterCancellationRequestCanSucceed() + { + asyncCall.StartDuplexStreamingCall(); + var responseStream = new ClientResponseStream(asyncCall); + + asyncCall.Cancel(); + Assert.IsTrue(fakeCall.IsCancelled); + + var readTask1 = responseStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, CreateResponsePayload()); + Assert.IsTrue(readTask1.Result); + Assert.AreEqual("response1", responseStream.Current); + + var readTask2 = responseStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.Cancelled)); + + AssertStreamingResponseError(asyncCall, fakeCall, readTask2, StatusCode.Cancelled); + } + + [Test] + public void DuplexStreaming_ReadStartedBeforeCancellationRequestCanSucceed() + { + asyncCall.StartDuplexStreamingCall(); + var responseStream = new ClientResponseStream(asyncCall); + + var readTask1 = responseStream.MoveNext(); // initiate the read before cancel request + asyncCall.Cancel(); + Assert.IsTrue(fakeCall.IsCancelled); + + fakeCall.ReceivedMessageHandler(true, CreateResponsePayload()); + Assert.IsTrue(readTask1.Result); + Assert.AreEqual("response1", responseStream.Current); + + var readTask2 = responseStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + fakeCall.ReceivedStatusOnClientHandler(true, CreateClientSideStatus(StatusCode.Cancelled)); + + AssertStreamingResponseError(asyncCall, fakeCall, readTask2, StatusCode.Cancelled); + } + ClientSideStatus CreateClientSideStatus(StatusCode statusCode) { return new ClientSideStatus(new Status(statusCode, ""), new Metadata()); From 51050562460bee6e0ecf831af037430a249f6d73 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 09:39:41 -0700 Subject: [PATCH 035/155] add one more client server test --- .../Grpc.Core.Tests/ClientServerTest.cs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs index 44be94c5ae6..d4a41f293eb 100644 --- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs @@ -239,6 +239,39 @@ namespace Grpc.Core.Tests Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode); } + [Test] + public async Task ClientStreamingCall_ServerSideReadAfterCancelNotificationReturnsNull() + { + var handlerStartedBarrier = new TaskCompletionSource(); + var cancelNotificationReceivedBarrier = new TaskCompletionSource(); + var successTcs = new TaskCompletionSource(); + + helper.ClientStreamingHandler = new ClientStreamingServerMethod(async (requestStream, context) => + { + + handlerStartedBarrier.SetResult(null); + + // wait for cancellation to be delivered. + context.CancellationToken.Register(() => cancelNotificationReceivedBarrier.SetResult(null)); + await cancelNotificationReceivedBarrier.Task; + + var moveNextResult = await requestStream.MoveNext(); + successTcs.SetResult(!moveNextResult ? "SUCCESS" : "FAIL"); + return ""; + }); + + var cts = new CancellationTokenSource(); + var call = Calls.AsyncClientStreamingCall(helper.CreateClientStreamingCall(new CallOptions(cancellationToken: cts.Token))); + + await handlerStartedBarrier.Task; + cts.Cancel(); + + var ex = Assert.ThrowsAsync(async () => await call.ResponseAsync); + Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode); + + Assert.AreEqual("SUCCESS", await successTcs.Task); + } + [Test] public async Task AsyncUnaryCall_EchoMetadata() { From 230c93279f753164db49e46db7204a1b8408ee09 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 09:40:18 -0700 Subject: [PATCH 036/155] fixup --- src/csharp/Grpc.Core.Tests/ClientServerTest.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs index d4a41f293eb..d92addbf548 100644 --- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs @@ -248,7 +248,6 @@ namespace Grpc.Core.Tests helper.ClientStreamingHandler = new ClientStreamingServerMethod(async (requestStream, context) => { - handlerStartedBarrier.SetResult(null); // wait for cancellation to be delivered. From 19abba3661c7d39231324eb815fec62d4d3187ea Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 10:06:29 -0700 Subject: [PATCH 037/155] Improve docs on GRPC_OP_RECV_CLOSE_ON_SERVER --- include/grpc/impl/codegen/grpc_types.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 4c7373006b2..fa287d52388 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -307,7 +307,9 @@ typedef enum { GRPC_OP_RECV_STATUS_ON_CLIENT, /** Receive close on the server: one and only one must be made on the server. - This op completes after the close has been received by the server. */ + This op completes after the close has been received by the server. + This operation always succeeds, meaning ops paired with this operation + will also appear to succeed, even though they may not have. */ GRPC_OP_RECV_CLOSE_ON_SERVER } grpc_op_type; From 8a1d8052eb8629643333ca4449180e27f65aa8dc Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 3 May 2016 10:44:56 -0700 Subject: [PATCH 038/155] Added bad_client test to check that the server rejects requests with too much metadata and refuses to send responses with too much metadata. --- Makefile | 24 + include/grpc/impl/codegen/slice_buffer.h | 5 +- .../chttp2/transport/chttp2_transport.c | 24 +- .../chttp2/transport/frame_rst_stream.c | 5 + .../ext/transport/chttp2/transport/internal.h | 15 +- .../ext/transport/chttp2/transport/parsing.c | 30 +- test/core/bad_client/bad_client.c | 41 +- test/core/bad_client/bad_client.h | 18 +- test/core/bad_client/gen_build_yaml.py | 3 +- test/core/bad_client/tests/badreq.c | 10 +- .../core/bad_client/tests/connection_prefix.c | 49 +- test/core/bad_client/tests/headers.c | 120 ++--- .../bad_client/tests/initial_settings_frame.c | 54 +- test/core/bad_client/tests/large_metadata.c | 478 ++++++++++++++++++ .../bad_client/tests/large_metadata.headers | 106 ++++ .../tests/server_registered_method.c | 22 +- test/core/bad_client/tests/simple_request.c | 24 +- test/core/bad_client/tests/unknown_frame.c | 2 +- tools/codegen/core/gen_header_frame.py | 7 +- tools/run_tests/sources_and_headers.json | 17 + tools/run_tests/tests.json | 21 + vsprojects/buildtests_c.sln | 28 + .../large_metadata_bad_client_test.vcxproj | 202 ++++++++ ...e_metadata_bad_client_test.vcxproj.filters | 24 + 24 files changed, 1145 insertions(+), 184 deletions(-) create mode 100644 test/core/bad_client/tests/large_metadata.c create mode 100644 test/core/bad_client/tests/large_metadata.headers create mode 100644 vsprojects/vcxproj/test/large_metadata_bad_client_test/large_metadata_bad_client_test.vcxproj create mode 100644 vsprojects/vcxproj/test/large_metadata_bad_client_test/large_metadata_bad_client_test.vcxproj.filters diff --git a/Makefile b/Makefile index 922e0b0568f..e85a5266fec 100644 --- a/Makefile +++ b/Makefile @@ -1089,6 +1089,7 @@ connection_prefix_bad_client_test: $(BINDIR)/$(CONFIG)/connection_prefix_bad_cli head_of_line_blocking_bad_client_test: $(BINDIR)/$(CONFIG)/head_of_line_blocking_bad_client_test headers_bad_client_test: $(BINDIR)/$(CONFIG)/headers_bad_client_test initial_settings_frame_bad_client_test: $(BINDIR)/$(CONFIG)/initial_settings_frame_bad_client_test +large_metadata_bad_client_test: $(BINDIR)/$(CONFIG)/large_metadata_bad_client_test server_registered_method_bad_client_test: $(BINDIR)/$(CONFIG)/server_registered_method_bad_client_test simple_request_bad_client_test: $(BINDIR)/$(CONFIG)/simple_request_bad_client_test unknown_frame_bad_client_test: $(BINDIR)/$(CONFIG)/unknown_frame_bad_client_test @@ -1318,6 +1319,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/head_of_line_blocking_bad_client_test \ $(BINDIR)/$(CONFIG)/headers_bad_client_test \ $(BINDIR)/$(CONFIG)/initial_settings_frame_bad_client_test \ + $(BINDIR)/$(CONFIG)/large_metadata_bad_client_test \ $(BINDIR)/$(CONFIG)/server_registered_method_bad_client_test \ $(BINDIR)/$(CONFIG)/simple_request_bad_client_test \ $(BINDIR)/$(CONFIG)/unknown_frame_bad_client_test \ @@ -1656,6 +1658,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/headers_bad_client_test || ( echo test headers_bad_client_test failed ; exit 1 ) $(E) "[RUN] Testing initial_settings_frame_bad_client_test" $(Q) $(BINDIR)/$(CONFIG)/initial_settings_frame_bad_client_test || ( echo test initial_settings_frame_bad_client_test failed ; exit 1 ) + $(E) "[RUN] Testing large_metadata_bad_client_test" + $(Q) $(BINDIR)/$(CONFIG)/large_metadata_bad_client_test || ( echo test large_metadata_bad_client_test failed ; exit 1 ) $(E) "[RUN] Testing server_registered_method_bad_client_test" $(Q) $(BINDIR)/$(CONFIG)/server_registered_method_bad_client_test || ( echo test server_registered_method_bad_client_test failed ; exit 1 ) $(E) "[RUN] Testing simple_request_bad_client_test" @@ -13100,6 +13104,26 @@ ifneq ($(NO_DEPS),true) endif +LARGE_METADATA_BAD_CLIENT_TEST_SRC = \ + test/core/bad_client/tests/large_metadata.c \ + +LARGE_METADATA_BAD_CLIENT_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LARGE_METADATA_BAD_CLIENT_TEST_SRC)))) + + +$(BINDIR)/$(CONFIG)/large_metadata_bad_client_test: $(LARGE_METADATA_BAD_CLIENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libbad_client_test.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(LARGE_METADATA_BAD_CLIENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libbad_client_test.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) -o $(BINDIR)/$(CONFIG)/large_metadata_bad_client_test + +$(OBJDIR)/$(CONFIG)/test/core/bad_client/tests/large_metadata.o: $(LIBDIR)/$(CONFIG)/libbad_client_test.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_large_metadata_bad_client_test: $(LARGE_METADATA_BAD_CLIENT_TEST_OBJS:.o=.dep) + +ifneq ($(NO_DEPS),true) +-include $(LARGE_METADATA_BAD_CLIENT_TEST_OBJS:.o=.dep) +endif + + SERVER_REGISTERED_METHOD_BAD_CLIENT_TEST_SRC = \ test/core/bad_client/tests/server_registered_method.c \ diff --git a/include/grpc/impl/codegen/slice_buffer.h b/include/grpc/impl/codegen/slice_buffer.h index 8ca51baa471..7858021600a 100644 --- a/include/grpc/impl/codegen/slice_buffer.h +++ b/include/grpc/impl/codegen/slice_buffer.h @@ -42,9 +42,8 @@ extern "C" { #define GRPC_SLICE_BUFFER_INLINE_ELEMENTS 8 -/* Represents an expandable array of slices, to be interpreted as a single item - TODO(ctiller): inline some small number of elements into the struct, to - avoid per-call allocations */ +/* Represents an expandable array of slices, to be interpreted as a + single item. */ typedef struct { /* slices in the array */ gpr_slice *slices; diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 6ee5e0680c7..221c7c80505 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -522,7 +522,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; *t->accepting_stream = s; grpc_chttp2_stream_map_add(&t->parsing_stream_map, s->global.id, s); - s->global.in_stream_map = 1; + s->global.in_stream_map = true; } grpc_chttp2_run_with_global_lock(exec_ctx, t, s, finish_init_stream_locked, @@ -838,7 +838,7 @@ static void maybe_start_some_streams( grpc_chttp2_stream_map_add( &TRANSPORT_FROM_GLOBAL(transport_global)->new_stream_map, stream_global->id, STREAM_FROM_GLOBAL(stream_global)); - stream_global->in_stream_map = 1; + stream_global->in_stream_map = true; transport_global->concurrent_stream_count++; grpc_chttp2_become_writable(transport_global, stream_global); } @@ -944,8 +944,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; if (metadata_size > metadata_peer_limit) { gpr_log(GPR_DEBUG, - "initial metadata size exceeds peer limit (%lu vs. %lu)", - metadata_size, metadata_peer_limit); + "to-be-sent initial metadata size exceeds peer limit " + "(%lu vs. %lu)", metadata_size, metadata_peer_limit); cancel_from_api(exec_ctx, transport_global, stream_global, GRPC_STATUS_RESOURCE_EXHAUSTED); } else { @@ -998,8 +998,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; if (metadata_size > metadata_peer_limit) { gpr_log(GPR_DEBUG, - "trailing metadata size exceeds peer limit (%lu vs. %lu)", - metadata_size, metadata_peer_limit); + "to-be-sent trailing metadata size exceeds peer limit " + "(%lu vs. %lu)", metadata_size, metadata_peer_limit); cancel_from_api(exec_ctx, transport_global, stream_global, GRPC_STATUS_RESOURCE_EXHAUSTED); } else { @@ -1259,7 +1259,7 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, s = grpc_chttp2_stream_map_delete(&t->new_stream_map, id); } GPR_ASSERT(s); - s->global.in_stream_map = 0; + s->global.in_stream_map = false; if (t->parsing.incoming_stream == &s->parsing) { t->parsing.incoming_stream = NULL; grpc_chttp2_parsing_become_skip_parser(exec_ctx, &t->parsing); @@ -1339,7 +1339,7 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, GRPC_MDSTR_GRPC_MESSAGE, grpc_mdstr_from_slice(gpr_slice_ref(*slice)))); } - stream_global->published_trailing_metadata = 1; + stream_global->published_trailing_metadata = true; grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } if (slice) { @@ -1369,13 +1369,13 @@ void grpc_chttp2_mark_stream_closed( } grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); if (close_reads && !stream_global->read_closed) { - stream_global->read_closed = 1; - stream_global->published_initial_metadata = 1; - stream_global->published_trailing_metadata = 1; + stream_global->read_closed = true; + stream_global->published_initial_metadata = true; + stream_global->published_trailing_metadata = true; decrement_active_streams_locked(exec_ctx, transport_global, stream_global); } if (close_writes && !stream_global->write_closed) { - stream_global->write_closed = 1; + stream_global->write_closed = true; if (TRANSPORT_FROM_GLOBAL(transport_global)->executor.writing_active) { GRPC_CHTTP2_STREAM_REF(stream_global, "finish_writes"); grpc_chttp2_list_add_closed_waiting_for_writing(transport_global, diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index 22467e9ddd2..7f01105e3ef 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -45,15 +45,20 @@ gpr_slice grpc_chttp2_rst_stream_create(uint32_t id, uint32_t code, stats->framing_bytes += frame_size; uint8_t *p = GPR_SLICE_START_PTR(slice); + // Frame size. *p++ = 0; *p++ = 0; *p++ = 4; + // Frame type. *p++ = GRPC_CHTTP2_FRAME_RST_STREAM; + // Flags. *p++ = 0; + // Stream ID. *p++ = (uint8_t)(id >> 24); *p++ = (uint8_t)(id >> 16); *p++ = (uint8_t)(id >> 8); *p++ = (uint8_t)(id); + // Error code. *p++ = (uint8_t)(code >> 24); *p++ = (uint8_t)(code >> 16); *p++ = (uint8_t)(code >> 8); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index d8f17a30fc1..2f3714b2bb4 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -419,24 +419,21 @@ typedef struct { /** number of streams that are currently being read */ gpr_refcount active_streams; - /** when the application requests writes be closed, the write_closed is - 'queued'; when the close is flow controlled into the send path, we are - 'sending' it; when the write has been performed it is 'sent' */ + /** Is this stream closed for writing. */ bool write_closed; - /** is this stream reading half-closed (boolean) */ + /** Is this stream reading half-closed. */ bool read_closed; - /** are all published incoming byte streams closed */ + /** Are all published incoming byte streams closed. */ bool all_incoming_byte_streams_finished; - /** is this stream in the stream map? (boolean) */ + /** Is this stream in the stream map. */ bool in_stream_map; - /** has this stream seen an error? if 1, then pending incoming frames - can be thrown away */ + /** Has this stream seen an error. + If true, then pending incoming frames can be thrown away. */ bool seen_error; bool exceeded_metadata_size; bool published_initial_metadata; bool published_trailing_metadata; - bool faked_trailing_metadata; grpc_chttp2_incoming_metadata_buffer received_initial_metadata; grpc_chttp2_incoming_metadata_buffer received_trailing_metadata; diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index f1018733379..29c26fb3bc5 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -634,10 +634,17 @@ static void on_initial_header(void *tp, grpc_mdelem *md) { GRPC_MDELEM_LENGTH(md); grpc_chttp2_transport_global *transport_global = &TRANSPORT_FROM_PARSING(transport_parsing)->global; - if (new_size > transport_global->settings - [GRPC_LOCAL_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]) { - stream_parsing->seen_error = true; - stream_parsing->exceeded_metadata_size = true; + const size_t metadata_size_limit = + transport_global->settings[GRPC_LOCAL_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; + if (new_size > metadata_size_limit) { + if (!stream_parsing->exceeded_metadata_size) { + gpr_log(GPR_DEBUG, + "received initial metadata size exceeds limit (%lu vs. %lu)", + new_size, metadata_size_limit); + stream_parsing->seen_error = true; + stream_parsing->exceeded_metadata_size = true; + } GRPC_MDELEM_UNREF(md); } else { grpc_chttp2_incoming_metadata_buffer_add( @@ -673,10 +680,17 @@ static void on_trailing_header(void *tp, grpc_mdelem *md) { GRPC_MDELEM_LENGTH(md); grpc_chttp2_transport_global *transport_global = &TRANSPORT_FROM_PARSING(transport_parsing)->global; - if (new_size > transport_global->settings - [GRPC_LOCAL_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]) { - stream_parsing->seen_error = true; - stream_parsing->exceeded_metadata_size = true; + const size_t metadata_size_limit = + transport_global->settings[GRPC_LOCAL_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; + if (new_size > metadata_size_limit) { + if (!stream_parsing->exceeded_metadata_size) { + gpr_log(GPR_DEBUG, + "received trailing metadata size exceeds limit (%lu vs. %lu)", + new_size, metadata_size_limit); + stream_parsing->seen_error = true; + stream_parsing->exceeded_metadata_size = true; + } GRPC_MDELEM_UNREF(md); } else { grpc_chttp2_incoming_metadata_buffer_add( diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c index cd5b5412492..aa9125dc7a9 100644 --- a/test/core/bad_client/bad_client.c +++ b/test/core/bad_client/bad_client.c @@ -75,9 +75,23 @@ static void server_setup_transport(void *ts, grpc_transport *transport) { grpc_exec_ctx_finish(&exec_ctx); } -void grpc_run_bad_client_test(grpc_bad_client_server_side_validator validator, - const char *client_payload, - size_t client_payload_length, uint32_t flags) { +typedef struct { + grpc_bad_client_client_stream_validator validator; + gpr_slice_buffer incoming; + gpr_event read_done; +} read_args; + +static void read_done(grpc_exec_ctx *exec_ctx, void *arg, bool success) { + read_args *a = arg; + a->validator(&a->incoming); + gpr_event_set(&a->read_done, (void *)1); +} + +void grpc_run_bad_client_test( + grpc_bad_client_server_side_validator server_validator, + grpc_bad_client_client_stream_validator client_validator, + const char *client_payload, + size_t client_payload_length, uint32_t flags) { grpc_endpoint_pair sfd; thd_args a; gpr_thd_id id; @@ -108,7 +122,7 @@ void grpc_run_bad_client_test(grpc_bad_client_server_side_validator validator, a.cq = grpc_completion_queue_create(NULL); gpr_event_init(&a.done_thd); gpr_event_init(&a.done_write); - a.validator = validator; + a.validator = server_validator; grpc_server_register_completion_queue(a.server, a.cq, NULL); a.registered_method = grpc_server_register_method(a.server, GRPC_BAD_CLIENT_REGISTERED_METHOD, @@ -151,8 +165,23 @@ void grpc_run_bad_client_test(grpc_bad_client_server_side_validator validator, GPR_ASSERT(gpr_event_wait(&a.done_thd, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); - /* Shutdown */ - if (sfd.client) { + if (sfd.client != NULL) { + // Validate client stream, if requested. + if (client_validator != NULL) { + read_args args; + args.validator = client_validator; + gpr_slice_buffer_init(&args.incoming); + gpr_event_init(&args.read_done); + grpc_closure read_done_closure; + grpc_closure_init(&read_done_closure, read_done, &args); + grpc_endpoint_read(&exec_ctx, sfd.client, &args.incoming, + &read_done_closure); + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(gpr_event_wait(&args.read_done, + GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + gpr_slice_buffer_destroy(&args.incoming); + } + // Shutdown. grpc_endpoint_shutdown(&exec_ctx, sfd.client); grpc_endpoint_destroy(&exec_ctx, sfd.client); grpc_exec_ctx_finish(&exec_ctx); diff --git a/test/core/bad_client/bad_client.h b/test/core/bad_client/bad_client.h index 19ddba83bf0..b6e8a6dd5b0 100644 --- a/test/core/bad_client/bad_client.h +++ b/test/core/bad_client/bad_client.h @@ -44,18 +44,24 @@ typedef void (*grpc_bad_client_server_side_validator)(grpc_server *server, grpc_completion_queue *cq, void *registered_method); +typedef void (*grpc_bad_client_client_stream_validator)( + gpr_slice_buffer *incoming); + #define GRPC_BAD_CLIENT_DISCONNECT 1 /* Test runner. Create a server, and send client_payload to it as bytes from a client. - Execute validator in a separate thread to assert that the bytes are + Execute server_validator in a separate thread to assert that the bytes are handled as expected. */ -void grpc_run_bad_client_test(grpc_bad_client_server_side_validator validator, - const char *client_payload, - size_t client_payload_length, uint32_t flags); +void grpc_run_bad_client_test( + grpc_bad_client_server_side_validator server_validator, + grpc_bad_client_client_stream_validator client_validator, + const char *client_payload, size_t client_payload_length, uint32_t flags); -#define GRPC_RUN_BAD_CLIENT_TEST(validator, payload, flags) \ - grpc_run_bad_client_test(validator, payload, sizeof(payload) - 1, flags) +#define GRPC_RUN_BAD_CLIENT_TEST(server_validator, client_validator, \ + payload, flags) \ + grpc_run_bad_client_test(server_validator, client_validator, \ + payload, sizeof(payload) - 1, flags) #endif /* GRPC_TEST_CORE_BAD_CLIENT_BAD_CLIENT_H */ diff --git a/test/core/bad_client/gen_build_yaml.py b/test/core/bad_client/gen_build_yaml.py index d49858ed6d8..fb86525b1a1 100755 --- a/test/core/bad_client/gen_build_yaml.py +++ b/test/core/bad_client/gen_build_yaml.py @@ -29,7 +29,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Generates the appropriate build.json data for all the end2end tests.""" +"""Generates the appropriate build.json data for all the bad_client tests.""" import collections @@ -45,6 +45,7 @@ BAD_CLIENT_TESTS = { 'headers': default_test_options._replace(cpu_cost=0.2), 'initial_settings_frame': default_test_options._replace(cpu_cost=0.2), 'head_of_line_blocking': default_test_options, + 'large_metadata': default_test_options, 'server_registered_method': default_test_options, 'simple_request': default_test_options, 'window_overflow': default_test_options, diff --git a/test/core/bad_client/tests/badreq.c b/test/core/bad_client/tests/badreq.c index b17e3b35ee4..5d9ffef3f20 100644 --- a/test/core/bad_client/tests/badreq.c +++ b/test/core/bad_client/tests/badreq.c @@ -56,7 +56,7 @@ int main(int argc, char **argv) { /* invalid content type */ GRPC_RUN_BAD_CLIENT_TEST( - verifier, PFX_STR + verifier, NULL, PFX_STR "\x00\x00\xc2\x01\x04\x00\x00\x00\x01" "\x10\x05:path\x08/foo/bar" "\x10\x07:scheme\x04http" @@ -71,7 +71,7 @@ int main(int argc, char **argv) { /* invalid te */ GRPC_RUN_BAD_CLIENT_TEST( - verifier, PFX_STR + verifier, NULL, PFX_STR "\x00\x00\xcb\x01\x04\x00\x00\x00\x01" "\x10\x05:path\x08/foo/bar" "\x10\x07:scheme\x04http" @@ -88,7 +88,7 @@ int main(int argc, char **argv) { /* two path headers */ GRPC_RUN_BAD_CLIENT_TEST( - verifier, PFX_STR + verifier, NULL, PFX_STR "\x00\x00\xd9\x01\x04\x00\x00\x00\x01" "\x10\x05:path\x08/foo/bar" "\x10\x05:path\x08/foo/bah" @@ -105,7 +105,7 @@ int main(int argc, char **argv) { /* bad accept-encoding algorithm */ GRPC_RUN_BAD_CLIENT_TEST( - verifier, PFX_STR + verifier, NULL, PFX_STR "\x00\x00\xd2\x01\x04\x00\x00\x00\x01" "\x10\x05:path\x08/foo/bar" "\x10\x07:scheme\x04http" @@ -121,7 +121,7 @@ int main(int argc, char **argv) { /* bad grpc-encoding algorithm */ GRPC_RUN_BAD_CLIENT_TEST( - verifier, PFX_STR + verifier, NULL, PFX_STR "\x00\x00\xf5\x01\x04\x00\x00\x00\x01" "\x10\x05:path\x08/foo/bar" "\x10\x07:scheme\x04http" diff --git a/test/core/bad_client/tests/connection_prefix.c b/test/core/bad_client/tests/connection_prefix.c index 9a30aad0e93..bc5ed2e3934 100644 --- a/test/core/bad_client/tests/connection_prefix.c +++ b/test/core/bad_client/tests/connection_prefix.c @@ -46,29 +46,30 @@ static void verifier(grpc_server *server, grpc_completion_queue *cq, int main(int argc, char **argv) { grpc_test_init(argc, argv); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "X", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRIX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI X", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI *X", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * X", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTPX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/X", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2X", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.X", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0X", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0\rX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0\r\nX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0\r\n\rX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0\r\n\r\nX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0\r\n\r\nSX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0\r\n\r\nSMX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0\r\n\r\nSM\rX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0\r\n\r\nSM\r\nX", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, "PRI * HTTP/2.0\r\n\r\nSM\r\n\rX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "X", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRIX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI X", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI *X", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * X", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTPX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/X", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2X", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.X", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0X", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0\rX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0\r\nX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0\r\n\rX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0\r\n\r\nX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0\r\n\r\nSX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0\r\n\r\nSMX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0\r\n\r\nSM\rX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0\r\n\r\nSM\r\nX", 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, "PRI * HTTP/2.0\r\n\r\nSM\r\n\rX", + 0); return 0; } diff --git a/test/core/bad_client/tests/headers.c b/test/core/bad_client/tests/headers.c index 4c1a76743e3..f872e5006a7 100644 --- a/test/core/bad_client/tests/headers.c +++ b/test/core/bad_client/tests/headers.c @@ -51,249 +51,251 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); /* partial http2 header prefixes */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x01", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x01\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x01\x04", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x04", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x01\x05", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x05", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x01\x04\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x04\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x01\x04\x00\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, + PFX_STR "\x00\x00\x00\x01\x04\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x01\x04\x00\x00\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, + PFX_STR "\x00\x00\x00\x01\x04\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x04\x00\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x04\x00\x00\x00\x01", GRPC_BAD_CLIENT_DISCONNECT); /* test adding prioritization data */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x01\x01\x24\x00\x00\x00\x01" "\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x02\x01\x24\x00\x00\x00\x01" "\x00\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x03\x01\x24\x00\x00\x00\x01" "\x00\x00\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x04\x01\x24\x00\x00\x00\x01" "\x00\x00\x00\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x05\x01\x24\x00\x00\x00\x01" "", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x05\x01\x24\x00\x00\x00\x01" "\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x05\x01\x24\x00\x00\x00\x01" "\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x05\x01\x24\x00\x00\x00\x01" "\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x05\x01\x24\x00\x00\x00\x01" "\x00\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x05\x01\x24\x00\x00\x00\x01" "\x00\x00\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); /* test looking up an invalid index */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x01\x01\x04\x00\x00\x00\x01" "\xfe", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x04\x01\x04\x00\x00\x00\x01" "\x7f\x7f\x01" "a", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x04\x01\x04\x00\x00\x00\x01" "\x0f\x7f\x01" "a", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x04\x01\x04\x00\x00\x00\x01" "\x1f\x7f\x01" "a", 0); /* test nvr, not indexed in static table */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x03\x01\x04\x00\x00\x00\x01" "\x01\x01" "a", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x03\x01\x04\x00\x00\x00\x01" "\x11\x01" "a", GRPC_BAD_CLIENT_DISCONNECT); /* illegal op code */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x01\x01\x04\x00\x00\x00\x01" "\x80", 0); /* parse some long indices */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x02\x01\x04\x00\x00\x00\x01" "\xff\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x03\x01\x04\x00\x00\x00\x01" "\xff\x80\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x04\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x05\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x06\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80\x80\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x07\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80\x80\x80\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x08\x01\x04\x00\x00\x00\x01" "\xff", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x08\x01\x04\x00\x00\x00\x01" "\xff\x80", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x08\x01\x04\x00\x00\x00\x01" "\xff\x80\x80", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x08\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x08\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80\x80", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x08\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80\x80\x80", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x08\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80\x80\x80\x80", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x08\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80\x80\x80\x80\x00", 0); /* overflow on byte 4 */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x06\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80\x80\x7f", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x06\x01\x04\x00\x00\x00\x01" "\xff\xff\xff\xff\xff\x0f", GRPC_BAD_CLIENT_DISCONNECT); /* overflow after byte 4 */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x08\x01\x04\x00\x00\x00\x01" "\xff\x80\x80\x80\x80\x80\x80\x02", 0); /* end of headers mid-opcode */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x01\x01\x04\x00\x00\x00\x01" "\x01", GRPC_BAD_CLIENT_DISCONNECT); /* dynamic table size update: set to default */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x03\x01\x04\x00\x00\x00\x01" "\x3f\xe1\x1f", GRPC_BAD_CLIENT_DISCONNECT); /* dynamic table size update: set too large */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x03\x01\x04\x00\x00\x00\x01" "\x3f\xf1\x1f", 0); /* dynamic table size update: set twice */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x04\x01\x04\x00\x00\x00\x01" "\x20\x3f\xe1\x1f", GRPC_BAD_CLIENT_DISCONNECT); /* dynamic table size update: set thrice */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x03\x01\x04\x00\x00\x00\x01" "\x20\x20\x20", 0); /* non-ending header followed by continuation frame */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x00\x00\x00\x00\x01" "\x00\x00\x00\x09\x04\x00\x00\x00\x01", GRPC_BAD_CLIENT_DISCONNECT); /* non-ending header followed by non-continuation frame */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x00\x00\x00\x00\x01" "\x00\x00\x00\x00\x04\x00\x00\x00\x01", 0); /* non-ending header followed by a continuation frame for a different stream */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x04\x00\x00\x00\x01" "\x00\x00\x00\x01\x00\x00\x00\x00\x03" "\x00\x00\x00\x09\x04\x00\x00\x00\x01", 0); /* opening with a continuation frame */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x09\x04\x00\x00\x00\x01", 0); /* three header frames */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x04\x00\x00\x00\x01" "\x00\x00\x00\x01\x04\x00\x00\x00\x01" "\x00\x00\x00\x01\x04\x00\x00\x00\x01", GRPC_BAD_CLIENT_DISCONNECT); /* an invalid header found with fuzzing */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x01\x39\x67\xed\x1d\x64", GRPC_BAD_CLIENT_DISCONNECT); /* a badly encoded timeout value */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x19\x01\x04\x00\x00\x00\x01" "\x10\x0cgrpc-timeout\x0a" "15 seconds", GRPC_BAD_CLIENT_DISCONNECT); /* a badly encoded timeout value: twice (catches caching) */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x19\x01\x04\x00\x00\x00\x01" "\x10\x0cgrpc-timeout\x0a" "15 seconds" diff --git a/test/core/bad_client/tests/initial_settings_frame.c b/test/core/bad_client/tests/initial_settings_frame.c index 63a770df91f..b84b67a7e58 100644 --- a/test/core/bad_client/tests/initial_settings_frame.c +++ b/test/core/bad_client/tests/initial_settings_frame.c @@ -50,70 +50,72 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); /* various partial prefixes */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x06", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x06", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x06", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x06", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x06", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x06", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x04", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x04", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x04\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x04\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x04\x01", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x04\x01", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x04\xff", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x04\xff", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x04\x00\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x04\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x04\x00\x00\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, + PFX_STR "\x00\x00\x00\x04\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR "\x00\x00\x00\x04\x00\x00\x00\x00", + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, + PFX_STR "\x00\x00\x00\x04\x00\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); /* must not send frames with stream id != 0 */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x04\x00\x00\x00\x00\x01", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x04\x00\x40\x00\x00\x00", 0); /* settings frame must be a multiple of six bytes long */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x01\x04\x00\x00\x00\x00\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x02\x04\x00\x00\x00\x00\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x03\x04\x00\x00\x00\x00\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x04\x04\x00\x00\x00\x00\x00", 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x05\x04\x00\x00\x00\x00\x00", 0); /* some settings values are illegal */ /* max frame size = 0 */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR ONE_SETTING_HDR "\x00\x05\x00\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR ONE_SETTING_HDR "\x00\x06\xff\xff\xff\xff", GRPC_BAD_CLIENT_DISCONNECT); /* update intiial window size */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR ONE_SETTING_HDR "\x00\x04\x00\x01\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); /* ack with data */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x04\x00\x00\x00\x00\x00" "\x00\x00\x01\x04\x01\x00\x00\x00\x00", 0); /* settings frame with invalid flags */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x04\x10\x00\x00\x00\x00", 0); /* unknown settings should be ignored */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR ONE_SETTING_HDR "\x00\x99\x00\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); diff --git a/test/core/bad_client/tests/large_metadata.c b/test/core/bad_client/tests/large_metadata.c new file mode 100644 index 00000000000..95932205cfc --- /dev/null +++ b/test/core/bad_client/tests/large_metadata.c @@ -0,0 +1,478 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/core/bad_client/bad_client.h" + +#include + +#include +#include "src/core/lib/surface/server.h" +#include "test/core/end2end/cq_verifier.h" + +#define PFX_TOO_MUCH_METADATA_FROM_CLIENT_STR \ + "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" \ + /* settings frame */ \ + "\x00\x00\x00\x04\x00\x00\x00\x00\x00" \ + /* headers: generated from large_metadata.headers in this directory */ \ + "\x00""5{\x01\x05\x00\x00\x00\x01" \ + "\x10\x05:path\x08/foo/bar" \ + "\x10\x07:scheme\x04http" \ + "\x10\x07:method\x04POST" \ + "\x10\x0a:authority\x09localhost" \ + "\x10\x0c""content-type\x10""application/grpc" \ + "\x10\x14grpc-accept-encoding\x15identity,deflate,gzip" \ + "\x10\x02te\x08trailers" \ + "\x10\x0auser-agent\"bad-client grpc-c/0.12.0.0 (linux)" \ + "\x10\x0duser-header00~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header01~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header02~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header03~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header04~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header05~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header06~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header07~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header08~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header09~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header10~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header11~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header12~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header13~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header14~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header15~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header16~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header17~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header18~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header19~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header20~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header21~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header22~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header23~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header24~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header25~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header26~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header27~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header28~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header29~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header30~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header31~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header32~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header33~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header34~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header35~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header36~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header37~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header38~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header39~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header40~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header41~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header42~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header43~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header44~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header45~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header46~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header47~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header48~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header49~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header50~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header51~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header52~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header53~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header54~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header55~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header56~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header57~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header58~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header59~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header60~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header61~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header62~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header63~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header64~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header65~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header66~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header67~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header68~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header69~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header70~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header71~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header72~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header73~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header74~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header75~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header76~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header77~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header78~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header79~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header80~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header81~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header82~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header83~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header84~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header85~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header86~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header87~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header88~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header89~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header90~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header91~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header92~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header93~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" \ + "\x10\x0duser-header94~aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ + "aaaaaaaa" + +#define PFX_TOO_MUCH_METADATA_FROM_SERVER_STR \ + "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" \ + /* settings frame: sets MAX_HEADER_LIST_SIZE to 16K */ \ + "\x00\x00\x06\x04\x00\x00\x00\x00\x00\x00\x06\x00\x00\x40\x00" \ + /* headers: generated from simple_request.headers in this directory */ \ + "\x00\x00\xc9\x01\x04\x00\x00\x00\x01" \ + "\x10\x05:path\x08/foo/bar" \ + "\x10\x07:scheme\x04http" \ + "\x10\x07:method\x04POST" \ + "\x10\x0a:authority\x09localhost" \ + "\x10\x0c" \ + "content-type\x10" \ + "application/grpc" \ + "\x10\x14grpc-accept-encoding\x15" \ + "deflate,identity,gzip" \ + "\x10\x02te\x08trailers" \ + "\x10\x0auser-agent\"bad-client grpc-c/0.12.0.0 (linux)" + +static void *tag(intptr_t t) { return (void *)t; } + +static void server_verifier(grpc_server *server, grpc_completion_queue *cq, + void *registered_method) { + grpc_call_error error; + grpc_call *s; + grpc_call_details call_details; + cq_verifier *cqv = cq_verifier_create(cq); + grpc_metadata_array request_metadata_recv; + + grpc_call_details_init(&call_details); + grpc_metadata_array_init(&request_metadata_recv); + + error = grpc_server_request_call(server, &s, &call_details, + &request_metadata_recv, cq, cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + cq_expect_completion(cqv, tag(101), 1); + cq_verify(cqv); + + GPR_ASSERT(0 == strcmp(call_details.host, "localhost")); + GPR_ASSERT(0 == strcmp(call_details.method, "/foo/bar")); + + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + grpc_call_destroy(s); + cq_verifier_destroy(cqv); +} + +static void server_verifier_sends_too_much_metadata( + grpc_server *server, grpc_completion_queue *cq, void *registered_method) { + grpc_call_error error; + grpc_call *s; + grpc_call_details call_details; + cq_verifier *cqv = cq_verifier_create(cq); + grpc_metadata_array request_metadata_recv; + + grpc_call_details_init(&call_details); + grpc_metadata_array_init(&request_metadata_recv); + + error = grpc_server_request_call(server, &s, &call_details, + &request_metadata_recv, cq, cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + cq_expect_completion(cqv, tag(101), 1); + cq_verify(cqv); + + GPR_ASSERT(0 == strcmp(call_details.host, "localhost")); + GPR_ASSERT(0 == strcmp(call_details.method, "/foo/bar")); + + const size_t metadata_value_size = 16 * 1024; + grpc_metadata meta; + meta.key = "key"; + meta.value = gpr_malloc(metadata_value_size + 1); + memset((char *)meta.value, 'a', metadata_value_size); + ((char *)meta.value)[metadata_value_size] = 0; + meta.value_length = metadata_value_size; + + grpc_op op; + op.op = GRPC_OP_SEND_INITIAL_METADATA; + op.data.send_initial_metadata.count = 1; + op.data.send_initial_metadata.metadata = &meta; + op.flags = 0; + op.reserved = NULL; + error = grpc_call_start_batch(s, &op, 1, tag(102), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + cq_expect_completion(cqv, tag(102), 0); // Operation fails. + cq_verify(cqv); + + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + grpc_call_destroy(s); + cq_verifier_destroy(cqv); +} + +static void client_validator(gpr_slice_buffer *incoming) { + // Get last frame from incoming slice buffer. + gpr_slice_buffer last_frame_buffer; + gpr_slice_buffer_init(&last_frame_buffer); + gpr_slice_buffer_trim_end(incoming, 13, &last_frame_buffer); + GPR_ASSERT(last_frame_buffer.count == 1); + gpr_slice last_frame = last_frame_buffer.slices[0]; + // Construct expected frame. + gpr_slice expected = gpr_slice_malloc(13); + uint8_t *p = GPR_SLICE_START_PTR(expected); + // Length. + *p++ = 0; + *p++ = 0; + *p++ = 4; + // Frame type (RST_STREAM). + *p++ = 3; + // Flags. + *p++ = 0; + // Stream ID. + *p++ = 0; + *p++ = 0; + *p++ = 0; + *p++ = 1; + // Payload (error code). + *p++ = 0; + *p++ = 0; + *p++ = 0; + *p++ = 11; + // Compare actual and expected. + GPR_ASSERT(gpr_slice_cmp(last_frame, expected) == 0); +} + +int main(int argc, char **argv) { + grpc_test_init(argc, argv); + + // Test sending more metadata than the server will accept. + GRPC_RUN_BAD_CLIENT_TEST(server_verifier, client_validator, + PFX_TOO_MUCH_METADATA_FROM_CLIENT_STR, 0); + + // Test sending more metadata than the client will accept. + GRPC_RUN_BAD_CLIENT_TEST(server_verifier_sends_too_much_metadata, + client_validator, + PFX_TOO_MUCH_METADATA_FROM_SERVER_STR, 0); + + return 0; +} diff --git a/test/core/bad_client/tests/large_metadata.headers b/test/core/bad_client/tests/large_metadata.headers new file mode 100644 index 00000000000..75de3ef1001 --- /dev/null +++ b/test/core/bad_client/tests/large_metadata.headers @@ -0,0 +1,106 @@ +# headers used in simple_request.c +# use tools/codegen/core/gen_header_frame.py --set_end_stream to generate +# the binary strings contained in the source code +:path: /foo/bar +:scheme: http +:method: POST +:authority: localhost +content-type: application/grpc +grpc-accept-encoding: identity,deflate,gzip +te: trailers +user-agent: bad-client grpc-c/0.12.0.0 (linux) +user-header00: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header01: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header02: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header03: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header04: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header05: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header06: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header07: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header08: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header09: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header10: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header11: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header12: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header13: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header14: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header15: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header16: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header17: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header18: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header19: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header20: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header21: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header22: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header23: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header24: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header25: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header26: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header27: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header28: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header29: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header30: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header31: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header32: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header33: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header34: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header35: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header36: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header37: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header38: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header39: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header40: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header41: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header42: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header43: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header44: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header45: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header46: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header47: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header48: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header49: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header50: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header51: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header52: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header53: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header54: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header55: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header56: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header57: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header58: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header59: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header60: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header61: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header62: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header63: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header64: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header65: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header66: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header67: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header68: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header69: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header70: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header71: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header72: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header73: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header74: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header75: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header76: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header77: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header78: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header79: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header80: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header81: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header82: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header83: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header84: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header85: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header86: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header87: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header88: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header89: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header90: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header91: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header92: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header93: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +user-header94: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/test/core/bad_client/tests/server_registered_method.c b/test/core/bad_client/tests/server_registered_method.c index 60d3b890b22..6216553a614 100644 --- a/test/core/bad_client/tests/server_registered_method.c +++ b/test/core/bad_client/tests/server_registered_method.c @@ -111,43 +111,43 @@ int main(int argc, char **argv) { /* body generated with * tools/codegen/core/gen_server_registered_method_bad_client_test_body.py */ - GRPC_RUN_BAD_CLIENT_TEST(verifier_fails, + GRPC_RUN_BAD_CLIENT_TEST(verifier_fails, NULL, PFX_STR "\x00\x00\x00\x00\x00\x00\x00\x00\x01", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier_fails, + GRPC_RUN_BAD_CLIENT_TEST(verifier_fails, NULL, PFX_STR "\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier_fails, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier_fails, NULL, PFX_STR "\x00\x00\x02\x00\x00\x00\x00\x00\x01\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); - GRPC_RUN_BAD_CLIENT_TEST(verifier_fails, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier_fails, NULL, PFX_STR "\x00\x00\x03\x00\x00\x00\x00\x00\x01\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); GRPC_RUN_BAD_CLIENT_TEST( - verifier_fails, + verifier_fails, NULL, PFX_STR "\x00\x00\x04\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); GRPC_RUN_BAD_CLIENT_TEST( - verifier_succeeds, + verifier_succeeds, NULL, PFX_STR "\x00\x00\x05\x00\x01\x00\x00\x00\x01\x00\x00\x00\x00\x00", 0); GRPC_RUN_BAD_CLIENT_TEST( - verifier_fails, + verifier_fails, NULL, PFX_STR "\x00\x00\x05\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x01", GRPC_BAD_CLIENT_DISCONNECT); GRPC_RUN_BAD_CLIENT_TEST( - verifier_succeeds, + verifier_succeeds, NULL, PFX_STR "\x00\x00\x06\x00\x01\x00\x00\x00\x01\x00\x00\x00\x00\x01\x00", 0); GRPC_RUN_BAD_CLIENT_TEST( - verifier_fails, + verifier_fails, NULL, PFX_STR "\x00\x00\x05\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x02", GRPC_BAD_CLIENT_DISCONNECT); GRPC_RUN_BAD_CLIENT_TEST( - verifier_fails, + verifier_fails, NULL, PFX_STR "\x00\x00\x06\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x02\x00", GRPC_BAD_CLIENT_DISCONNECT); GRPC_RUN_BAD_CLIENT_TEST( - verifier_succeeds, PFX_STR + verifier_succeeds, NULL, PFX_STR "\x00\x00\x07\x00\x01\x00\x00\x00\x01\x00\x00\x00\x00\x02\x00\x00", 0); diff --git a/test/core/bad_client/tests/simple_request.c b/test/core/bad_client/tests/simple_request.c index 3ae6eb3592d..25bbe968e4a 100644 --- a/test/core/bad_client/tests/simple_request.c +++ b/test/core/bad_client/tests/simple_request.c @@ -139,42 +139,42 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); /* basic request: check that things are working */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR, 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR_UNUSUAL, 0); - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR_UNUSUAL2, 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR, 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR_UNUSUAL, 0); + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR_UNUSUAL2, 0); /* push an illegal data frame */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x05\x00\x00\x00\x00\x00\x01" "\x34\x00\x00\x00\x00", 0); /* push a data frame with bad flags */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x00\x02\x00\x00\x00\x01", 0); /* push a window update with a bad length */ - GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, + GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, NULL, PFX_STR "\x00\x00\x01\x08\x00\x00\x00\x00\x01", 0); /* push a window update with bad flags */ - GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, + GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, NULL, PFX_STR "\x00\x00\x00\x08\x10\x00\x00\x00\x01", 0); /* push a window update with bad data */ - GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, PFX_STR + GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, NULL, PFX_STR "\x00\x00\x04\x08\x00\x00\x00\x00\x01" "\xff\xff\xff\xff", 0); /* push a short goaway */ - GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, + GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, NULL, PFX_STR "\x00\x00\x04\x07\x00\x00\x00\x00\x00", 0); /* disconnect before sending goaway */ - GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, + GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, NULL, PFX_STR "\x00\x01\x12\x07\x00\x00\x00\x00\x00", GRPC_BAD_CLIENT_DISCONNECT); /* push a rst_stream with a bad length */ - GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, + GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, NULL, PFX_STR "\x00\x00\x01\x03\x00\x00\x00\x00\x01", 0); /* push a rst_stream with bad flags */ - GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, + GRPC_RUN_BAD_CLIENT_TEST(failure_verifier, NULL, PFX_STR "\x00\x00\x00\x03\x10\x00\x00\x00\x01", 0); return 0; diff --git a/test/core/bad_client/tests/unknown_frame.c b/test/core/bad_client/tests/unknown_frame.c index f3870a18138..4f483d21f2e 100644 --- a/test/core/bad_client/tests/unknown_frame.c +++ b/test/core/bad_client/tests/unknown_frame.c @@ -51,7 +51,7 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); /* test adding prioritization data */ - GRPC_RUN_BAD_CLIENT_TEST(verifier, + GRPC_RUN_BAD_CLIENT_TEST(verifier, NULL, PFX_STR "\x00\x00\x00\x88\x00\x00\x00\x00\x01", GRPC_BAD_CLIENT_DISCONNECT); diff --git a/tools/codegen/core/gen_header_frame.py b/tools/codegen/core/gen_header_frame.py index 96e6c67fa66..ee476267f23 100755 --- a/tools/codegen/core/gen_header_frame.py +++ b/tools/codegen/core/gen_header_frame.py @@ -38,6 +38,8 @@ import json import sys +set_end_stream = len(sys.argv) > 1 and sys.argv[1] == '--set_end_stream' + # parse input, fill in vals vals = [] for line in sys.stdin: @@ -65,6 +67,9 @@ for key, value in vals: payload_bytes.append(payload_line) # fill in header +flags = 0x04 # END_HEADERS +if set_end_stream: + flags |= 0x01 # END_STREAM payload_bytes[0].extend([ (payload_len >> 16) & 0xff, (payload_len >> 8) & 0xff, @@ -72,7 +77,7 @@ payload_bytes[0].extend([ # header frame 0x01, # flags - 0x04, + flags, # stream id 0x00, 0x00, diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index f546f3b9950..e20d808fefb 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -3387,6 +3387,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "bad_client_test", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "large_metadata_bad_client_test", + "src": [ + "test/core/bad_client/tests/large_metadata.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "bad_client_test", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 0fd77854d22..4a6c1373275 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -2713,6 +2713,27 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "large_metadata_bad_client_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index bdae4475456..be8b5d40ace 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -1095,6 +1095,18 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "initial_settings_frame_bad_ {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "large_metadata_bad_client_test", "vcxproj\test\large_metadata_bad_client_test\large_metadata_bad_client_test.vcxproj", "{B706A9EC-7982-0DBC-495D-07B165F6CF56}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {BA67B418-B699-E41A-9CC4-0279C49481A5} = {BA67B418-B699-E41A-9CC4-0279C49481A5} + {0A7E7F92-FDEA-40F1-A9EC-3BA484F98BBF} = {0A7E7F92-FDEA-40F1-A9EC-3BA484F98BBF} + {46CEDFFF-9692-456A-AA24-38B5D6BCF4C5} = {46CEDFFF-9692-456A-AA24-38B5D6BCF4C5} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "server_registered_method_bad_client_test", "vcxproj\test\server_registered_method_bad_client_test\server_registered_method_bad_client_test.vcxproj", "{B4E7CD82-988A-BD3A-29F8-8590D3A8BC28}" ProjectSection(myProperties) = preProject lib = "False" @@ -3087,6 +3099,22 @@ Global {6756895E-05BF-8CC7-58F2-868DF0C0300C}.Release-DLL|Win32.Build.0 = Release|Win32 {6756895E-05BF-8CC7-58F2-868DF0C0300C}.Release-DLL|x64.ActiveCfg = Release|x64 {6756895E-05BF-8CC7-58F2-868DF0C0300C}.Release-DLL|x64.Build.0 = Release|x64 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Debug|Win32.ActiveCfg = Debug|Win32 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Debug|x64.ActiveCfg = Debug|x64 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Release|Win32.ActiveCfg = Release|Win32 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Release|x64.ActiveCfg = Release|x64 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Debug|Win32.Build.0 = Debug|Win32 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Debug|x64.Build.0 = Debug|x64 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Release|Win32.Build.0 = Release|Win32 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Release|x64.Build.0 = Release|x64 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Debug-DLL|x64.Build.0 = Debug|x64 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Release-DLL|Win32.Build.0 = Release|Win32 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Release-DLL|x64.ActiveCfg = Release|x64 + {B706A9EC-7982-0DBC-495D-07B165F6CF56}.Release-DLL|x64.Build.0 = Release|x64 {B4E7CD82-988A-BD3A-29F8-8590D3A8BC28}.Debug|Win32.ActiveCfg = Debug|Win32 {B4E7CD82-988A-BD3A-29F8-8590D3A8BC28}.Debug|x64.ActiveCfg = Debug|x64 {B4E7CD82-988A-BD3A-29F8-8590D3A8BC28}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/test/large_metadata_bad_client_test/large_metadata_bad_client_test.vcxproj b/vsprojects/vcxproj/test/large_metadata_bad_client_test/large_metadata_bad_client_test.vcxproj new file mode 100644 index 00000000000..0a14694b757 --- /dev/null +++ b/vsprojects/vcxproj/test/large_metadata_bad_client_test/large_metadata_bad_client_test.vcxproj @@ -0,0 +1,202 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {B706A9EC-7982-0DBC-495D-07B165F6CF56} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + large_metadata_bad_client_test + static + Debug + static + Debug + + + large_metadata_bad_client_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {BA67B418-B699-E41A-9CC4-0279C49481A5} + + + {0A7E7F92-FDEA-40F1-A9EC-3BA484F98BBF} + + + {46CEDFFF-9692-456A-AA24-38B5D6BCF4C5} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/large_metadata_bad_client_test/large_metadata_bad_client_test.vcxproj.filters b/vsprojects/vcxproj/test/large_metadata_bad_client_test/large_metadata_bad_client_test.vcxproj.filters new file mode 100644 index 00000000000..5eb9a5e7cb9 --- /dev/null +++ b/vsprojects/vcxproj/test/large_metadata_bad_client_test/large_metadata_bad_client_test.vcxproj.filters @@ -0,0 +1,24 @@ + + + + + test\core\bad_client\tests + + + + + + {6c1eb0cb-9d82-f961-7220-1f6edc913666} + + + {79d5006f-93a1-aa0e-2568-37aa63eef567} + + + {dbde5995-24a0-2332-4bee-0540ed3aa848} + + + {5cf4a13f-ae24-fd98-eb59-b5301f30367c} + + + + From 38281cfa59097c7006a6514fca791a7b50b7ff8a Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 3 May 2016 10:51:49 -0700 Subject: [PATCH 039/155] Ruby: improve server error handling, fix a reference error --- src/ruby/lib/grpc/generic/rpc_server.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index 7f3a38a9f46..9a2af049471 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -332,15 +332,13 @@ module GRPC # the current thread to terminate it. def run_till_terminated GRPC.trap_signals - stopped = false t = Thread.new do run - stopped = true end + t.abort_on_exception = true wait_till_running - loop do + until running_state == :stopped sleep SIGNAL_CHECK_PERIOD - break if stopped break unless GRPC.handle_signals end stop @@ -416,7 +414,7 @@ module GRPC GRPC.logger.warn("NOT AVAILABLE: too many jobs_waiting: #{an_rpc}") noop = proc { |x| x } c = ActiveCall.new(an_rpc.call, @cq, noop, noop, an_rpc.deadline) - c.send_status(StatusCodes::RESOURCE_EXHAUSTED, '') + c.send_status(GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED, '') nil end @@ -427,7 +425,7 @@ module GRPC GRPC.logger.warn("UNIMPLEMENTED: #{an_rpc}") noop = proc { |x| x } c = ActiveCall.new(an_rpc.call, @cq, noop, noop, an_rpc.deadline) - c.send_status(StatusCodes::UNIMPLEMENTED, '') + c.send_status(GRPC::Core::StatusCodes::UNIMPLEMENTED, '') nil end @@ -443,7 +441,12 @@ module GRPC unless active_call.nil? @pool.schedule(active_call) do |ac| c, mth = ac - rpc_descs[mth].run_server_method(c, rpc_handlers[mth]) + begin + rpc_descs[mth].run_server_method(c, rpc_handlers[mth]) + rescue StandardError => e + c.send_status(code = GRPC::Core::StatusCodes::INTERNAL, + details = "Server handler failed") + end end end rescue Core::CallError, RuntimeError => e From 59dfee880016e74edc9ebcadaf1ea624a4d457be Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 3 May 2016 11:33:25 -0700 Subject: [PATCH 040/155] Fixed rubocop issues with newest changes --- src/ruby/.rubocop.yml | 4 ++-- src/ruby/lib/grpc/generic/rpc_server.rb | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ruby/.rubocop.yml b/src/ruby/.rubocop.yml index d13ce426553..34bb4775435 100644 --- a/src/ruby/.rubocop.yml +++ b/src/ruby/.rubocop.yml @@ -11,10 +11,10 @@ AllCops: - 'pb/test/**/*' Metrics/CyclomaticComplexity: - Max: 8 + Max: 9 Metrics/PerceivedComplexity: - Max: 8 + Max: 9 Metrics/ClassLength: Max: 250 diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index 9a2af049471..a0f4071adc8 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -443,9 +443,9 @@ module GRPC c, mth = ac begin rpc_descs[mth].run_server_method(c, rpc_handlers[mth]) - rescue StandardError => e - c.send_status(code = GRPC::Core::StatusCodes::INTERNAL, - details = "Server handler failed") + rescue StandardError + c.send_status(GRPC::Core::StatusCodes::INTERNAL, + 'Server handler failed') end end end From e4d2748f2fec4b189cdb7d13e25df0be95888ba2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 3 May 2016 12:19:33 -0700 Subject: [PATCH 041/155] Fix async_end2end_test flow control Completion queues + flow control + single threading is hard. We need a read outstanding on a call to grant flow control tokens to the remote end. To do that we need to request a read *before* we wait for the write to be finished, otherwise, in the case of a large write we'll block waiting for flow control tokens. Built on #6402 --- test/cpp/end2end/async_end2end_test.cc | 131 ++++++++++++++----------- 1 file changed, 73 insertions(+), 58 deletions(-) diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 4de181b9011..0232a9fa317 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -281,10 +281,11 @@ class AsyncEnd2endTest : public ::testing::TestWithParam { send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(3)); - Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); - response_reader->Finish(&recv_response, &recv_status, tag(4)); - Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(3, true) + .Expect(4, true) + .Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); @@ -345,12 +346,9 @@ TEST_P(AsyncEnd2endTest, AsyncNextRpc) { send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(3)); - Verifier(GetParam().disable_blocking) - .Expect(3, true) - .Verify(cq_.get(), std::chrono::system_clock::time_point::max()); - response_reader->Finish(&recv_response, &recv_status, tag(4)); Verifier(GetParam().disable_blocking) + .Expect(3, true) .Expect(4, true) .Verify(cq_.get(), std::chrono::system_clock::time_point::max()); @@ -384,31 +382,35 @@ TEST_P(AsyncEnd2endTest, SimpleClientStreaming) { .Verify(cq_.get()); cli_stream->Write(send_request, tag(3)); - Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); - srv_stream.Read(&recv_request, tag(4)); - Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(3, true) + .Expect(4, true) + .Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); cli_stream->Write(send_request, tag(5)); - Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); - srv_stream.Read(&recv_request, tag(6)); - Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(5, true) + .Expect(6, true) + .Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); cli_stream->WritesDone(tag(7)); - Verifier(GetParam().disable_blocking).Expect(7, true).Verify(cq_.get()); - srv_stream.Read(&recv_request, tag(8)); - Verifier(GetParam().disable_blocking).Expect(8, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(7, true) + .Expect(8, false) + .Verify(cq_.get()); send_response.set_message(recv_request.message()); srv_stream.Finish(send_response, Status::OK, tag(9)); - Verifier(GetParam().disable_blocking).Expect(9, true).Verify(cq_.get()); - cli_stream->Finish(&recv_status, tag(10)); - Verifier(GetParam().disable_blocking).Expect(10, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(9, true) + .Expect(10, true) + .Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); @@ -442,24 +444,27 @@ TEST_P(AsyncEnd2endTest, SimpleServerStreaming) { send_response.set_message(recv_request.message()); srv_stream.Write(send_response, tag(3)); - Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); - cli_stream->Read(&recv_response, tag(4)); - Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(3, true) + .Expect(4, true) + .Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); srv_stream.Write(send_response, tag(5)); - Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); - cli_stream->Read(&recv_response, tag(6)); - Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(5, true) + .Expect(6, true) + .Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); srv_stream.Finish(Status::OK, tag(7)); - Verifier(GetParam().disable_blocking).Expect(7, true).Verify(cq_.get()); - cli_stream->Read(&recv_response, tag(8)); - Verifier(GetParam().disable_blocking).Expect(8, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(7, true) + .Expect(8, false) + .Verify(cq_.get()); cli_stream->Finish(&recv_status, tag(9)); Verifier(GetParam().disable_blocking).Expect(9, true).Verify(cq_.get()); @@ -493,31 +498,35 @@ TEST_P(AsyncEnd2endTest, SimpleBidiStreaming) { .Verify(cq_.get()); cli_stream->Write(send_request, tag(3)); - Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); - srv_stream.Read(&recv_request, tag(4)); - Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(3, true) + .Expect(4, true) + .Verify(cq_.get()); EXPECT_EQ(send_request.message(), recv_request.message()); send_response.set_message(recv_request.message()); srv_stream.Write(send_response, tag(5)); - Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); - cli_stream->Read(&recv_response, tag(6)); - Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(5, true) + .Expect(6, true) + .Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); cli_stream->WritesDone(tag(7)); - Verifier(GetParam().disable_blocking).Expect(7, true).Verify(cq_.get()); - srv_stream.Read(&recv_request, tag(8)); - Verifier(GetParam().disable_blocking).Expect(8, false).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(7, true) + .Expect(8, false) + .Verify(cq_.get()); srv_stream.Finish(Status::OK, tag(9)); - Verifier(GetParam().disable_blocking).Expect(9, true).Verify(cq_.get()); - cli_stream->Finish(&recv_status, tag(10)); - Verifier(GetParam().disable_blocking).Expect(10, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(9, true) + .Expect(10, true) + .Verify(cq_.get()); EXPECT_TRUE(recv_status.ok()); } @@ -562,11 +571,11 @@ TEST_P(AsyncEnd2endTest, ClientInitialMetadataRpc) { send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(3)); - - Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); - response_reader->Finish(&recv_response, &recv_status, tag(4)); - Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(3, true) + .Expect(4, true) + .Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); @@ -612,10 +621,11 @@ TEST_P(AsyncEnd2endTest, ServerInitialMetadataRpc) { send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(5)); - Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); - response_reader->Finish(&recv_response, &recv_status, tag(6)); - Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(5, true) + .Expect(6, true) + .Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); @@ -652,11 +662,13 @@ TEST_P(AsyncEnd2endTest, ServerTrailingMetadataRpc) { srv_ctx.AddTrailingMetadata(meta1.first, meta1.second); srv_ctx.AddTrailingMetadata(meta2.first, meta2.second); response_writer.Finish(send_response, Status::OK, tag(4)); + response_reader->Finish(&recv_response, &recv_status, tag(5)); - Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(4, true) + .Expect(5, true) + .Verify(cq_.get()); - response_reader->Finish(&recv_response, &recv_status, tag(5)); - Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); auto server_trailing_metadata = cli_ctx.GetServerTrailingMetadata(); @@ -730,11 +742,13 @@ TEST_P(AsyncEnd2endTest, MetadataRpc) { srv_ctx.AddTrailingMetadata(meta5.first, meta5.second); srv_ctx.AddTrailingMetadata(meta6.first, meta6.second); response_writer.Finish(send_response, Status::OK, tag(5)); + response_reader->Finish(&recv_response, &recv_status, tag(6)); - Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(5, true) + .Expect(6, true) + .Verify(cq_.get()); - response_reader->Finish(&recv_response, &recv_status, tag(6)); - Verifier(GetParam().disable_blocking).Expect(6, true).Verify(cq_.get()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); auto server_trailing_metadata = cli_ctx.GetServerTrailingMetadata(); @@ -807,12 +821,13 @@ TEST_P(AsyncEnd2endTest, ServerCheckDone) { send_response.set_message(recv_request.message()); response_writer.Finish(send_response, Status::OK, tag(3)); - Verifier(GetParam().disable_blocking).Expect(3, true).Verify(cq_.get()); - Verifier(GetParam().disable_blocking).Expect(5, true).Verify(cq_.get()); - EXPECT_FALSE(srv_ctx.IsCancelled()); - response_reader->Finish(&recv_response, &recv_status, tag(4)); - Verifier(GetParam().disable_blocking).Expect(4, true).Verify(cq_.get()); + Verifier(GetParam().disable_blocking) + .Expect(3, true) + .Expect(4, true) + .Expect(5, true) + .Verify(cq_.get()); + EXPECT_FALSE(srv_ctx.IsCancelled()); EXPECT_EQ(send_response.message(), recv_response.message()); EXPECT_TRUE(recv_status.ok()); From 7cae1f7198abd0f7048c8cc4e5946afbacc36b0b Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 3 May 2016 12:23:08 -0700 Subject: [PATCH 042/155] Remove some unneeded code from an earlier iteration of this code. --- test/core/end2end/cq_verifier.c | 8 -------- test/core/end2end/cq_verifier.h | 1 - 2 files changed, 9 deletions(-) diff --git a/test/core/end2end/cq_verifier.c b/test/core/end2end/cq_verifier.c index 5f1a33242ee..77afe588d79 100644 --- a/test/core/end2end/cq_verifier.c +++ b/test/core/end2end/cq_verifier.c @@ -107,14 +107,6 @@ int contains_metadata(grpc_metadata_array *array, const char *key, return has_metadata(array->metadata, array->count, key, value); } -int contains_metadata_key(grpc_metadata_array *array, const char *key) { - for (size_t i = 0; i < array->count; ++i) { - if (strcmp(array->metadata[i].key, key) == 0) - return 1; - } - return 0; -} - static gpr_slice merge_slices(gpr_slice *slices, size_t nslices) { size_t i; size_t len = 0; diff --git a/test/core/end2end/cq_verifier.h b/test/core/end2end/cq_verifier.h index a54065950d2..b3e07c45a58 100644 --- a/test/core/end2end/cq_verifier.h +++ b/test/core/end2end/cq_verifier.h @@ -62,6 +62,5 @@ void cq_expect_completion(cq_verifier *v, void *tag, int success); int byte_buffer_eq_string(grpc_byte_buffer *byte_buffer, const char *string); int contains_metadata(grpc_metadata_array *array, const char *key, const char *value); -int contains_metadata_key(grpc_metadata_array *array, const char *key); #endif /* GRPC_TEST_CORE_END2END_CQ_VERIFIER_H */ From a4edeb337229b7468a7ccbef6cb9d3c135b21b41 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 3 May 2016 12:24:16 -0700 Subject: [PATCH 043/155] Fix build breakage in bad_client tests. --- test/core/bad_client/tests/head_of_line_blocking.c | 2 +- test/core/bad_client/tests/window_overflow.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/bad_client/tests/head_of_line_blocking.c b/test/core/bad_client/tests/head_of_line_blocking.c index 53cd4537b2f..e4051bb6685 100644 --- a/test/core/bad_client/tests/head_of_line_blocking.c +++ b/test/core/bad_client/tests/head_of_line_blocking.c @@ -144,7 +144,7 @@ int main(int argc, char **argv) { addbuf(hdr, sizeof(hdr)); addbuf(msg, FRAME_SIZE); } - grpc_run_bad_client_test(verifier, g_buffer, g_count, 0); + grpc_run_bad_client_test(verifier, NULL, g_buffer, g_count, 0); gpr_free(g_buffer); return 0; diff --git a/test/core/bad_client/tests/window_overflow.c b/test/core/bad_client/tests/window_overflow.c index 16f11e7dd0a..0d17dbe448e 100644 --- a/test/core/bad_client/tests/window_overflow.c +++ b/test/core/bad_client/tests/window_overflow.c @@ -105,7 +105,7 @@ int main(int argc, char **argv) { addbuf(message, sizeof(message)); } } - grpc_run_bad_client_test(verifier, g_buffer, g_count, 0); + grpc_run_bad_client_test(verifier, NULL, g_buffer, g_count, 0); gpr_free(g_buffer); return 0; From b6a9016fc234714632a20cb25dc2e822a72243f8 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 09:41:15 -0700 Subject: [PATCH 044/155] add tests for AsyncCallServer --- .../Grpc.Core.Tests/Grpc.Core.Tests.csproj | 2 + .../Internal/AsyncCallServerTest.cs | 134 +++++++++++++ .../Grpc.Core.Tests/Internal/AsyncCallTest.cs | 134 +------------ .../Internal/FakeNativeCall.cs | 177 ++++++++++++++++++ .../Grpc.Core/Internal/AsyncCallServer.cs | 9 + 5 files changed, 324 insertions(+), 132 deletions(-) create mode 100644 src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs create mode 100644 src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 0cd059c2327..47131fc454e 100644 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -84,6 +84,8 @@ + + diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs new file mode 100644 index 00000000000..8c178657a1b --- /dev/null +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs @@ -0,0 +1,134 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Threading.Tasks; + +using Grpc.Core.Internal; +using NUnit.Framework; + +namespace Grpc.Core.Internal.Tests +{ + /// + /// Uses fake native call to test interaction of AsyncCallServer wrapping code with C core in different situations. + /// + public class AsyncCallServerTest + { + Server server; + FakeNativeCall fakeCall; + AsyncCallServer asyncCallServer; + + [SetUp] + public void Init() + { + var environment = GrpcEnvironment.AddRef(); + server = new Server(); + + fakeCall = new FakeNativeCall(); + asyncCallServer = new AsyncCallServer( + Marshallers.StringMarshaller.Serializer, Marshallers.StringMarshaller.Deserializer, + environment, + server); + asyncCallServer.InitializeForTesting(fakeCall); + } + + [TearDown] + public void Cleanup() + { + GrpcEnvironment.Release(); + } + + [Test] + public void CancelNotificationAfterStartDisposes() + { + var finishedTask = asyncCallServer.ServerSideCallAsync(); + var requestStream = new ServerRequestStream(asyncCallServer); + var responseStream = new ServerResponseStream(asyncCallServer); + + // Finishing requestStream is needed for dispose to happen. + var moveNextTask = requestStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + Assert.IsFalse(moveNextTask.Result); + + fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); + AssertDisposed(asyncCallServer, fakeCall, finishedTask); + } + + [Test] + public void ReadAfterCancelNotificationCanSucceed() + { + var finishedTask = asyncCallServer.ServerSideCallAsync(); + var requestStream = new ServerRequestStream(asyncCallServer); + var responseStream = new ServerResponseStream(asyncCallServer); + + fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); + + // Check that startin a read after cancel notification has been processed is legal. + var moveNextTask = requestStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + Assert.IsFalse(moveNextTask.Result); + + AssertDisposed(asyncCallServer, fakeCall, finishedTask); + } + + + // TODO: read completion failure ... + + // TODO: + + + + // TODO: write fails... + + // TODO: write completion fails... + + // TODO: cancellation delivered... + + // TODO: cancel notification in the middle of a read... + + // TODO: cancel notification in the middle of a write... + + // TODO: cancellation delivered... + + // TODO: what does writing status do to reads? + + static void AssertDisposed(AsyncCallServer asyncCallServer, FakeNativeCall fakeCall, Task finishedTask) + { + Assert.IsTrue(fakeCall.IsDisposed); + Assert.IsTrue(finishedTask.IsCompleted); + Assert.DoesNotThrow(() => finishedTask.Wait()); + } + } +} diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs index ed2d22815b6..abe9d4a2e62 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs @@ -42,7 +42,7 @@ using NUnit.Framework; namespace Grpc.Core.Internal.Tests { /// - /// Uses fake native call to test interaction of wrapping code with C core in different situations. + /// Uses fake native call to test interaction of AsyncCall wrapping code with C core in different situations. /// public class AsyncCallTest { @@ -480,139 +480,9 @@ namespace Grpc.Core.Internal.Tests Assert.IsTrue(fakeCall.IsDisposed); var ex = Assert.ThrowsAsync(async () => await moveNextTask); + Assert.AreEqual(expectedStatusCode, ex.Status.StatusCode); Assert.AreEqual(expectedStatusCode, asyncCall.GetStatus().StatusCode); Assert.AreEqual(0, asyncCall.GetTrailers().Count); } - - internal class FakeNativeCall : INativeCall - { - public UnaryResponseClientHandler UnaryResponseClientHandler - { - get; - set; - } - - public ReceivedStatusOnClientHandler ReceivedStatusOnClientHandler - { - get; - set; - } - - public ReceivedMessageHandler ReceivedMessageHandler - { - get; - set; - } - - public ReceivedResponseHeadersHandler ReceivedResponseHeadersHandler - { - get; - set; - } - - public SendCompletionHandler SendCompletionHandler - { - get; - set; - } - - public ReceivedCloseOnServerHandler ReceivedCloseOnServerHandler - { - get; - set; - } - - public bool IsCancelled - { - get; - set; - } - - public bool IsDisposed - { - get; - set; - } - - public void Cancel() - { - IsCancelled = true; - } - - public void CancelWithStatus(Status status) - { - IsCancelled = true; - } - - public string GetPeer() - { - return "PEER"; - } - - public void StartUnary(UnaryResponseClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags) - { - UnaryResponseClientHandler = callback; - } - - public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags) - { - throw new NotImplementedException(); - } - - public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray) - { - UnaryResponseClientHandler = callback; - } - - public void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags) - { - ReceivedStatusOnClientHandler = callback; - } - - public void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray) - { - ReceivedStatusOnClientHandler = callback; - } - - public void StartReceiveMessage(ReceivedMessageHandler callback) - { - ReceivedMessageHandler = callback; - } - - public void StartReceiveInitialMetadata(ReceivedResponseHeadersHandler callback) - { - ReceivedResponseHeadersHandler = callback; - } - - public void StartSendInitialMetadata(SendCompletionHandler callback, MetadataArraySafeHandle metadataArray) - { - SendCompletionHandler = callback; - } - - public void StartSendMessage(SendCompletionHandler callback, byte[] payload, WriteFlags writeFlags, bool sendEmptyInitialMetadata) - { - SendCompletionHandler = callback; - } - - public void StartSendCloseFromClient(SendCompletionHandler callback) - { - SendCompletionHandler = callback; - } - - public void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata) - { - SendCompletionHandler = callback; - } - - public void StartServerSide(ReceivedCloseOnServerHandler callback) - { - ReceivedCloseOnServerHandler = callback; - } - - public void Dispose() - { - IsDisposed = true; - } - } } } diff --git a/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs b/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs new file mode 100644 index 00000000000..441bf9660be --- /dev/null +++ b/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs @@ -0,0 +1,177 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Threading.Tasks; + +using Grpc.Core.Internal; +using NUnit.Framework; + +namespace Grpc.Core.Internal.Tests +{ + /// + /// For testing purposes. + /// + internal class FakeNativeCall : INativeCall + { + public UnaryResponseClientHandler UnaryResponseClientHandler + { + get; + set; + } + + public ReceivedStatusOnClientHandler ReceivedStatusOnClientHandler + { + get; + set; + } + + public ReceivedMessageHandler ReceivedMessageHandler + { + get; + set; + } + + public ReceivedResponseHeadersHandler ReceivedResponseHeadersHandler + { + get; + set; + } + + public SendCompletionHandler SendCompletionHandler + { + get; + set; + } + + public ReceivedCloseOnServerHandler ReceivedCloseOnServerHandler + { + get; + set; + } + + public bool IsCancelled + { + get; + set; + } + + public bool IsDisposed + { + get; + set; + } + + public void Cancel() + { + IsCancelled = true; + } + + public void CancelWithStatus(Status status) + { + IsCancelled = true; + } + + public string GetPeer() + { + return "PEER"; + } + + public void StartUnary(UnaryResponseClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags) + { + UnaryResponseClientHandler = callback; + } + + public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags) + { + throw new NotImplementedException(); + } + + public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray) + { + UnaryResponseClientHandler = callback; + } + + public void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags) + { + ReceivedStatusOnClientHandler = callback; + } + + public void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray) + { + ReceivedStatusOnClientHandler = callback; + } + + public void StartReceiveMessage(ReceivedMessageHandler callback) + { + ReceivedMessageHandler = callback; + } + + public void StartReceiveInitialMetadata(ReceivedResponseHeadersHandler callback) + { + ReceivedResponseHeadersHandler = callback; + } + + public void StartSendInitialMetadata(SendCompletionHandler callback, MetadataArraySafeHandle metadataArray) + { + SendCompletionHandler = callback; + } + + public void StartSendMessage(SendCompletionHandler callback, byte[] payload, WriteFlags writeFlags, bool sendEmptyInitialMetadata) + { + SendCompletionHandler = callback; + } + + public void StartSendCloseFromClient(SendCompletionHandler callback) + { + SendCompletionHandler = callback; + } + + public void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata) + { + SendCompletionHandler = callback; + } + + public void StartServerSide(ReceivedCloseOnServerHandler callback) + { + ReceivedCloseOnServerHandler = callback; + } + + public void Dispose() + { + IsDisposed = true; + } + } +} diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs index cce480b2c4a..efcf4ea7fea 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs @@ -64,6 +64,15 @@ namespace Grpc.Core.Internal InitializeInternal(call); } + /// + /// Only for testing purposes. + /// + public void InitializeForTesting(INativeCall call) + { + server.AddCallReference(this); + InitializeInternal(call); + } + /// /// Starts a server side call. /// From ce60d8e7a4f9a23495d0d2edb22dadbb78ca8089 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 09:52:40 -0700 Subject: [PATCH 045/155] add AsyncCalServer tests to sanity tests --- .../Grpc.Core.Tests/Internal/AsyncCallServerTest.cs | 8 +++----- src/csharp/tests.json | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs index 8c178657a1b..a1671c97d5d 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs @@ -83,7 +83,7 @@ namespace Grpc.Core.Internal.Tests Assert.IsFalse(moveNextTask.Result); fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); - AssertDisposed(asyncCallServer, fakeCall, finishedTask); + AssertFinished(asyncCallServer, fakeCall, finishedTask); } [Test] @@ -100,14 +100,12 @@ namespace Grpc.Core.Internal.Tests fakeCall.ReceivedMessageHandler(true, null); Assert.IsFalse(moveNextTask.Result); - AssertDisposed(asyncCallServer, fakeCall, finishedTask); + AssertFinished(asyncCallServer, fakeCall, finishedTask); } // TODO: read completion failure ... - // TODO: - // TODO: write fails... @@ -124,7 +122,7 @@ namespace Grpc.Core.Internal.Tests // TODO: what does writing status do to reads? - static void AssertDisposed(AsyncCallServer asyncCallServer, FakeNativeCall fakeCall, Task finishedTask) + static void AssertFinished(AsyncCallServer asyncCallServer, FakeNativeCall fakeCall, Task finishedTask) { Assert.IsTrue(fakeCall.IsDisposed); Assert.IsTrue(finishedTask.IsCompleted); diff --git a/src/csharp/tests.json b/src/csharp/tests.json index f733352a310..f6af3408d51 100644 --- a/src/csharp/tests.json +++ b/src/csharp/tests.json @@ -1,5 +1,6 @@ { "Grpc.Core.Tests": [ + "Grpc.Core.Internal.Tests.AsyncCallServerTest", "Grpc.Core.Internal.Tests.AsyncCallTest", "Grpc.Core.Internal.Tests.ChannelArgsSafeHandleTest", "Grpc.Core.Internal.Tests.CompletionQueueEventTest", From b32e29f0a2f2cbe858e040190f2fd69237007f9a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 12:55:42 -0700 Subject: [PATCH 046/155] make SendStatusFromServer independent on WriteAsync --- .../Grpc.Core/Internal/AsyncCallBase.cs | 9 +++------ .../Grpc.Core/Internal/AsyncCallServer.cs | 19 ++++++++++--------- .../Grpc.Core/Internal/ServerCallHandler.cs | 12 +++++------- .../Internal/ServerResponseStream.cs | 7 ------- 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs index abacfabadb6..18dbe87734b 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs @@ -69,6 +69,7 @@ namespace Grpc.Core.Internal protected AsyncCompletionDelegate sendCompletionDelegate; // Completion of a pending send or sendclose if not null. protected TaskCompletionSource streamingReadTcs; // Completion of a pending streaming read if not null. + protected TaskCompletionSource sendStatusFromServerTcs; protected bool readingDone; // True if last read (i.e. read with null payload) was already received. protected bool halfcloseRequested; // True if send close have been initiated. @@ -328,22 +329,18 @@ namespace Grpc.Core.Internal /// protected void HandleSendStatusFromServerFinished(bool success) { - AsyncCompletionDelegate origCompletionDelegate = null; lock (myLock) { - origCompletionDelegate = sendCompletionDelegate; - sendCompletionDelegate = null; - ReleaseResourcesIfPossible(); } if (!success) { - FireCompletion(origCompletionDelegate, null, new InvalidOperationException("Error sending status from server.")); + sendStatusFromServerTcs.SetException(new InvalidOperationException("Error sending status from server.")); } else { - FireCompletion(origCompletionDelegate, null, null); + sendStatusFromServerTcs.SetResult(null); } } diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs index efcf4ea7fea..94f49bd8f2a 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs @@ -136,24 +136,24 @@ namespace Grpc.Core.Internal } /// - /// Sends call result status, also indicating server is done with streaming responses. - /// Only one pending send action is allowed at any given time. - /// completionDelegate is called when the operation finishes. + /// Sends call result status, indicating we are done with writes. + /// Sending a status different from StatusCode.OK will also implicitly cancel the call. /// - public void StartSendStatusFromServer(Status status, Metadata trailers, AsyncCompletionDelegate completionDelegate) + public Task SendStatusFromServerAsync(Status status, Metadata trailers) { lock (myLock) { - GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null"); - CheckSendingAllowed(allowFinished: false); + GrpcPreconditions.CheckState(started); + GrpcPreconditions.CheckState(!disposed); + GrpcPreconditions.CheckState(!halfcloseRequested, "Can only send status from server once."); using (var metadataArray = MetadataArraySafeHandle.Create(trailers)) { call.StartSendStatusFromServer(HandleSendStatusFromServerFinished, status, metadataArray, !initialMetadataSent); } halfcloseRequested = true; - readingDone = true; - sendCompletionDelegate = completionDelegate; + sendStatusFromServerTcs = new TaskCompletionSource(); + return sendStatusFromServerTcs.Task; } } @@ -198,12 +198,13 @@ namespace Grpc.Core.Internal /// private void HandleFinishedServerside(bool success, bool cancelled) { + // NOTE: because this event is a result of batch containing GRPC_OP_RECV_CLOSE_ON_SERVER, + // success will be always set to true. lock (myLock) { finished = true; ReleaseResourcesIfPossible(); } - // TODO(jtattermusch): handle error if (cancelled) { diff --git a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs index 1f83e51548e..bf9df9f783b 100644 --- a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs +++ b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs @@ -93,7 +93,7 @@ namespace Grpc.Core.Internal } try { - await responseStream.WriteStatusAsync(status, context.ResponseTrailers).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -149,7 +149,7 @@ namespace Grpc.Core.Internal try { - await responseStream.WriteStatusAsync(status, context.ResponseTrailers).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -209,7 +209,7 @@ namespace Grpc.Core.Internal try { - await responseStream.WriteStatusAsync(status, context.ResponseTrailers).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -260,7 +260,7 @@ namespace Grpc.Core.Internal } try { - await responseStream.WriteStatusAsync(status, context.ResponseTrailers).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -282,9 +282,7 @@ namespace Grpc.Core.Internal asyncCall.Initialize(newRpc.Call); var finishedTask = asyncCall.ServerSideCallAsync(); - var responseStream = new ServerResponseStream(asyncCall); - - await responseStream.WriteStatusAsync(new Status(StatusCode.Unimplemented, ""), Metadata.Empty).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(new Status(StatusCode.Unimplemented, ""), Metadata.Empty).ConfigureAwait(false); await finishedTask.ConfigureAwait(false); } } diff --git a/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs b/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs index 03e39efc024..ecfee0bfddb 100644 --- a/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs +++ b/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs @@ -57,13 +57,6 @@ namespace Grpc.Core.Internal return taskSource.Task; } - public Task WriteStatusAsync(Status status, Metadata trailers) - { - var taskSource = new AsyncCompletionTaskSource(); - call.StartSendStatusFromServer(status, trailers, taskSource.CompletionDelegate); - return taskSource.Task; - } - public Task WriteResponseHeadersAsync(Metadata responseHeaders) { var taskSource = new AsyncCompletionTaskSource(); From 9a9813bc4b89b58400adbcfdb4eab6a12165f563 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 12:57:41 -0700 Subject: [PATCH 047/155] add more tests --- .../Internal/AsyncCallServerTest.cs | 83 ++++++++++++++++--- .../Internal/FakeNativeCall.cs | 8 +- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs index a1671c97d5d..169de5a7800 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs @@ -75,7 +75,6 @@ namespace Grpc.Core.Internal.Tests { var finishedTask = asyncCallServer.ServerSideCallAsync(); var requestStream = new ServerRequestStream(asyncCallServer); - var responseStream = new ServerResponseStream(asyncCallServer); // Finishing requestStream is needed for dispose to happen. var moveNextTask = requestStream.MoveNext(); @@ -91,7 +90,6 @@ namespace Grpc.Core.Internal.Tests { var finishedTask = asyncCallServer.ServerSideCallAsync(); var requestStream = new ServerRequestStream(asyncCallServer); - var responseStream = new ServerResponseStream(asyncCallServer); fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); @@ -103,24 +101,89 @@ namespace Grpc.Core.Internal.Tests AssertFinished(asyncCallServer, fakeCall, finishedTask); } + [Test] + public void ReadCompletionFailureClosesRequestStream() + { + var finishedTask = asyncCallServer.ServerSideCallAsync(); + var requestStream = new ServerRequestStream(asyncCallServer); + + // if a read completion's success==false, the request stream will silently finish + // and we rely on C core cancelling the call. + var moveNextTask = requestStream.MoveNext(); + fakeCall.ReceivedMessageHandler(false, null); + Assert.IsFalse(moveNextTask.Result); + + fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); + AssertFinished(asyncCallServer, fakeCall, finishedTask); + } + + [Test] + public void WriteAfterCancelNotificationFails() + { + var finishedTask = asyncCallServer.ServerSideCallAsync(); + var requestStream = new ServerRequestStream(asyncCallServer); + var responseStream = new ServerResponseStream(asyncCallServer); + + fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); + + // TODO(jtattermusch): should we throw a different exception type instead? + Assert.Throws(typeof(InvalidOperationException), () => responseStream.WriteAsync("request1")); + + // Finishing requestStream is needed for dispose to happen. + var moveNextTask = requestStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + Assert.IsFalse(moveNextTask.Result); + + AssertFinished(asyncCallServer, fakeCall, finishedTask); + } + + [Test] + public void WriteCompletionFailureThrows() + { + var finishedTask = asyncCallServer.ServerSideCallAsync(); + var requestStream = new ServerRequestStream(asyncCallServer); + var responseStream = new ServerResponseStream(asyncCallServer); + + var writeTask = responseStream.WriteAsync("request1"); + fakeCall.SendCompletionHandler(false); + // TODO(jtattermusch): should we throw a different exception type instead? + Assert.ThrowsAsync(typeof(InvalidOperationException), async () => await writeTask); - // TODO: read completion failure ... + // Finishing requestStream is needed for dispose to happen. + var moveNextTask = requestStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + Assert.IsFalse(moveNextTask.Result); + fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); + AssertFinished(asyncCallServer, fakeCall, finishedTask); + } - // TODO: write fails... + [Test] + public void WriteAndWriteStatusCanRunConcurrently() + { + var finishedTask = asyncCallServer.ServerSideCallAsync(); + var requestStream = new ServerRequestStream(asyncCallServer); + var responseStream = new ServerResponseStream(asyncCallServer); - // TODO: write completion fails... + var writeTask = responseStream.WriteAsync("request1"); + var writeStatusTask = asyncCallServer.SendStatusFromServerAsync(Status.DefaultSuccess, new Metadata()); - // TODO: cancellation delivered... + fakeCall.SendCompletionHandler(true); + fakeCall.SendStatusFromServerHandler(true); - // TODO: cancel notification in the middle of a read... + Assert.DoesNotThrowAsync(async () => await writeTask); + Assert.DoesNotThrowAsync(async () => await writeStatusTask); - // TODO: cancel notification in the middle of a write... + // Finishing requestStream is needed for dispose to happen. + var moveNextTask = requestStream.MoveNext(); + fakeCall.ReceivedMessageHandler(true, null); + Assert.IsFalse(moveNextTask.Result); - // TODO: cancellation delivered... + fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); - // TODO: what does writing status do to reads? + AssertFinished(asyncCallServer, fakeCall, finishedTask); + } static void AssertFinished(AsyncCallServer asyncCallServer, FakeNativeCall fakeCall, Task finishedTask) { diff --git a/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs b/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs index 441bf9660be..1bec258ca29 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs @@ -76,6 +76,12 @@ namespace Grpc.Core.Internal.Tests set; } + public SendCompletionHandler SendStatusFromServerHandler + { + get; + set; + } + public ReceivedCloseOnServerHandler ReceivedCloseOnServerHandler { get; @@ -161,7 +167,7 @@ namespace Grpc.Core.Internal.Tests public void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata) { - SendCompletionHandler = callback; + SendStatusFromServerHandler = callback; } public void StartServerSide(ReceivedCloseOnServerHandler callback) From 2624cfb511d54b7d41f9eeab4073d2e6c9def9ad Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 13:04:33 -0700 Subject: [PATCH 048/155] fixup --- src/csharp/Grpc.Core/Internal/AsyncCallServer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs index 94f49bd8f2a..44f2988e21e 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs @@ -152,6 +152,7 @@ namespace Grpc.Core.Internal call.StartSendStatusFromServer(HandleSendStatusFromServerFinished, status, metadataArray, !initialMetadataSent); } halfcloseRequested = true; + initialMetadataSent = true; sendStatusFromServerTcs = new TaskCompletionSource(); return sendStatusFromServerTcs.Task; } From f21f465bceaca0bfd23a377d56fa19fa4794fede Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 15:29:45 -0700 Subject: [PATCH 049/155] make AsyncCallServerTest finish correctly --- .../Grpc.Core.Tests/Internal/AsyncCallServerTest.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs index 169de5a7800..0b6981f871d 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs @@ -54,7 +54,14 @@ namespace Grpc.Core.Internal.Tests public void Init() { var environment = GrpcEnvironment.AddRef(); - server = new Server(); + + // Create a fake server just so we have an instance to refer to. + // The server won't actually be used at all. + server = new Server() + { + Ports = { { "localhost", 0, ServerCredentials.Insecure } } + }; + server.Start(); fakeCall = new FakeNativeCall(); asyncCallServer = new AsyncCallServer( @@ -67,6 +74,7 @@ namespace Grpc.Core.Internal.Tests [TearDown] public void Cleanup() { + server.ShutdownAsync().Wait(); GrpcEnvironment.Release(); } From b0de7163ade5b290850cae660349dbcab86f99ba Mon Sep 17 00:00:00 2001 From: yang-g Date: Tue, 3 May 2016 15:48:19 -0700 Subject: [PATCH 050/155] Add proto comments to generated _pb2.py files --- examples/python/helloworld/helloworld_pb2.py | 73 ++----- .../python/route_guide/route_guide_pb2.py | 189 ++++++------------ src/compiler/python_generator.cc | 47 +++-- 3 files changed, 114 insertions(+), 195 deletions(-) diff --git a/examples/python/helloworld/helloworld_pb2.py b/examples/python/helloworld/helloworld_pb2.py index 1b2674e4c8e..1ee80e4034c 100644 --- a/examples/python/helloworld/helloworld_pb2.py +++ b/examples/python/helloworld/helloworld_pb2.py @@ -1,6 +1,8 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: helloworld.proto +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -17,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='helloworld.proto', package='helloworld', syntax='proto3', - serialized_pb=b'\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2I\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x42\x18\n\x10io.grpc.examples\xa2\x02\x03HLWb\x06proto3' + serialized_pb=_b('\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2I\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x42\x36\n\x1bio.grpc.examples.helloworldB\x0fHelloWorldProtoP\x01\xa2\x02\x03HLWb\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -34,7 +36,7 @@ _HELLOREQUEST = _descriptor.Descriptor( _descriptor.FieldDescriptor( name='name', full_name='helloworld.HelloRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -65,7 +67,7 @@ _HELLOREPLY = _descriptor.Descriptor( _descriptor.FieldDescriptor( name='message', full_name='helloworld.HelloReply.message', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -104,69 +106,28 @@ _sym_db.RegisterMessage(HelloReply) DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\020io.grpc.examples\242\002\003HLW') +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW')) import abc +import six from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities +from grpc.beta import interfaces as beta_interfaces from grpc.framework.common import cardinality from grpc.framework.interfaces.face import utilities as face_utilities -class EarlyAdopterGreeterServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def SayHello(self, request, context): - raise NotImplementedError() -class EarlyAdopterGreeterServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterGreeterStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def SayHello(self, request): - raise NotImplementedError() - SayHello.async = None -def early_adopter_create_Greeter_server(servicer, port, private_key=None, certificate_chain=None): - import helloworld_pb2 - import helloworld_pb2 - method_service_descriptions = { - "SayHello": alpha_utilities.unary_unary_service_description( - servicer.SayHello, - helloworld_pb2.HelloRequest.FromString, - helloworld_pb2.HelloReply.SerializeToString, - ), - } - return early_adopter_implementations.server("helloworld.Greeter", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_Greeter_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import helloworld_pb2 - import helloworld_pb2 - method_invocation_descriptions = { - "SayHello": alpha_utilities.unary_unary_invocation_description( - helloworld_pb2.HelloRequest.SerializeToString, - helloworld_pb2.HelloReply.FromString, - ), - } - return early_adopter_implementations.stub("helloworld.Greeter", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) class BetaGreeterServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod + """The greeting service definition. + """ def SayHello(self, request, context): - raise NotImplementedError() + """Sends a greeting + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) class BetaGreeterStub(object): - """The interface to which stubs will conform.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod + """The greeting service definition. + """ def SayHello(self, request, timeout): + """Sends a greeting + """ raise NotImplementedError() SayHello.future = None diff --git a/examples/python/route_guide/route_guide_pb2.py b/examples/python/route_guide/route_guide_pb2.py index d4d9f8dcd5d..81d5d075274 100644 --- a/examples/python/route_guide/route_guide_pb2.py +++ b/examples/python/route_guide/route_guide_pb2.py @@ -1,6 +1,8 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: route_guide.proto +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -17,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='route_guide.proto', package='routeguide', syntax='proto3', - serialized_pb=b'\n\x11route_guide.proto\x12\nrouteguide\",\n\x05Point\x12\x10\n\x08latitude\x18\x01 \x01(\x05\x12\x11\n\tlongitude\x18\x02 \x01(\x05\"I\n\tRectangle\x12\x1d\n\x02lo\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x1d\n\x02hi\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"<\n\x07\x46\x65\x61ture\x12\x0c\n\x04name\x18\x01 \x01(\t\x12#\n\x08location\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"A\n\tRouteNote\x12#\n\x08location\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x0f\n\x07message\x18\x02 \x01(\t\"b\n\x0cRouteSummary\x12\x13\n\x0bpoint_count\x18\x01 \x01(\x05\x12\x15\n\rfeature_count\x18\x02 \x01(\x05\x12\x10\n\x08\x64istance\x18\x03 \x01(\x05\x12\x14\n\x0c\x65lapsed_time\x18\x04 \x01(\x05\x32\x85\x02\n\nRouteGuide\x12\x36\n\nGetFeature\x12\x11.routeguide.Point\x1a\x13.routeguide.Feature\"\x00\x12>\n\x0cListFeatures\x12\x15.routeguide.Rectangle\x1a\x13.routeguide.Feature\"\x00\x30\x01\x12>\n\x0bRecordRoute\x12\x11.routeguide.Point\x1a\x18.routeguide.RouteSummary\"\x00(\x01\x12?\n\tRouteChat\x12\x15.routeguide.RouteNote\x1a\x15.routeguide.RouteNote\"\x00(\x01\x30\x01\x42\x0f\n\x07\x65x.grpc\xa2\x02\x03RTGb\x06proto3' + serialized_pb=_b('\n\x11route_guide.proto\x12\nrouteguide\",\n\x05Point\x12\x10\n\x08latitude\x18\x01 \x01(\x05\x12\x11\n\tlongitude\x18\x02 \x01(\x05\"I\n\tRectangle\x12\x1d\n\x02lo\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x1d\n\x02hi\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"<\n\x07\x46\x65\x61ture\x12\x0c\n\x04name\x18\x01 \x01(\t\x12#\n\x08location\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"A\n\tRouteNote\x12#\n\x08location\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x0f\n\x07message\x18\x02 \x01(\t\"b\n\x0cRouteSummary\x12\x13\n\x0bpoint_count\x18\x01 \x01(\x05\x12\x15\n\rfeature_count\x18\x02 \x01(\x05\x12\x10\n\x08\x64istance\x18\x03 \x01(\x05\x12\x14\n\x0c\x65lapsed_time\x18\x04 \x01(\x05\x32\x85\x02\n\nRouteGuide\x12\x36\n\nGetFeature\x12\x11.routeguide.Point\x1a\x13.routeguide.Feature\"\x00\x12>\n\x0cListFeatures\x12\x15.routeguide.Rectangle\x1a\x13.routeguide.Feature\"\x00\x30\x01\x12>\n\x0bRecordRoute\x12\x11.routeguide.Point\x1a\x18.routeguide.RouteSummary\"\x00(\x01\x12?\n\tRouteChat\x12\x15.routeguide.RouteNote\x1a\x15.routeguide.RouteNote\"\x00(\x01\x30\x01\x42\x36\n\x1bio.grpc.examples.routeguideB\x0fRouteGuideProtoP\x01\xa2\x02\x03RTGb\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -110,7 +112,7 @@ _FEATURE = _descriptor.Descriptor( _descriptor.FieldDescriptor( name='name', full_name='routeguide.Feature.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -155,7 +157,7 @@ _ROUTENOTE = _descriptor.Descriptor( _descriptor.FieldDescriptor( name='message', full_name='routeguide.RouteNote.message', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -274,149 +276,86 @@ _sym_db.RegisterMessage(RouteSummary) DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\007ex.grpc\242\002\003RTG') +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.routeguideB\017RouteGuideProtoP\001\242\002\003RTG')) import abc +import six from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities +from grpc.beta import interfaces as beta_interfaces from grpc.framework.common import cardinality from grpc.framework.interfaces.face import utilities as face_utilities -class EarlyAdopterRouteGuideServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetFeature(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListFeatures(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def RecordRoute(self, request_iterator, context): - raise NotImplementedError() - @abc.abstractmethod - def RouteChat(self, request_iterator, context): - raise NotImplementedError() -class EarlyAdopterRouteGuideServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterRouteGuideStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetFeature(self, request): - raise NotImplementedError() - GetFeature.async = None - @abc.abstractmethod - def ListFeatures(self, request): - raise NotImplementedError() - ListFeatures.async = None - @abc.abstractmethod - def RecordRoute(self, request_iterator): - raise NotImplementedError() - RecordRoute.async = None - @abc.abstractmethod - def RouteChat(self, request_iterator): - raise NotImplementedError() - RouteChat.async = None -def early_adopter_create_RouteGuide_server(servicer, port, private_key=None, certificate_chain=None): - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - method_service_descriptions = { - "GetFeature": alpha_utilities.unary_unary_service_description( - servicer.GetFeature, - route_guide_pb2.Point.FromString, - route_guide_pb2.Feature.SerializeToString, - ), - "ListFeatures": alpha_utilities.unary_stream_service_description( - servicer.ListFeatures, - route_guide_pb2.Rectangle.FromString, - route_guide_pb2.Feature.SerializeToString, - ), - "RecordRoute": alpha_utilities.stream_unary_service_description( - servicer.RecordRoute, - route_guide_pb2.Point.FromString, - route_guide_pb2.RouteSummary.SerializeToString, - ), - "RouteChat": alpha_utilities.stream_stream_service_description( - servicer.RouteChat, - route_guide_pb2.RouteNote.FromString, - route_guide_pb2.RouteNote.SerializeToString, - ), - } - return early_adopter_implementations.server("routeguide.RouteGuide", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_RouteGuide_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - import route_guide_pb2 - method_invocation_descriptions = { - "GetFeature": alpha_utilities.unary_unary_invocation_description( - route_guide_pb2.Point.SerializeToString, - route_guide_pb2.Feature.FromString, - ), - "ListFeatures": alpha_utilities.unary_stream_invocation_description( - route_guide_pb2.Rectangle.SerializeToString, - route_guide_pb2.Feature.FromString, - ), - "RecordRoute": alpha_utilities.stream_unary_invocation_description( - route_guide_pb2.Point.SerializeToString, - route_guide_pb2.RouteSummary.FromString, - ), - "RouteChat": alpha_utilities.stream_stream_invocation_description( - route_guide_pb2.RouteNote.SerializeToString, - route_guide_pb2.RouteNote.FromString, - ), - } - return early_adopter_implementations.stub("routeguide.RouteGuide", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) class BetaRouteGuideServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod + """Interface exported by the server. + """ def GetFeature(self, request, context): - raise NotImplementedError() - @abc.abstractmethod + """A simple RPC. + + Obtains the feature at a given position. + + A feature with an empty name is returned if there's no feature at the given + position. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) def ListFeatures(self, request, context): - raise NotImplementedError() - @abc.abstractmethod + """A server-to-client streaming RPC. + + Obtains the Features available within the given Rectangle. Results are + streamed rather than returned at once (e.g. in a response message with a + repeated field), as the rectangle may cover a large area and contain a + huge number of features. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) def RecordRoute(self, request_iterator, context): - raise NotImplementedError() - @abc.abstractmethod + """A client-to-server streaming RPC. + + Accepts a stream of Points on a route being traversed, returning a + RouteSummary when traversal is completed. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) def RouteChat(self, request_iterator, context): - raise NotImplementedError() + """A Bidirectional streaming RPC. + + Accepts a stream of RouteNotes sent while a route is being traversed, + while receiving other RouteNotes (e.g. from other users). + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) class BetaRouteGuideStub(object): - """The interface to which stubs will conform.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod + """Interface exported by the server. + """ def GetFeature(self, request, timeout): + """A simple RPC. + + Obtains the feature at a given position. + + A feature with an empty name is returned if there's no feature at the given + position. + """ raise NotImplementedError() GetFeature.future = None - @abc.abstractmethod def ListFeatures(self, request, timeout): + """A server-to-client streaming RPC. + + Obtains the Features available within the given Rectangle. Results are + streamed rather than returned at once (e.g. in a response message with a + repeated field), as the rectangle may cover a large area and contain a + huge number of features. + """ raise NotImplementedError() - @abc.abstractmethod def RecordRoute(self, request_iterator, timeout): + """A client-to-server streaming RPC. + + Accepts a stream of Points on a route being traversed, returning a + RouteSummary when traversal is completed. + """ raise NotImplementedError() RecordRoute.future = None - @abc.abstractmethod def RouteChat(self, request_iterator, timeout): + """A Bidirectional streaming RPC. + + Accepts a stream of RouteNotes sent while a route is being traversed, + while receiving other RouteNotes (e.g. from other users). + """ raise NotImplementedError() def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 59137e1c923..8e76e6dce6f 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -182,18 +182,40 @@ bool GetModuleAndMessagePath(const Descriptor* type, return true; } +// Get all comments (leading, leading_detached, trailing) and print them as a +// docstring. Any leading space of a line will be removed, but the line wrapping +// will not be changed. +template +static void PrintAllComments(const DescriptorType* desc, Printer* printer) { + std::vector comments; + grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_LEADING_DETACHED, + &comments); + grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_LEADING, + &comments); + grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_TRAILING, + &comments); + if (comments.empty()) { + return; + } + printer->Print("\"\"\""); + for (auto it = comments.begin(); it != comments.end(); ++it) { + size_t start_pos = it->find_first_not_of(' '); + if (start_pos != grpc::string::npos) { + printer->Print(it->c_str() + start_pos); + } + printer->Print("\n"); + } + printer->Print("\"\"\"\n"); +} + bool PrintBetaServicer(const ServiceDescriptor* service, Printer* out) { - grpc::string doc = ""; - map dict = ListToDict({ - "Service", service->name(), - "Documentation", doc, - }); out->Print("\n"); - out->Print(dict, "class Beta$Service$Servicer(object):\n"); + out->Print("class Beta$Service$Servicer(object):\n", "Service", + service->name()); { IndentScope raii_class_indent(out); - out->Print(dict, "\"\"\"$Documentation$\"\"\"\n"); + PrintAllComments(service, out); for (int i = 0; i < service->method_count(); ++i) { auto meth = service->method(i); grpc::string arg_name = meth->client_streaming() ? @@ -202,6 +224,7 @@ bool PrintBetaServicer(const ServiceDescriptor* service, "Method", meth->name(), "ArgName", arg_name); { IndentScope raii_method_indent(out); + PrintAllComments(meth, out); out->Print("context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)\n"); } } @@ -211,16 +234,11 @@ bool PrintBetaServicer(const ServiceDescriptor* service, bool PrintBetaStub(const ServiceDescriptor* service, Printer* out) { - grpc::string doc = "The interface to which stubs will conform."; - map dict = ListToDict({ - "Service", service->name(), - "Documentation", doc, - }); out->Print("\n"); - out->Print(dict, "class Beta$Service$Stub(object):\n"); + out->Print("class Beta$Service$Stub(object):\n", "Service", service->name()); { IndentScope raii_class_indent(out); - out->Print(dict, "\"\"\"$Documentation$\"\"\"\n"); + PrintAllComments(service, out); for (int i = 0; i < service->method_count(); ++i) { const MethodDescriptor* meth = service->method(i); grpc::string arg_name = meth->client_streaming() ? @@ -229,6 +247,7 @@ bool PrintBetaStub(const ServiceDescriptor* service, out->Print(methdict, "def $Method$(self, $ArgName$, timeout):\n"); { IndentScope raii_method_indent(out); + PrintAllComments(meth, out); out->Print("raise NotImplementedError()\n"); } if (!meth->server_streaming()) { From 99341e30cf17889d3063e76a2eaa05c98021645b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 29 Apr 2016 14:47:39 -0700 Subject: [PATCH 051/155] move helper script for dockerization --- tools/{jenkins => run_tests/dockerize}/build_and_run_docker.sh | 0 .../dockerize}/build_docker_and_run_tests.sh | 0 tools/{jenkins => run_tests/dockerize}/build_interop_image.sh | 0 .../dockerize}/build_interop_stress_image.sh | 0 tools/{jenkins => run_tests/dockerize}/docker_run.sh | 0 tools/{jenkins => run_tests/dockerize}/docker_run_tests.sh | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename tools/{jenkins => run_tests/dockerize}/build_and_run_docker.sh (100%) rename tools/{jenkins => run_tests/dockerize}/build_docker_and_run_tests.sh (100%) rename tools/{jenkins => run_tests/dockerize}/build_interop_image.sh (100%) rename tools/{jenkins => run_tests/dockerize}/build_interop_stress_image.sh (100%) rename tools/{jenkins => run_tests/dockerize}/docker_run.sh (100%) rename tools/{jenkins => run_tests/dockerize}/docker_run_tests.sh (100%) diff --git a/tools/jenkins/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh similarity index 100% rename from tools/jenkins/build_and_run_docker.sh rename to tools/run_tests/dockerize/build_and_run_docker.sh diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh similarity index 100% rename from tools/jenkins/build_docker_and_run_tests.sh rename to tools/run_tests/dockerize/build_docker_and_run_tests.sh diff --git a/tools/jenkins/build_interop_image.sh b/tools/run_tests/dockerize/build_interop_image.sh similarity index 100% rename from tools/jenkins/build_interop_image.sh rename to tools/run_tests/dockerize/build_interop_image.sh diff --git a/tools/jenkins/build_interop_stress_image.sh b/tools/run_tests/dockerize/build_interop_stress_image.sh similarity index 100% rename from tools/jenkins/build_interop_stress_image.sh rename to tools/run_tests/dockerize/build_interop_stress_image.sh diff --git a/tools/jenkins/docker_run.sh b/tools/run_tests/dockerize/docker_run.sh similarity index 100% rename from tools/jenkins/docker_run.sh rename to tools/run_tests/dockerize/docker_run.sh diff --git a/tools/jenkins/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh similarity index 100% rename from tools/jenkins/docker_run_tests.sh rename to tools/run_tests/dockerize/docker_run_tests.sh From 26ca776e3ee4b81203cc4b97a956cbe9f6c00268 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 29 Apr 2016 14:51:35 -0700 Subject: [PATCH 052/155] update working directories --- tools/run_tests/dockerize/build_and_run_docker.sh | 2 +- tools/run_tests/dockerize/build_docker_and_run_tests.sh | 2 +- tools/run_tests/dockerize/build_interop_image.sh | 2 +- tools/run_tests/dockerize/build_interop_stress_image.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index 92dbbc6f380..1ef34b2f960 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -33,7 +33,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. git_root=$(pwd) cd - diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index 5779e63db97..c2ea6f2c6eb 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -33,7 +33,7 @@ set -ex -cd $(dirname $0)/../.. +cd $(dirname $0)/../../.. git_root=$(pwd) cd - diff --git a/tools/run_tests/dockerize/build_interop_image.sh b/tools/run_tests/dockerize/build_interop_image.sh index d2ba97c3de8..48a216a124b 100755 --- a/tools/run_tests/dockerize/build_interop_image.sh +++ b/tools/run_tests/dockerize/build_interop_image.sh @@ -40,7 +40,7 @@ set -x # BUILD_INTEROP_DOCKER_EXTRA_ARGS - optional args to be passed to the # docker run command -cd `dirname $0`/../.. +cd `dirname $0`/../../.. GRPC_ROOT=`pwd` MOUNT_ARGS="-v $GRPC_ROOT:/var/local/jenkins/grpc:ro" diff --git a/tools/run_tests/dockerize/build_interop_stress_image.sh b/tools/run_tests/dockerize/build_interop_stress_image.sh index 31ffa752ab5..4407c8da90c 100755 --- a/tools/run_tests/dockerize/build_interop_stress_image.sh +++ b/tools/run_tests/dockerize/build_interop_stress_image.sh @@ -44,7 +44,7 @@ set -x # BUILD_INTEROP_DOCKER_EXTRA_ARGS - optional args to be passed to the # docker run command -cd `dirname $0`/../.. +cd `dirname $0`/../../.. GRPC_ROOT=`pwd` MOUNT_ARGS="-v $GRPC_ROOT:/var/local/jenkins/grpc:ro" From 9835d4b1f7a4a065117e865e98c6f7b01e8eda96 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 29 Apr 2016 15:05:05 -0700 Subject: [PATCH 053/155] update locations of dockerization scripts --- tools/jenkins/run_fuzzer.sh | 4 ++-- tools/run_tests/artifact_targets.py | 4 ++-- tools/run_tests/distribtest_targets.py | 4 ++-- tools/run_tests/package_targets.py | 4 ++-- tools/run_tests/run_interop_tests.py | 2 +- tools/run_tests/run_stress_tests.py | 2 +- tools/run_tests/run_tests.py | 4 ++-- tools/run_tests/stress_test/configs/asan.json | 2 +- tools/run_tests/stress_test/configs/go.json | 2 +- tools/run_tests/stress_test/configs/java.json | 2 +- tools/run_tests/stress_test/configs/node-cxx.json | 4 ++-- tools/run_tests/stress_test/configs/node.json | 2 +- tools/run_tests/stress_test/configs/opt-tsan-asan.json | 6 +++--- tools/run_tests/stress_test/configs/opt.json | 2 +- tools/run_tests/stress_test/configs/tsan.json | 2 +- 15 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tools/jenkins/run_fuzzer.sh b/tools/jenkins/run_fuzzer.sh index 3f25a93319d..cfa7acefab9 100755 --- a/tools/jenkins/run_fuzzer.sh +++ b/tools/jenkins/run_fuzzer.sh @@ -33,14 +33,14 @@ set -ex export RUN_COMMAND="tools/fuzzer/build_and_run_fuzzer.sh $1" -export DOCKER_RUN_SCRIPT=tools/jenkins/docker_run.sh +export DOCKER_RUN_SCRIPT=tools/run_tests/dockerize/docker_run.sh export DOCKERFILE_DIR=tools/dockerfile/test/fuzzer export OUTPUT_DIR=fuzzer_output runtime=${runtime:-3600} jobs=${jobs:-3} -tools/jenkins/build_and_run_docker.sh \ +tools/run_tests/dockerize/build_and_run_docker.sh \ -e RUN_COMMAND="$RUN_COMMAND" \ -e OUTPUT_DIR="$OUTPUT_DIR" \ -e config="$config" \ diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index e61c46d8b5a..3e08c1d62b3 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -43,10 +43,10 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, for k,v in environ.iteritems(): docker_args += ['-e', '%s=%s' % (k, v)] docker_env = {'DOCKERFILE_DIR': dockerfile_dir, - 'DOCKER_RUN_SCRIPT': 'tools/jenkins/docker_run.sh', + 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', 'OUTPUT_DIR': 'artifacts'} jobspec = jobset.JobSpec( - cmdline=['tools/jenkins/build_and_run_docker.sh'] + docker_args, + cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, environ=docker_env, shortname='build_artifact.%s' % (name), timeout_seconds=30*60, diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index 34cc1cd7101..ae918be21d2 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -44,9 +44,9 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, for k,v in environ.iteritems(): docker_args += ['-e', '%s=%s' % (k, v)] docker_env = {'DOCKERFILE_DIR': dockerfile_dir, - 'DOCKER_RUN_SCRIPT': 'tools/jenkins/docker_run.sh'} + 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh'} jobspec = jobset.JobSpec( - cmdline=['tools/jenkins/build_and_run_docker.sh'] + docker_args, + cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, environ=docker_env, shortname='distribtest.%s' % (name), timeout_seconds=30*60, diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py index 87bc4865ce3..820b539b59f 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/package_targets.py @@ -42,10 +42,10 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, for k,v in environ.iteritems(): docker_args += ['-e', '%s=%s' % (k, v)] docker_env = {'DOCKERFILE_DIR': dockerfile_dir, - 'DOCKER_RUN_SCRIPT': 'tools/jenkins/docker_run.sh', + 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', 'OUTPUT_DIR': 'artifacts'} jobspec = jobset.JobSpec( - cmdline=['tools/jenkins/build_and_run_docker.sh'] + docker_args, + cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, environ=docker_env, shortname='build_package.%s' % (name), timeout_seconds=30*60, diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 758be9304d6..e813473421d 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -542,7 +542,7 @@ def build_interop_image_jobspec(language, tag=None): env['BUILD_INTEROP_DOCKER_EXTRA_ARGS'] = \ '-v %s:/root/.composer/auth.json:ro' % host_file build_job = jobset.JobSpec( - cmdline=['tools/jenkins/build_interop_image.sh'], + cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], environ=env, shortname='build_docker_%s' % (language), timeout_seconds=30*60) diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py index 0ba8f51c583..e42ee24ffb0 100755 --- a/tools/run_tests/run_stress_tests.py +++ b/tools/run_tests/run_stress_tests.py @@ -195,7 +195,7 @@ def build_interop_stress_image_jobspec(language, tag=None): tag = 'grpc_interop_stress_%s:%s' % (language.safename, uuid.uuid4()) env = {'INTEROP_IMAGE': tag, 'BASE_NAME': 'grpc_interop_stress_%s' % language.safename} - build_job = jobset.JobSpec(cmdline=['tools/jenkins/build_interop_stress_image.sh'], + build_job = jobset.JobSpec(cmdline=['tools/run_tests/dockerize/build_interop_stress_image.sh'], environ=env, shortname='build_docker_%s' % (language), timeout_seconds=30 * 60) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index d50d5aac95e..37291f4d3fb 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -912,13 +912,13 @@ if args.use_docker: env = os.environ.copy() env['RUN_TESTS_COMMAND'] = run_tests_cmd env['DOCKERFILE_DIR'] = dockerfile_dir - env['DOCKER_RUN_SCRIPT'] = 'tools/jenkins/docker_run_tests.sh' + env['DOCKER_RUN_SCRIPT'] = 'tools/run_tests/dockerize/docker_run_tests.sh' if args.xml_report: env['XML_REPORT'] = args.xml_report if not args.travis: env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. - subprocess.check_call(['tools/jenkins/build_docker_and_run_tests.sh'], + subprocess.check_call(['tools/run_tests/dockerize/build_docker_and_run_tests.sh'], shell=True, env=env) sys.exit(0) diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json index cb9f55763b1..7ae11ccbf1e 100644 --- a/tools/run_tests/stress_test/configs/asan.json +++ b/tools/run_tests/stress_test/configs/asan.json @@ -1,7 +1,7 @@ { "dockerImages": { "grpc_stress_cxx_asan" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_cxx", "buildType": "asan" } diff --git a/tools/run_tests/stress_test/configs/go.json b/tools/run_tests/stress_test/configs/go.json index 36b465e763e..f1b2b523d3c 100644 --- a/tools/run_tests/stress_test/configs/go.json +++ b/tools/run_tests/stress_test/configs/go.json @@ -1,7 +1,7 @@ { "dockerImages": { "grpc_stress_go" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_go" } }, diff --git a/tools/run_tests/stress_test/configs/java.json b/tools/run_tests/stress_test/configs/java.json index 275384c066d..2ce6c007804 100644 --- a/tools/run_tests/stress_test/configs/java.json +++ b/tools/run_tests/stress_test/configs/java.json @@ -1,7 +1,7 @@ { "dockerImages": { "grpc_stress_java" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_java" } }, diff --git a/tools/run_tests/stress_test/configs/node-cxx.json b/tools/run_tests/stress_test/configs/node-cxx.json index c4245bf9df2..094c1236e71 100644 --- a/tools/run_tests/stress_test/configs/node-cxx.json +++ b/tools/run_tests/stress_test/configs/node-cxx.json @@ -1,12 +1,12 @@ { "dockerImages": { "grpc_stress_cxx_opt" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_cxx", "buildType": "opt" }, "grpc_stress_node": { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_node" } }, diff --git a/tools/run_tests/stress_test/configs/node.json b/tools/run_tests/stress_test/configs/node.json index 7a48c56a5e2..85eb9e00031 100644 --- a/tools/run_tests/stress_test/configs/node.json +++ b/tools/run_tests/stress_test/configs/node.json @@ -1,7 +1,7 @@ { "dockerImages": { "grpc_stress_node" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_node" } }, diff --git a/tools/run_tests/stress_test/configs/opt-tsan-asan.json b/tools/run_tests/stress_test/configs/opt-tsan-asan.json index 936d15169e1..fcb3678c02f 100644 --- a/tools/run_tests/stress_test/configs/opt-tsan-asan.json +++ b/tools/run_tests/stress_test/configs/opt-tsan-asan.json @@ -1,17 +1,17 @@ { "dockerImages": { "grpc_stress_cxx_opt" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_cxx", "buildType": "opt" }, "grpc_stress_cxx_tsan": { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_cxx", "buildType": "tsan" }, "grpc_stress_cxx_asan": { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_cxx", "buildType": "asan" } diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json index f45b8240487..5e0e930d453 100644 --- a/tools/run_tests/stress_test/configs/opt.json +++ b/tools/run_tests/stress_test/configs/opt.json @@ -1,7 +1,7 @@ { "dockerImages": { "grpc_stress_cxx_opt" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_cxx", "buildType": "opt" } diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json index 6ef3bdf7ea7..abc759c79da 100644 --- a/tools/run_tests/stress_test/configs/tsan.json +++ b/tools/run_tests/stress_test/configs/tsan.json @@ -1,7 +1,7 @@ { "dockerImages": { "grpc_stress_cxx_tsan" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_cxx", "buildType": "tsan" } From 080ddf09ab9c037bea940a32d0075ee1b3d346d4 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 29 Apr 2016 15:07:42 -0700 Subject: [PATCH 054/155] add a readme file --- tools/jenkins/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 tools/jenkins/README.md diff --git a/tools/jenkins/README.md b/tools/jenkins/README.md new file mode 100644 index 00000000000..8e06b68466c --- /dev/null +++ b/tools/jenkins/README.md @@ -0,0 +1 @@ +Scripts invoked by Jenkins (our CI platform) to run gRPC test suites. From 9c8351257569e3c79d1d900c16cf942a8183fe95 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 15:58:06 -0700 Subject: [PATCH 055/155] update remaining stresstest configs --- tools/run_tests/stress_test/configs/csharp.json | 2 +- tools/run_tests/stress_test/configs/ruby.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/stress_test/configs/csharp.json b/tools/run_tests/stress_test/configs/csharp.json index b7090696b4f..406949557dc 100644 --- a/tools/run_tests/stress_test/configs/csharp.json +++ b/tools/run_tests/stress_test/configs/csharp.json @@ -1,7 +1,7 @@ { "dockerImages": { "grpc_stress_csharp" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_csharp" } }, diff --git a/tools/run_tests/stress_test/configs/ruby.json b/tools/run_tests/stress_test/configs/ruby.json index a323fa72b6b..7e2afcbb69e 100644 --- a/tools/run_tests/stress_test/configs/ruby.json +++ b/tools/run_tests/stress_test/configs/ruby.json @@ -1,7 +1,7 @@ { "dockerImages": { "grpc_stress_ruby" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_ruby" } }, From ed4d89e1f51fa06cc5e8cb231547b837ca1d1fa8 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 16:01:38 -0700 Subject: [PATCH 056/155] Remove trailing space. --- include/grpc/impl/codegen/grpc_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index fa287d52388..7b20cc14d43 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -307,7 +307,7 @@ typedef enum { GRPC_OP_RECV_STATUS_ON_CLIENT, /** Receive close on the server: one and only one must be made on the server. - This op completes after the close has been received by the server. + This op completes after the close has been received by the server. This operation always succeeds, meaning ops paired with this operation will also appear to succeed, even though they may not have. */ GRPC_OP_RECV_CLOSE_ON_SERVER From 1717bff3f47939b83fece86075fb9b9aa178ebb3 Mon Sep 17 00:00:00 2001 From: yang-g Date: Tue, 3 May 2016 16:04:54 -0700 Subject: [PATCH 057/155] add a dummy service with no rpc to test codegen coverage --- src/proto/grpc/testing/echo.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/proto/grpc/testing/echo.proto b/src/proto/grpc/testing/echo.proto index 0eef53a92a1..c596aabfcc4 100644 --- a/src/proto/grpc/testing/echo.proto +++ b/src/proto/grpc/testing/echo.proto @@ -45,3 +45,7 @@ service EchoTestService { service UnimplementedService { rpc Unimplemented(EchoRequest) returns (EchoResponse); } + +// A service without any rpc defined to test coverage. +service NoRpcService { +} From 1bdd5319b077956214b95e730e844f079baa4ddf Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 2 May 2016 19:57:04 -0700 Subject: [PATCH 058/155] Python dockerfiles and config files for stress testing --- .../Dockerfile.template | 45 ++++++++ .../grpc_interop_stress_python/Dockerfile | 103 ++++++++++++++++++ .../build_interop_stress.sh | 46 ++++++++ .../run_tests/stress_test/configs/python.json | 98 +++++++++++++++++ tools/run_tests/stress_test/run_on_gke.py | 13 ++- 5 files changed, 301 insertions(+), 4 deletions(-) create mode 100644 templates/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile.template create mode 100644 tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile create mode 100755 tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh create mode 100644 tools/run_tests/stress_test/configs/python.json diff --git a/templates/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile.template new file mode 100644 index 00000000000..27e9eeec5a8 --- /dev/null +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile.template @@ -0,0 +1,45 @@ +%YAML 1.2 +--- | + # Copyright 2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM debian:jessie + + <%include file="../../apt_get_basic.include"/> + <%include file="../../ccache_setup.include"/> + <%include file="../../cxx_deps.include"/> + <%include file="../../gcp_api_libraries.include"/> + <%include file="../../python_deps.include"/> + + RUN pip install coverage + RUN pip install oauth2client + + # Define the default command. + CMD ["bash"] + diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile new file mode 100644 index 00000000000..606b7654576 --- /dev/null +++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile @@ -0,0 +1,103 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + + +#==================== +# Python dependencies + +# Install dependencies + +RUN apt-get update && apt-get install -y \ + python-all-dev \ + python3-all-dev \ + python-pip + +# Install Python packages from PyPI +RUN pip install pip --upgrade +RUN pip install virtualenv +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 tox + + +RUN pip install coverage +RUN pip install oauth2client + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh new file mode 100755 index 00000000000..e65332f2f30 --- /dev/null +++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds Python interop server and client in a base image. +set -e + +mkdir -p /var/local/git +git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc + +# copy service account keys if available +cp -r /var/local/jenkins/service_account $HOME || true + +cd /var/local/git/grpc + +tools/run_tests/run_tests.py -l python -c opt --build_only + +# Build c++ interop client +make metrics_client -j + diff --git a/tools/run_tests/stress_test/configs/python.json b/tools/run_tests/stress_test/configs/python.json new file mode 100644 index 00000000000..ea5b5ba2ec4 --- /dev/null +++ b/tools/run_tests/stress_test/configs/python.json @@ -0,0 +1,98 @@ +{ + "dockerImages": { + "grpc_stress_python" : { + "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "dockerFileDir": "grpc_interop_stress_python" + } + }, + + "clientTemplates": { + "baseTemplates": { + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", + "pollIntervalSecs": 60, + "clientArgs": { + "num_channels_per_server":5, + "num_stubs_per_channel":10, + "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", + "metrics_port": 8081 + }, + "metricsPort": 8081, + "metricsArgs": { + "metrics_server_address": "localhost:8081", + "total_only": "true" + }, + "env": { + "PYTHONPATH": "/var/local/git/grpc/src/python/gens:/var/local/git/grpc/src/python/grpcio", + "LD_LIBRARY_PATH":"/var/local/git/grpc/libs/opt" + } + } + }, + "templates": { + "python_client": { + "baseTemplate": "default", + "stressClientCmd": [ + "python", + "/var/local/git/grpc/src/python/grpcio/tests/stress/client.py" + ], + "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] + } + } + }, + + "serverTemplates": { + "baseTemplates":{ + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", + "serverPort": 8080, + "serverArgs": { + "port": 8080 + }, + "env": { + "PYTHONPATH": "/var/local/git/grpc/src/python/gens:/var/local/git/grpc/src/python/grpcio", + "LD_LIBRARY_PATH":"/var/local/git/grpc/libs/opt" + } + } + }, + "templates": { + "python_server": { + "baseTemplate": "default", + "stressServerCmd": [ + "python", + "/var/local/git/grpc/src/python/grpcio/tests/interop/server.py" + ] + } + } + }, + + "testMatrix": { + "serverPodSpecs": { + "python-stress-server": { + "serverTemplate": "python_server", + "dockerImage": "grpc_stress_python", + "numInstances": 1 + } + }, + + "clientPodSpecs": { + "python-stress-client": { + "clientTemplate": "python_client", + "dockerImage": "grpc_stress_python", + "numInstances": 5, + "serverPodSpec": "python-stress-server" + } + } + }, + + "globalSettings": { + "buildDockerImages": true, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8011, + "datasetIdNamePrefix": "stress_test_python", + "summaryTableId": "summary", + "qpsTableId": "qps", + "podWarmupSecs": 60 + } +} + diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index d4f1c4ad3dc..583e58316f4 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -69,7 +69,7 @@ class ClientTemplate: def __init__(self, name, stress_client_cmd, metrics_client_cmd, metrics_port, wrapper_script_path, poll_interval_secs, client_args_dict, - metrics_args_dict, will_run_forever): + metrics_args_dict, will_run_forever, env_dict): self.name = name self.stress_client_cmd = stress_client_cmd self.metrics_client_cmd = metrics_client_cmd @@ -79,19 +79,21 @@ class ClientTemplate: self.client_args_dict = client_args_dict self.metrics_args_dict = metrics_args_dict self.will_run_forever = will_run_forever + self.env_dict = env_dict class ServerTemplate: """ Contains all the common settings used by a stress server """ def __init__(self, name, server_cmd, wrapper_script_path, server_port, - server_args_dict, will_run_forever): + server_args_dict, will_run_forever, env_dict): self.name = name self.server_cmd = server_cmd self.wrapper_script_path = wrapper_script_path self.server_port = server_port self.server_args_dict = server_args_dict self.will_run_forever = will_run_forever + self.env_dict = env_dict class DockerImage: @@ -240,6 +242,7 @@ class Gke: # server_pod_spec.template.wrapper_script_path) are are injected into the # container via environment variables server_env = self.gke_env.copy() + server_env.update(server_pod_spec.template.env_dict) server_env.update({ 'STRESS_TEST_IMAGE_TYPE': 'SERVER', 'STRESS_TEST_CMD': server_pod_spec.template.server_cmd, @@ -283,6 +286,7 @@ class Gke: # client_pod_spec.template.wrapper_script_path) are are injected into the # container via environment variables client_env = self.gke_env.copy() + client_env.update(client_pod_spec.template.env_dict) client_env.update({ 'STRESS_TEST_IMAGE_TYPE': 'CLIENT', 'STRESS_TEST_CMD': client_pod_spec.template.stress_client_cmd, @@ -425,7 +429,8 @@ class Config: template_name, stress_client_cmd, metrics_client_cmd, temp_dict['metricsPort'], temp_dict['wrapperScriptPath'], temp_dict['pollIntervalSecs'], temp_dict['clientArgs'].copy(), - temp_dict['metricsArgs'].copy(), temp_dict.get('willRunForever', 1)) + temp_dict['metricsArgs'].copy(), temp_dict.get('willRunForever', 1), + temp_dict.get('env', {}).copy()) return client_templates_dict @@ -461,7 +466,7 @@ class Config: server_templates_dict[template_name] = ServerTemplate( template_name, stress_server_cmd, temp_dict['wrapperScriptPath'], temp_dict['serverPort'], temp_dict['serverArgs'].copy(), - temp_dict.get('willRunForever', 1)) + temp_dict.get('willRunForever', 1), temp_dict.get('env', {}).copy()) return server_templates_dict From f023800ff1cc3e78fdfd53f37eaf0a1f31a4d88e Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 3 May 2016 16:40:43 -0700 Subject: [PATCH 059/155] Correct csharp kubernetes proxy port (so that it doesn't collide with other configs when running on the same machine) --- tools/run_tests/stress_test/configs/csharp.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/stress_test/configs/csharp.json b/tools/run_tests/stress_test/configs/csharp.json index b7090696b4f..306fe115c72 100644 --- a/tools/run_tests/stress_test/configs/csharp.json +++ b/tools/run_tests/stress_test/configs/csharp.json @@ -80,7 +80,7 @@ "buildDockerImages": true, "pollIntervalSecs": 60, "testDurationSecs": 7200, - "kubernetesProxyPort": 8001, + "kubernetesProxyPort": 8009, "datasetIdNamePrefix": "stress_test_csharp", "summaryTableId": "summary", "qpsTableId": "qps", From d8c0d385f45a953d683203f24bda90e679d98f25 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 3 May 2016 17:12:45 -0700 Subject: [PATCH 060/155] Fix a bug in stress client code --- src/python/grpcio/tests/stress/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/tests/stress/client.py b/src/python/grpcio/tests/stress/client.py index a733741b736..e2e016760c8 100644 --- a/src/python/grpcio/tests/stress/client.py +++ b/src/python/grpcio/tests/stress/client.py @@ -117,7 +117,10 @@ def run_test(args): for runner in runners: runner.start() try: - raise exception_queue.get(block=True, timeout=args.test_duration_secs) + timeout_secs = args.test_duration_secs + if timeout_secs < 0: + timeout_secs = None + raise exception_queue.get(block=True, timeout=timeout_secs) except Queue.Empty: # No exceptions thrown, success pass From 739e86c394040031ca6ac116b84e2975fb5fe83a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 17:24:33 -0700 Subject: [PATCH 061/155] finishing serverside request stream should not be required for disposal --- .../Internal/AsyncCallServerTest.cs | 34 ++++++------------- .../Grpc.Core/Internal/AsyncCallBase.cs | 7 +--- .../Grpc.Core/Internal/AsyncCallServer.cs | 14 ++++---- 3 files changed, 20 insertions(+), 35 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs index 0b6981f871d..058371521d6 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs @@ -80,16 +80,24 @@ namespace Grpc.Core.Internal.Tests [Test] public void CancelNotificationAfterStartDisposes() + { + var finishedTask = asyncCallServer.ServerSideCallAsync(); + fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); + AssertFinished(asyncCallServer, fakeCall, finishedTask); + } + + [Test] + public void CancelNotificationAfterStartDisposesAfterPendingReadFinishes() { var finishedTask = asyncCallServer.ServerSideCallAsync(); var requestStream = new ServerRequestStream(asyncCallServer); - // Finishing requestStream is needed for dispose to happen. var moveNextTask = requestStream.MoveNext(); + + fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); fakeCall.ReceivedMessageHandler(true, null); Assert.IsFalse(moveNextTask.Result); - fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); AssertFinished(asyncCallServer, fakeCall, finishedTask); } @@ -101,9 +109,8 @@ namespace Grpc.Core.Internal.Tests fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); - // Check that startin a read after cancel notification has been processed is legal. + // Check that starting a read after cancel notification has been processed is legal. var moveNextTask = requestStream.MoveNext(); - fakeCall.ReceivedMessageHandler(true, null); Assert.IsFalse(moveNextTask.Result); AssertFinished(asyncCallServer, fakeCall, finishedTask); @@ -136,12 +143,6 @@ namespace Grpc.Core.Internal.Tests // TODO(jtattermusch): should we throw a different exception type instead? Assert.Throws(typeof(InvalidOperationException), () => responseStream.WriteAsync("request1")); - - // Finishing requestStream is needed for dispose to happen. - var moveNextTask = requestStream.MoveNext(); - fakeCall.ReceivedMessageHandler(true, null); - Assert.IsFalse(moveNextTask.Result); - AssertFinished(asyncCallServer, fakeCall, finishedTask); } @@ -149,7 +150,6 @@ namespace Grpc.Core.Internal.Tests public void WriteCompletionFailureThrows() { var finishedTask = asyncCallServer.ServerSideCallAsync(); - var requestStream = new ServerRequestStream(asyncCallServer); var responseStream = new ServerResponseStream(asyncCallServer); var writeTask = responseStream.WriteAsync("request1"); @@ -157,13 +157,7 @@ namespace Grpc.Core.Internal.Tests // TODO(jtattermusch): should we throw a different exception type instead? Assert.ThrowsAsync(typeof(InvalidOperationException), async () => await writeTask); - // Finishing requestStream is needed for dispose to happen. - var moveNextTask = requestStream.MoveNext(); - fakeCall.ReceivedMessageHandler(true, null); - Assert.IsFalse(moveNextTask.Result); - fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); - AssertFinished(asyncCallServer, fakeCall, finishedTask); } @@ -171,7 +165,6 @@ namespace Grpc.Core.Internal.Tests public void WriteAndWriteStatusCanRunConcurrently() { var finishedTask = asyncCallServer.ServerSideCallAsync(); - var requestStream = new ServerRequestStream(asyncCallServer); var responseStream = new ServerResponseStream(asyncCallServer); var writeTask = responseStream.WriteAsync("request1"); @@ -183,11 +176,6 @@ namespace Grpc.Core.Internal.Tests Assert.DoesNotThrowAsync(async () => await writeTask); Assert.DoesNotThrowAsync(async () => await writeStatusTask); - // Finishing requestStream is needed for dispose to happen. - var moveNextTask = requestStream.MoveNext(); - fakeCall.ReceivedMessageHandler(true, null); - Assert.IsFalse(moveNextTask.Result); - fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true); AssertFinished(asyncCallServer, fakeCall, finishedTask); diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs index 18dbe87734b..42234dcac21 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs @@ -155,7 +155,7 @@ namespace Grpc.Core.Internal { lock (myLock) { - CheckReadingAllowed(); + GrpcPreconditions.CheckState(started); if (readingDone) { // the last read that returns null or throws an exception is idempotent @@ -224,11 +224,6 @@ namespace Grpc.Core.Internal GrpcPreconditions.CheckState(sendCompletionDelegate == null, "Only one write can be pending at a time"); } - protected virtual void CheckReadingAllowed() - { - GrpcPreconditions.CheckState(started); - } - protected void CheckNotCancelled() { if (cancelRequested) diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs index 44f2988e21e..eafe2ccab87 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs @@ -183,12 +183,6 @@ namespace Grpc.Core.Internal get { return false; } } - protected override void CheckReadingAllowed() - { - base.CheckReadingAllowed(); - GrpcPreconditions.CheckArgument(!cancelRequested); - } - protected override void OnAfterReleaseResources() { server.RemoveCallReference(this); @@ -204,6 +198,14 @@ namespace Grpc.Core.Internal lock (myLock) { finished = true; + if (streamingReadTcs == null) + { + // if there's no pending read, readingDone=true will dispose now. + // if there is a pending read, we will dispose once that read finishes. + readingDone = true; + streamingReadTcs = new TaskCompletionSource(); + streamingReadTcs.SetResult(default(TRequest)); + } ReleaseResourcesIfPossible(); } From 274bcc8f0b50bfef7fe1dd5cc33ebaa6b13edba0 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 3 May 2016 17:34:54 -0700 Subject: [PATCH 062/155] Added dummy cronet api implementation so we can build on Jenkins. --- BUILD | 12 +++ Makefile | 7 ++ binding.gyp | 3 + build.yaml | 10 ++ config.m4 | 5 + gRPC.podspec | 8 ++ grpc.def | 1 + grpc.gemspec | 6 ++ package.xml | 6 ++ .../client/secure/cronet_channel_create.c | 3 - .../cronet/transport/cronet_api_dummy.c | 91 +++++++++++++++++++ .../cronet/transport/cronet_transport.c | 6 +- .../grpcio/grpc/_cython/imports.generated.c | 2 + .../grpcio/grpc/_cython/imports.generated.h | 4 + src/python/grpcio/grpc_core_dependencies.py | 3 + src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 + src/ruby/ext/grpc/rb_grpc_imports.generated.h | 4 + .../core/surface/public_headers_must_be_c89.c | 1 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 6 ++ tools/run_tests/sources_and_headers.json | 22 ++++- vsprojects/vcxproj/grpc/grpc.vcxproj | 9 ++ vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 39 ++++++++ 23 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 src/core/ext/transport/cronet/transport/cronet_api_dummy.c diff --git a/BUILD b/BUILD index b4b10b535e8..2e1d762f079 100644 --- a/BUILD +++ b/BUILD @@ -285,6 +285,8 @@ cc_library( "src/core/ext/client_config/subchannel_call_holder.h", "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", + "include/grpc/support/port_platform.h", + "third_party/objective_c/Cronet/cronet_c_for_grpc.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/ext/census/aggregation.h", @@ -439,6 +441,9 @@ cc_library( "src/core/ext/client_config/uri_parser.c", "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c", "src/core/ext/transport/chttp2/client/insecure/channel_create.c", + "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", + "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", + "src/core/ext/transport/cronet/transport/cronet_transport.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/ext/lb_policy/pick_first/pick_first.c", @@ -482,6 +487,7 @@ cc_library( "include/grpc/impl/codegen/sync_posix.h", "include/grpc/impl/codegen/sync_win32.h", "include/grpc/impl/codegen/time.h", + "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "include/grpc/census.h", @@ -1456,6 +1462,9 @@ objc_library( "src/core/ext/client_config/uri_parser.c", "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c", "src/core/ext/transport/chttp2/client/insecure/channel_create.c", + "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", + "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", + "src/core/ext/transport/cronet/transport/cronet_transport.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/ext/lb_policy/pick_first/pick_first.c", @@ -1499,6 +1508,7 @@ objc_library( "include/grpc/impl/codegen/sync_posix.h", "include/grpc/impl/codegen/sync_win32.h", "include/grpc/impl/codegen/time.h", + "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "include/grpc/census.h", @@ -1626,6 +1636,8 @@ objc_library( "src/core/ext/client_config/subchannel_call_holder.h", "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", + "include/grpc/support/port_platform.h", + "third_party/objective_c/Cronet/cronet_c_for_grpc.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/ext/census/aggregation.h", diff --git a/Makefile b/Makefile index 922e0b0568f..c6fd3b40f78 100644 --- a/Makefile +++ b/Makefile @@ -2623,6 +2623,9 @@ LIBGRPC_SRC = \ src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ + src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ + src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ + src/core/ext/transport/cronet/transport/cronet_transport.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ @@ -2669,6 +2672,7 @@ PUBLIC_HEADERS_C += \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_win32.h \ include/grpc/impl/codegen/time.h \ + include/grpc/grpc_cronet.h \ include/grpc/grpc_security.h \ include/grpc/grpc_security_constants.h \ include/grpc/census.h \ @@ -14313,6 +14317,9 @@ ifneq ($(OPENSSL_DEP),) # otherwise parallel compilation will fail if a source is compiled first. src/core/ext/transport/chttp2/client/secure/secure_channel_create.c: $(OPENSSL_DEP) src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c: $(OPENSSL_DEP) +src/core/ext/transport/cronet/client/secure/cronet_channel_create.c: $(OPENSSL_DEP) +src/core/ext/transport/cronet/transport/cronet_api_dummy.c: $(OPENSSL_DEP) +src/core/ext/transport/cronet/transport/cronet_transport.c: $(OPENSSL_DEP) src/core/lib/http/httpcli_security_connector.c: $(OPENSSL_DEP) src/core/lib/security/b64.c: $(OPENSSL_DEP) src/core/lib/security/client_auth_filter.c: $(OPENSSL_DEP) diff --git a/binding.gyp b/binding.gyp index 4314ab7243d..12a745ffb01 100644 --- a/binding.gyp +++ b/binding.gyp @@ -709,6 +709,9 @@ 'src/core/ext/client_config/uri_parser.c', 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', + 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', + 'src/core/ext/transport/cronet/transport/cronet_api_dummy.c', + 'src/core/ext/transport/cronet/transport/cronet_transport.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/build.yaml b/build.yaml index 441752dc3dc..a7ab412722c 100644 --- a/build.yaml +++ b/build.yaml @@ -399,6 +399,7 @@ filegroups: - grpc_client_config - name: grpc_secure public_headers: + - include/grpc/grpc_cronet.h - include/grpc/grpc_security.h - include/grpc/grpc_security_constants.h headers: @@ -546,6 +547,14 @@ filegroups: - grpc_transport_chttp2 - grpc_base - grpc_secure +- name: grpc_transport_cronet_client_secure + headers: + - include/grpc/support/port_platform.h + - third_party/objective_c/Cronet/cronet_c_for_grpc.h + src: + - src/core/ext/transport/cronet/client/secure/cronet_channel_create.c + - src/core/ext/transport/cronet/transport/cronet_api_dummy.c + - src/core/ext/transport/cronet/transport/cronet_transport.c - name: nanopb headers: - third_party/nanopb/pb.h @@ -733,6 +742,7 @@ libs: - grpc_transport_chttp2_client_secure - grpc_transport_chttp2_server_insecure - grpc_transport_chttp2_client_insecure + - grpc_transport_cronet_client_secure - grpc_lb_policy_grpclb - grpc_lb_policy_pick_first - grpc_lb_policy_round_robin diff --git a/config.m4 b/config.m4 index 74f9ad242ab..5259e679ba9 100644 --- a/config.m4 +++ b/config.m4 @@ -228,6 +228,9 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ + src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ + src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ + src/core/ext/transport/cronet/transport/cronet_transport.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ @@ -566,6 +569,8 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/server/insecure) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/server/secure) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/transport) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/cronet/client/secure) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/cronet/transport) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/channel) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/compression) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/debug) diff --git a/gRPC.podspec b/gRPC.podspec index 77d35bd2c79..f57ba745194 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -287,6 +287,8 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_call_holder.h', 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', + 'include/grpc/support/port_platform.h', + 'third_party/objective_c/Cronet/cronet_c_for_grpc.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', 'third_party/nanopb/pb.h', @@ -324,6 +326,7 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_posix.h', 'include/grpc/impl/codegen/sync_win32.h', 'include/grpc/impl/codegen/time.h', + 'include/grpc/grpc_cronet.h', 'include/grpc/grpc_security.h', 'include/grpc/grpc_security_constants.h', 'include/grpc/census.h', @@ -473,6 +476,9 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/uri_parser.c', 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', + 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', + 'src/core/ext/transport/cronet/transport/cronet_api_dummy.c', + 'src/core/ext/transport/cronet/transport/cronet_transport.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', @@ -629,6 +635,8 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_call_holder.h', 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', + 'include/grpc/support/port_platform.h', + 'third_party/objective_c/Cronet/cronet_c_for_grpc.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', 'third_party/nanopb/pb.h', diff --git a/grpc.def b/grpc.def index 61948ed1b89..09a94a6cd0c 100644 --- a/grpc.def +++ b/grpc.def @@ -87,6 +87,7 @@ EXPORTS grpc_header_nonbin_value_is_legal grpc_is_binary_header grpc_call_error_to_string + grpc_cronet_secure_channel_create grpc_auth_property_iterator_next grpc_auth_context_property_iterator grpc_auth_context_peer_identity diff --git a/grpc.gemspec b/grpc.gemspec index e68cd81da7a..488f4657cdc 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -168,6 +168,7 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/impl/codegen/sync_posix.h ) s.files += %w( include/grpc/impl/codegen/sync_win32.h ) s.files += %w( include/grpc/impl/codegen/time.h ) + s.files += %w( include/grpc/grpc_cronet.h ) s.files += %w( include/grpc/grpc_security.h ) s.files += %w( include/grpc/grpc_security_constants.h ) s.files += %w( include/grpc/census.h ) @@ -295,6 +296,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/client_config/subchannel_call_holder.h ) s.files += %w( src/core/ext/client_config/subchannel_index.h ) s.files += %w( src/core/ext/client_config/uri_parser.h ) + s.files += %w( include/grpc/support/port_platform.h ) + s.files += %w( third_party/objective_c/Cronet/cronet_c_for_grpc.h ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.h ) s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h ) s.files += %w( third_party/nanopb/pb.h ) @@ -453,6 +456,9 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/client_config/uri_parser.c ) s.files += %w( src/core/ext/transport/chttp2/server/insecure/server_chttp2.c ) s.files += %w( src/core/ext/transport/chttp2/client/insecure/channel_create.c ) + s.files += %w( src/core/ext/transport/cronet/client/secure/cronet_channel_create.c ) + s.files += %w( src/core/ext/transport/cronet/transport/cronet_api_dummy.c ) + s.files += %w( src/core/ext/transport/cronet/transport/cronet_transport.c ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.c ) s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c ) s.files += %w( third_party/nanopb/pb_common.c ) diff --git a/package.xml b/package.xml index ffb1c56ed64..e8fd375eb60 100644 --- a/package.xml +++ b/package.xml @@ -175,6 +175,7 @@ + @@ -302,6 +303,8 @@ + + @@ -460,6 +463,9 @@ + + + diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c index a6cb1f70a70..df1acddcc08 100644 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c +++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c @@ -33,8 +33,6 @@ #include -#ifdef GRPC_COMPILE_WITH_CRONET - #include #include @@ -69,4 +67,3 @@ GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( return grpc_channel_create(&exec_ctx, target, args, GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); } -#endif // GRPC_COMPILE_WITH_CRONET diff --git a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c new file mode 100644 index 00000000000..200f9f7daaa --- /dev/null +++ b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c @@ -0,0 +1,91 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* This file has empty implementation of all the functions exposed by the cronet +library, so we can build it in all environments */ + +#include + +#include + +#include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" + +#ifdef GRPC_COMPILE_WITH_CRONET + /* link with the real CRONET library in the build system */ +#else + /* Dummy implementation of cronet API just to test for build-ability */ +cronet_bidirectional_stream* cronet_bidirectional_stream_create( + cronet_engine* engine, + void* annotation, + cronet_bidirectional_stream_callback* callback) { + GPR_ASSERT(0); + return NULL; +} + +int cronet_bidirectional_stream_destroy(cronet_bidirectional_stream* stream) { + GPR_ASSERT(0); + return 0; +} + +int cronet_bidirectional_stream_start( + cronet_bidirectional_stream* stream, + const char* url, + int priority, + const char* method, + const cronet_bidirectional_stream_header_array* headers, + bool end_of_stream) { + GPR_ASSERT(0); + return 0; +} + +int cronet_bidirectional_stream_read(cronet_bidirectional_stream* stream, + char* buffer, + int capacity) { + GPR_ASSERT(0); + return 0; +} + +int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream, + const char* buffer, + int count, + bool end_of_stream) { + GPR_ASSERT(0); + return 0; +} + +int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream) { + GPR_ASSERT(0); + return 0; +} + +#endif /* GRPC_COMPILE_WITH_CRONET */ diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index d337e84606b..64bd5f5778d 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -49,8 +49,6 @@ #include "src/core/lib/transport/transport_impl.h" #include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" -#ifdef GRPC_COMPILE_WITH_CRONET - #define GRPC_HEADER_SIZE_IN_BYTES 5 // Global flag that gets set with GRPC_TRACE env variable @@ -613,7 +611,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs) { + grpc_stream *gs, void *and_free_memory) { if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "Destroy stream"); } @@ -623,6 +621,7 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, gpr_free(s->write_buffer); gpr_free(s->url); gpr_mu_destroy(&s->recv_mu); + if (and_free_memory) { gpr_free(and_free_memory); } } static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { @@ -637,4 +636,3 @@ const grpc_transport_vtable grpc_cronet_vtable = { sizeof(stream_obj), "cronet_http", init_stream, set_pollset_do_nothing, perform_stream_op, NULL, destroy_stream, destroy_transport, NULL}; -#endif // GRPC_COMPILE_WITH_CRONET diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c index f0a40dbb35c..09551472b5f 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.c +++ b/src/python/grpcio/grpc/_cython/imports.generated.c @@ -125,6 +125,7 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; +grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -395,6 +396,7 @@ void pygrpc_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); + grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h index d5e810b7cf4..54c8aaad13e 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.h +++ b/src/python/grpcio/grpc/_cython/imports.generated.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -325,6 +326,9 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import +typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); +extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; +#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index dab62530aac..5314329c2cb 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -222,6 +222,9 @@ CORE_SOURCE_FILES = [ 'src/core/ext/client_config/uri_parser.c', 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', + 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', + 'src/core/ext/transport/cronet/transport/cronet_api_dummy.c', + 'src/core/ext/transport/cronet/transport/cronet_transport.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index bc43f9d36b5..cebbe8c40fe 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -125,6 +125,7 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; +grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -391,6 +392,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); + grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index b67361ca25b..d7ea6c574c8 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -325,6 +326,9 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import +typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); +extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; +#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 0eede6c23bd..65f3e1738a4 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 034d9c6e6f4..a582d76a58f 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -785,6 +785,7 @@ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_win32.h \ include/grpc/impl/codegen/time.h \ +include/grpc/grpc_cronet.h \ include/grpc/grpc_security.h \ include/grpc/grpc_security_constants.h \ include/grpc/census.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 1b1453f7ea2..16dcd9b79a7 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -785,6 +785,7 @@ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_win32.h \ include/grpc/impl/codegen/time.h \ +include/grpc/grpc_cronet.h \ include/grpc/grpc_security.h \ include/grpc/grpc_security_constants.h \ include/grpc/census.h \ @@ -912,6 +913,8 @@ src/core/ext/client_config/subchannel.h \ src/core/ext/client_config/subchannel_call_holder.h \ src/core/ext/client_config/subchannel_index.h \ src/core/ext/client_config/uri_parser.h \ +include/grpc/support/port_platform.h \ +third_party/objective_c/Cronet/cronet_c_for_grpc.h \ src/core/ext/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h \ third_party/nanopb/pb.h \ @@ -1070,6 +1073,9 @@ src/core/ext/client_config/subchannel_index.c \ src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ +src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ +src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ +src/core/ext/transport/cronet/transport/cronet_transport.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index f546f3b9950..e22318d2387 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4140,7 +4140,8 @@ "grpc_transport_chttp2_client_insecure", "grpc_transport_chttp2_client_secure", "grpc_transport_chttp2_server_insecure", - "grpc_transport_chttp2_server_secure" + "grpc_transport_chttp2_server_secure", + "grpc_transport_cronet_client_secure" ], "headers": [], "language": "c", @@ -6012,6 +6013,7 @@ "tsi" ], "headers": [ + "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "src/core/lib/security/auth_filters.h", @@ -6027,6 +6029,7 @@ "language": "c", "name": "grpc_secure", "src": [ + "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "src/core/lib/http/httpcli_security_connector.c", @@ -6262,6 +6265,23 @@ "third_party": false, "type": "filegroup" }, + { + "deps": [], + "headers": [ + "include/grpc/support/port_platform.h", + "third_party/objective_c/Cronet/cronet_c_for_grpc.h" + ], + "language": "c", + "name": "grpc_transport_cronet_client_secure", + "src": [ + "include/grpc/support/port_platform.h", + "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", + "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", + "src/core/ext/transport/cronet/transport/cronet_transport.c" + ], + "third_party": false, + "type": "filegroup" + }, { "deps": [], "headers": [ diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 4eec05a3b1f..cbf854875a6 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -292,6 +292,7 @@ + @@ -421,6 +422,8 @@ + + @@ -727,6 +730,12 @@ + + + + + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 17c88c4805e..02297b7746a 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -439,6 +439,15 @@ src\core\ext\transport\chttp2\client\insecure + + src\core\ext\transport\cronet\client\secure + + + src\core\ext\transport\cronet\transport + + + src\core\ext\transport\cronet\transport + src\core\ext\lb_policy\grpclb @@ -573,6 +582,9 @@ include\grpc\impl\codegen + + include\grpc + include\grpc @@ -956,6 +968,12 @@ src\core\ext\client_config + + include\grpc\support + + + third_party\objective_c\Cronet + src\core\ext\lb_policy\grpclb @@ -1007,6 +1025,9 @@ {def748f5-ed2a-a9bb-40d9-c31d00f0e13b} + + {31de82ea-dc6c-73fb-a640-979b8a7b240c} + {d538af37-07b2-062b-fa2a-d9f882cb2737} @@ -1088,6 +1109,18 @@ {6f34254e-e69f-c9b4-156d-5024bade5408} + + {1e9c85e9-5522-7ef8-0017-7e19990a6194} + + + {d0530883-75d9-b5f7-d594-26735a70ac7b} + + + {4fa6fe90-b7a8-5c8f-d629-db1e68d89eed} + + + {31518af8-5860-6d0d-ff78-4059fce29ec2} + {5b2ded3f-84a5-f6b4-2060-286c7d1dc945} @@ -1130,6 +1163,12 @@ {93d6596d-330c-1d27-6f84-3c840e57869e} + + {3a56a516-857e-d2aa-95cc-11685baf4e8c} + + + {a165c6e3-0776-6f40-7351-d7865668e220} + From 6504c02568da17113c8ebce408245054ca19f0a3 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 May 2016 17:46:49 -0700 Subject: [PATCH 063/155] remove useless code and todos --- src/csharp/Grpc.Core/Internal/ServerCallHandler.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs index bf9df9f783b..00d82d51e82 100644 --- a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs +++ b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs @@ -80,8 +80,6 @@ namespace Grpc.Core.Internal { GrpcPreconditions.CheckArgument(await requestStream.MoveNext().ConfigureAwait(false)); var request = requestStream.Current; - // TODO(jtattermusch): we need to read the full stream so that native callhandle gets deallocated. - GrpcPreconditions.CheckArgument(!await requestStream.MoveNext().ConfigureAwait(false)); var result = await handler(request, context).ConfigureAwait(false); status = context.Status; await responseStream.WriteAsync(result).ConfigureAwait(false); @@ -136,8 +134,6 @@ namespace Grpc.Core.Internal { GrpcPreconditions.CheckArgument(await requestStream.MoveNext().ConfigureAwait(false)); var request = requestStream.Current; - // TODO(jtattermusch): we need to read the full stream so that native callhandle gets deallocated. - GrpcPreconditions.CheckArgument(!await requestStream.MoveNext().ConfigureAwait(false)); await handler(request, responseStream, context).ConfigureAwait(false); status = context.Status; } @@ -298,7 +294,6 @@ namespace Grpc.Core.Internal return rpcException.Status; } - // TODO(jtattermusch): what is the right status code here? return new Status(StatusCode.Unknown, "Exception was thrown by handler."); } From a93c4353e79b2913769cc78cecd1da15a6b0e089 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 3 May 2016 21:23:58 -0700 Subject: [PATCH 064/155] clang-format fixes --- .../cronet/transport/cronet_api_dummy.c | 22 +++++++------------ .../cronet/transport/cronet_transport.c | 4 +++- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c index 200f9f7daaa..687026c9fde 100644 --- a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c +++ b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c @@ -41,12 +41,11 @@ library, so we can build it in all environments */ #include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" #ifdef GRPC_COMPILE_WITH_CRONET - /* link with the real CRONET library in the build system */ +/* link with the real CRONET library in the build system */ #else - /* Dummy implementation of cronet API just to test for build-ability */ +/* Dummy implementation of cronet API just to test for build-ability */ cronet_bidirectional_stream* cronet_bidirectional_stream_create( - cronet_engine* engine, - void* annotation, + cronet_engine* engine, void* annotation, cronet_bidirectional_stream_callback* callback) { GPR_ASSERT(0); return NULL; @@ -58,26 +57,21 @@ int cronet_bidirectional_stream_destroy(cronet_bidirectional_stream* stream) { } int cronet_bidirectional_stream_start( - cronet_bidirectional_stream* stream, - const char* url, - int priority, - const char* method, - const cronet_bidirectional_stream_header_array* headers, + cronet_bidirectional_stream* stream, const char* url, int priority, + const char* method, const cronet_bidirectional_stream_header_array* headers, bool end_of_stream) { GPR_ASSERT(0); return 0; } int cronet_bidirectional_stream_read(cronet_bidirectional_stream* stream, - char* buffer, - int capacity) { + char* buffer, int capacity) { GPR_ASSERT(0); return 0; } int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream, - const char* buffer, - int count, + const char* buffer, int count, bool end_of_stream) { GPR_ASSERT(0); return 0; @@ -88,4 +82,4 @@ int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream) { return 0; } -#endif /* GRPC_COMPILE_WITH_CRONET */ +#endif /* GRPC_COMPILE_WITH_CRONET */ diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 64bd5f5778d..5da4b873fbf 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -621,7 +621,9 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, gpr_free(s->write_buffer); gpr_free(s->url); gpr_mu_destroy(&s->recv_mu); - if (and_free_memory) { gpr_free(and_free_memory); } + if (and_free_memory) { + gpr_free(and_free_memory); + } } static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { From e5c1a154a4b3b2a3afb4efc0447a74e410cdb074 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 3 May 2016 21:49:21 -0700 Subject: [PATCH 065/155] Fixed a compiler warning that only shows up on linux --- src/core/ext/transport/cronet/transport/cronet_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 5da4b873fbf..5bb085195c6 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -218,7 +218,7 @@ static void on_write_completed(cronet_bidirectional_stream *stream, static void process_recv_message(stream_obj *s, const uint8_t *recv_data) { gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes); uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); - memcpy(dst_p, recv_data, s->total_read_bytes); + memcpy(dst_p, recv_data, (size_t)s->total_read_bytes); gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice); grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0); *s->recv_message = (grpc_byte_buffer *)&s->sbs; From 823fa1d77a80cda46e93280ab7f90149a8951297 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 3 May 2016 22:19:11 -0700 Subject: [PATCH 066/155] Exclude 1byte tests from msan --- test/core/end2end/gen_build_yaml.py | 8 +- tools/run_tests/tests.json | 128 +++++++++++++++++++++------- 2 files changed, 100 insertions(+), 36 deletions(-) diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index cffe5995bcd..3e10ad580fc 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -39,9 +39,9 @@ import hashlib FixtureOptions = collections.namedtuple( 'FixtureOptions', - 'fullstack includes_proxy dns_resolver secure platforms ci_mac tracing') + 'fullstack includes_proxy dns_resolver secure platforms ci_mac tracing exclude_configs') default_unsecure_fixture_options = FixtureOptions( - True, False, True, False, ['windows', 'linux', 'mac', 'posix'], True, False) + True, False, True, False, ['windows', 'linux', 'mac', 'posix'], True, False, []) socketpair_unsecure_fixture_options = default_unsecure_fixture_options._replace(fullstack=False, dns_resolver=False) default_secure_fixture_options = default_unsecure_fixture_options._replace(secure=True) uds_fixture_options = default_unsecure_fixture_options._replace(dns_resolver=False, platforms=['linux', 'mac', 'posix']) @@ -60,7 +60,7 @@ END2END_FIXTURES = { 'h2_proxy': default_unsecure_fixture_options._replace(includes_proxy=True, ci_mac=False), 'h2_sockpair_1byte': socketpair_unsecure_fixture_options._replace( - ci_mac=False), + ci_mac=False, exclude_configs=['msan']), 'h2_sockpair': socketpair_unsecure_fixture_options._replace(ci_mac=False), 'h2_sockpair+trace': socketpair_unsecure_fixture_options._replace( ci_mac=False, tracing=True), @@ -246,7 +246,7 @@ def main(): { 'name': '%s_nosec_test' % f, 'args': [t], - 'exclude_configs': [], + 'exclude_configs': END2END_FIXTURES[f].exclude_configs, 'platforms': END2END_FIXTURES[f].platforms, 'ci_platforms': (END2END_FIXTURES[f].platforms if END2END_FIXTURES[f].ci_mac else without( diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index cf1154426f4..4cc407e39ed 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -21531,7 +21531,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21552,7 +21554,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21573,7 +21577,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21594,7 +21600,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21615,7 +21623,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21636,7 +21646,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21657,7 +21669,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21678,7 +21692,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21699,7 +21715,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21720,7 +21738,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21741,7 +21761,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21762,7 +21784,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21783,7 +21807,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21804,7 +21830,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21825,7 +21853,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21846,7 +21876,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21867,7 +21899,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21888,7 +21922,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21909,7 +21945,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21930,7 +21968,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21951,7 +21991,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21972,7 +22014,9 @@ "posix" ], "cpu_cost": 0.1, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -21993,7 +22037,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -22014,7 +22060,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -22035,7 +22083,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -22056,7 +22106,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -22077,7 +22129,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -22098,7 +22152,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -22119,7 +22175,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -22140,7 +22198,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -22161,7 +22221,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", @@ -22182,7 +22244,9 @@ "posix" ], "cpu_cost": 1.0, - "exclude_configs": [], + "exclude_configs": [ + "msan" + ], "flaky": false, "language": "c", "name": "h2_sockpair_1byte_nosec_test", From e6d03b828da478a7b4205c422bd95aa21f8270d4 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 4 May 2016 09:49:42 -0700 Subject: [PATCH 067/155] Fix the build path in config --- tools/run_tests/stress_test/configs/python.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/stress_test/configs/python.json b/tools/run_tests/stress_test/configs/python.json index ea5b5ba2ec4..4f85de1d5f6 100644 --- a/tools/run_tests/stress_test/configs/python.json +++ b/tools/run_tests/stress_test/configs/python.json @@ -1,7 +1,7 @@ { "dockerImages": { "grpc_stress_python" : { - "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", "dockerFileDir": "grpc_interop_stress_python" } }, From bc846725376821d62ac35bd21fc542fb4c712930 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 4 May 2016 10:53:50 -0700 Subject: [PATCH 068/155] Use unlimited default for max header size until receiving new settings from the peer. This both complies with the RFC and ensures that if the user sets a higher limit than the 16K default, we won't incorrectly reject data sent before settings are exchanged. Also fix proxy tests. --- .../chttp2/transport/chttp2_transport.c | 4 ++++ .../chttp2/transport/frame_settings.c | 3 +-- src/core/lib/iomgr/tcp_posix.c | 4 ++-- test/core/end2end/fixtures/h2_proxy.c | 12 +++++++----- test/core/end2end/fixtures/h2_ssl_proxy.c | 19 +++++++++++-------- test/core/end2end/fixtures/proxy.c | 7 ++++--- test/core/end2end/fixtures/proxy.h | 9 ++++++--- test/core/end2end/tests/large_metadata.c | 10 +++++----- 8 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index f5359f2a816..44b2a7a59f7 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -56,6 +56,8 @@ #define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024) #define MAX_WINDOW 0x7fffffffu +#define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024) + #define MAX_CLIENT_STREAM_ID 0x7fffffffu int grpc_http_trace = 0; @@ -311,6 +313,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, 0); } push_setting(t, GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, DEFAULT_WINDOW); + push_setting(t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, + DEFAULT_MAX_HEADER_LIST_SIZE); if (channel_args) { for (i = 0; i < channel_args->num_args; i++) { diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c index 7fa66247e4d..a3c1e15f35e 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.c +++ b/src/core/ext/transport/chttp2/transport/frame_settings.c @@ -44,7 +44,6 @@ #include "src/core/ext/transport/chttp2/transport/http2_errors.h" #include "src/core/lib/debug/trace.h" -#define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024) #define MAX_MAX_HEADER_LIST_SIZE (1024 * 1024 * 1024) /* HTTP/2 mandated initial connection settings */ @@ -63,7 +62,7 @@ const grpc_chttp2_setting_parameters GRPC_CHTTP2_FLOW_CONTROL_ERROR}, {"MAX_FRAME_SIZE", 16384, 16384, 16777215, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_CHTTP2_PROTOCOL_ERROR}, - {"MAX_HEADER_LIST_SIZE", DEFAULT_MAX_HEADER_LIST_SIZE, 0, + {"MAX_HEADER_LIST_SIZE", MAX_MAX_HEADER_LIST_SIZE, 0, MAX_MAX_HEADER_LIST_SIZE, GRPC_CHTTP2_CLAMP_INVALID_VALUE, GRPC_CHTTP2_PROTOCOL_ERROR}, }; diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 7210aef5d5c..e2869224f1f 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -164,7 +164,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, int success) { for (i = 0; i < tcp->incoming_buffer->count; i++) { char *dump = gpr_dump_slice(tcp->incoming_buffer->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); - gpr_log(GPR_DEBUG, "READ %p: %s", tcp, dump); + gpr_log(GPR_DEBUG, "READ %p (peer=%s): %s", tcp, tcp->peer_string, dump); gpr_free(dump); } } @@ -398,7 +398,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, for (i = 0; i < buf->count; i++) { char *data = gpr_dump_slice(buf->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); - gpr_log(GPR_DEBUG, "WRITE %p: %s", tcp, data); + gpr_log(GPR_DEBUG, "WRITE %p (peer=%s): %s", tcp, tcp->peer_string, data); gpr_free(data); } } diff --git a/test/core/end2end/fixtures/h2_proxy.c b/test/core/end2end/fixtures/h2_proxy.c index 863673a4e01..8c50eeb5d5b 100644 --- a/test/core/end2end/fixtures/h2_proxy.c +++ b/test/core/end2end/fixtures/h2_proxy.c @@ -55,14 +55,16 @@ typedef struct fullstack_fixture_data { grpc_end2end_proxy *proxy; } fullstack_fixture_data; -static grpc_server *create_proxy_server(const char *port) { - grpc_server *s = grpc_server_create(NULL, NULL); +static grpc_server *create_proxy_server(const char *port, + grpc_channel_args *server_args) { + grpc_server *s = grpc_server_create(server_args, NULL); GPR_ASSERT(grpc_server_add_insecure_http2_port(s, port)); return s; } -static grpc_channel *create_proxy_client(const char *target) { - return grpc_insecure_channel_create(target, NULL, NULL); +static grpc_channel *create_proxy_client(const char *target, + grpc_channel_args *client_args) { + return grpc_insecure_channel_create(target, client_args, NULL); } static const grpc_end2end_proxy_def proxy_def = {create_proxy_server, @@ -74,7 +76,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_fullstack( fullstack_fixture_data *ffd = gpr_malloc(sizeof(fullstack_fixture_data)); memset(&f, 0, sizeof(f)); - ffd->proxy = grpc_end2end_proxy_create(&proxy_def); + ffd->proxy = grpc_end2end_proxy_create(&proxy_def, client_args, server_args); f.fixture_data = ffd; f.cq = grpc_completion_queue_create(NULL); diff --git a/test/core/end2end/fixtures/h2_ssl_proxy.c b/test/core/end2end/fixtures/h2_ssl_proxy.c index 1403b760f50..0ec6ad437ee 100644 --- a/test/core/end2end/fixtures/h2_ssl_proxy.c +++ b/test/core/end2end/fixtures/h2_ssl_proxy.c @@ -54,8 +54,9 @@ typedef struct fullstack_secure_fixture_data { grpc_end2end_proxy *proxy; } fullstack_secure_fixture_data; -static grpc_server *create_proxy_server(const char *port) { - grpc_server *s = grpc_server_create(NULL, NULL); +static grpc_server *create_proxy_server(const char *port, + grpc_channel_args *server_args) { + grpc_server *s = grpc_server_create(server_args, NULL); grpc_ssl_pem_key_cert_pair pem_cert_key_pair = {test_server1_key, test_server1_cert}; grpc_server_credentials *ssl_creds = @@ -65,18 +66,20 @@ static grpc_server *create_proxy_server(const char *port) { return s; } -static grpc_channel *create_proxy_client(const char *target) { +static grpc_channel *create_proxy_client(const char *target, + grpc_channel_args *client_args) { grpc_channel *channel; grpc_channel_credentials *ssl_creds = grpc_ssl_credentials_create(NULL, NULL, NULL); grpc_arg ssl_name_override = {GRPC_ARG_STRING, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, {"foo.test.google.fr"}}; - grpc_channel_args client_args; - client_args.num_args = 1; - client_args.args = &ssl_name_override; - channel = grpc_secure_channel_create(ssl_creds, target, &client_args, NULL); + grpc_channel_args *new_client_args = + grpc_channel_args_copy_and_add(client_args, &ssl_name_override, 1); + channel = grpc_secure_channel_create(ssl_creds, target, new_client_args, + NULL); grpc_channel_credentials_release(ssl_creds); + grpc_channel_args_destroy(new_client_args); return channel; } @@ -90,7 +93,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_secure_fullstack( gpr_malloc(sizeof(fullstack_secure_fixture_data)); memset(&f, 0, sizeof(f)); - ffd->proxy = grpc_end2end_proxy_create(&proxy_def); + ffd->proxy = grpc_end2end_proxy_create(&proxy_def, client_args, server_args); f.fixture_data = ffd; f.cq = grpc_completion_queue_create(NULL); diff --git a/test/core/end2end/fixtures/proxy.c b/test/core/end2end/fixtures/proxy.c index a6487a17ac2..ff413ffd652 100644 --- a/test/core/end2end/fixtures/proxy.c +++ b/test/core/end2end/fixtures/proxy.c @@ -90,7 +90,8 @@ static void thread_main(void *arg); static void request_call(grpc_end2end_proxy *proxy); grpc_end2end_proxy *grpc_end2end_proxy_create( - const grpc_end2end_proxy_def *def) { + const grpc_end2end_proxy_def *def, + grpc_channel_args *client_args, grpc_channel_args *server_args) { gpr_thd_options opt = gpr_thd_options_default(); int proxy_port = grpc_pick_unused_port_or_die(); int server_port = grpc_pick_unused_port_or_die(); @@ -105,8 +106,8 @@ grpc_end2end_proxy *grpc_end2end_proxy_create( proxy->server_port); proxy->cq = grpc_completion_queue_create(NULL); - proxy->server = def->create_server(proxy->proxy_port); - proxy->client = def->create_client(proxy->server_port); + proxy->server = def->create_server(proxy->proxy_port, server_args); + proxy->client = def->create_client(proxy->server_port, client_args); grpc_server_register_completion_queue(proxy->server, proxy->cq, NULL); grpc_server_start(proxy->server); diff --git a/test/core/end2end/fixtures/proxy.h b/test/core/end2end/fixtures/proxy.h index c1cf01d39a4..89f95f09f90 100644 --- a/test/core/end2end/fixtures/proxy.h +++ b/test/core/end2end/fixtures/proxy.h @@ -41,12 +41,15 @@ typedef struct grpc_end2end_proxy grpc_end2end_proxy; typedef struct grpc_end2end_proxy_def { - grpc_server *(*create_server)(const char *port); - grpc_channel *(*create_client)(const char *target); + grpc_server *(*create_server)(const char *port, + grpc_channel_args *server_args); + grpc_channel *(*create_client)(const char *target, + grpc_channel_args *client_args); } grpc_end2end_proxy_def; grpc_end2end_proxy *grpc_end2end_proxy_create( - const grpc_end2end_proxy_def *def); + const grpc_end2end_proxy_def *def, + grpc_channel_args *client_args, grpc_channel_args *server_args); void grpc_end2end_proxy_destroy(grpc_end2end_proxy *proxy); const char *grpc_end2end_proxy_get_client_target(grpc_end2end_proxy *proxy); diff --git a/test/core/end2end/tests/large_metadata.c b/test/core/end2end/tests/large_metadata.c index b78d5b8292c..6d3074a94ba 100644 --- a/test/core/end2end/tests/large_metadata.c +++ b/test/core/end2end/tests/large_metadata.c @@ -97,7 +97,7 @@ static void end_test(grpc_end2end_test_fixture *f) { grpc_completion_queue_destroy(f->cq); } -/* Request with a large amount of metadata. */ +// Request with a large amount of metadata. static void test_request_with_large_metadata(grpc_end2end_test_config config) { grpc_call *c; grpc_call *s; @@ -141,7 +141,7 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { grpc_metadata_array_init(&request_metadata_recv); grpc_call_details_init(&call_details); - /* Client: send request. */ + // Client: send request. op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; op->data.send_initial_metadata.count = 1; @@ -182,7 +182,7 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { cq_expect_completion(cqv, tag(101), 1); cq_verify(cqv); - /* Server: send initial metadata and receive request. */ + // Server: send initial metadata and receive request. op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; op->data.send_initial_metadata.count = 0; @@ -200,8 +200,8 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { cq_expect_completion(cqv, tag(102), 1); cq_verify(cqv); - /* Server: receive close and send status. This should trigger - completion of request on client. */ + // Server: receive close and send status. This should trigger + // completion of request on client. op = ops; op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; op->data.recv_close_on_server.cancelled = &was_cancelled; From 2df7d409144af95c99cdabefcc7a3ad841c464d0 Mon Sep 17 00:00:00 2001 From: Michael Yeh Date: Wed, 4 May 2016 10:59:03 -0700 Subject: [PATCH 069/155] Modified sample async client to be non-blocking. A reader thread is spawned to handle all the RPC results. --- .../cpp/helloworld/greeter_async_client.cc | 93 ++++++++++++------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/examples/cpp/helloworld/greeter_async_client.cc b/examples/cpp/helloworld/greeter_async_client.cc index 35014267feb..ca5a7defb0c 100644 --- a/examples/cpp/helloworld/greeter_async_client.cc +++ b/examples/cpp/helloworld/greeter_async_client.cc @@ -36,6 +36,7 @@ #include #include +#include #include "helloworld.grpc.pb.h" @@ -55,71 +56,91 @@ class GreeterClient { // Assambles the client's payload, sends it and presents the response back // from the server. - std::string SayHello(const std::string& user) { + void SayHello(const std::string& user) { // Data we are sending to the server. HelloRequest request; request.set_name(user); - // Container for the data we expect from the server. - HelloReply reply; - - // Context for the client. It could be used to convey extra information to - // the server and/or tweak certain RPC behaviors. - ClientContext context; - - // The producer-consumer queue we use to communicate asynchronously with the - // gRPC runtime. - CompletionQueue cq; - - // Storage for the status of the RPC upon completion. - Status status; + AsyncClientCall* call = new AsyncClientCall; // stub_->AsyncSayHello() perform the RPC call, returning an instance we // store in "rpc". Because we are using the asynchronous API, we need the - // hold on to the "rpc" instance in order to get updates on the ongoig RPC. - std::unique_ptr > rpc( - stub_->AsyncSayHello(&context, request, &cq)); + // hold on to the "rpc" instance in order to get updates on the ongoing RPC. + call->response_reader = stub_->AsyncSayHello(&call->context, request, &cq_); + // Request that, upon completion of the RPC, "reply" be updated with the // server's response; "status" with the indication of whether the operation - // was successful. Tag the request with the integer 1. - rpc->Finish(&reply, &status, (void*)1); + // was successful. Tag the request with the memory address of call object. + call->response_reader->Finish(&call->reply, &call->status, (void*)call); + + } + void AsyncCompleteRpc() + { void* got_tag; bool ok = false; + // Block until the next result is available in the completion queue "cq". - cq.Next(&got_tag, &ok); - - // Verify that the result from "cq" corresponds, by its tag, our previous - // request. - GPR_ASSERT(got_tag == (void*)1); - // ... and that the request was completed successfully. Note that "ok" - // corresponds solely to the request for updates introduced by Finish(). - GPR_ASSERT(ok); - - // Act upon the status of the actual RPC. - if (status.ok()) { - return reply.message(); - } else { - return "RPC failed"; + while (cq_.Next(&got_tag, &ok)) + { + AsyncClientCall* call = static_cast(got_tag); + if (ok) + std::cout << "Greeter received: " << call->reply.message() << std::endl; + else + std::cout << "RPC failed" << std::endl; + delete call; } } private: + + // Structure for keeping state and data information + struct AsyncClientCall + { + // Container for the data we expect from the server. + HelloReply reply; + + // Context for the client. It could be used to convey extra information to + // the server and/or tweak certain RPC behaviors. + ClientContext context; + + // Storage for the status of the RPC upon completion. + Status status; + + + std::unique_ptr> response_reader; + }; + // Out of the passed in Channel comes the stub, stored here, our view of the // server's exposed services. std::unique_ptr stub_; + + // The producer-consumer queue we use to communicate asynchronously with the + // gRPC runtime. + CompletionQueue cq_; }; int main(int argc, char** argv) { + + // Instantiate the client. It requires a channel, out of which the actual RPCs // are created. This channel models a connection to an endpoint (in this case, // localhost at port 50051). We indicate that the channel isn't authenticated // (use of InsecureChannelCredentials()). GreeterClient greeter(grpc::CreateChannel( "localhost:50051", grpc::InsecureChannelCredentials())); - std::string user("world"); - std::string reply = greeter.SayHello(user); // The actual RPC call! - std::cout << "Greeter received: " << reply << std::endl; + + // Spawn reader thread that loops indefinitely + std::thread thread_ = std::thread(&GreeterClient::AsyncCompleteRpc, &greeter); + + for (int i = 0; i < 100; i++) + { + std::string user("world " + std::to_string(i)); + greeter.SayHello(user); // The actual RPC call! + } + + std::cout << "Press control-c to quit" << std::endl << std::endl; + thread_.join(); //blocks forever return 0; } From 2df502e5181d98752d193e55824b07722a16c594 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 4 May 2016 11:22:24 -0700 Subject: [PATCH 070/155] Fix max_message_length test, which was broken by the previous fix to the proxy tests. --- test/core/end2end/tests/max_message_length.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/core/end2end/tests/max_message_length.c b/test/core/end2end/tests/max_message_length.c index b5dbc1382a8..4f572789d95 100644 --- a/test/core/end2end/tests/max_message_length.c +++ b/test/core/end2end/tests/max_message_length.c @@ -53,7 +53,10 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, grpc_channel_args *server_args) { grpc_end2end_test_fixture f; gpr_log(GPR_INFO, "%s/%s", test_name, config.name); - f = config.create_fixture(client_args, server_args); + // We intentionally do not pass the client and server args to + // create_fixture(), since we don't want the limit enforced on the + // proxy, only on the backend server. + f = config.create_fixture(NULL, NULL); config.init_server(&f, server_args); config.init_client(&f, client_args); return f; From 470cbd1c6e0f1e4d49265a1ed403ffb1219fffa6 Mon Sep 17 00:00:00 2001 From: Michael Yeh Date: Wed, 4 May 2016 12:39:20 -0700 Subject: [PATCH 071/155] Created a new sample async client that is non-blocking. A reader thread is spawned to handle all the RPC results. --- examples/cpp/helloworld/Makefile | 7 +- .../cpp/helloworld/greeter_async_client.cc | 93 +++++------ .../cpp/helloworld/greeter_async_client2.cc | 148 ++++++++++++++++++ 3 files changed, 189 insertions(+), 59 deletions(-) create mode 100644 examples/cpp/helloworld/greeter_async_client2.cc diff --git a/examples/cpp/helloworld/Makefile b/examples/cpp/helloworld/Makefile index 470b83573e4..58a82dbb831 100644 --- a/examples/cpp/helloworld/Makefile +++ b/examples/cpp/helloworld/Makefile @@ -41,7 +41,7 @@ PROTOS_PATH = ../../protos vpath %.proto $(PROTOS_PATH) -all: system-check greeter_client greeter_server greeter_async_client greeter_async_server +all: system-check greeter_client greeter_server greeter_async_client greeter_async_client2 greeter_async_server greeter_client: helloworld.pb.o helloworld.grpc.pb.o greeter_client.o $(CXX) $^ $(LDFLAGS) -o $@ @@ -52,6 +52,9 @@ greeter_server: helloworld.pb.o helloworld.grpc.pb.o greeter_server.o greeter_async_client: helloworld.pb.o helloworld.grpc.pb.o greeter_async_client.o $(CXX) $^ $(LDFLAGS) -o $@ +greeter_async_client2: helloworld.pb.o helloworld.grpc.pb.o greeter_async_client2.o + $(CXX) $^ $(LDFLAGS) -o $@ + greeter_async_server: helloworld.pb.o helloworld.grpc.pb.o greeter_async_server.o $(CXX) $^ $(LDFLAGS) -o $@ @@ -64,7 +67,7 @@ greeter_async_server: helloworld.pb.o helloworld.grpc.pb.o greeter_async_server. $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< clean: - rm -f *.o *.pb.cc *.pb.h greeter_client greeter_server greeter_async_client greeter_async_server + rm -f *.o *.pb.cc *.pb.h greeter_client greeter_server greeter_async_client greeter_async_client2 greeter_async_server # The following is to test your system and ensure a smoother experience. diff --git a/examples/cpp/helloworld/greeter_async_client.cc b/examples/cpp/helloworld/greeter_async_client.cc index ca5a7defb0c..35014267feb 100644 --- a/examples/cpp/helloworld/greeter_async_client.cc +++ b/examples/cpp/helloworld/greeter_async_client.cc @@ -36,7 +36,6 @@ #include #include -#include #include "helloworld.grpc.pb.h" @@ -56,91 +55,71 @@ class GreeterClient { // Assambles the client's payload, sends it and presents the response back // from the server. - void SayHello(const std::string& user) { + std::string SayHello(const std::string& user) { // Data we are sending to the server. HelloRequest request; request.set_name(user); - AsyncClientCall* call = new AsyncClientCall; + // Container for the data we expect from the server. + HelloReply reply; + + // Context for the client. It could be used to convey extra information to + // the server and/or tweak certain RPC behaviors. + ClientContext context; + + // The producer-consumer queue we use to communicate asynchronously with the + // gRPC runtime. + CompletionQueue cq; + + // Storage for the status of the RPC upon completion. + Status status; // stub_->AsyncSayHello() perform the RPC call, returning an instance we // store in "rpc". Because we are using the asynchronous API, we need the - // hold on to the "rpc" instance in order to get updates on the ongoing RPC. - call->response_reader = stub_->AsyncSayHello(&call->context, request, &cq_); - + // hold on to the "rpc" instance in order to get updates on the ongoig RPC. + std::unique_ptr > rpc( + stub_->AsyncSayHello(&context, request, &cq)); // Request that, upon completion of the RPC, "reply" be updated with the // server's response; "status" with the indication of whether the operation - // was successful. Tag the request with the memory address of call object. - call->response_reader->Finish(&call->reply, &call->status, (void*)call); - - } - void AsyncCompleteRpc() - { + // was successful. Tag the request with the integer 1. + rpc->Finish(&reply, &status, (void*)1); void* got_tag; bool ok = false; - // Block until the next result is available in the completion queue "cq". - while (cq_.Next(&got_tag, &ok)) - { - AsyncClientCall* call = static_cast(got_tag); - if (ok) - std::cout << "Greeter received: " << call->reply.message() << std::endl; - else - std::cout << "RPC failed" << std::endl; - delete call; + cq.Next(&got_tag, &ok); + + // Verify that the result from "cq" corresponds, by its tag, our previous + // request. + GPR_ASSERT(got_tag == (void*)1); + // ... and that the request was completed successfully. Note that "ok" + // corresponds solely to the request for updates introduced by Finish(). + GPR_ASSERT(ok); + + // Act upon the status of the actual RPC. + if (status.ok()) { + return reply.message(); + } else { + return "RPC failed"; } } private: - - // Structure for keeping state and data information - struct AsyncClientCall - { - // Container for the data we expect from the server. - HelloReply reply; - - // Context for the client. It could be used to convey extra information to - // the server and/or tweak certain RPC behaviors. - ClientContext context; - - // Storage for the status of the RPC upon completion. - Status status; - - - std::unique_ptr> response_reader; - }; - // Out of the passed in Channel comes the stub, stored here, our view of the // server's exposed services. std::unique_ptr stub_; - - // The producer-consumer queue we use to communicate asynchronously with the - // gRPC runtime. - CompletionQueue cq_; }; int main(int argc, char** argv) { - - // Instantiate the client. It requires a channel, out of which the actual RPCs // are created. This channel models a connection to an endpoint (in this case, // localhost at port 50051). We indicate that the channel isn't authenticated // (use of InsecureChannelCredentials()). GreeterClient greeter(grpc::CreateChannel( "localhost:50051", grpc::InsecureChannelCredentials())); - - // Spawn reader thread that loops indefinitely - std::thread thread_ = std::thread(&GreeterClient::AsyncCompleteRpc, &greeter); - - for (int i = 0; i < 100; i++) - { - std::string user("world " + std::to_string(i)); - greeter.SayHello(user); // The actual RPC call! - } - - std::cout << "Press control-c to quit" << std::endl << std::endl; - thread_.join(); //blocks forever + std::string user("world"); + std::string reply = greeter.SayHello(user); // The actual RPC call! + std::cout << "Greeter received: " << reply << std::endl; return 0; } diff --git a/examples/cpp/helloworld/greeter_async_client2.cc b/examples/cpp/helloworld/greeter_async_client2.cc new file mode 100644 index 00000000000..cdbdd1f3063 --- /dev/null +++ b/examples/cpp/helloworld/greeter_async_client2.cc @@ -0,0 +1,148 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include +#include + +#include "helloworld.grpc.pb.h" + +using grpc::Channel; +using grpc::ClientAsyncResponseReader; +using grpc::ClientContext; +using grpc::CompletionQueue; +using grpc::Status; +using helloworld::HelloRequest; +using helloworld::HelloReply; +using helloworld::Greeter; + +class GreeterClient { + public: + explicit GreeterClient(std::shared_ptr channel) + : stub_(Greeter::NewStub(channel)) {} + + // Assambles the client's payload, sends it and presents the response back + // from the server. + void SayHello(const std::string& user) { + // Data we are sending to the server. + HelloRequest request; + request.set_name(user); + + AsyncClientCall* call = new AsyncClientCall; + + // stub_->AsyncSayHello() perform the RPC call, returning an instance we + // store in "rpc". Because we are using the asynchronous API, we need the + // hold on to the "rpc" instance in order to get updates on the ongoing RPC. + call->response_reader = stub_->AsyncSayHello(&call->context, request, &cq_); + + + // Request that, upon completion of the RPC, "reply" be updated with the + // server's response; "status" with the indication of whether the operation + // was successful. Tag the request with the memory address of call object. + call->response_reader->Finish(&call->reply, &call->status, (void*)call); + + } + + // Loop while listening for completed responses + void AsyncCompleteRpc() + { + void* got_tag; + bool ok = false; + + // Block until the next result is available in the completion queue "cq". + while (cq_.Next(&got_tag, &ok)) + { + AsyncClientCall* call = static_cast(got_tag); + if (ok) + std::cout << "Greeter received: " << call->reply.message() << std::endl; + else + std::cout << "RPC failed" << std::endl; + delete call; + } + } + + private: + + // struct for keeping state and data information + struct AsyncClientCall + { + // Container for the data we expect from the server. + HelloReply reply; + + // Context for the client. It could be used to convey extra information to + // the server and/or tweak certain RPC behaviors. + ClientContext context; + + // Storage for the status of the RPC upon completion. + Status status; + + + std::unique_ptr> response_reader; + }; + + // Out of the passed in Channel comes the stub, stored here, our view of the + // server's exposed services. + std::unique_ptr stub_; + + // The producer-consumer queue we use to communicate asynchronously with the + // gRPC runtime. + CompletionQueue cq_; +}; + +int main(int argc, char** argv) { + + + // Instantiate the client. It requires a channel, out of which the actual RPCs + // are created. This channel models a connection to an endpoint (in this case, + // localhost at port 50051). We indicate that the channel isn't authenticated + // (use of InsecureChannelCredentials()). + GreeterClient greeter(grpc::CreateChannel( + "localhost:50051", grpc::InsecureChannelCredentials())); + + // Spawn reader thread that loops indefinitely + std::thread thread_ = std::thread(&GreeterClient::AsyncCompleteRpc, &greeter); + + for (int i = 0; i < 100; i++) + { + std::string user("world " + std::to_string(i)); + greeter.SayHello(user); // The actual RPC call! + } + + std::cout << "Press control-c to quit" << std::endl << std::endl; + thread_.join(); //blocks forever + + return 0; +} From 2b6a2ab59a47c166652c7eb3e483eb8a75634d19 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 4 May 2016 12:54:14 -0700 Subject: [PATCH 072/155] Make namespacing of executables exposed by grpc-tools packages consistent between Node and Ruby --- src/node/tools/package.json | 4 ++-- src/ruby/tools/bin/{protoc.rb => grpc_tools_ruby_protoc.rb} | 0 ...c_grpc_ruby_plugin.rb => grpc_tools_ruby_protoc_plugin.rb} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/ruby/tools/bin/{protoc.rb => grpc_tools_ruby_protoc.rb} (100%) rename src/ruby/tools/bin/{protoc_grpc_ruby_plugin.rb => grpc_tools_ruby_protoc_plugin.rb} (100%) diff --git a/src/node/tools/package.json b/src/node/tools/package.json index d98ed0b1fc3..62155edd687 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -16,8 +16,8 @@ } ], "bin": { - "grpc-tools-protoc": "./bin/protoc.js", - "grpc-tools-plugin": "./bin/protoc_plugin.js" + "grpc_tools_node_protoc": "./bin/protoc.js", + "grpc_tools_node_protoc_plugin": "./bin/protoc_plugin.js" }, "scripts": { "install": "./node_modules/.bin/node-pre-gyp install" diff --git a/src/ruby/tools/bin/protoc.rb b/src/ruby/tools/bin/grpc_tools_ruby_protoc.rb similarity index 100% rename from src/ruby/tools/bin/protoc.rb rename to src/ruby/tools/bin/grpc_tools_ruby_protoc.rb diff --git a/src/ruby/tools/bin/protoc_grpc_ruby_plugin.rb b/src/ruby/tools/bin/grpc_tools_ruby_protoc_plugin.rb similarity index 100% rename from src/ruby/tools/bin/protoc_grpc_ruby_plugin.rb rename to src/ruby/tools/bin/grpc_tools_ruby_protoc_plugin.rb From d5d8f8fbf94fab16b5c2ecc84cdefffed1a44b4e Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 4 May 2016 13:22:26 -0700 Subject: [PATCH 073/155] Added all transitive header dependencies in build.yaml to fix the failing check. --- BUILD | 68 ++++++++++++ build.yaml | 49 +++++++++ gRPC.podspec | 68 ++++++++++++ grpc.gemspec | 34 ++++++ package.xml | 34 ++++++ tools/doxygen/Doxyfile.core.internal | 34 ++++++ tools/run_tests/sources_and_headers.json | 100 +++++++++++++++++- vsprojects/vcxproj/grpc/grpc.vcxproj | 34 ++++++ vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 105 +++++++++++++++++++ 9 files changed, 525 insertions(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 2e1d762f079..9a14f30c361 100644 --- a/BUILD +++ b/BUILD @@ -285,7 +285,41 @@ cc_library( "src/core/ext/client_config/subchannel_call_holder.h", "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", + "include/grpc/byte_buffer.h", + "include/grpc/grpc.h", + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/compression_types.h", + "include/grpc/impl/codegen/connectivity_state.h", + "include/grpc/impl/codegen/grpc_types.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/propagation_bits.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/status.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "include/grpc/status.h", + "include/grpc/support/alloc.h", + "include/grpc/support/atm.h", + "include/grpc/support/host_port.h", + "include/grpc/support/log.h", "include/grpc/support/port_platform.h", + "include/grpc/support/slice.h", + "include/grpc/support/slice_buffer.h", + "include/grpc/support/string_util.h", + "include/grpc/support/sync.h", + "include/grpc/support/time.h", + "include/grpc/support/useful.h", + "src/core/lib/support/string.h", "third_party/objective_c/Cronet/cronet_c_for_grpc.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", @@ -1636,7 +1670,41 @@ objc_library( "src/core/ext/client_config/subchannel_call_holder.h", "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", + "include/grpc/byte_buffer.h", + "include/grpc/grpc.h", + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/compression_types.h", + "include/grpc/impl/codegen/connectivity_state.h", + "include/grpc/impl/codegen/grpc_types.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/propagation_bits.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/status.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "include/grpc/status.h", + "include/grpc/support/alloc.h", + "include/grpc/support/atm.h", + "include/grpc/support/host_port.h", + "include/grpc/support/log.h", "include/grpc/support/port_platform.h", + "include/grpc/support/slice.h", + "include/grpc/support/slice_buffer.h", + "include/grpc/support/string_util.h", + "include/grpc/support/sync.h", + "include/grpc/support/time.h", + "include/grpc/support/useful.h", + "src/core/lib/support/string.h", "third_party/objective_c/Cronet/cronet_c_for_grpc.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", diff --git a/build.yaml b/build.yaml index a7ab412722c..ff4e7f0f90b 100644 --- a/build.yaml +++ b/build.yaml @@ -549,7 +549,56 @@ filegroups: - grpc_secure - name: grpc_transport_cronet_client_secure headers: + - include/grpc/byte_buffer.h + - include/grpc/grpc.h + - include/grpc/impl/codegen/alloc.h + - include/grpc/impl/codegen/atm.h + - include/grpc/impl/codegen/atm_gcc_atomic.h + - include/grpc/impl/codegen/atm_gcc_sync.h + - include/grpc/impl/codegen/atm_win32.h + - include/grpc/impl/codegen/byte_buffer.h + - include/grpc/impl/codegen/compression_types.h + - include/grpc/impl/codegen/connectivity_state.h + - include/grpc/impl/codegen/grpc_types.h + - include/grpc/impl/codegen/log.h + - include/grpc/impl/codegen/port_platform.h + - include/grpc/impl/codegen/propagation_bits.h + - include/grpc/impl/codegen/slice.h + - include/grpc/impl/codegen/slice_buffer.h + - include/grpc/impl/codegen/status.h + - include/grpc/impl/codegen/sync.h + - include/grpc/impl/codegen/sync_generic.h + - include/grpc/impl/codegen/sync_posix.h + - include/grpc/impl/codegen/sync_win32.h + - include/grpc/impl/codegen/time.h + - include/grpc/status.h + - include/grpc/support/alloc.h + - include/grpc/support/atm.h + - include/grpc/support/host_port.h + - include/grpc/support/log.h - include/grpc/support/port_platform.h + - include/grpc/support/slice.h + - include/grpc/support/slice_buffer.h + - include/grpc/support/string_util.h + - include/grpc/support/sync.h + - include/grpc/support/time.h + - include/grpc/support/useful.h + - src/core/ext/transport/chttp2/transport/incoming_metadata.h + - src/core/lib/channel/channel_stack.h + - src/core/lib/channel/context.h + - src/core/lib/debug/trace.h + - src/core/lib/iomgr/closure.h + - src/core/lib/iomgr/exec_ctx.h + - src/core/lib/iomgr/pollset.h + - src/core/lib/iomgr/pollset_set.h + - src/core/lib/support/string.h + - src/core/lib/surface/channel.h + - src/core/lib/surface/channel_stack_type.h + - src/core/lib/transport/byte_stream.h + - src/core/lib/transport/metadata.h + - src/core/lib/transport/metadata_batch.h + - src/core/lib/transport/transport.h + - src/core/lib/transport/transport_impl.h - third_party/objective_c/Cronet/cronet_c_for_grpc.h src: - src/core/ext/transport/cronet/client/secure/cronet_channel_create.c diff --git a/gRPC.podspec b/gRPC.podspec index f57ba745194..c3c3a65128c 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -287,7 +287,41 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_call_holder.h', 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', + 'include/grpc/byte_buffer.h', + 'include/grpc/grpc.h', + 'include/grpc/impl/codegen/alloc.h', + 'include/grpc/impl/codegen/atm.h', + 'include/grpc/impl/codegen/atm_gcc_atomic.h', + 'include/grpc/impl/codegen/atm_gcc_sync.h', + 'include/grpc/impl/codegen/atm_win32.h', + 'include/grpc/impl/codegen/byte_buffer.h', + 'include/grpc/impl/codegen/compression_types.h', + 'include/grpc/impl/codegen/connectivity_state.h', + 'include/grpc/impl/codegen/grpc_types.h', + 'include/grpc/impl/codegen/log.h', + 'include/grpc/impl/codegen/port_platform.h', + 'include/grpc/impl/codegen/propagation_bits.h', + 'include/grpc/impl/codegen/slice.h', + 'include/grpc/impl/codegen/slice_buffer.h', + 'include/grpc/impl/codegen/status.h', + 'include/grpc/impl/codegen/sync.h', + 'include/grpc/impl/codegen/sync_generic.h', + 'include/grpc/impl/codegen/sync_posix.h', + 'include/grpc/impl/codegen/sync_win32.h', + 'include/grpc/impl/codegen/time.h', + 'include/grpc/status.h', + 'include/grpc/support/alloc.h', + 'include/grpc/support/atm.h', + 'include/grpc/support/host_port.h', + 'include/grpc/support/log.h', 'include/grpc/support/port_platform.h', + 'include/grpc/support/slice.h', + 'include/grpc/support/slice_buffer.h', + 'include/grpc/support/string_util.h', + 'include/grpc/support/sync.h', + 'include/grpc/support/time.h', + 'include/grpc/support/useful.h', + 'src/core/lib/support/string.h', 'third_party/objective_c/Cronet/cronet_c_for_grpc.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', @@ -635,7 +669,41 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_call_holder.h', 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', + 'include/grpc/byte_buffer.h', + 'include/grpc/grpc.h', + 'include/grpc/impl/codegen/alloc.h', + 'include/grpc/impl/codegen/atm.h', + 'include/grpc/impl/codegen/atm_gcc_atomic.h', + 'include/grpc/impl/codegen/atm_gcc_sync.h', + 'include/grpc/impl/codegen/atm_win32.h', + 'include/grpc/impl/codegen/byte_buffer.h', + 'include/grpc/impl/codegen/compression_types.h', + 'include/grpc/impl/codegen/connectivity_state.h', + 'include/grpc/impl/codegen/grpc_types.h', + 'include/grpc/impl/codegen/log.h', + 'include/grpc/impl/codegen/port_platform.h', + 'include/grpc/impl/codegen/propagation_bits.h', + 'include/grpc/impl/codegen/slice.h', + 'include/grpc/impl/codegen/slice_buffer.h', + 'include/grpc/impl/codegen/status.h', + 'include/grpc/impl/codegen/sync.h', + 'include/grpc/impl/codegen/sync_generic.h', + 'include/grpc/impl/codegen/sync_posix.h', + 'include/grpc/impl/codegen/sync_win32.h', + 'include/grpc/impl/codegen/time.h', + 'include/grpc/status.h', + 'include/grpc/support/alloc.h', + 'include/grpc/support/atm.h', + 'include/grpc/support/host_port.h', + 'include/grpc/support/log.h', 'include/grpc/support/port_platform.h', + 'include/grpc/support/slice.h', + 'include/grpc/support/slice_buffer.h', + 'include/grpc/support/string_util.h', + 'include/grpc/support/sync.h', + 'include/grpc/support/time.h', + 'include/grpc/support/useful.h', + 'src/core/lib/support/string.h', 'third_party/objective_c/Cronet/cronet_c_for_grpc.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', diff --git a/grpc.gemspec b/grpc.gemspec index 488f4657cdc..cde01942b71 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -296,7 +296,41 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/client_config/subchannel_call_holder.h ) s.files += %w( src/core/ext/client_config/subchannel_index.h ) s.files += %w( src/core/ext/client_config/uri_parser.h ) + s.files += %w( include/grpc/byte_buffer.h ) + s.files += %w( include/grpc/grpc.h ) + s.files += %w( include/grpc/impl/codegen/alloc.h ) + s.files += %w( include/grpc/impl/codegen/atm.h ) + s.files += %w( include/grpc/impl/codegen/atm_gcc_atomic.h ) + s.files += %w( include/grpc/impl/codegen/atm_gcc_sync.h ) + s.files += %w( include/grpc/impl/codegen/atm_win32.h ) + s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) + s.files += %w( include/grpc/impl/codegen/compression_types.h ) + s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) + s.files += %w( include/grpc/impl/codegen/grpc_types.h ) + s.files += %w( include/grpc/impl/codegen/log.h ) + s.files += %w( include/grpc/impl/codegen/port_platform.h ) + s.files += %w( include/grpc/impl/codegen/propagation_bits.h ) + s.files += %w( include/grpc/impl/codegen/slice.h ) + s.files += %w( include/grpc/impl/codegen/slice_buffer.h ) + s.files += %w( include/grpc/impl/codegen/status.h ) + s.files += %w( include/grpc/impl/codegen/sync.h ) + s.files += %w( include/grpc/impl/codegen/sync_generic.h ) + s.files += %w( include/grpc/impl/codegen/sync_posix.h ) + s.files += %w( include/grpc/impl/codegen/sync_win32.h ) + s.files += %w( include/grpc/impl/codegen/time.h ) + s.files += %w( include/grpc/status.h ) + s.files += %w( include/grpc/support/alloc.h ) + s.files += %w( include/grpc/support/atm.h ) + s.files += %w( include/grpc/support/host_port.h ) + s.files += %w( include/grpc/support/log.h ) s.files += %w( include/grpc/support/port_platform.h ) + s.files += %w( include/grpc/support/slice.h ) + s.files += %w( include/grpc/support/slice_buffer.h ) + s.files += %w( include/grpc/support/string_util.h ) + s.files += %w( include/grpc/support/sync.h ) + s.files += %w( include/grpc/support/time.h ) + s.files += %w( include/grpc/support/useful.h ) + s.files += %w( src/core/lib/support/string.h ) s.files += %w( third_party/objective_c/Cronet/cronet_c_for_grpc.h ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.h ) s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h ) diff --git a/package.xml b/package.xml index e8fd375eb60..012ce6ab2ef 100644 --- a/package.xml +++ b/package.xml @@ -303,7 +303,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 16dcd9b79a7..6bc5e78e620 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -913,7 +913,41 @@ src/core/ext/client_config/subchannel.h \ src/core/ext/client_config/subchannel_call_holder.h \ src/core/ext/client_config/subchannel_index.h \ src/core/ext/client_config/uri_parser.h \ +include/grpc/byte_buffer.h \ +include/grpc/grpc.h \ +include/grpc/impl/codegen/alloc.h \ +include/grpc/impl/codegen/atm.h \ +include/grpc/impl/codegen/atm_gcc_atomic.h \ +include/grpc/impl/codegen/atm_gcc_sync.h \ +include/grpc/impl/codegen/atm_win32.h \ +include/grpc/impl/codegen/byte_buffer.h \ +include/grpc/impl/codegen/compression_types.h \ +include/grpc/impl/codegen/connectivity_state.h \ +include/grpc/impl/codegen/grpc_types.h \ +include/grpc/impl/codegen/log.h \ +include/grpc/impl/codegen/port_platform.h \ +include/grpc/impl/codegen/propagation_bits.h \ +include/grpc/impl/codegen/slice.h \ +include/grpc/impl/codegen/slice_buffer.h \ +include/grpc/impl/codegen/status.h \ +include/grpc/impl/codegen/sync.h \ +include/grpc/impl/codegen/sync_generic.h \ +include/grpc/impl/codegen/sync_posix.h \ +include/grpc/impl/codegen/sync_win32.h \ +include/grpc/impl/codegen/time.h \ +include/grpc/status.h \ +include/grpc/support/alloc.h \ +include/grpc/support/atm.h \ +include/grpc/support/host_port.h \ +include/grpc/support/log.h \ include/grpc/support/port_platform.h \ +include/grpc/support/slice.h \ +include/grpc/support/slice_buffer.h \ +include/grpc/support/string_util.h \ +include/grpc/support/sync.h \ +include/grpc/support/time.h \ +include/grpc/support/useful.h \ +src/core/lib/support/string.h \ third_party/objective_c/Cronet/cronet_c_for_grpc.h \ src/core/ext/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index e22318d2387..426a9ba91ee 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -6268,16 +6268,114 @@ { "deps": [], "headers": [ + "include/grpc/byte_buffer.h", + "include/grpc/grpc.h", + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/compression_types.h", + "include/grpc/impl/codegen/connectivity_state.h", + "include/grpc/impl/codegen/grpc_types.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/propagation_bits.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/status.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "include/grpc/status.h", + "include/grpc/support/alloc.h", + "include/grpc/support/atm.h", + "include/grpc/support/host_port.h", + "include/grpc/support/log.h", "include/grpc/support/port_platform.h", + "include/grpc/support/slice.h", + "include/grpc/support/slice_buffer.h", + "include/grpc/support/string_util.h", + "include/grpc/support/sync.h", + "include/grpc/support/time.h", + "include/grpc/support/useful.h", + "src/core/ext/transport/chttp2/transport/incoming_metadata.h", + "src/core/lib/channel/channel_stack.h", + "src/core/lib/channel/context.h", + "src/core/lib/debug/trace.h", + "src/core/lib/iomgr/closure.h", + "src/core/lib/iomgr/exec_ctx.h", + "src/core/lib/iomgr/pollset.h", + "src/core/lib/iomgr/pollset_set.h", + "src/core/lib/support/string.h", + "src/core/lib/surface/channel.h", + "src/core/lib/surface/channel_stack_type.h", + "src/core/lib/transport/byte_stream.h", + "src/core/lib/transport/metadata.h", + "src/core/lib/transport/metadata_batch.h", + "src/core/lib/transport/transport.h", + "src/core/lib/transport/transport_impl.h", "third_party/objective_c/Cronet/cronet_c_for_grpc.h" ], "language": "c", "name": "grpc_transport_cronet_client_secure", "src": [ + "include/grpc/byte_buffer.h", + "include/grpc/grpc.h", + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/compression_types.h", + "include/grpc/impl/codegen/connectivity_state.h", + "include/grpc/impl/codegen/grpc_types.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/propagation_bits.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/status.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "include/grpc/status.h", + "include/grpc/support/alloc.h", + "include/grpc/support/atm.h", + "include/grpc/support/host_port.h", + "include/grpc/support/log.h", "include/grpc/support/port_platform.h", + "include/grpc/support/slice.h", + "include/grpc/support/slice_buffer.h", + "include/grpc/support/string_util.h", + "include/grpc/support/sync.h", + "include/grpc/support/time.h", + "include/grpc/support/useful.h", + "src/core/ext/transport/chttp2/transport/incoming_metadata.h", "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", - "src/core/ext/transport/cronet/transport/cronet_transport.c" + "src/core/ext/transport/cronet/transport/cronet_transport.c", + "src/core/lib/channel/channel_stack.h", + "src/core/lib/channel/context.h", + "src/core/lib/debug/trace.h", + "src/core/lib/iomgr/closure.h", + "src/core/lib/iomgr/exec_ctx.h", + "src/core/lib/iomgr/pollset.h", + "src/core/lib/iomgr/pollset_set.h", + "src/core/lib/support/string.h", + "src/core/lib/surface/channel.h", + "src/core/lib/surface/channel_stack_type.h", + "src/core/lib/transport/byte_stream.h", + "src/core/lib/transport/metadata.h", + "src/core/lib/transport/metadata_batch.h", + "src/core/lib/transport/transport.h", + "src/core/lib/transport/transport_impl.h" ], "third_party": false, "type": "filegroup" diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index cbf854875a6..4f0157e63fd 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -422,7 +422,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 02297b7746a..43f36e3856d 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -968,9 +968,111 @@ src\core\ext\client_config + + include\grpc + + + include\grpc + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc + + + include\grpc\support + + + include\grpc\support + + + include\grpc\support + + + include\grpc\support + include\grpc\support + + include\grpc\support + + + include\grpc\support + + + include\grpc\support + + + include\grpc\support + + + include\grpc\support + + + include\grpc\support + + + src\core\lib\support + third_party\objective_c\Cronet @@ -1145,6 +1247,9 @@ {c4661d64-349f-01c1-1ba8-0602f9047595} + + {27f30339-d694-40f5-db07-4b89b9aeea73} + {a21971fb-304f-da08-b1b2-7bd8df8ac373} From 22b338e2109745a27f1f84de8490bc8148df1a79 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 4 May 2016 13:27:56 -0700 Subject: [PATCH 074/155] Apply clang-format diffs. --- .../chttp2/transport/chttp2_transport.c | 14 +++-- .../chttp2/transport/incoming_metadata.h | 2 +- .../ext/transport/chttp2/transport/parsing.c | 8 +-- src/core/lib/transport/metadata.h | 4 +- src/core/lib/transport/metadata_batch.c | 4 +- test/core/bad_client/bad_client.c | 7 +-- test/core/bad_client/bad_client.h | 8 +-- test/core/bad_client/tests/large_metadata.c | 57 +++++++++++-------- test/core/end2end/fixtures/h2_ssl_proxy.c | 4 +- test/core/end2end/fixtures/proxy.c | 6 +- test/core/end2end/fixtures/proxy.h | 6 +- test/core/end2end/tests/large_metadata.c | 11 ++-- 12 files changed, 71 insertions(+), 60 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 44b2a7a59f7..737a6ccbf18 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -948,15 +948,16 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, stream_global->send_initial_metadata_finished = add_closure_barrier(on_complete); stream_global->send_initial_metadata = op->send_initial_metadata; - const size_t metadata_size = grpc_metadata_batch_size( - op->send_initial_metadata); + const size_t metadata_size = + grpc_metadata_batch_size(op->send_initial_metadata); const size_t metadata_peer_limit = transport_global->settings[GRPC_PEER_SETTINGS] [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; if (metadata_size > metadata_peer_limit) { gpr_log(GPR_DEBUG, "to-be-sent initial metadata size exceeds peer limit " - "(%lu vs. %lu)", metadata_size, metadata_peer_limit); + "(%lu vs. %lu)", + metadata_size, metadata_peer_limit); cancel_from_api(exec_ctx, transport_global, stream_global, GRPC_STATUS_RESOURCE_EXHAUSTED); } else { @@ -1002,15 +1003,16 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, stream_global->send_trailing_metadata_finished = add_closure_barrier(on_complete); stream_global->send_trailing_metadata = op->send_trailing_metadata; - const size_t metadata_size = grpc_metadata_batch_size( - op->send_trailing_metadata); + const size_t metadata_size = + grpc_metadata_batch_size(op->send_trailing_metadata); const size_t metadata_peer_limit = transport_global->settings[GRPC_PEER_SETTINGS] [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; if (metadata_size > metadata_peer_limit) { gpr_log(GPR_DEBUG, "to-be-sent trailing metadata size exceeds peer limit " - "(%lu vs. %lu)", metadata_size, metadata_peer_limit); + "(%lu vs. %lu)", + metadata_size, metadata_peer_limit); cancel_from_api(exec_ctx, transport_global, stream_global, GRPC_STATUS_RESOURCE_EXHAUSTED); } else { diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.h b/src/core/ext/transport/chttp2/transport/incoming_metadata.h index 7db5db8de05..df4343b93ed 100644 --- a/src/core/ext/transport/chttp2/transport/incoming_metadata.h +++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.h @@ -42,7 +42,7 @@ typedef struct { size_t capacity; gpr_timespec deadline; int published; - size_t size; /* total size of metadata */ + size_t size; // total size of metadata } grpc_chttp2_incoming_metadata_buffer; /** assumes everything initially zeroed */ diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 95169567249..4bd374b7fa7 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -639,8 +639,8 @@ static void on_initial_header(void *tp, grpc_mdelem *md) { gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), *cached_timeout)); GRPC_MDELEM_UNREF(md); } else { - const size_t new_size = stream_parsing->metadata_buffer[0].size + - GRPC_MDELEM_LENGTH(md); + const size_t new_size = + stream_parsing->metadata_buffer[0].size + GRPC_MDELEM_LENGTH(md); grpc_chttp2_transport_global *transport_global = &TRANSPORT_FROM_PARSING(transport_parsing)->global; const size_t metadata_size_limit = @@ -685,8 +685,8 @@ static void on_trailing_header(void *tp, grpc_mdelem *md) { stream_parsing->seen_error = true; } - const size_t new_size = stream_parsing->metadata_buffer[1].size + - GRPC_MDELEM_LENGTH(md); + const size_t new_size = + stream_parsing->metadata_buffer[1].size + GRPC_MDELEM_LENGTH(md); grpc_chttp2_transport_global *transport_global = &TRANSPORT_FROM_PARSING(transport_parsing)->global; const size_t metadata_size_limit = diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 4ecbbd1b1b8..6d82f4d6819 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -148,8 +148,8 @@ const char *grpc_mdstr_as_c_string(grpc_mdstr *s); #define GRPC_MDSTR_LENGTH(s) (GPR_SLICE_LENGTH(s->slice)) /* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */ -#define GRPC_MDELEM_LENGTH(e) (GRPC_MDSTR_LENGTH((e)->key) + \ - GRPC_MDSTR_LENGTH((e)->value) + 32) +#define GRPC_MDELEM_LENGTH(e) \ + (GRPC_MDSTR_LENGTH((e)->key) + GRPC_MDSTR_LENGTH((e)->value) + 32) int grpc_mdstr_is_legal_header(grpc_mdstr *s); int grpc_mdstr_is_legal_nonbin_header(grpc_mdstr *s); diff --git a/src/core/lib/transport/metadata_batch.c b/src/core/lib/transport/metadata_batch.c index 4e1cd8e2c1c..c0afc715bcc 100644 --- a/src/core/lib/transport/metadata_batch.c +++ b/src/core/lib/transport/metadata_batch.c @@ -195,8 +195,8 @@ int grpc_metadata_batch_is_empty(grpc_metadata_batch *batch) { size_t grpc_metadata_batch_size(grpc_metadata_batch *batch) { size_t size = 0; - for (grpc_linked_mdelem* elem = batch->list.head; - elem != NULL; elem = elem->next) { + for (grpc_linked_mdelem* elem = batch->list.head; elem != NULL; + elem = elem->next) { size += GRPC_MDELEM_LENGTH(elem->md); } return size; diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c index aa9125dc7a9..e5820688ef1 100644 --- a/test/core/bad_client/bad_client.c +++ b/test/core/bad_client/bad_client.c @@ -90,8 +90,7 @@ static void read_done(grpc_exec_ctx *exec_ctx, void *arg, bool success) { void grpc_run_bad_client_test( grpc_bad_client_server_side_validator server_validator, grpc_bad_client_client_stream_validator client_validator, - const char *client_payload, - size_t client_payload_length, uint32_t flags) { + const char *client_payload, size_t client_payload_length, uint32_t flags) { grpc_endpoint_pair sfd; thd_args a; gpr_thd_id id; @@ -177,8 +176,8 @@ void grpc_run_bad_client_test( grpc_endpoint_read(&exec_ctx, sfd.client, &args.incoming, &read_done_closure); grpc_exec_ctx_finish(&exec_ctx); - GPR_ASSERT(gpr_event_wait(&args.read_done, - GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + GPR_ASSERT( + gpr_event_wait(&args.read_done, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); gpr_slice_buffer_destroy(&args.incoming); } // Shutdown. diff --git a/test/core/bad_client/bad_client.h b/test/core/bad_client/bad_client.h index b6e8a6dd5b0..ecd6721a78a 100644 --- a/test/core/bad_client/bad_client.h +++ b/test/core/bad_client/bad_client.h @@ -59,9 +59,9 @@ void grpc_run_bad_client_test( grpc_bad_client_client_stream_validator client_validator, const char *client_payload, size_t client_payload_length, uint32_t flags); -#define GRPC_RUN_BAD_CLIENT_TEST(server_validator, client_validator, \ - payload, flags) \ - grpc_run_bad_client_test(server_validator, client_validator, \ - payload, sizeof(payload) - 1, flags) +#define GRPC_RUN_BAD_CLIENT_TEST(server_validator, client_validator, payload, \ + flags) \ + grpc_run_bad_client_test(server_validator, client_validator, payload, \ + sizeof(payload) - 1, flags) #endif /* GRPC_TEST_CORE_BAD_CLIENT_BAD_CLIENT_H */ diff --git a/test/core/bad_client/tests/large_metadata.c b/test/core/bad_client/tests/large_metadata.c index 95932205cfc..b3521439f52 100644 --- a/test/core/bad_client/tests/large_metadata.c +++ b/test/core/bad_client/tests/large_metadata.c @@ -40,16 +40,19 @@ #include "test/core/end2end/cq_verifier.h" #define PFX_TOO_MUCH_METADATA_FROM_CLIENT_STR \ - "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" \ - /* settings frame */ \ - "\x00\x00\x00\x04\x00\x00\x00\x00\x00" \ - /* headers: generated from large_metadata.headers in this directory */ \ - "\x00""5{\x01\x05\x00\x00\x00\x01" \ + "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" /* settings frame */ \ + "\x00\x00\x00\x04\x00\x00\x00\x00\x00" /* headers: generated from \ + large_metadata.headers in this \ + directory */ \ + "\x00" \ + "5{\x01\x05\x00\x00\x00\x01" \ "\x10\x05:path\x08/foo/bar" \ "\x10\x07:scheme\x04http" \ "\x10\x07:method\x04POST" \ "\x10\x0a:authority\x09localhost" \ - "\x10\x0c""content-type\x10""application/grpc" \ + "\x10\x0c" \ + "content-type\x10" \ + "application/grpc" \ "\x10\x14grpc-accept-encoding\x15identity,deflate,gzip" \ "\x10\x02te\x08trailers" \ "\x10\x0auser-agent\"bad-client grpc-c/0.12.0.0 (linux)" \ @@ -339,22 +342,27 @@ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ "aaaaaaaa" -#define PFX_TOO_MUCH_METADATA_FROM_SERVER_STR \ - "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" \ - /* settings frame: sets MAX_HEADER_LIST_SIZE to 16K */ \ - "\x00\x00\x06\x04\x00\x00\x00\x00\x00\x00\x06\x00\x00\x40\x00" \ - /* headers: generated from simple_request.headers in this directory */ \ - "\x00\x00\xc9\x01\x04\x00\x00\x00\x01" \ - "\x10\x05:path\x08/foo/bar" \ - "\x10\x07:scheme\x04http" \ - "\x10\x07:method\x04POST" \ - "\x10\x0a:authority\x09localhost" \ - "\x10\x0c" \ - "content-type\x10" \ - "application/grpc" \ - "\x10\x14grpc-accept-encoding\x15" \ - "deflate,identity,gzip" \ - "\x10\x02te\x08trailers" \ +#define PFX_TOO_MUCH_METADATA_FROM_SERVER_STR \ + "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" /* settings frame: sets \ + MAX_HEADER_LIST_SIZE to 16K */ \ + "\x00\x00\x06\x04\x00\x00\x00\x00\x00\x00\x06\x00\x00\x40\x00" /* headers: \ + generated \ + from \ + simple_request.headers \ + in this \ + directory \ + */ \ + "\x00\x00\xc9\x01\x04\x00\x00\x00\x01" \ + "\x10\x05:path\x08/foo/bar" \ + "\x10\x07:scheme\x04http" \ + "\x10\x07:method\x04POST" \ + "\x10\x0a:authority\x09localhost" \ + "\x10\x0c" \ + "content-type\x10" \ + "application/grpc" \ + "\x10\x14grpc-accept-encoding\x15" \ + "deflate,identity,gzip" \ + "\x10\x02te\x08trailers" \ "\x10\x0auser-agent\"bad-client grpc-c/0.12.0.0 (linux)" static void *tag(intptr_t t) { return (void *)t; } @@ -385,8 +393,9 @@ static void server_verifier(grpc_server *server, grpc_completion_queue *cq, cq_verifier_destroy(cqv); } -static void server_verifier_sends_too_much_metadata( - grpc_server *server, grpc_completion_queue *cq, void *registered_method) { +static void server_verifier_sends_too_much_metadata(grpc_server *server, + grpc_completion_queue *cq, + void *registered_method) { grpc_call_error error; grpc_call *s; grpc_call_details call_details; diff --git a/test/core/end2end/fixtures/h2_ssl_proxy.c b/test/core/end2end/fixtures/h2_ssl_proxy.c index 0ec6ad437ee..151a86cb8f5 100644 --- a/test/core/end2end/fixtures/h2_ssl_proxy.c +++ b/test/core/end2end/fixtures/h2_ssl_proxy.c @@ -76,8 +76,8 @@ static grpc_channel *create_proxy_client(const char *target, {"foo.test.google.fr"}}; grpc_channel_args *new_client_args = grpc_channel_args_copy_and_add(client_args, &ssl_name_override, 1); - channel = grpc_secure_channel_create(ssl_creds, target, new_client_args, - NULL); + channel = + grpc_secure_channel_create(ssl_creds, target, new_client_args, NULL); grpc_channel_credentials_release(ssl_creds); grpc_channel_args_destroy(new_client_args); return channel; diff --git a/test/core/end2end/fixtures/proxy.c b/test/core/end2end/fixtures/proxy.c index ff413ffd652..f6e01ec41c5 100644 --- a/test/core/end2end/fixtures/proxy.c +++ b/test/core/end2end/fixtures/proxy.c @@ -89,9 +89,9 @@ typedef struct { static void thread_main(void *arg); static void request_call(grpc_end2end_proxy *proxy); -grpc_end2end_proxy *grpc_end2end_proxy_create( - const grpc_end2end_proxy_def *def, - grpc_channel_args *client_args, grpc_channel_args *server_args) { +grpc_end2end_proxy *grpc_end2end_proxy_create(const grpc_end2end_proxy_def *def, + grpc_channel_args *client_args, + grpc_channel_args *server_args) { gpr_thd_options opt = gpr_thd_options_default(); int proxy_port = grpc_pick_unused_port_or_die(); int server_port = grpc_pick_unused_port_or_die(); diff --git a/test/core/end2end/fixtures/proxy.h b/test/core/end2end/fixtures/proxy.h index 89f95f09f90..75b75d13319 100644 --- a/test/core/end2end/fixtures/proxy.h +++ b/test/core/end2end/fixtures/proxy.h @@ -47,9 +47,9 @@ typedef struct grpc_end2end_proxy_def { grpc_channel_args *client_args); } grpc_end2end_proxy_def; -grpc_end2end_proxy *grpc_end2end_proxy_create( - const grpc_end2end_proxy_def *def, - grpc_channel_args *client_args, grpc_channel_args *server_args); +grpc_end2end_proxy *grpc_end2end_proxy_create(const grpc_end2end_proxy_def *def, + grpc_channel_args *client_args, + grpc_channel_args *server_args); void grpc_end2end_proxy_destroy(grpc_end2end_proxy *proxy); const char *grpc_end2end_proxy_get_client_target(grpc_end2end_proxy *proxy); diff --git a/test/core/end2end/tests/large_metadata.c b/test/core/end2end/tests/large_metadata.c index 6d3074a94ba..dd29552b975 100644 --- a/test/core/end2end/tests/large_metadata.c +++ b/test/core/end2end/tests/large_metadata.c @@ -107,11 +107,12 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { gpr_timespec deadline = five_seconds_time(); grpc_metadata meta; const size_t large_size = 64 * 1024; - grpc_arg arg = { GRPC_ARG_INTEGER, GRPC_ARG_MAX_METADATA_SIZE, - { .integer=(int)large_size + 1024 } }; - grpc_channel_args args = { 1, &arg }; - grpc_end2end_test_fixture f = begin_test( - config, "test_request_with_large_metadata", &args, &args); + grpc_arg arg = {GRPC_ARG_INTEGER, + GRPC_ARG_MAX_METADATA_SIZE, + {.integer=(int)large_size + 1024}}; + grpc_channel_args args = {1, &arg}; + grpc_end2end_test_fixture f = + begin_test(config, "test_request_with_large_metadata", &args, &args); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; From 305ffd4847617c7206b0b4ccec0fcd8977b2e095 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 4 May 2016 10:26:24 -0700 Subject: [PATCH 075/155] make SendStatusFromServer optionally send a message as well --- .../Internal/FakeNativeCall.cs | 3 +- .../Grpc.Core/Internal/AsyncCallServer.cs | 3 +- .../Grpc.Core/Internal/CallSafeHandle.cs | 7 +++-- src/csharp/Grpc.Core/Internal/INativeCall.cs | 2 +- .../Grpc.Core/Internal/NativeMethods.cs | 18 ++++++----- src/csharp/ext/grpc_csharp_ext.c | 30 +++++++++++++------ 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs b/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs index 1bec258ca29..909112a47c5 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs @@ -165,7 +165,8 @@ namespace Grpc.Core.Internal.Tests SendCompletionHandler = callback; } - public void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata) + public void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata, + byte[] optionalPayload, WriteFlags writeFlags) { SendStatusFromServerHandler = callback; } diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs index eafe2ccab87..c5d900b1346 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs @@ -149,7 +149,8 @@ namespace Grpc.Core.Internal using (var metadataArray = MetadataArraySafeHandle.Create(trailers)) { - call.StartSendStatusFromServer(HandleSendStatusFromServerFinished, status, metadataArray, !initialMetadataSent); + call.StartSendStatusFromServer(HandleSendStatusFromServerFinished, status, metadataArray, !initialMetadataSent, + null, new WriteFlags()); } halfcloseRequested = true; initialMetadataSent = true; diff --git a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs index 500653ba5d3..244b97d4a47 100644 --- a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs +++ b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs @@ -135,13 +135,16 @@ namespace Grpc.Core.Internal } } - public void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata) + public void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata, + byte[] optionalPayload, WriteFlags writeFlags) { using (completionQueue.NewScope()) { var ctx = BatchContextSafeHandle.Create(); + var optionalPayloadLength = optionalPayload != null ? new UIntPtr((ulong)optionalPayload.Length) : UIntPtr.Zero; completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success)); - Native.grpcsharp_call_send_status_from_server(this, ctx, status.StatusCode, status.Detail, metadataArray, sendEmptyInitialMetadata).CheckOk(); + Native.grpcsharp_call_send_status_from_server(this, ctx, status.StatusCode, status.Detail, metadataArray, sendEmptyInitialMetadata, + optionalPayload, optionalPayloadLength, writeFlags).CheckOk(); } } diff --git a/src/csharp/Grpc.Core/Internal/INativeCall.cs b/src/csharp/Grpc.Core/Internal/INativeCall.cs index cbef5991396..cd3719cb50c 100644 --- a/src/csharp/Grpc.Core/Internal/INativeCall.cs +++ b/src/csharp/Grpc.Core/Internal/INativeCall.cs @@ -78,7 +78,7 @@ namespace Grpc.Core.Internal void StartSendCloseFromClient(SendCompletionHandler callback); - void StartSendStatusFromServer(SendCompletionHandler callback, Grpc.Core.Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata); + void StartSendStatusFromServer(SendCompletionHandler callback, Grpc.Core.Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata, byte[] optionalPayload, Grpc.Core.WriteFlags writeFlags); void StartServerSide(ReceivedCloseOnServerHandler callback); } diff --git a/src/csharp/Grpc.Core/Internal/NativeMethods.cs b/src/csharp/Grpc.Core/Internal/NativeMethods.cs index 9ee0ba3bc05..c277c73ef04 100644 --- a/src/csharp/Grpc.Core/Internal/NativeMethods.cs +++ b/src/csharp/Grpc.Core/Internal/NativeMethods.cs @@ -421,20 +421,21 @@ namespace Grpc.Core.Internal public delegate GRPCCallError grpcsharp_call_cancel_delegate(CallSafeHandle call); public delegate GRPCCallError grpcsharp_call_cancel_with_status_delegate(CallSafeHandle call, StatusCode status, string description); public delegate GRPCCallError grpcsharp_call_start_unary_delegate(CallSafeHandle call, - BatchContextSafeHandle ctx, byte[] send_buffer, UIntPtr send_buffer_len, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags); + BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags); public delegate GRPCCallError grpcsharp_call_start_client_streaming_delegate(CallSafeHandle call, BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray); public delegate GRPCCallError grpcsharp_call_start_server_streaming_delegate(CallSafeHandle call, - BatchContextSafeHandle ctx, byte[] send_buffer, UIntPtr send_buffer_len, + BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags); public delegate GRPCCallError grpcsharp_call_start_duplex_streaming_delegate(CallSafeHandle call, BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray); public delegate GRPCCallError grpcsharp_call_send_message_delegate(CallSafeHandle call, - BatchContextSafeHandle ctx, byte[] send_buffer, UIntPtr send_buffer_len, WriteFlags writeFlags, bool sendEmptyInitialMetadata); + BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, bool sendEmptyInitialMetadata); public delegate GRPCCallError grpcsharp_call_send_close_from_client_delegate(CallSafeHandle call, BatchContextSafeHandle ctx); public delegate GRPCCallError grpcsharp_call_send_status_from_server_delegate(CallSafeHandle call, - BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata); + BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata, + byte[] optionalSendBuffer, UIntPtr optionalSendBufferLen, WriteFlags writeFlags); public delegate GRPCCallError grpcsharp_call_recv_message_delegate(CallSafeHandle call, BatchContextSafeHandle ctx); public delegate GRPCCallError grpcsharp_call_recv_initial_metadata_delegate(CallSafeHandle call, @@ -593,7 +594,7 @@ namespace Grpc.Core.Internal [DllImport("grpc_csharp_ext.dll")] public static extern GRPCCallError grpcsharp_call_start_unary(CallSafeHandle call, - BatchContextSafeHandle ctx, byte[] send_buffer, UIntPtr send_buffer_len, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags); + BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags); [DllImport("grpc_csharp_ext.dll")] public static extern GRPCCallError grpcsharp_call_start_client_streaming(CallSafeHandle call, @@ -601,7 +602,7 @@ namespace Grpc.Core.Internal [DllImport("grpc_csharp_ext.dll")] public static extern GRPCCallError grpcsharp_call_start_server_streaming(CallSafeHandle call, - BatchContextSafeHandle ctx, byte[] send_buffer, UIntPtr send_buffer_len, + BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags); [DllImport("grpc_csharp_ext.dll")] @@ -610,7 +611,7 @@ namespace Grpc.Core.Internal [DllImport("grpc_csharp_ext.dll")] public static extern GRPCCallError grpcsharp_call_send_message(CallSafeHandle call, - BatchContextSafeHandle ctx, byte[] send_buffer, UIntPtr send_buffer_len, WriteFlags writeFlags, bool sendEmptyInitialMetadata); + BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, bool sendEmptyInitialMetadata); [DllImport("grpc_csharp_ext.dll")] public static extern GRPCCallError grpcsharp_call_send_close_from_client(CallSafeHandle call, @@ -618,7 +619,8 @@ namespace Grpc.Core.Internal [DllImport("grpc_csharp_ext.dll")] public static extern GRPCCallError grpcsharp_call_send_status_from_server(CallSafeHandle call, - BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata); + BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata, + byte[] optionalSendBuffer, UIntPtr optionalSendBufferLen, WriteFlags writeFlags); [DllImport("grpc_csharp_ext.dll")] public static extern GRPCCallError grpcsharp_call_recv_message(CallSafeHandle call, diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c index aeef8a79e99..5b8ff9b819b 100644 --- a/src/csharp/ext/grpc_csharp_ext.c +++ b/src/csharp/ext/grpc_csharp_ext.c @@ -715,10 +715,11 @@ grpcsharp_call_send_close_from_client(grpc_call *call, GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_send_status_from_server( grpc_call *call, grpcsharp_batch_context *ctx, grpc_status_code status_code, const char *status_details, grpc_metadata_array *trailing_metadata, - int32_t send_empty_initial_metadata) { + int32_t send_empty_initial_metadata, const char* optional_send_buffer, + size_t optional_send_buffer_len, uint32_t write_flags) { /* TODO: don't use magic number */ - grpc_op ops[2]; - size_t nops = send_empty_initial_metadata ? 2 : 1; + grpc_op ops[3]; + size_t nops = 1; ops[0].op = GRPC_OP_SEND_STATUS_FROM_SERVER; ops[0].data.send_status_from_server.status = status_code; ops[0].data.send_status_from_server.status_details = @@ -731,12 +732,23 @@ GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_send_status_from_server( ctx->send_status_from_server.trailing_metadata.metadata; ops[0].flags = 0; ops[0].reserved = NULL; - ops[1].op = GRPC_OP_SEND_INITIAL_METADATA; - ops[1].data.send_initial_metadata.count = 0; - ops[1].data.send_initial_metadata.metadata = NULL; - ops[1].flags = 0; - ops[1].reserved = NULL; - + if (optional_send_buffer) { + ops[nops].op = GRPC_OP_SEND_MESSAGE; + ctx->send_message = string_to_byte_buffer(optional_send_buffer, + optional_send_buffer_len); + ops[nops].data.send_message = ctx->send_message; + ops[nops].flags = write_flags; + ops[nops].reserved = NULL; + nops ++; + } + if (send_empty_initial_metadata) { + ops[nops].op = GRPC_OP_SEND_INITIAL_METADATA; + ops[nops].data.send_initial_metadata.count = 0; + ops[nops].data.send_initial_metadata.metadata = NULL; + ops[nops].flags = 0; + ops[nops].reserved = NULL; + nops++; + } return grpc_call_start_batch(call, ops, nops, ctx, NULL); } From 14e8dee2dd1fc2db7c2dfbf092fc7d99a296062a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 4 May 2016 12:59:23 -0700 Subject: [PATCH 076/155] use just one response batch for unary response serverside calls --- .../Internal/AsyncCallServerTest.cs | 2 +- .../Grpc.Core/Internal/AsyncCallServer.cs | 11 +++++-- .../Grpc.Core/Internal/ServerCallHandler.cs | 32 +++++++++---------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs index 058371521d6..0e204761f61 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs @@ -168,7 +168,7 @@ namespace Grpc.Core.Internal.Tests var responseStream = new ServerResponseStream(asyncCallServer); var writeTask = responseStream.WriteAsync("request1"); - var writeStatusTask = asyncCallServer.SendStatusFromServerAsync(Status.DefaultSuccess, new Metadata()); + var writeStatusTask = asyncCallServer.SendStatusFromServerAsync(Status.DefaultSuccess, new Metadata(), null); fakeCall.SendCompletionHandler(true); fakeCall.SendStatusFromServerHandler(true); diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs index c5d900b1346..b1566b44a7c 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs @@ -139,8 +139,11 @@ namespace Grpc.Core.Internal /// Sends call result status, indicating we are done with writes. /// Sending a status different from StatusCode.OK will also implicitly cancel the call. /// - public Task SendStatusFromServerAsync(Status status, Metadata trailers) + public Task SendStatusFromServerAsync(Status status, Metadata trailers, Tuple optionalWrite) { + byte[] payload = optionalWrite != null ? UnsafeSerialize(optionalWrite.Item1) : null; + var writeFlags = optionalWrite != null ? optionalWrite.Item2 : default(WriteFlags); + lock (myLock) { GrpcPreconditions.CheckState(started); @@ -150,11 +153,15 @@ namespace Grpc.Core.Internal using (var metadataArray = MetadataArraySafeHandle.Create(trailers)) { call.StartSendStatusFromServer(HandleSendStatusFromServerFinished, status, metadataArray, !initialMetadataSent, - null, new WriteFlags()); + payload, writeFlags); } halfcloseRequested = true; initialMetadataSent = true; sendStatusFromServerTcs = new TaskCompletionSource(); + if (optionalWrite != null) + { + streamingWritesCounter++; + } return sendStatusFromServerTcs.Task; } } diff --git a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs index 00d82d51e82..bbbefd06990 100644 --- a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs +++ b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs @@ -75,14 +75,15 @@ namespace Grpc.Core.Internal var responseStream = new ServerResponseStream(asyncCall); Status status; + Tuple responseTuple = null; var context = HandlerUtils.NewContext(newRpc, asyncCall.Peer, responseStream, asyncCall.CancellationToken); try { GrpcPreconditions.CheckArgument(await requestStream.MoveNext().ConfigureAwait(false)); var request = requestStream.Current; - var result = await handler(request, context).ConfigureAwait(false); + var response = await handler(request, context).ConfigureAwait(false); status = context.Status; - await responseStream.WriteAsync(result).ConfigureAwait(false); + responseTuple = Tuple.Create(response, HandlerUtils.GetWriteFlags(context.WriteOptions)); } catch (Exception e) { @@ -91,7 +92,7 @@ namespace Grpc.Core.Internal } try { - await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, responseTuple).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -145,7 +146,7 @@ namespace Grpc.Core.Internal try { - await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, null).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -183,19 +184,13 @@ namespace Grpc.Core.Internal var responseStream = new ServerResponseStream(asyncCall); Status status; + Tuple responseTuple = null; var context = HandlerUtils.NewContext(newRpc, asyncCall.Peer, responseStream, asyncCall.CancellationToken); try { - var result = await handler(requestStream, context).ConfigureAwait(false); + var response = await handler(requestStream, context).ConfigureAwait(false); status = context.Status; - try - { - await responseStream.WriteAsync(result).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - status = Status.DefaultCancelled; - } + responseTuple = Tuple.Create(response, HandlerUtils.GetWriteFlags(context.WriteOptions)); } catch (Exception e) { @@ -205,7 +200,7 @@ namespace Grpc.Core.Internal try { - await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, responseTuple).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -256,7 +251,7 @@ namespace Grpc.Core.Internal } try { - await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, null).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -278,7 +273,7 @@ namespace Grpc.Core.Internal asyncCall.Initialize(newRpc.Call); var finishedTask = asyncCall.ServerSideCallAsync(); - await asyncCall.SendStatusFromServerAsync(new Status(StatusCode.Unimplemented, ""), Metadata.Empty).ConfigureAwait(false); + await asyncCall.SendStatusFromServerAsync(new Status(StatusCode.Unimplemented, ""), Metadata.Empty, null).ConfigureAwait(false); await finishedTask.ConfigureAwait(false); } } @@ -297,6 +292,11 @@ namespace Grpc.Core.Internal return new Status(StatusCode.Unknown, "Exception was thrown by handler."); } + public static WriteFlags GetWriteFlags(WriteOptions writeOptions) + { + return writeOptions != null ? writeOptions.Flags : default(WriteFlags); + } + public static ServerCallContext NewContext(ServerRpcNew newRpc, string peer, ServerResponseStream serverResponseStream, CancellationToken cancellationToken) where TRequest : class where TResponse : class From 2901ea55ed17b16107bda30091a8ca0b84f6a926 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 4 May 2016 13:42:04 -0700 Subject: [PATCH 077/155] improve serverside handlers --- .../Grpc.Core/Internal/ServerCallHandler.cs | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs index bbbefd06990..85b7a4b01e9 100644 --- a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs +++ b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs @@ -87,16 +87,20 @@ namespace Grpc.Core.Internal } catch (Exception e) { - Logger.Error(e, "Exception occured in handler."); + if (!(e is RpcException)) + { + Logger.Warning(e, "Exception occured in handler."); + } status = HandlerUtils.StatusFromException(e); } try { await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, responseTuple).ConfigureAwait(false); } - catch (OperationCanceledException) + catch (Exception) { - // Call has been already cancelled. + asyncCall.Cancel(); + throw; } await finishedTask.ConfigureAwait(false); } @@ -140,7 +144,10 @@ namespace Grpc.Core.Internal } catch (Exception e) { - Logger.Error(e, "Exception occured in handler."); + if (!(e is RpcException)) + { + Logger.Warning(e, "Exception occured in handler."); + } status = HandlerUtils.StatusFromException(e); } @@ -148,9 +155,10 @@ namespace Grpc.Core.Internal { await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, null).ConfigureAwait(false); } - catch (OperationCanceledException) + catch (Exception) { - // Call has been already cancelled. + asyncCall.Cancel(); + throw; } await finishedTask.ConfigureAwait(false); } @@ -194,7 +202,10 @@ namespace Grpc.Core.Internal } catch (Exception e) { - Logger.Error(e, "Exception occured in handler."); + if (!(e is RpcException)) + { + Logger.Warning(e, "Exception occured in handler."); + } status = HandlerUtils.StatusFromException(e); } @@ -202,9 +213,10 @@ namespace Grpc.Core.Internal { await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, responseTuple).ConfigureAwait(false); } - catch (OperationCanceledException) + catch (Exception) { - // Call has been already cancelled. + asyncCall.Cancel(); + throw; } await finishedTask.ConfigureAwait(false); } @@ -246,16 +258,20 @@ namespace Grpc.Core.Internal } catch (Exception e) { - Logger.Error(e, "Exception occured in handler."); + if (!(e is RpcException)) + { + Logger.Warning(e, "Exception occured in handler."); + } status = HandlerUtils.StatusFromException(e); } try { await asyncCall.SendStatusFromServerAsync(status, context.ResponseTrailers, null).ConfigureAwait(false); } - catch (OperationCanceledException) + catch (Exception) { - // Call has been already cancelled. + asyncCall.Cancel(); + throw; } await finishedTask.ConfigureAwait(false); } From d039a98a94cc2f1d8769a3a11eea73494cc287f4 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Tue, 5 Apr 2016 15:15:03 -0700 Subject: [PATCH 078/155] Add Python grpcio protoc package --- setup.py | 1 + src/python/grpcio/grpc/__init__.py | 3 +- tools/distrib/python/bazel_deps.sh | 39 ++++++ tools/distrib/python/grpcio_protoc/.gitignore | 7 ++ .../python/grpcio_protoc/grpc/__init__.py | 30 +++++ .../grpcio_protoc/grpc/protoc/__init__.py | 29 +++++ .../protoc/grpc_python_protoc_compiler.py | 38 ++++++ .../grpc/protoc/grpc_python_protoc_plugin.py | 38 ++++++ .../python/grpcio_protoc/grpc/protoc/main.h | 33 +++++ .../grpcio_protoc/grpc/protoc/protoc.pyx | 39 ++++++ .../grpc/protoc/protoc_plugin.pyx | 39 ++++++ .../python/grpcio_protoc/protoc_deps.py | 32 +++++ .../python/grpcio_protoc/protoc_lib_deps.py | 32 +++++ tools/distrib/python/grpcio_protoc/setup.py | 102 +++++++++++++++ tools/distrib/python/grpcio_tools/MANIFEST.in | 5 + tools/distrib/python/mk_grpcio_protoc.py | 118 ++++++++++++++++++ tools/dockerfile/bazel/Dockerfile | 53 ++++++++ 17 files changed, 637 insertions(+), 1 deletion(-) create mode 100755 tools/distrib/python/bazel_deps.sh create mode 100644 tools/distrib/python/grpcio_protoc/.gitignore create mode 100644 tools/distrib/python/grpcio_protoc/grpc/__init__.py create mode 100644 tools/distrib/python/grpcio_protoc/grpc/protoc/__init__.py create mode 100644 tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_compiler.py create mode 100644 tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_plugin.py create mode 100644 tools/distrib/python/grpcio_protoc/grpc/protoc/main.h create mode 100644 tools/distrib/python/grpcio_protoc/grpc/protoc/protoc.pyx create mode 100644 tools/distrib/python/grpcio_protoc/grpc/protoc/protoc_plugin.pyx create mode 100644 tools/distrib/python/grpcio_protoc/protoc_deps.py create mode 100644 tools/distrib/python/grpcio_protoc/protoc_lib_deps.py create mode 100644 tools/distrib/python/grpcio_protoc/setup.py create mode 100644 tools/distrib/python/grpcio_tools/MANIFEST.in create mode 100755 tools/distrib/python/mk_grpcio_protoc.py create mode 100644 tools/dockerfile/bazel/Dockerfile diff --git a/setup.py b/setup.py index cd0d3a1a51b..af9eb685348 100644 --- a/setup.py +++ b/setup.py @@ -236,6 +236,7 @@ setup_arguments = { 'ext_modules': CYTHON_EXTENSION_MODULES, 'packages': list(PACKAGES), 'package_dir': PACKAGE_DIRECTORIES, + 'namespace_packages': ['grpc'], 'package_data': PACKAGE_DATA, 'install_requires': INSTALL_REQUIRES, 'setup_requires': SETUP_REQUIRES, diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 70865191060..b844a14c48d 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -27,4 +27,5 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +__import__('pkg_resources').declare_namespace(__name__) diff --git a/tools/distrib/python/bazel_deps.sh b/tools/distrib/python/bazel_deps.sh new file mode 100755 index 00000000000..4619c008875 --- /dev/null +++ b/tools/distrib/python/bazel_deps.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +cd $(dirname $0)/../../../ + +docker build -t bazel `realpath ./tools/dockerfile/bazel/` +docker run -v "`realpath .`:/src/grpc/" \ + -w /src/grpc/third_party/protobuf \ + bazel \ + bazel query 'deps('$1')' + diff --git a/tools/distrib/python/grpcio_protoc/.gitignore b/tools/distrib/python/grpcio_protoc/.gitignore new file mode 100644 index 00000000000..979704d970b --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/.gitignore @@ -0,0 +1,7 @@ +build/ +protobuf/ +grpc_plugin/ +grpc_root/ +*.c +*.cpp +*.egg-info diff --git a/tools/distrib/python/grpcio_protoc/grpc/__init__.py b/tools/distrib/python/grpcio_protoc/grpc/__init__.py new file mode 100644 index 00000000000..70ac5edd483 --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/grpc/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +__import__('pkg_resources').declare_namespace(__name__) diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/__init__.py b/tools/distrib/python/grpcio_protoc/grpc/protoc/__init__.py new file mode 100644 index 00000000000..d5ad73a74ab --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/grpc/protoc/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_compiler.py b/tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_compiler.py new file mode 100644 index 00000000000..5395f2ac105 --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_compiler.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys + +from grpc.protoc import protoc + + +if __name__ == '__main__': + protoc.run_main(sys.argv) diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_plugin.py b/tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_plugin.py new file mode 100644 index 00000000000..3b279e29769 --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_plugin.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys + +from grpc.protoc import protoc_plugin + + +if __name__ == '__main__': + protoc_plugin.run_main(sys.argv) diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/main.h b/tools/distrib/python/grpcio_protoc/grpc/protoc/main.h new file mode 100644 index 00000000000..fc406a9260c --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/grpc/protoc/main.h @@ -0,0 +1,33 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +// We declare `main` here since we want access to it from Cython as an extern +// but *without* triggering a dllimport declspec when on Windows. +int main(int argc, char *argv[]); diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/protoc.pyx b/tools/distrib/python/grpcio_protoc/grpc/protoc/protoc.pyx new file mode 100644 index 00000000000..d987ee1ceaa --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/grpc/protoc/protoc.pyx @@ -0,0 +1,39 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from libc cimport stdlib + +cdef extern from "grpc/protoc/main.h": + int main(int argc, char *argv[]) + +def run_main(list args not None): + cdef char **argv = stdlib.malloc(len(args)*sizeof(char *)) + for i in range(len(args)): + argv[i] = args[i] + return main(len(args), argv) diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/protoc_plugin.pyx b/tools/distrib/python/grpcio_protoc/grpc/protoc/protoc_plugin.pyx new file mode 100644 index 00000000000..d987ee1ceaa --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/grpc/protoc/protoc_plugin.pyx @@ -0,0 +1,39 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from libc cimport stdlib + +cdef extern from "grpc/protoc/main.h": + int main(int argc, char *argv[]) + +def run_main(list args not None): + cdef char **argv = stdlib.malloc(len(args)*sizeof(char *)) + for i in range(len(args)): + argv[i] = args[i] + return main(len(args), argv) diff --git a/tools/distrib/python/grpcio_protoc/protoc_deps.py b/tools/distrib/python/grpcio_protoc/protoc_deps.py new file mode 100644 index 00000000000..5be28a77d8b --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/protoc_deps.py @@ -0,0 +1,32 @@ + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# AUTO-GENERATED BY mk_grpcio_protoc.py! +CC_FILES=['google/protobuf/compiler/main.cc', 'google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] diff --git a/tools/distrib/python/grpcio_protoc/protoc_lib_deps.py b/tools/distrib/python/grpcio_protoc/protoc_lib_deps.py new file mode 100644 index 00000000000..00a8f31069c --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/protoc_lib_deps.py @@ -0,0 +1,32 @@ + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# AUTO-GENERATED BY mk_grpcio_protoc.py! +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] diff --git a/tools/distrib/python/grpcio_protoc/setup.py b/tools/distrib/python/grpcio_protoc/setup.py new file mode 100644 index 00000000000..d462b6b213d --- /dev/null +++ b/tools/distrib/python/grpcio_protoc/setup.py @@ -0,0 +1,102 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from distutils import extension +import os +import os.path +import sys + +import setuptools +from setuptools.command import build_ext + +# TODO(atash) add flag to disable Cython use + +os.chdir(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.abspath('.')) + +import protoc_deps +import protoc_lib_deps + +def protoc_ext_module(): + protoc_sources = [ + os.path.join('third_party/protobuf/src', cc_file) + for cc_file in protoc_deps.CC_FILES] + protoc_ext = extension.Extension( + name='grpc.protoc.protoc', + sources=['grpc/protoc/protoc.pyx'] + protoc_sources, + include_dirs=['.', 'third_party/protobuf/src'], + language='c++', + define_macros=[('HAVE_PTHREAD', 1)], + extra_compile_args=['-lpthread', '-frtti'], + ) + return protoc_ext + +def plugin_ext_module(): + plugin_sources = [ + 'grpc_root/src/compiler/python_generator.cc', + 'grpc_root/src/compiler/python_plugin.cc'] + [ + os.path.join('third_party/protobuf/src', cc_file) + for cc_file in protoc_lib_deps.CC_FILES] + plugin_ext = extension.Extension( + name='grpc.protoc.protoc_plugin', + sources=['grpc/protoc/protoc_plugin.pyx'] + plugin_sources, + include_dirs=[ + '.', + 'grpc_root', + 'grpc_root/include', + 'third_party/protobuf/src', + ], + language='c++', + define_macros=[('HAVE_PTHREAD', 1)], + extra_compile_args=['-lpthread', '-std=c++11'], + ) + return plugin_ext + +def maybe_cythonize(exts): + from Cython import Build + return Build.cythonize(exts) + +setuptools.setup( + name='grpcio_protoc', + version='0.14.0rc1', + license='', + ext_modules=maybe_cythonize([ + protoc_ext_module(), + plugin_ext_module(), + ]), + scripts=[ + 'grpc/protoc/grpc_python_protoc_compiler.py', + 'grpc/protoc/grpc_python_protoc_plugin.py', + ], + packages=setuptools.find_packages('.'), + namespace_packages=['grpc'], + install_requires=[ + 'protobuf>=3.0.0a3', + ], +) diff --git a/tools/distrib/python/grpcio_tools/MANIFEST.in b/tools/distrib/python/grpcio_tools/MANIFEST.in new file mode 100644 index 00000000000..e6ab312f09c --- /dev/null +++ b/tools/distrib/python/grpcio_tools/MANIFEST.in @@ -0,0 +1,5 @@ +include protoc_deps.py +include protoc_lib_deps.py +graft grpc +graft grpc_root +graft third_party diff --git a/tools/distrib/python/mk_grpcio_protoc.py b/tools/distrib/python/mk_grpcio_protoc.py new file mode 100755 index 00000000000..c09a77982d7 --- /dev/null +++ b/tools/distrib/python/mk_grpcio_protoc.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import os.path +import shutil +import subprocess + +DEPS_FILE_CONTENT=""" +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# AUTO-GENERATED BY mk_grpcio_protoc.py! +CC_FILES={} +""" + +# Bazel query result prefix for expected source files in protobuf. +PROTOBUF_CC_PREFIX = '//:src/' + +os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', '..', '..')) + +GRPC_PYTHON_ROOT = os.path.abspath('tools/distrib/python/grpcio_protoc') + +GRPC_PROTOBUF = os.path.abspath('third_party/protobuf/src') +GRPC_PROTOC_PLUGINS = os.path.abspath('src/compiler') +GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, + 'third_party/protobuf/src') +GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT, + 'grpc_root/src/compiler') +GRPC_PYTHON_PROTOC_DEPS = os.path.join(GRPC_PYTHON_ROOT, + 'protoc_deps.py') +GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT, + 'protoc_lib_deps.py') + +GRPC_INCLUDE = os.path.abspath('include') +GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include') + +for tree in [GRPC_PYTHON_PROTOBUF, + GRPC_PYTHON_PROTOC_PLUGINS, + GRPC_PYTHON_INCLUDE]: + try: + shutil.rmtree(tree) + except Exception as _: + pass +shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF) +shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS) +shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE) + +def write_deps(query, out_filename): + output = subprocess.check_output(['tools/distrib/python/bazel_deps.sh', query]) + output = output.splitlines() + cc_files = [ + name for name in output + if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX)] + cc_files = [cc_file[len(PROTOBUF_CC_PREFIX):] for cc_file in cc_files] + + deps_file_content = DEPS_FILE_CONTENT.format(cc_files) + + with open(out_filename, 'w') as deps_file: + deps_file.write(deps_file_content) + +write_deps('//:protoc', GRPC_PYTHON_PROTOC_DEPS) +write_deps('//:protoc_lib', GRPC_PYTHON_PROTOC_LIB_DEPS) diff --git a/tools/dockerfile/bazel/Dockerfile b/tools/dockerfile/bazel/Dockerfile new file mode 100644 index 00000000000..af31adb94df --- /dev/null +++ b/tools/dockerfile/bazel/Dockerfile @@ -0,0 +1,53 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM ubuntu:wily +RUN apt-get update +RUN apt-get -y install software-properties-common python-software-properties +RUN add-apt-repository ppa:webupd8team/java +RUN apt-get update +RUN apt-get -y install \ + vim \ + wget \ + openjdk-8-jdk \ + pkg-config \ + zip \ + g++ \ + zlib1g-dev \ + unzip \ + git + +RUN git clone https://github.com/bazelbuild/bazel.git /bazel +RUN cd /bazel && \ + ./compile.sh + +RUN cp /bazel/output/bazel /bin/bazel + +# ensure the installation has been extracted +RUN bazel From 58d24c259a637ccffdee09fac6bc0123f81e4fc0 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Fri, 29 Apr 2016 14:11:38 -0700 Subject: [PATCH 079/155] Add sanity test for grpcio protoc package --- .../test/sanity/Dockerfile.template | 12 +++- tools/distrib/python/bazel_deps.sh | 17 +++-- tools/distrib/python/check_grpcio_tools.py | 53 +++++++++++++++ .../.gitignore | 0 .../grpc/__init__.py | 0 .../grpc/protoc/__init__.py | 0 .../protoc/grpc_python_protoc_compiler.py | 0 .../grpc/protoc/grpc_python_protoc_plugin.py | 0 .../grpc/protoc/main.h | 0 .../grpc/protoc/protoc.pyx | 0 .../grpc/protoc/protoc_plugin.pyx | 0 .../protoc_deps.py | 2 +- .../protoc_lib_deps.py | 2 +- .../{grpcio_protoc => grpcio_tools}/setup.py | 2 +- ..._grpcio_protoc.py => make_grpcio_tools.py} | 65 ++++++++++++------- tools/dockerfile/bazel/Dockerfile | 5 +- tools/dockerfile/test/sanity/Dockerfile | 9 +++ tools/run_tests/sanity/sanity_tests.yaml | 1 + 18 files changed, 131 insertions(+), 37 deletions(-) create mode 100755 tools/distrib/python/check_grpcio_tools.py rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/.gitignore (100%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/grpc/__init__.py (100%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/grpc/protoc/__init__.py (100%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/grpc/protoc/grpc_python_protoc_compiler.py (100%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/grpc/protoc/grpc_python_protoc_plugin.py (100%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/grpc/protoc/main.h (100%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/grpc/protoc/protoc.pyx (100%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/grpc/protoc/protoc_plugin.pyx (100%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/protoc_deps.py (99%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/protoc_lib_deps.py (99%) rename tools/distrib/python/{grpcio_protoc => grpcio_tools}/setup.py (99%) rename tools/distrib/python/{mk_grpcio_protoc.py => make_grpcio_tools.py} (73%) diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template index 8d6f52db54e..8e2140e648b 100644 --- a/templates/tools/dockerfile/test/sanity/Dockerfile.template +++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template @@ -43,7 +43,17 @@ python-virtualenv ${"\\"} python-lxml RUN pip install simplejson mako - + + #====================================== + # More sanity test dependencies (bazel) + RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" > \ + /etc/apt/sources.list.d/backports.list + RUN apt-get update + RUN apt-get -t jessie-backports install -y openjdk-8-jdk + RUN git clone https://github.com/bazelbuild/bazel.git /bazel + RUN cd /bazel && ./compile.sh + RUN ln -s /bazel/output/bazel /bin/ + #=================== # Docker "inception" # Note this is quite the ugly hack. diff --git a/tools/distrib/python/bazel_deps.sh b/tools/distrib/python/bazel_deps.sh index 4619c008875..de3ee079708 100755 --- a/tools/distrib/python/bazel_deps.sh +++ b/tools/distrib/python/bazel_deps.sh @@ -31,9 +31,16 @@ cd $(dirname $0)/../../../ -docker build -t bazel `realpath ./tools/dockerfile/bazel/` -docker run -v "`realpath .`:/src/grpc/" \ - -w /src/grpc/third_party/protobuf \ - bazel \ +# First check if bazel is installed on the machine. If it is, then we don't need +# to invoke the docker bazel. +if [ "bazel version" ] +then + cd third_party/protobuf bazel query 'deps('$1')' - +else + docker build -t bazel `realpath ./tools/dockerfile/bazel/` + docker run -v "`realpath .`:/src/grpc/" \ + -w /src/grpc/third_party/protobuf \ + bazel \ + bazel query 'deps('$1')' +fi diff --git a/tools/distrib/python/check_grpcio_tools.py b/tools/distrib/python/check_grpcio_tools.py new file mode 100755 index 00000000000..fec218fdabd --- /dev/null +++ b/tools/distrib/python/check_grpcio_tools.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import cStringIO + +import make_grpcio_tools as make + +OUT_OF_DATE_MESSAGE = """file {} is out of date + +Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?""" + +check_protoc_deps_file = cStringIO.StringIO() +check_protoc_lib_deps_file = cStringIO.StringIO() +make.write_deps(make.BAZEL_DEPS_PROTOC_QUERY, check_protoc_deps_file) +make.write_deps(make.BAZEL_DEPS_PROTOC_LIB_QUERY, check_protoc_lib_deps_file) + +with open(make.GRPC_PYTHON_PROTOC_DEPS, 'r') as protoc_deps_file: + if protoc_deps_file.read() != check_protoc_deps_file.getvalue(): + print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_DEPS)) + raise SystemExit(1) + +with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file: + if protoc_lib_deps_file.read() != check_protoc_lib_deps_file.getvalue(): + print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_LIB_DEPS)) + raise SystemExit(1) diff --git a/tools/distrib/python/grpcio_protoc/.gitignore b/tools/distrib/python/grpcio_tools/.gitignore similarity index 100% rename from tools/distrib/python/grpcio_protoc/.gitignore rename to tools/distrib/python/grpcio_tools/.gitignore diff --git a/tools/distrib/python/grpcio_protoc/grpc/__init__.py b/tools/distrib/python/grpcio_tools/grpc/__init__.py similarity index 100% rename from tools/distrib/python/grpcio_protoc/grpc/__init__.py rename to tools/distrib/python/grpcio_tools/grpc/__init__.py diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/__init__.py b/tools/distrib/python/grpcio_tools/grpc/protoc/__init__.py similarity index 100% rename from tools/distrib/python/grpcio_protoc/grpc/protoc/__init__.py rename to tools/distrib/python/grpcio_tools/grpc/protoc/__init__.py diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_compiler.py b/tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_compiler.py similarity index 100% rename from tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_compiler.py rename to tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_compiler.py diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_plugin.py b/tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_plugin.py similarity index 100% rename from tools/distrib/python/grpcio_protoc/grpc/protoc/grpc_python_protoc_plugin.py rename to tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_plugin.py diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/main.h b/tools/distrib/python/grpcio_tools/grpc/protoc/main.h similarity index 100% rename from tools/distrib/python/grpcio_protoc/grpc/protoc/main.h rename to tools/distrib/python/grpcio_tools/grpc/protoc/main.h diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/protoc.pyx b/tools/distrib/python/grpcio_tools/grpc/protoc/protoc.pyx similarity index 100% rename from tools/distrib/python/grpcio_protoc/grpc/protoc/protoc.pyx rename to tools/distrib/python/grpcio_tools/grpc/protoc/protoc.pyx diff --git a/tools/distrib/python/grpcio_protoc/grpc/protoc/protoc_plugin.pyx b/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_plugin.pyx similarity index 100% rename from tools/distrib/python/grpcio_protoc/grpc/protoc/protoc_plugin.pyx rename to tools/distrib/python/grpcio_tools/grpc/protoc/protoc_plugin.pyx diff --git a/tools/distrib/python/grpcio_protoc/protoc_deps.py b/tools/distrib/python/grpcio_tools/protoc_deps.py similarity index 99% rename from tools/distrib/python/grpcio_protoc/protoc_deps.py rename to tools/distrib/python/grpcio_tools/protoc_deps.py index 5be28a77d8b..2ce5838c478 100644 --- a/tools/distrib/python/grpcio_protoc/protoc_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_deps.py @@ -28,5 +28,5 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# AUTO-GENERATED BY mk_grpcio_protoc.py! +# AUTO-GENERATED BY mk_grpcio_tools.py! CC_FILES=['google/protobuf/compiler/main.cc', 'google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] diff --git a/tools/distrib/python/grpcio_protoc/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py similarity index 99% rename from tools/distrib/python/grpcio_protoc/protoc_lib_deps.py rename to tools/distrib/python/grpcio_tools/protoc_lib_deps.py index 00a8f31069c..d4fe0eeaa24 100644 --- a/tools/distrib/python/grpcio_protoc/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -28,5 +28,5 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# AUTO-GENERATED BY mk_grpcio_protoc.py! +# AUTO-GENERATED BY mk_grpcio_tools.py! CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] diff --git a/tools/distrib/python/grpcio_protoc/setup.py b/tools/distrib/python/grpcio_tools/setup.py similarity index 99% rename from tools/distrib/python/grpcio_protoc/setup.py rename to tools/distrib/python/grpcio_tools/setup.py index d462b6b213d..3fa61320c8a 100644 --- a/tools/distrib/python/grpcio_protoc/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -83,7 +83,7 @@ def maybe_cythonize(exts): return Build.cythonize(exts) setuptools.setup( - name='grpcio_protoc', + name='grpcio_tools', version='0.14.0rc1', license='', ext_modules=maybe_cythonize([ diff --git a/tools/distrib/python/mk_grpcio_protoc.py b/tools/distrib/python/make_grpcio_tools.py similarity index 73% rename from tools/distrib/python/mk_grpcio_protoc.py rename to tools/distrib/python/make_grpcio_tools.py index c09a77982d7..178dd6a85db 100755 --- a/tools/distrib/python/mk_grpcio_protoc.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -64,20 +64,21 @@ DEPS_FILE_CONTENT=""" # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# AUTO-GENERATED BY mk_grpcio_protoc.py! +# AUTO-GENERATED BY mk_grpcio_tools.py! CC_FILES={} """ # Bazel query result prefix for expected source files in protobuf. PROTOBUF_CC_PREFIX = '//:src/' -os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), - '..', '..', '..')) +GRPC_ROOT = os.path.abspath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', '..', '..')) -GRPC_PYTHON_ROOT = os.path.abspath('tools/distrib/python/grpcio_protoc') +GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools/distrib/python/grpcio_tools') -GRPC_PROTOBUF = os.path.abspath('third_party/protobuf/src') -GRPC_PROTOC_PLUGINS = os.path.abspath('src/compiler') +GRPC_PROTOBUF = os.path.join(GRPC_ROOT, 'third_party/protobuf/src') +GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src/compiler') GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party/protobuf/src') GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT, @@ -87,32 +88,46 @@ GRPC_PYTHON_PROTOC_DEPS = os.path.join(GRPC_PYTHON_ROOT, GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT, 'protoc_lib_deps.py') -GRPC_INCLUDE = os.path.abspath('include') +GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include') GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include') -for tree in [GRPC_PYTHON_PROTOBUF, - GRPC_PYTHON_PROTOC_PLUGINS, - GRPC_PYTHON_INCLUDE]: - try: - shutil.rmtree(tree) - except Exception as _: - pass -shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF) -shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS) -shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE) - -def write_deps(query, out_filename): - output = subprocess.check_output(['tools/distrib/python/bazel_deps.sh', query]) +BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh') +BAZEL_DEPS_PROTOC_QUERY = '//:protoc' +BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib' + + +def write_deps(query, out_file): + """Write the result of the bazel query `query` against protobuf to + `out_file`.""" + output = subprocess.check_output([BAZEL_DEPS, query]) output = output.splitlines() cc_files = [ name for name in output if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX)] cc_files = [cc_file[len(PROTOBUF_CC_PREFIX):] for cc_file in cc_files] - deps_file_content = DEPS_FILE_CONTENT.format(cc_files) + out_file.write(deps_file_content) + + +def main(): + os.chdir(GRPC_ROOT) + + for tree in [GRPC_PYTHON_PROTOBUF, + GRPC_PYTHON_PROTOC_PLUGINS, + GRPC_PYTHON_INCLUDE]: + try: + shutil.rmtree(tree) + except Exception as _: + pass + shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF) + shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS) + shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE) + + with open(GRPC_PYTHON_PROTOC_DEPS, 'w') as deps_file: + write_deps(BAZEL_DEPS_PROTOC_QUERY, deps_file) + with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file: + write_deps(BAZEL_DEPS_PROTOC_LIB_QUERY, deps_file) - with open(out_filename, 'w') as deps_file: - deps_file.write(deps_file_content) +if __name__ == '__main__': + main() -write_deps('//:protoc', GRPC_PYTHON_PROTOC_DEPS) -write_deps('//:protoc_lib', GRPC_PYTHON_PROTOC_LIB_DEPS) diff --git a/tools/dockerfile/bazel/Dockerfile b/tools/dockerfile/bazel/Dockerfile index af31adb94df..2a80a4d4d53 100644 --- a/tools/dockerfile/bazel/Dockerfile +++ b/tools/dockerfile/bazel/Dockerfile @@ -44,10 +44,9 @@ RUN apt-get -y install \ git RUN git clone https://github.com/bazelbuild/bazel.git /bazel -RUN cd /bazel && \ - ./compile.sh +RUN cd /bazel && ./compile.sh -RUN cp /bazel/output/bazel /bin/bazel +RUN ln -s /bazel/output/bazel /bin/ # ensure the installation has been extracted RUN bazel diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 3146a922b7e..43b2a0c3d96 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -75,6 +75,15 @@ RUN apt-get update && apt-get install -y \ python-lxml RUN pip install simplejson mako +#====================================== +# More sanity test dependencies (bazel) +RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list +RUN apt-get update +RUN apt-get -t jessie-backports install -y openjdk-8-jdk +RUN git clone https://github.com/bazelbuild/bazel.git /bazel +RUN cd /bazel && ./compile.sh +RUN ln -s /bazel/output/bazel /bin/ + #=================== # Docker "inception" # Note this is quite the ugly hack. diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index efc21e6591b..c5945c602f4 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -10,3 +10,4 @@ - script: tools/distrib/check_trailing_newlines.sh - script: tools/distrib/check_nanopb_output.sh - script: tools/distrib/check_include_guards.py +- script: tools/distrib/python/check_grpcio_tools.py From af3158350ecaa86ff727af39d16a4a4599e9de3d Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Fri, 29 Apr 2016 15:40:29 -0700 Subject: [PATCH 080/155] Integrate Python protoc Linux artifact builders --- tools/dockerfile/grpc_artifact_linux_x64/Dockerfile | 9 +++++++++ tools/dockerfile/grpc_artifact_linux_x86/Dockerfile | 9 +++++++++ tools/run_tests/build_artifact_python.sh | 9 ++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile index d048b725c85..5be2d370618 100644 --- a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile @@ -37,6 +37,7 @@ RUN apt-get update && apt-get install -y \ autotools-dev \ build-essential \ bzip2 \ + clang \ curl \ gcc \ gcc-multilib \ @@ -66,6 +67,14 @@ RUN /bin/bash -l -c "nvm install 4 && npm install -g node-pre-gyp" ################## # Python dependencies +# Install bazel +RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list +RUN apt-get update +RUN apt-get -t jessie-backports install -y openjdk-8-jdk +RUN git clone https://github.com/bazelbuild/bazel.git /bazel +RUN cd /bazel && ./compile.sh +RUN ln -s /bazel/output/bazel /bin/ + RUN apt-get update && apt-get install -y \ python-all-dev \ python3-all-dev \ diff --git a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile index 46bc9f8f524..bacd899ceda 100644 --- a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile @@ -37,6 +37,7 @@ RUN apt-get update && apt-get install -y \ autotools-dev \ build-essential \ bzip2 \ + clang \ curl \ gcc \ gcc-multilib \ @@ -66,6 +67,14 @@ RUN /bin/bash -l -c "nvm install 4 && npm install -g node-pre-gyp" ################## # Python dependencies +# Install bazel +RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list +RUN apt-get update +RUN apt-get -t jessie-backports install -y openjdk-8-jdk +RUN git clone https://github.com/bazelbuild/bazel.git /bazel +RUN cd /bazel && ./compile.sh +RUN ln -s /bazel/output/bazel /bin/ + RUN apt-get update && apt-get install -y \ python-all-dev \ python3-all-dev \ diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 1f23f9fade8..454f4727598 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -62,6 +62,13 @@ ${SETARCH_CMD} python setup.py \ ${SETARCH_CMD} python setup.py \ bdist_wheel -mkdir -p artifacts +# Build gRPC tools package +python tools/distrib/python/make_grpcio_tools.py +# Build with clang since there's a bug in GCC 4.x where some constant +# expressions are treated as non-constant in the presence of the fwrapv flag +# (fixed in at most GCC 5.3). +CC=clang python tools/distrib/python/grpcio_tools/setup.py bdist_wheel +mkdir -p artifacts cp -r dist/* artifacts +cp -r tools/distrib/python/grpcio_tools/dist/* artifacts From 0e25c8d71a4ac49d284dcfa926bc04dcac577ba9 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Fri, 29 Apr 2016 16:54:17 -0700 Subject: [PATCH 081/155] Tightly integrate the Python plugin with its protoc This grossly simplifies the protoc invocation to: python -m grpc.protoc.compiler --python_out=... --grpc_python_out=... [...] --plugin=protoc-gen-python-grpc=grpc_python_protoc_plugin... [...] --- tools/distrib/python/check_grpcio_tools.py | 7 ---- ..._python_protoc_compiler.py => compiler.py} | 4 +- .../grpc/protoc/grpc_python_protoc_plugin.py | 38 ------------------ .../python/grpcio_tools/grpc/protoc/main.cc | 25 ++++++++++++ .../{protoc.pyx => protoc_compiler.pyx} | 0 .../grpc/protoc/protoc_plugin.pyx | 39 ------------------- .../python/grpcio_tools/protoc_deps.py | 32 --------------- .../python/grpcio_tools/protoc_lib_deps.py | 2 +- tools/distrib/python/grpcio_tools/setup.py | 30 +++----------- tools/distrib/python/make_grpcio_tools.py | 7 +--- 10 files changed, 34 insertions(+), 150 deletions(-) rename tools/distrib/python/grpcio_tools/grpc/protoc/{grpc_python_protoc_compiler.py => compiler.py} (95%) delete mode 100644 tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_plugin.py create mode 100644 tools/distrib/python/grpcio_tools/grpc/protoc/main.cc rename tools/distrib/python/grpcio_tools/grpc/protoc/{protoc.pyx => protoc_compiler.pyx} (100%) delete mode 100644 tools/distrib/python/grpcio_tools/grpc/protoc/protoc_plugin.pyx delete mode 100644 tools/distrib/python/grpcio_tools/protoc_deps.py diff --git a/tools/distrib/python/check_grpcio_tools.py b/tools/distrib/python/check_grpcio_tools.py index fec218fdabd..f9f869e9902 100755 --- a/tools/distrib/python/check_grpcio_tools.py +++ b/tools/distrib/python/check_grpcio_tools.py @@ -37,16 +37,9 @@ OUT_OF_DATE_MESSAGE = """file {} is out of date Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?""" -check_protoc_deps_file = cStringIO.StringIO() check_protoc_lib_deps_file = cStringIO.StringIO() -make.write_deps(make.BAZEL_DEPS_PROTOC_QUERY, check_protoc_deps_file) make.write_deps(make.BAZEL_DEPS_PROTOC_LIB_QUERY, check_protoc_lib_deps_file) -with open(make.GRPC_PYTHON_PROTOC_DEPS, 'r') as protoc_deps_file: - if protoc_deps_file.read() != check_protoc_deps_file.getvalue(): - print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_DEPS)) - raise SystemExit(1) - with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file: if protoc_lib_deps_file.read() != check_protoc_lib_deps_file.getvalue(): print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_LIB_DEPS)) diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_compiler.py b/tools/distrib/python/grpcio_tools/grpc/protoc/compiler.py similarity index 95% rename from tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_compiler.py rename to tools/distrib/python/grpcio_tools/grpc/protoc/compiler.py index 5395f2ac105..caafc544b2b 100644 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_compiler.py +++ b/tools/distrib/python/grpcio_tools/grpc/protoc/compiler.py @@ -31,8 +31,8 @@ import sys -from grpc.protoc import protoc +from grpc.protoc import protoc_compiler if __name__ == '__main__': - protoc.run_main(sys.argv) + protoc_compiler.run_main(sys.argv) diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_plugin.py b/tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_plugin.py deleted file mode 100644 index 3b279e29769..00000000000 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/grpc_python_protoc_plugin.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -from grpc.protoc import protoc_plugin - - -if __name__ == '__main__': - protoc_plugin.run_main(sys.argv) diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc b/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc new file mode 100644 index 00000000000..2a0bc6d2d1a --- /dev/null +++ b/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc @@ -0,0 +1,25 @@ +#include +#include + +#include "src/compiler/python_generator.h" + +#include "grpc/protoc/main.h" + +int main(int argc, char* argv[]) { + google::protobuf::compiler::CommandLineInterface cli; + cli.AllowPlugins("protoc-"); + + // Proto2 Python + google::protobuf::compiler::python::Generator py_generator; + cli.RegisterGenerator("--python_out", &py_generator, + "Generate Python source file."); + + // gRPC Python + grpc_python_generator::GeneratorConfiguration grpc_py_config; + grpc_py_config.beta_package_root = "grpc.beta"; + grpc_python_generator::PythonGrpcGenerator grpc_py_generator(grpc_py_config); + cli.RegisterGenerator("--grpc_python_out", &grpc_py_generator, + "Generate Python source file."); + + return cli.Run(argc, argv); +} diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/protoc.pyx b/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx similarity index 100% rename from tools/distrib/python/grpcio_tools/grpc/protoc/protoc.pyx rename to tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_plugin.pyx b/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_plugin.pyx deleted file mode 100644 index d987ee1ceaa..00000000000 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_plugin.pyx +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2016, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from libc cimport stdlib - -cdef extern from "grpc/protoc/main.h": - int main(int argc, char *argv[]) - -def run_main(list args not None): - cdef char **argv = stdlib.malloc(len(args)*sizeof(char *)) - for i in range(len(args)): - argv[i] = args[i] - return main(len(args), argv) diff --git a/tools/distrib/python/grpcio_tools/protoc_deps.py b/tools/distrib/python/grpcio_tools/protoc_deps.py deleted file mode 100644 index 2ce5838c478..00000000000 --- a/tools/distrib/python/grpcio_tools/protoc_deps.py +++ /dev/null @@ -1,32 +0,0 @@ - -# Copyright 2016, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# AUTO-GENERATED BY mk_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/main.cc', 'google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index d4fe0eeaa24..9f31172170d 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -28,5 +28,5 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# AUTO-GENERATED BY mk_grpcio_tools.py! +# AUTO-GENERATED BY make_grpcio_tools.py! CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 3fa61320c8a..59093aa5163 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -40,32 +40,17 @@ from setuptools.command import build_ext os.chdir(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.abspath('.')) -import protoc_deps import protoc_lib_deps def protoc_ext_module(): - protoc_sources = [ - os.path.join('third_party/protobuf/src', cc_file) - for cc_file in protoc_deps.CC_FILES] - protoc_ext = extension.Extension( - name='grpc.protoc.protoc', - sources=['grpc/protoc/protoc.pyx'] + protoc_sources, - include_dirs=['.', 'third_party/protobuf/src'], - language='c++', - define_macros=[('HAVE_PTHREAD', 1)], - extra_compile_args=['-lpthread', '-frtti'], - ) - return protoc_ext - -def plugin_ext_module(): plugin_sources = [ - 'grpc_root/src/compiler/python_generator.cc', - 'grpc_root/src/compiler/python_plugin.cc'] + [ + 'grpc/protoc/main.cc', + 'grpc_root/src/compiler/python_generator.cc'] + [ os.path.join('third_party/protobuf/src', cc_file) for cc_file in protoc_lib_deps.CC_FILES] plugin_ext = extension.Extension( - name='grpc.protoc.protoc_plugin', - sources=['grpc/protoc/protoc_plugin.pyx'] + plugin_sources, + name='grpc.protoc.protoc_compiler', + sources=['grpc/protoc/protoc_compiler.pyx'] + plugin_sources, include_dirs=[ '.', 'grpc_root', @@ -74,7 +59,7 @@ def plugin_ext_module(): ], language='c++', define_macros=[('HAVE_PTHREAD', 1)], - extra_compile_args=['-lpthread', '-std=c++11'], + extra_compile_args=['-lpthread', '-frtti', '-std=c++11'], ) return plugin_ext @@ -88,12 +73,7 @@ setuptools.setup( license='', ext_modules=maybe_cythonize([ protoc_ext_module(), - plugin_ext_module(), ]), - scripts=[ - 'grpc/protoc/grpc_python_protoc_compiler.py', - 'grpc/protoc/grpc_python_protoc_plugin.py', - ], packages=setuptools.find_packages('.'), namespace_packages=['grpc'], install_requires=[ diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index 178dd6a85db..27e7d5f0a4f 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -64,7 +64,7 @@ DEPS_FILE_CONTENT=""" # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# AUTO-GENERATED BY mk_grpcio_tools.py! +# AUTO-GENERATED BY make_grpcio_tools.py! CC_FILES={} """ @@ -83,8 +83,6 @@ GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party/protobuf/src') GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/src/compiler') -GRPC_PYTHON_PROTOC_DEPS = os.path.join(GRPC_PYTHON_ROOT, - 'protoc_deps.py') GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT, 'protoc_lib_deps.py') @@ -92,7 +90,6 @@ GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include') GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include') BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh') -BAZEL_DEPS_PROTOC_QUERY = '//:protoc' BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib' @@ -123,8 +120,6 @@ def main(): shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS) shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE) - with open(GRPC_PYTHON_PROTOC_DEPS, 'w') as deps_file: - write_deps(BAZEL_DEPS_PROTOC_QUERY, deps_file) with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file: write_deps(BAZEL_DEPS_PROTOC_LIB_QUERY, deps_file) From 176ac65faf22fae7cf3d1874a97fc16636ef1be3 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Tue, 3 May 2016 11:55:40 -0700 Subject: [PATCH 082/155] Don't truncate deps file on failure to find bazel --- tools/distrib/python/check_grpcio_tools.py | 5 ++--- tools/distrib/python/make_grpcio_tools.py | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/tools/distrib/python/check_grpcio_tools.py b/tools/distrib/python/check_grpcio_tools.py index f9f869e9902..baf2ff4effb 100755 --- a/tools/distrib/python/check_grpcio_tools.py +++ b/tools/distrib/python/check_grpcio_tools.py @@ -37,10 +37,9 @@ OUT_OF_DATE_MESSAGE = """file {} is out of date Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?""" -check_protoc_lib_deps_file = cStringIO.StringIO() -make.write_deps(make.BAZEL_DEPS_PROTOC_LIB_QUERY, check_protoc_lib_deps_file) +check_protoc_lib_deps_content = make.get_deps(make.BAZEL_DEPS_PROTOC_LIB_QUERY) with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file: - if protoc_lib_deps_file.read() != check_protoc_lib_deps_file.getvalue(): + if protoc_lib_deps_file.read() != check_protoc_lib_deps_content: print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_LIB_DEPS)) raise SystemExit(1) diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index 27e7d5f0a4f..50fbdbb14c0 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -33,6 +33,8 @@ import os import os.path import shutil import subprocess +import sys +import traceback DEPS_FILE_CONTENT=""" # Copyright 2016, Google Inc. @@ -93,7 +95,7 @@ BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh') BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib' -def write_deps(query, out_file): +def get_deps(query): """Write the result of the bazel query `query` against protobuf to `out_file`.""" output = subprocess.check_output([BAZEL_DEPS, query]) @@ -103,7 +105,7 @@ def write_deps(query, out_file): if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX)] cc_files = [cc_file[len(PROTOBUF_CC_PREFIX):] for cc_file in cc_files] deps_file_content = DEPS_FILE_CONTENT.format(cc_files) - out_file.write(deps_file_content) + return deps_file_content def main(): @@ -120,8 +122,18 @@ def main(): shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS) shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE) + try: + protoc_lib_deps_content = get_deps(BAZEL_DEPS_PROTOC_LIB_QUERY) + except Exception as error: + # We allow this script to succeed even if we couldn't get the dependencies, + # as then we can assume that even without a successful bazel run the + # dependencies currently in source control are 'good enough'. + sys.stderr.write("Got non-fatal error:\n") + traceback.print_exc(file=sys.stderr) + return + # If we successfully got the dependencies, truncate and rewrite the deps file. with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file: - write_deps(BAZEL_DEPS_PROTOC_LIB_QUERY, deps_file) + deps_file.write(protoc_lib_deps_content) if __name__ == '__main__': main() From 955887928f1ac6d9b39599d14b6b7a522db026d9 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Tue, 3 May 2016 12:05:13 -0700 Subject: [PATCH 083/155] Keep grpcio_tools version in sync with rest of project --- .../grpcio_tools/grpc_version.py.template | 34 +++++++++++++++++++ .../python/grpcio_tools/grpc_version.py | 32 +++++++++++++++++ tools/distrib/python/grpcio_tools/setup.py | 3 +- 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 templates/tools/distrib/python/grpcio_tools/grpc_version.py.template create mode 100644 tools/distrib/python/grpcio_tools/grpc_version.py diff --git a/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template b/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template new file mode 100644 index 00000000000..200905d4f3f --- /dev/null +++ b/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template @@ -0,0 +1,34 @@ +%YAML 1.2 +--- | + # Copyright 2015, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! + + VERSION='${settings.python_version.pep440()}' diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py new file mode 100644 index 00000000000..b8ae8e20b8b --- /dev/null +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -0,0 +1,32 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! + +VERSION='0.14.0.dev0' diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 59093aa5163..8530398b6d1 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -41,6 +41,7 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.abspath('.')) import protoc_lib_deps +import grpc_version def protoc_ext_module(): plugin_sources = [ @@ -69,7 +70,7 @@ def maybe_cythonize(exts): setuptools.setup( name='grpcio_tools', - version='0.14.0rc1', + version=grpc_version.VERSION, license='', ext_modules=maybe_cythonize([ protoc_ext_module(), From 616b2793fd20eb28c2be53831f967332659987d0 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Tue, 3 May 2016 12:33:02 -0700 Subject: [PATCH 084/155] Fix docker build --- tools/dockerfile/grpc_artifact_linux_x64/Dockerfile | 12 +++--------- tools/dockerfile/grpc_artifact_linux_x86/Dockerfile | 12 +++--------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile index 5be2d370618..4ae4ebdb06d 100644 --- a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile @@ -31,8 +31,10 @@ FROM debian:jessie +RUN apt-get update && apt-get install debian-keyring && apt-key update + # Install Git and basic packages. -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-key update && apt-get install -y \ autoconf \ autotools-dev \ build-essential \ @@ -67,14 +69,6 @@ RUN /bin/bash -l -c "nvm install 4 && npm install -g node-pre-gyp" ################## # Python dependencies -# Install bazel -RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list -RUN apt-get update -RUN apt-get -t jessie-backports install -y openjdk-8-jdk -RUN git clone https://github.com/bazelbuild/bazel.git /bazel -RUN cd /bazel && ./compile.sh -RUN ln -s /bazel/output/bazel /bin/ - RUN apt-get update && apt-get install -y \ python-all-dev \ python3-all-dev \ diff --git a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile index bacd899ceda..9c2fd52eee1 100644 --- a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile @@ -31,8 +31,10 @@ FROM 32bit/debian:jessie +RUN apt-get update && apt-get install debian-keyring && apt-key update + # Install Git and basic packages. -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-key update && apt-get install -y \ autoconf \ autotools-dev \ build-essential \ @@ -67,14 +69,6 @@ RUN /bin/bash -l -c "nvm install 4 && npm install -g node-pre-gyp" ################## # Python dependencies -# Install bazel -RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list -RUN apt-get update -RUN apt-get -t jessie-backports install -y openjdk-8-jdk -RUN git clone https://github.com/bazelbuild/bazel.git /bazel -RUN cd /bazel && ./compile.sh -RUN ln -s /bazel/output/bazel /bin/ - RUN apt-get update && apt-get install -y \ python-all-dev \ python3-all-dev \ From 010eb48d2169e13ab8afeae32c29ede7b84a0e4d Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Tue, 3 May 2016 15:59:40 -0700 Subject: [PATCH 085/155] Use manylinux --- setup.py | 3 +- .../python/grpcio_tools/grpc/protoc/main.cc | 2 +- .../python/grpcio_tools/grpc/protoc/main.h | 6 +- .../grpc/protoc/protoc_compiler.pyx | 4 +- tools/distrib/python/grpcio_tools/setup.py | 3 +- .../Dockerfile | 43 +++++++++++++++ .../Dockerfile | 43 +++++++++++++++ tools/run_tests/artifact_targets.py | 55 +++++++++++++++++-- tools/run_tests/build_artifact_python.sh | 49 +++++++++-------- 9 files changed, 173 insertions(+), 35 deletions(-) create mode 100644 tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile create mode 100644 tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile diff --git a/setup.py b/setup.py index af9eb685348..5cd26124f6d 100644 --- a/setup.py +++ b/setup.py @@ -236,7 +236,8 @@ setup_arguments = { 'ext_modules': CYTHON_EXTENSION_MODULES, 'packages': list(PACKAGES), 'package_dir': PACKAGE_DIRECTORIES, - 'namespace_packages': ['grpc'], + # TODO(atash): Figure out why auditwheel doesn't like namespace packages. + #'namespace_packages': ['grpc'], 'package_data': PACKAGE_DATA, 'install_requires': INSTALL_REQUIRES, 'setup_requires': SETUP_REQUIRES, diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc b/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc index 2a0bc6d2d1a..4487a6851ae 100644 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc +++ b/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc @@ -5,7 +5,7 @@ #include "grpc/protoc/main.h" -int main(int argc, char* argv[]) { +int protoc_main(int argc, char* argv[]) { google::protobuf::compiler::CommandLineInterface cli; cli.AllowPlugins("protoc-"); diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/main.h b/tools/distrib/python/grpcio_tools/grpc/protoc/main.h index fc406a9260c..ea2860ff020 100644 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/main.h +++ b/tools/distrib/python/grpcio_tools/grpc/protoc/main.h @@ -28,6 +28,6 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// We declare `main` here since we want access to it from Cython as an extern -// but *without* triggering a dllimport declspec when on Windows. -int main(int argc, char *argv[]); +// We declare `protoc_main` here since we want access to it from Cython as an +// extern but *without* triggering a dllimport declspec when on Windows. +int protoc_main(int argc, char *argv[]); diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx b/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx index d987ee1ceaa..af15f3db30e 100644 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx +++ b/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx @@ -30,10 +30,10 @@ from libc cimport stdlib cdef extern from "grpc/protoc/main.h": - int main(int argc, char *argv[]) + int protoc_main(int argc, char *argv[]) def run_main(list args not None): cdef char **argv = stdlib.malloc(len(args)*sizeof(char *)) for i in range(len(args)): argv[i] = args[i] - return main(len(args), argv) + return protoc_main(len(args), argv) diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 8530398b6d1..0281c01796d 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -76,7 +76,8 @@ setuptools.setup( protoc_ext_module(), ]), packages=setuptools.find_packages('.'), - namespace_packages=['grpc'], + # TODO(atash): Figure out why auditwheel doesn't like namespace packages. + #namespace_packages=['grpc'], install_requires=[ 'protobuf>=3.0.0a3', ], diff --git a/tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile new file mode 100644 index 00000000000..3e31a2b623d --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile @@ -0,0 +1,43 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Docker file for building gRPC manylinux Python artifacts. + +FROM quay.io/pypa/manylinux1_x86_64 + +# Update the package manager +RUN yum update -y + +################################### +# Install Python build requirements +RUN /opt/python/cp27-cp27m/bin/pip install cython +RUN /opt/python/cp27-cp27mu/bin/pip install cython +RUN /opt/python/cp34-cp34m/bin/pip install cython +RUN /opt/python/cp35-cp35m/bin/pip install cython + diff --git a/tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile new file mode 100644 index 00000000000..5fe62c28b73 --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile @@ -0,0 +1,43 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Docker file for building gRPC manylinux Python artifacts. + +FROM quay.io/pypa/manylinux1_i686 + +# Update the package manager +RUN yum update -y + +################################### +# Install Python build requirements +RUN /opt/python/cp27-cp27m/bin/pip install cython +RUN /opt/python/cp27-cp27mu/bin/pip install cython +RUN /opt/python/cp34-cp34m/bin/pip install cython +RUN /opt/python/cp35-cp35m/bin/pip install cython + diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 3e08c1d62b3..ec44e3746c9 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -84,12 +84,16 @@ python_version_arch_map = { class PythonArtifact: """Builds Python artifacts.""" - def __init__(self, platform, arch): - self.name = 'python_%s_%s' % (platform, arch) + def __init__(self, platform, arch, manylinux_build=None): + if manylinux_build: + self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build) + else: + self.name = 'python_%s_%s' % (platform, arch) self.platform = platform self.arch = arch self.labels = ['artifact', 'python', platform, arch] self.python_version = python_version_arch_map[arch] + self.manylinux_build = manylinux_build def pre_build_jobspecs(self): return [] @@ -99,8 +103,47 @@ class PythonArtifact: if self.platform == 'linux': if self.arch == 'x86': environ['SETARCH_CMD'] = 'linux32' + # Inside the manylinux container, the python installations are located in + # special places... + environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build) + environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build) + # Our docker image has all the prerequisites pip-installed already. + environ['SKIP_PIP_INSTALL'] = '1' + # Platform autodetection for the manylinux1 image breaks so we set the + # defines ourselves. + # TODO(atash) get better platform-detection support in core so we don't + # need to do this manually... + environ['CFLAGS'] = " ".join([ + '-DGPR_NO_AUTODETECT_PLATFORM', + '-DGPR_PLATFORM_STRING=\\"manylinux\\"', + '-DGPR_POSIX_CRASH_HANDLER=1', + '-DGPR_CPU_LINUX=1', + '-DGPR_GCC_ATOMIC=1', + '-DGPR_GCC_TLS=1', + '-DGPR_LINUX=1', + '-DGPR_LINUX_LOG=1', + #'-DGPR_LINUX_MULTIPOLL_WITH_EPOLL=1', + '-DGPR_POSIX_SOCKET=1', + '-DGPR_POSIX_WAKEUP_FD=1', + '-DGPR_POSIX_SOCKETADDR=1', + #'-DGPR_LINUX_EVENTFD=1', + #'-DGPR_LINUX_SOCKETUTILS=1', + '-DGPR_HAVE_UNIX_SOCKET=1', + '-DGPR_HAVE_IP_PKTINFO=1', + '-DGPR_HAVE_IPV6_RECVPKTINFO=1', + '-DGPR_LINUX_ENV=1', + '-DGPR_POSIX_FILE=1', + '-DGPR_POSIX_TMPFILE=1', + '-DGPR_POSIX_STRING=1', + '-DGPR_POSIX_SUBPROCESS=1', + '-DGPR_POSIX_SYNC=1', + '-DGPR_POSIX_TIME=1', + '-DGPR_GETPID_IN_UNISTD_H=1', + '-DGPR_HAVE_MSG_NOSIGNAL=1', + '-DGPR_ARCH_{arch}=1'.format(arch=('32' if self.arch == 'x86' else '64')), + ]) return create_docker_jobspec(self.name, - 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, + 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, 'tools/run_tests/build_artifact_python.sh', environ=environ) elif self.platform == 'windows': @@ -307,8 +350,10 @@ def targets(): for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64')] + - [PythonArtifact('linux', 'x86'), - PythonArtifact('linux', 'x64'), + [PythonArtifact('linux', 'x86', 'cp27-cp27m'), + PythonArtifact('linux', 'x86', 'cp27-cp27mu'), + PythonArtifact('linux', 'x64', 'cp27-cp27m'), + PythonArtifact('linux', 'x64', 'cp27-cp27mu'), PythonArtifact('macos', 'x64'), PythonArtifact('windows', 'x86'), PythonArtifact('windows', 'x64'), diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 454f4727598..920d255aeda 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -32,43 +32,48 @@ set -ex cd $(dirname $0)/../.. +export GRPC_PYTHON_USE_CUSTOM_BDIST=0 +export GRPC_PYTHON_BUILD_WITH_CYTHON=1 +export PYTHON=${PYTHON:-python} +export PIP=${PIP:-pip} +export AUDITWHEEL=${AUDITWHEEL:-auditwheel} + + if [ "$SKIP_PIP_INSTALL" == "" ] then - pip install --upgrade six + ${PIP} install --upgrade six # There's a bug in newer versions of setuptools (see # https://bitbucket.org/pypa/setuptools/issues/503/pkg_resources_vendorpackagingrequirementsi) - pip install --upgrade 'setuptools==18' - pip install -rrequirements.txt + ${PIP} pip install --upgrade 'setuptools==18' + ${PIP} install -rrequirements.txt fi -export GRPC_PYTHON_USE_CUSTOM_BDIST=0 -export GRPC_PYTHON_BUILD_WITH_CYTHON=1 - # Build the source distribution first because MANIFEST.in cannot override # exclusion of built shared objects among package resources (for some # inexplicable reason). -${SETARCH_CMD} python setup.py \ +${SETARCH_CMD} ${PYTHON} setup.py \ sdist -# The bdist_wheel_grpc_custom command is finicky about command output ordering -# and thus ought to be run in a shell command separate of others. Further, it -# trashes the actual bdist_wheel output, so it should be run first so that -# bdist_wheel may be run unmolested. -${SETARCH_CMD} python setup.py \ - build_tagged_ext - # Wheel has a bug where directories don't get excluded. # https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory -${SETARCH_CMD} python setup.py \ +${SETARCH_CMD} ${PYTHON} setup.py \ bdist_wheel # Build gRPC tools package -python tools/distrib/python/make_grpcio_tools.py -# Build with clang since there's a bug in GCC 4.x where some constant -# expressions are treated as non-constant in the presence of the fwrapv flag -# (fixed in at most GCC 5.3). -CC=clang python tools/distrib/python/grpcio_tools/setup.py bdist_wheel +${PYTHON} tools/distrib/python/make_grpcio_tools.py +CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \ + ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel mkdir -p artifacts -cp -r dist/* artifacts -cp -r tools/distrib/python/grpcio_tools/dist/* artifacts +if command -v ${AUDITWHEEL} +then + for wheel in dist/*.whl; do + ${AUDITWHEEL} repair $wheel -w artifacts/ + done + for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do + ${AUDITWHEEL} repair $wheel -w artifacts/ + done +else + cp -r dist/* artifacts + cp -r tools/distrib/python/grpcio_tools/dist/* artifacts +fi From e9407c2e74a3ca3819d78a8ffa4d20fb51a00a59 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 4 May 2016 06:26:15 -0700 Subject: [PATCH 086/155] Test binary Python distributions --- test/distrib/python/run_distrib_test.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/distrib/python/run_distrib_test.sh b/test/distrib/python/run_distrib_test.sh index 79893af8e54..73e471dd265 100755 --- a/test/distrib/python/run_distrib_test.sh +++ b/test/distrib/python/run_distrib_test.sh @@ -33,8 +33,7 @@ set -ex cd $(dirname $0) # Pick up the source dist archive whatever its version is -SDIST_ARCHIVE=$EXTERNAL_GIT_ROOT/input_artifacts/grpcio-*.tar.gz -BDIST_DIR="file://$EXTERNAL_GIT_ROOT/input_artifacts" +BDIST_ARCHIVES=$EXTERNAL_GIT_ROOT/input_artifacts/grpcio-*.whl if [ ! -f ${SDIST_ARCHIVE} ] then @@ -48,11 +47,13 @@ PYTHON=python2 which $PYTHON || PYTHON=python # TODO(jtattermusch): this shouldn't be required -$PIP install --upgrade six +$PIP install --upgrade six pip -GRPC_PYTHON_BINARIES_REPOSITORY="${BDIST_DIR}" \ - $PIP install \ - ${SDIST_ARCHIVE} +# At least one of the bdist packages has to succeed (whichever one matches the +# test machine, anyway). +for bdist in ${BDIST_ARCHIVES}; do + ($PIP install $bdist) || true +done $PYTHON distribtest.py From 88baf86d710a2234f6f83a04d46fbe162a4ef0d4 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 4 May 2016 06:33:15 -0700 Subject: [PATCH 087/155] Copy all generated Python distribs --- tools/run_tests/build_artifact_python.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 920d255aeda..35c3a48afae 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -73,7 +73,7 @@ then for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do ${AUDITWHEEL} repair $wheel -w artifacts/ done -else - cp -r dist/* artifacts - cp -r tools/distrib/python/grpcio_tools/dist/* artifacts fi + +cp -r dist/* artifacts +cp -r tools/distrib/python/grpcio_tools/dist/* artifacts From f837cc8f6304e1cd2a018d449fb4826e560cb282 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 4 May 2016 11:59:22 -0700 Subject: [PATCH 088/155] Fix missing symbols --- tools/run_tests/artifact_targets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index ec44e3746c9..72f4f8dd967 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -127,7 +127,9 @@ class PythonArtifact: '-DGPR_POSIX_WAKEUP_FD=1', '-DGPR_POSIX_SOCKETADDR=1', #'-DGPR_LINUX_EVENTFD=1', + '-DGPR_POSIX_NO_SPECIAL_WAKEUP_FD=1', #'-DGPR_LINUX_SOCKETUTILS=1', + '-DGPR_POSIX_SOCKETUTILS=1', '-DGPR_HAVE_UNIX_SOCKET=1', '-DGPR_HAVE_IP_PKTINFO=1', '-DGPR_HAVE_IPV6_RECVPKTINFO=1', From 916c960e593affc049ade727e86f78febeeb2420 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 4 May 2016 12:48:39 -0700 Subject: [PATCH 089/155] Fix Python distrib test --- test/distrib/python/run_distrib_test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/distrib/python/run_distrib_test.sh b/test/distrib/python/run_distrib_test.sh index 73e471dd265..e20767a885f 100755 --- a/test/distrib/python/run_distrib_test.sh +++ b/test/distrib/python/run_distrib_test.sh @@ -41,18 +41,18 @@ then exit 1 fi -PIP=pip2 -which $PIP || PIP=pip PYTHON=python2 +PIP=pip2 which $PYTHON || PYTHON=python +which $PIP || PIP=pip # TODO(jtattermusch): this shouldn't be required -$PIP install --upgrade six pip +${PIP} install --upgrade six pip # At least one of the bdist packages has to succeed (whichever one matches the # test machine, anyway). for bdist in ${BDIST_ARCHIVES}; do - ($PIP install $bdist) || true + ($PYTHON -m pip install $bdist) || true done $PYTHON distribtest.py From f9b89b181d243f8b3eaddeb000f751d71c257ad5 Mon Sep 17 00:00:00 2001 From: Michael Yeh Date: Wed, 4 May 2016 16:36:20 -0700 Subject: [PATCH 090/155] Updated to more closely mimic greeter_async_client.cc. Fixed typos in comments. --- .../cpp/helloworld/greeter_async_client.cc | 8 ++-- .../cpp/helloworld/greeter_async_client2.cc | 37 +++++++++++-------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/examples/cpp/helloworld/greeter_async_client.cc b/examples/cpp/helloworld/greeter_async_client.cc index 35014267feb..c1f5eb55f05 100644 --- a/examples/cpp/helloworld/greeter_async_client.cc +++ b/examples/cpp/helloworld/greeter_async_client.cc @@ -53,7 +53,7 @@ class GreeterClient { explicit GreeterClient(std::shared_ptr channel) : stub_(Greeter::NewStub(channel)) {} - // Assambles the client's payload, sends it and presents the response back + // Assembles the client's payload, sends it and presents the response back // from the server. std::string SayHello(const std::string& user) { // Data we are sending to the server. @@ -74,9 +74,9 @@ class GreeterClient { // Storage for the status of the RPC upon completion. Status status; - // stub_->AsyncSayHello() perform the RPC call, returning an instance we - // store in "rpc". Because we are using the asynchronous API, we need the - // hold on to the "rpc" instance in order to get updates on the ongoig RPC. + // stub_->AsyncSayHello() performs the RPC call, returning an instance we + // store in "rpc". Because we are using the asynchronous API, we need to + // hold on to the "rpc" instance in order to get updates on the ongoing RPC. std::unique_ptr > rpc( stub_->AsyncSayHello(&context, request, &cq)); diff --git a/examples/cpp/helloworld/greeter_async_client2.cc b/examples/cpp/helloworld/greeter_async_client2.cc index cdbdd1f3063..0902376074e 100644 --- a/examples/cpp/helloworld/greeter_async_client2.cc +++ b/examples/cpp/helloworld/greeter_async_client2.cc @@ -54,42 +54,49 @@ class GreeterClient { explicit GreeterClient(std::shared_ptr channel) : stub_(Greeter::NewStub(channel)) {} - // Assambles the client's payload, sends it and presents the response back - // from the server. + // Assembles the client's payload and sends it to the server. void SayHello(const std::string& user) { // Data we are sending to the server. HelloRequest request; request.set_name(user); + // Call object to store rpc data AsyncClientCall* call = new AsyncClientCall; - // stub_->AsyncSayHello() perform the RPC call, returning an instance we - // store in "rpc". Because we are using the asynchronous API, we need the - // hold on to the "rpc" instance in order to get updates on the ongoing RPC. + // stub_->AsyncSayHello() performs the RPC call, returning an instance to + // store in "call". Because we are using the asynchronous API, we need to + // hold on to the "call" instance in order to get updates on the ongoing RPC. call->response_reader = stub_->AsyncSayHello(&call->context, request, &cq_); // Request that, upon completion of the RPC, "reply" be updated with the // server's response; "status" with the indication of whether the operation - // was successful. Tag the request with the memory address of call object. + // was successful. Tag the request with the memory address of the call object. call->response_reader->Finish(&call->reply, &call->status, (void*)call); } - // Loop while listening for completed responses - void AsyncCompleteRpc() - { + // Loop while listening for completed responses. + // Prints out the response from the server. + void AsyncCompleteRpc() { void* got_tag; bool ok = false; // Block until the next result is available in the completion queue "cq". - while (cq_.Next(&got_tag, &ok)) - { + while (cq_.Next(&got_tag, &ok)) { + // The tag in this example is the memory location of the call object AsyncClientCall* call = static_cast(got_tag); - if (ok) + + // Verify that the request was completed successfully. Note that "ok" + // corresponds solely to the request for updates introduced by Finish(). + GPR_ASSERT(ok); + + if (call->status.ok()) std::cout << "Greeter received: " << call->reply.message() << std::endl; else std::cout << "RPC failed" << std::endl; + + // Once we're complete, deallocate the call object. delete call; } } @@ -97,8 +104,7 @@ class GreeterClient { private: // struct for keeping state and data information - struct AsyncClientCall - { + struct AsyncClientCall { // Container for the data we expect from the server. HelloReply reply; @@ -135,8 +141,7 @@ int main(int argc, char** argv) { // Spawn reader thread that loops indefinitely std::thread thread_ = std::thread(&GreeterClient::AsyncCompleteRpc, &greeter); - for (int i = 0; i < 100; i++) - { + for (int i = 0; i < 100; i++) { std::string user("world " + std::to_string(i)); greeter.SayHello(user); // The actual RPC call! } From 751fbb06f6d2040d49d6c3d89192eb08a1c6cff4 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 4 May 2016 16:12:56 -0700 Subject: [PATCH 091/155] Build grpcio tools on Windows --- tools/run_tests/artifact_targets.py | 3 ++- tools/run_tests/build_artifact_python.bat | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 72f4f8dd967..477bd468887 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -151,7 +151,8 @@ class PythonArtifact: elif self.platform == 'windows': return create_jobspec(self.name, ['tools\\run_tests\\build_artifact_python.bat', - self.python_version + self.python_version, + '32' if self.arch == 'x86' else '64' ], shell=True) else: diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat index 023d3945494..636ae0d393d 100644 --- a/tools/run_tests/build_artifact_python.bat +++ b/tools/run_tests/build_artifact_python.bat @@ -52,8 +52,21 @@ set GRPC_PYTHON_BUILD_WITH_CYTHON=1 python setup.py bdist_wheel +@rem Build gRPC Python tools +set PATH=C:\msys64\mingw%2\bin;%PATH% +set CC=C:\msys64\mingw%2\bin\g++.exe +set CFLAGS=-fno-wrapv +python tools\distrib\python\make_grpcio_tools.py +if %2 == 32 ( + python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 +) else ( + python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 -DMS_WIN64 +) +python tools\distrib\python\grpcio_tools\setup.py bdist_wheel + mkdir artifacts xcopy /Y /I /S dist\* artifacts\ || goto :error +xcopy /Y /I /S tools\distrib\python\grpcio_tools\dist\* artifacts\ || goto :error goto :EOF From 4e1ebfa17689e045b725e64bd75db044eb2abea6 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 4 May 2016 17:43:07 -0700 Subject: [PATCH 092/155] Updated template file --- templates/src/node/tools/package.json.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/src/node/tools/package.json.template b/templates/src/node/tools/package.json.template index 4f673c48d1c..69ad71a3b83 100644 --- a/templates/src/node/tools/package.json.template +++ b/templates/src/node/tools/package.json.template @@ -18,8 +18,8 @@ } ], "bin": { - "grpc-tools-protoc": "./bin/protoc.js", - "grpc-tools-plugin": "./bin/protoc_plugin.js" + "grpc_tools_node_protoc": "./bin/protoc.js", + "grpc_tools_node_protoc_plugin": "./bin/protoc_plugin.js" }, "scripts": { "install": "./node_modules/.bin/node-pre-gyp install" From e79b08b5ec06bf7ee1af19258f3e98386668899f Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 4 May 2016 18:09:20 -0700 Subject: [PATCH 093/155] RouteGuide example now works with moving between tabs. Fixes issue 6404. --- .../objective-c/route_guide/ViewControllers.m | 89 +++++++++++++------ 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/examples/objective-c/route_guide/ViewControllers.m b/examples/objective-c/route_guide/ViewControllers.m index 0b1a1cf4823..e32978240b6 100644 --- a/examples/objective-c/route_guide/ViewControllers.m +++ b/examples/objective-c/route_guide/ViewControllers.m @@ -80,19 +80,14 @@ static NSString * const kHostAddress = @"localhost:50051"; * Run the getFeature demo. Calls getFeature with a point known to have a feature and a point known * not to have a feature. */ -@interface GetFeatureViewController : UIViewController +@interface GetFeatureViewController : UIViewController { + RTGRouteGuide *service; +} @end @implementation GetFeatureViewController -- (void)viewDidLoad { - [super viewDidLoad]; - - // This only needs to be done once per host, before creating service objects for that host. - [GRPCCall useInsecureConnectionsForHost:kHostAddress]; - - RTGRouteGuide *service = [[RTGRouteGuide alloc] initWithHost:kHostAddress]; - +- (void)execRequest { void (^handler)(RTGFeature *response, NSError *error) = ^(RTGFeature *response, NSError *error) { if (response.name.length) { NSLog(@"Found feature called %@ at %@.", response.name, response.location); @@ -111,6 +106,19 @@ static NSString * const kHostAddress = @"localhost:50051"; [service getFeatureWithRequest:[RTGPoint message] handler:handler]; } +- (void)viewDidLoad { + [super viewDidLoad]; + + // This only needs to be done once per host, before creating service objects for that host. + [GRPCCall useInsecureConnectionsForHost:kHostAddress]; + + service = [[RTGRouteGuide alloc] initWithHost:kHostAddress]; +} + +- (void)viewDidAppear:(BOOL)animated { + [self execRequest]; +} + @end @@ -120,16 +128,15 @@ static NSString * const kHostAddress = @"localhost:50051"; * Run the listFeatures demo. Calls listFeatures with a rectangle containing all of the features in * the pre-generated database. Prints each response as it comes in. */ -@interface ListFeaturesViewController : UIViewController +@interface ListFeaturesViewController : UIViewController { + RTGRouteGuide *service; +} + @end @implementation ListFeaturesViewController -- (void)viewDidLoad { - [super viewDidLoad]; - - RTGRouteGuide *service = [[RTGRouteGuide alloc] initWithHost:kHostAddress]; - +- (void)execRequest { RTGRectangle *rectangle = [RTGRectangle message]; rectangle.lo.latitude = 405E6; rectangle.lo.longitude = -750E6; @@ -147,6 +154,16 @@ static NSString * const kHostAddress = @"localhost:50051"; }]; } +- (void)viewDidLoad { + [super viewDidLoad]; + + service = [[RTGRouteGuide alloc] initWithHost:kHostAddress]; +} + +- (void)viewDidAppear:(BOOL)animated { + [self execRequest]; +} + @end @@ -157,14 +174,15 @@ static NSString * const kHostAddress = @"localhost:50051"; * database with a variable delay in between. Prints the statistics when they are sent from the * server. */ -@interface RecordRouteViewController : UIViewController +@interface RecordRouteViewController : UIViewController { + RTGRouteGuide *service; +} + @end @implementation RecordRouteViewController -- (void)viewDidLoad { - [super viewDidLoad]; - +- (void)execRequest { NSString *dataBasePath = [NSBundle.mainBundle pathForResource:@"route_guide_db" ofType:@"json"]; NSData *dataBaseContent = [NSData dataWithContentsOfFile:dataBasePath]; @@ -178,8 +196,6 @@ static NSString * const kHostAddress = @"localhost:50051"; return location; }]; - RTGRouteGuide *service = [[RTGRouteGuide alloc] initWithHost:kHostAddress]; - [service recordRouteWithRequestsWriter:locations handler:^(RTGRouteSummary *response, NSError *error) { if (response) { @@ -193,6 +209,16 @@ static NSString * const kHostAddress = @"localhost:50051"; }]; } +- (void)viewDidLoad { + [super viewDidLoad]; + + service = [[RTGRouteGuide alloc] initWithHost:kHostAddress]; +} + +- (void)viewDidAppear:(BOOL)animated { + [self execRequest]; +} + @end @@ -202,14 +228,15 @@ static NSString * const kHostAddress = @"localhost:50051"; * Run the routeChat demo. Send some chat messages, and print any chat messages that are sent from * the server. */ -@interface RouteChatViewController : UIViewController +@interface RouteChatViewController : UIViewController { + RTGRouteGuide *service; +} + @end @implementation RouteChatViewController -- (void)viewDidLoad { - [super viewDidLoad]; - +- (void)execRequest { NSArray *notes = @[[RTGRouteNote noteWithMessage:@"First message" latitude:0 longitude:0], [RTGRouteNote noteWithMessage:@"Second message" latitude:0 longitude:1], [RTGRouteNote noteWithMessage:@"Third message" latitude:1 longitude:0], @@ -219,8 +246,6 @@ static NSString * const kHostAddress = @"localhost:50051"; return note; }]; - RTGRouteGuide *service = [[RTGRouteGuide alloc] initWithHost:kHostAddress]; - [service routeChatWithRequestsWriter:notesWriter eventHandler:^(BOOL done, RTGRouteNote *note, NSError *error) { if (note) { @@ -234,4 +259,14 @@ static NSString * const kHostAddress = @"localhost:50051"; }]; } +- (void)viewDidLoad { + [super viewDidLoad]; + + service = [[RTGRouteGuide alloc] initWithHost:kHostAddress]; +} + +- (void)viewDidAppear:(BOOL)animated { + [self execRequest]; +} + @end From d8b07cb3a1168bce8073e39ab64d877c2c7d185e Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Thu, 5 May 2016 03:55:55 +0200 Subject: [PATCH 094/155] Let's not compile grpc under Windows - it's taken care of already. --- src/ruby/ext/grpc/extconf.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 07f7bb93b8a..6d65db83067 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -78,9 +78,11 @@ output_dir = File.expand_path(RbConfig::CONFIG['topdir']) grpc_lib_dir = File.join(output_dir, 'libs', grpc_config) ENV['BUILDDIR'] = output_dir -puts 'Building internal gRPC into ' + grpc_lib_dir -system("make -j -C #{grpc_root} #{grpc_lib_dir}/libgrpc.a CONFIG=#{grpc_config}") -exit 1 unless $? == 0 +unless windows + puts 'Building internal gRPC into ' + grpc_lib_dir + system("make -j -C #{grpc_root} #{grpc_lib_dir}/libgrpc.a CONFIG=#{grpc_config}") + exit 1 unless $? == 0 +end $CFLAGS << ' -I' + File.join(grpc_root, 'include') $LDFLAGS << ' ' + File.join(grpc_lib_dir, 'libgrpc.a') unless windows From 88f56e5ce386b0292ded2f08aba2bc92a09369b1 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Thu, 5 May 2016 04:04:50 +0200 Subject: [PATCH 095/155] 'exception_info' is a bad word for Windows... --- src/ruby/ext/grpc/rb_call_credentials.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ruby/ext/grpc/rb_call_credentials.c b/src/ruby/ext/grpc/rb_call_credentials.c index 38bf1f7710f..615bf9415d5 100644 --- a/src/ruby/ext/grpc/rb_call_credentials.c +++ b/src/ruby/ext/grpc/rb_call_credentials.c @@ -35,7 +35,6 @@ #include "rb_grpc_imports.generated.h" #include "rb_call_credentials.h" -#include #include #include @@ -86,11 +85,11 @@ static VALUE grpc_rb_call_credentials_callback_rescue(VALUE args, rb_funcall(exception_object, rb_intern("backtrace"), 0), rb_intern("join"), 1, rb_str_new2("\n\tfrom ")); - VALUE exception_info = rb_funcall(exception_object, rb_intern("to_s"), 0); + VALUE rb_exception_info = rb_funcall(exception_object, rb_intern("to_s"), 0); const char *exception_classname = rb_obj_classname(exception_object); (void)args; gpr_log(GPR_INFO, "Call credentials callback failed: %s: %s\n%s", - exception_classname, StringValueCStr(exception_info), + exception_classname, StringValueCStr(rb_exception_info), StringValueCStr(backtrace)); rb_hash_aset(result, rb_str_new2("metadata"), Qnil); /* Currently only gives the exception class name. It should be possible get From 9fcdc8765fe301d6828dd70ad83e45e59db39176 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Thu, 5 May 2016 06:15:34 +0200 Subject: [PATCH 096/155] Cleaning up includes. --- src/ruby/ext/grpc/rb_byte_buffer.c | 3 +-- src/ruby/ext/grpc/rb_call.c | 3 +-- src/ruby/ext/grpc/rb_call_credentials.c | 1 + src/ruby/ext/grpc/rb_channel.c | 3 +-- src/ruby/ext/grpc/rb_channel_args.c | 3 +-- src/ruby/ext/grpc/rb_channel_credentials.c | 5 ++--- src/ruby/ext/grpc/rb_completion_queue.c | 2 +- src/ruby/ext/grpc/rb_event_thread.c | 2 +- src/ruby/ext/grpc/rb_grpc.c | 2 +- src/ruby/ext/grpc/rb_server.c | 3 +-- src/ruby/ext/grpc/rb_server_credentials.c | 3 +-- 11 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/ruby/ext/grpc/rb_byte_buffer.c b/src/ruby/ext/grpc/rb_byte_buffer.c index cba910d832a..1172691116c 100644 --- a/src/ruby/ext/grpc/rb_byte_buffer.c +++ b/src/ruby/ext/grpc/rb_byte_buffer.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_byte_buffer.h" -#include - #include #include #include diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c index 48c49a21e91..1b06273af9d 100644 --- a/src/ruby/ext/grpc/rb_call.c +++ b/src/ruby/ext/grpc/rb_call.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_call.h" -#include - #include #include diff --git a/src/ruby/ext/grpc/rb_call_credentials.c b/src/ruby/ext/grpc/rb_call_credentials.c index 615bf9415d5..79ca5b32ced 100644 --- a/src/ruby/ext/grpc/rb_call_credentials.c +++ b/src/ruby/ext/grpc/rb_call_credentials.c @@ -32,6 +32,7 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_call_credentials.h" diff --git a/src/ruby/ext/grpc/rb_channel.c b/src/ruby/ext/grpc/rb_channel.c index 984afad1073..013321ffc8a 100644 --- a/src/ruby/ext/grpc/rb_channel.c +++ b/src/ruby/ext/grpc/rb_channel.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_channel.h" -#include - #include #include #include diff --git a/src/ruby/ext/grpc/rb_channel_args.c b/src/ruby/ext/grpc/rb_channel_args.c index 2ffb8f41dae..87c0e0a7055 100644 --- a/src/ruby/ext/grpc/rb_channel_args.c +++ b/src/ruby/ext/grpc/rb_channel_args.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_channel_args.h" -#include - #include #include "rb_grpc.h" diff --git a/src/ruby/ext/grpc/rb_channel_credentials.c b/src/ruby/ext/grpc/rb_channel_credentials.c index 09bd3093a94..cbb23885aa6 100644 --- a/src/ruby/ext/grpc/rb_channel_credentials.c +++ b/src/ruby/ext/grpc/rb_channel_credentials.c @@ -31,14 +31,13 @@ * */ +#include + #include -#include #include "rb_grpc_imports.generated.h" #include "rb_channel_credentials.h" -#include - #include #include #include diff --git a/src/ruby/ext/grpc/rb_completion_queue.c b/src/ruby/ext/grpc/rb_completion_queue.c index 2a2eee190cd..4bb615f8bec 100644 --- a/src/ruby/ext/grpc/rb_completion_queue.c +++ b/src/ruby/ext/grpc/rb_completion_queue.c @@ -32,10 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_completion_queue.h" -#include #include #include diff --git a/src/ruby/ext/grpc/rb_event_thread.c b/src/ruby/ext/grpc/rb_event_thread.c index 2649a1087f2..9e85bbcfbf2 100644 --- a/src/ruby/ext/grpc/rb_event_thread.c +++ b/src/ruby/ext/grpc/rb_event_thread.c @@ -32,12 +32,12 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_event_thread.h" #include -#include #include #include #include diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c index acb47b00558..06a07ac6463 100644 --- a/src/ruby/ext/grpc/rb_grpc.c +++ b/src/ruby/ext/grpc/rb_grpc.c @@ -32,11 +32,11 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_grpc.h" #include -#include #include #include diff --git a/src/ruby/ext/grpc/rb_server.c b/src/ruby/ext/grpc/rb_server.c index 96e60c67763..2b3acaaf59c 100644 --- a/src/ruby/ext/grpc/rb_server.c +++ b/src/ruby/ext/grpc/rb_server.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_server.h" -#include - #include #include #include "rb_call.h" diff --git a/src/ruby/ext/grpc/rb_server_credentials.c b/src/ruby/ext/grpc/rb_server_credentials.c index b2d7280a30a..3b0fb6c910d 100644 --- a/src/ruby/ext/grpc/rb_server_credentials.c +++ b/src/ruby/ext/grpc/rb_server_credentials.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_server_credentials.h" -#include - #include #include From 74b41c0513c3c5fe8863342dd1097f6bf186896e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 May 2016 07:38:03 -0700 Subject: [PATCH 097/155] fix C# distrib test after getting rid of grpc.native.csharp package --- test/distrib/csharp/DistribTest/DistribTest.csproj | 6 +++--- test/distrib/csharp/DistribTest/packages.config | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/test/distrib/csharp/DistribTest/DistribTest.csproj b/test/distrib/csharp/DistribTest/DistribTest.csproj index 7605495f0f6..1acb34d1b2e 100644 --- a/test/distrib/csharp/DistribTest/DistribTest.csproj +++ b/test/distrib/csharp/DistribTest/DistribTest.csproj @@ -113,12 +113,12 @@ - + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + @@ -129,4 +129,4 @@ --> - \ No newline at end of file + diff --git a/test/distrib/csharp/DistribTest/packages.config b/test/distrib/csharp/DistribTest/packages.config index aca09f600cb..69630193d02 100644 --- a/test/distrib/csharp/DistribTest/packages.config +++ b/test/distrib/csharp/DistribTest/packages.config @@ -6,11 +6,10 @@ - - \ No newline at end of file + From 4ead664c33bc4c613fc7c33f73e7bcf08cc37428 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 May 2016 08:01:40 -0700 Subject: [PATCH 098/155] small addendum for python distribtests --- test/distrib/python/run_distrib_test.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/distrib/python/run_distrib_test.sh b/test/distrib/python/run_distrib_test.sh index e20767a885f..6196e540c82 100755 --- a/test/distrib/python/run_distrib_test.sh +++ b/test/distrib/python/run_distrib_test.sh @@ -34,6 +34,7 @@ cd $(dirname $0) # Pick up the source dist archive whatever its version is BDIST_ARCHIVES=$EXTERNAL_GIT_ROOT/input_artifacts/grpcio-*.whl +TOOLS_BDIST_ARCHIVES=$EXTERNAL_GIT_ROOT/input_artifacts/grpcio_tools-*.whl if [ ! -f ${SDIST_ARCHIVE} ] then @@ -51,9 +52,12 @@ ${PIP} install --upgrade six pip # At least one of the bdist packages has to succeed (whichever one matches the # test machine, anyway). -for bdist in ${BDIST_ARCHIVES}; do +for bdist in ${BDIST_ARCHIVES} ${TOOLS_BDIST_ARCHIVES}; do ($PYTHON -m pip install $bdist) || true done -$PYTHON distribtest.py +# TODO(jtattermusch): add a .proto file to the distribtest, generate python +# code from it and then use the generated code from distribtest.py +$PYTHON -m grpc.protoc.compiler +$PYTHON distribtest.py From 5deda3db97828454da88cb2e1d463c9e0dff7263 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 5 May 2016 12:34:19 -0700 Subject: [PATCH 099/155] Temporary fix for plugin initialization problem --- src/core/ext/client_config/resolver_registry.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core/ext/client_config/resolver_registry.c b/src/core/ext/client_config/resolver_registry.c index 07f29bcb27a..e7a4abd568c 100644 --- a/src/core/ext/client_config/resolver_registry.c +++ b/src/core/ext/client_config/resolver_registry.c @@ -47,7 +47,6 @@ static int g_number_of_resolvers = 0; static char *g_default_resolver_prefix; void grpc_resolver_registry_init(const char *default_resolver_prefix) { - g_number_of_resolvers = 0; g_default_resolver_prefix = gpr_strdup(default_resolver_prefix); } @@ -57,6 +56,13 @@ void grpc_resolver_registry_shutdown(void) { grpc_resolver_factory_unref(g_all_of_the_resolvers[i]); } gpr_free(g_default_resolver_prefix); + // FIXME(ctiller): this should live in grpc_resolver_registry_init, + // however that would have the client_config plugin call this AFTER we start + // registering resolvers from third party plugins, and so they'd never show + // up. + // We likely need some kind of dependency system for plugins.... what form + // that takes is TBD. + g_number_of_resolvers = 0; } void grpc_register_resolver_type(grpc_resolver_factory *factory) { From f2f707ca5503249e637542d7b3015c3fef2e3023 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 5 May 2016 13:04:20 -0700 Subject: [PATCH 100/155] More clang-format fixes. --- src/core/lib/transport/metadata_batch.c | 2 +- test/core/bad_client/bad_client.h | 4 ++-- test/core/end2end/tests/large_metadata.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/lib/transport/metadata_batch.c b/src/core/lib/transport/metadata_batch.c index c0afc715bcc..e4398abeb78 100644 --- a/src/core/lib/transport/metadata_batch.c +++ b/src/core/lib/transport/metadata_batch.c @@ -195,7 +195,7 @@ int grpc_metadata_batch_is_empty(grpc_metadata_batch *batch) { size_t grpc_metadata_batch_size(grpc_metadata_batch *batch) { size_t size = 0; - for (grpc_linked_mdelem* elem = batch->list.head; elem != NULL; + for (grpc_linked_mdelem *elem = batch->list.head; elem != NULL; elem = elem->next) { size += GRPC_MDELEM_LENGTH(elem->md); } diff --git a/test/core/bad_client/bad_client.h b/test/core/bad_client/bad_client.h index ecd6721a78a..c8b2a4122f0 100644 --- a/test/core/bad_client/bad_client.h +++ b/test/core/bad_client/bad_client.h @@ -60,8 +60,8 @@ void grpc_run_bad_client_test( const char *client_payload, size_t client_payload_length, uint32_t flags); #define GRPC_RUN_BAD_CLIENT_TEST(server_validator, client_validator, payload, \ - flags) \ - grpc_run_bad_client_test(server_validator, client_validator, payload, \ + flags) \ + grpc_run_bad_client_test(server_validator, client_validator, payload, \ sizeof(payload) - 1, flags) #endif /* GRPC_TEST_CORE_BAD_CLIENT_BAD_CLIENT_H */ diff --git a/test/core/end2end/tests/large_metadata.c b/test/core/end2end/tests/large_metadata.c index dd29552b975..ae1f68a2b45 100644 --- a/test/core/end2end/tests/large_metadata.c +++ b/test/core/end2end/tests/large_metadata.c @@ -109,7 +109,7 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { const size_t large_size = 64 * 1024; grpc_arg arg = {GRPC_ARG_INTEGER, GRPC_ARG_MAX_METADATA_SIZE, - {.integer=(int)large_size + 1024}}; + {.integer = (int)large_size + 1024}}; grpc_channel_args args = {1, &arg}; grpc_end2end_test_fixture f = begin_test(config, "test_request_with_large_metadata", &args, &args); From 7809f2b78a5abf3ecb2ade16f3be907388c559e8 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Thu, 5 May 2016 20:38:42 +0000 Subject: [PATCH 101/155] Raise parallelism in grpc._cython._channel_test The underlying bug in Core that this test was written to isolate was fixed weeks ago. --- src/python/grpcio/tests/unit/_cython/_channel_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio/tests/unit/_cython/_channel_test.py b/src/python/grpcio/tests/unit/_cython/_channel_test.py index b414f8e6f69..931cd9083ec 100644 --- a/src/python/grpcio/tests/unit/_cython/_channel_test.py +++ b/src/python/grpcio/tests/unit/_cython/_channel_test.py @@ -33,8 +33,7 @@ import unittest from grpc._cython import cygrpc -# TODO(nathaniel): This should be at least one hundred. Why not one thousand? -_PARALLELISM = 4 +from tests.unit.framework.common import test_constants def _channel_and_completion_queue(): @@ -61,7 +60,7 @@ def _create_loop_destroy(): def _in_parallel(behavior, arguments): threads = tuple( threading.Thread(target=behavior, args=arguments) - for _ in range(_PARALLELISM)) + for _ in range(test_constants.PARALLELISM)) for thread in threads: thread.start() for thread in threads: From b836b5bfee46e9d1ee211dba31a9b22c03339b57 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 6 May 2016 00:55:59 +0200 Subject: [PATCH 102/155] Adding missing copyright. --- .../python/grpcio_tools/grpc/protoc/main.cc | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc b/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc index 4487a6851ae..c9936a3a6b4 100644 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc +++ b/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc @@ -1,3 +1,32 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #include #include From d7b162f6015bb57e67c61dc99617ac8bad7e8adc Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 4 May 2016 15:09:26 -0700 Subject: [PATCH 103/155] add some more scenarios --- .../run_tests/performance/scenario_config.py | 823 +++++++----------- 1 file changed, 319 insertions(+), 504 deletions(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index ddbe2375691..52efe8b86b3 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -69,6 +69,75 @@ DEEP=100 WIDE=64 +def _get_secargs(is_secure): + if is_secure: + return SECURE_SECARGS + else: + return None + + +def _ping_pong_scenario(name, rpc_type, + client_type, server_type, + secure=True, + use_generic_payload=False, + use_unconstrained_client=False, + server_language=None, + server_core_limit=0, + async_server_threads=0, + warmup_seconds=WARMUP_SECONDS): + """Creates a basic ping pong scenario.""" + scenario = { + 'name': name, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': client_type, + 'security_params': _get_secargs(secure), + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': rpc_type, + 'load_params': { + 'closed_loop': {} + }, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': server_type, + 'security_params': _get_secargs(secure), + 'core_limit': server_core_limit, + 'async_server_threads': async_server_threads, + }, + 'warmup_seconds': warmup_seconds, + 'benchmark_seconds': BENCHMARK_SECONDS + } + if use_generic_payload: + if server_type != 'ASYNC_GENERIC_SERVER': + raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') + scenario['client_config']['payload_config'] = EMPTY_GENERIC_PAYLOAD + scenario['server_config']['payload_config'] = EMPTY_GENERIC_PAYLOAD + else: + # For proto payload, only the client should get the config. + scenario['client_config']['payload_config'] = EMPTY_PROTO_PAYLOAD + + if use_unconstrained_client: + scenario['num_clients'] = 0 # use as many client as available. + # TODO(jtattermusch): for SYNC_CLIENT, this will create 100*64 threads + # and that's probably too much (at least for wrapped languages). + scenario['client_config']['outstanding_rpcs_per_channel'] = DEEP + scenario['client_config']['client_channels'] = WIDE + scenario['client_config']['async_client_threads'] = 0 + else: + scenario['client_config']['outstanding_rpcs_per_channel'] = 1 + scenario['client_config']['client_channels'] = 1 + scenario['client_config']['async_client_threads'] = 1 + + if server_language: + # the SERVER_LANGUAGE field is recognized by run_performance_tests.py + scenario['SERVER_LANGUAGE'] = server_language + return scenario + + class CXXLanguage: def __init__(self): @@ -83,205 +152,59 @@ class CXXLanguage: def scenarios(self): # TODO(ctiller): add 70% load latency test for secure in [True, False]: - if secure: - secstr = 'secure' - secargs = SECURE_SECARGS - else: - secstr = 'insecure' - secargs = None - - yield { - 'name': 'cpp_generic_async_streaming_ping_pong_%s' - % secstr, - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'STREAMING', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_GENERIC_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_GENERIC_SERVER', - 'security_params': secargs, - 'core_limit': 1, - 'async_server_threads': 1, - 'payload_config': EMPTY_GENERIC_PAYLOAD, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } - yield { - 'name': 'cpp_generic_async_streaming_qps_unconstrained_%s' - % secstr, - 'num_servers': 1, - 'num_clients': 0, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': DEEP, - 'client_channels': WIDE, - 'async_client_threads': 0, - 'rpc_type': 'STREAMING', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_GENERIC_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_GENERIC_SERVER', - 'security_params': secargs, - 'core_limit': SINGLE_MACHINE_CORES/2, - 'async_server_threads': 0, - 'payload_config': EMPTY_GENERIC_PAYLOAD, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } - yield { - 'name': 'cpp_generic_async_streaming_qps_one_server_core_%s' - % secstr, - 'num_servers': 1, - 'num_clients': 0, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': DEEP, - 'client_channels': WIDE, - 'async_client_threads': 0, - 'rpc_type': 'STREAMING', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_GENERIC_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_GENERIC_SERVER', - 'security_params': secargs, - 'core_limit': 1, - 'async_server_threads': 1, - 'payload_config': EMPTY_GENERIC_PAYLOAD, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } - yield { - 'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s' - % secstr, - 'num_servers': 1, - 'num_clients': 0, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': DEEP, - 'client_channels': WIDE, - 'async_client_threads': 0, - 'rpc_type': 'STREAMING', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_SERVER', - 'security_params': secargs, - 'core_limit': SINGLE_MACHINE_CORES/2, - 'async_server_threads': 0, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } - yield { - 'name': 'cpp_protobuf_async_streaming_ping_pong_%s' - % secstr, - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'STREAMING', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_SERVER', - 'security_params': secargs, - 'core_limit': 1, - 'async_server_threads': 1, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } - yield { - 'name': 'cpp_protobuf_sync_unary_ping_pong_%s' - % secstr, - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'SYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 0, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'SYNC_SERVER', - 'security_params': secargs, - 'core_limit': 1, - 'async_server_threads': 0, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } - yield { - 'name': 'cpp_protobuf_async_unary_ping_pong_%s' - % secstr, - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_SERVER', - 'security_params': secargs, - 'core_limit': 1, - 'async_server_threads': 1, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } + secstr = 'secure' if secure else 'insecure' + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True, server_core_limit=1, async_server_threads=1, + secure=secure) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + server_core_limit=1, async_server_threads=1, + secure=secure) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + server_core_limit=1, async_server_threads=1, + secure=secure) + + yield _ping_pong_scenario( + 'cpp_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_core_limit=1, async_server_threads=1, + secure=secure) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + server_core_limit=SINGLE_MACHINE_CORES/2, + use_unconstrained_client=True, + secure=secure) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + server_core_limit=SINGLE_MACHINE_CORES/2, + use_unconstrained_client=True, + secure=secure) + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_unconstrained_client=True, use_generic_payload=True, + server_core_limit=SINGLE_MACHINE_CORES/2, + secure=secure) + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_unconstrained_client=True, use_generic_payload=True, + server_core_limit=1, async_server_threads=1, + secure=secure) def __str__(self): return 'c++' @@ -299,113 +222,42 @@ class CSharpLanguage: return 100 def scenarios(self): - secargs = SECURE_SECARGS - yield { - 'name': 'csharp_generic_async_streaming_ping_pong', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'STREAMING', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_GENERIC_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_GENERIC_SERVER', - 'security_params': secargs, - 'core_limit': 0, - 'async_server_threads': 0, - 'payload_config': EMPTY_GENERIC_PAYLOAD, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } - yield { - 'name': 'csharp_protobuf_async_unary_ping_pong', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_SERVER', - 'security_params': secargs, - 'core_limit': 0, - 'async_server_threads': 0, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } - yield { - 'name': 'csharp_protobuf_sync_to_async_unary_ping_pong', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'SYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_SERVER', - 'security_params': secargs, - 'core_limit': 0, - 'async_server_threads': 0, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } - yield { - 'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'SYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'SYNC_SERVER', - 'security_params': secargs, - 'core_limit': 1, - 'async_server_threads': 1, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS, - 'SERVER_LANGUAGE': 'c++' # recognized by run_performance_tests.py - } + yield _ping_pong_scenario( + 'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True) + + yield _ping_pong_scenario( + 'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + use_unconstrained_client=True) + + yield _ping_pong_scenario( + 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + use_unconstrained_client=True) + + yield _ping_pong_scenario( + 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1) + + yield _ping_pong_scenario( + 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1) def __str__(self): return 'csharp' @@ -424,34 +276,43 @@ class NodeLanguage: return 200 def scenarios(self): - # TODO(jtattermusch): add more scenarios - secargs = SECURE_SECARGS - yield { - 'name': 'node_protobuf_unary_ping_pong', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_SERVER', - 'security_params': secargs, - 'core_limit': 0, - 'async_server_threads': 1, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'node_generic_async_streaming_ping_pong', rpc_type='STREAMING', + # client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + # use_generic_payload=True) + + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'node_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'node_protobuf_unary_ping_pong', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + use_unconstrained_client=True) + + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'node_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + # use_unconstrained_client=True) + + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'node_to_cpp_protobuf_async_unary_ping_pong', rpc_type='UNARY', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + # server_language='c++', server_core_limit=1, async_server_threads=1) + + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'node_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + # server_language='c++', server_core_limit=1, async_server_threads=1) def __str__(self): return 'node' @@ -468,114 +329,49 @@ class PythonLanguage: return 500 def scenarios(self): - yield { - 'name': 'python_to_cpp_protobuf_streaming_ping_pong', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': SECURE_SECARGS, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'STREAMING', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'SYNC_SERVER', - 'security_params': SECURE_SECARGS, - 'core_limit': 0, - 'async_server_threads': 1, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS, - 'SERVER_LANGUAGE': 'c++' - } - yield { - 'name': 'python_protobuf_sync_unary_ping_pong', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'SYNC_CLIENT', - 'security_params': SECURE_SECARGS, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'SYNC_SERVER', - 'security_params': SECURE_SECARGS, - 'core_limit': 0, - 'async_server_threads': 1, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS, - } - yield { - 'name': 'python_protobuf_async_unary_ping_pong', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': SECURE_SECARGS, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'SYNC_SERVER', - 'security_params': SECURE_SECARGS, - 'core_limit': 0, - 'async_server_threads': 1, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS, - } - yield { - 'name': 'python_to_cpp_single_channel_throughput', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'ASYNC_CLIENT', - 'security_params': SECURE_SECARGS, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'STREAMING', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': BIG_GENERIC_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'ASYNC_GENERIC_SERVER', - 'security_params': SECURE_SECARGS, - 'core_limit': SINGLE_MACHINE_CORES/2, - 'async_server_threads': 1, - 'payload_config': BIG_GENERIC_PAYLOAD, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS, - 'SERVER_LANGUAGE': 'c++' - } - + # TODO(jtattermusch): this scenario reports QPS 0.0 + yield _ping_pong_scenario( + 'python_generic_async_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True) + + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'python_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') + + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'python_protobuf_async_unary_ping_pong', rpc_type='UNARY', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER') + + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', + # client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + # use_unconstrained_client=True) + + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'python_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + # use_unconstrained_client=True) + + yield _ping_pong_scenario( + 'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1) + + # TODO(jtattermusch): make this scenario work + #yield _ping_pong_scenario( + # 'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + # client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + # server_language='c++', server_core_limit=1, async_server_threads=1) + def __str__(self): return 'python' @@ -592,34 +388,35 @@ class RubyLanguage: return 300 def scenarios(self): - # TODO(jtattermusch): add more scenarios - secargs = SECURE_SECARGS - yield { - 'name': 'ruby_protobuf_unary_ping_pong', - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'SYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'SYNC_SERVER', - 'security_params': secargs, - 'core_limit': 0, - 'async_server_threads': 1, - }, - 'warmup_seconds': WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } + yield _ping_pong_scenario( + 'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER') + + yield _ping_pong_scenario( + 'ruby_protobuf_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER') + + # TODO: scenario reports QPS of 0.0 + #yield _ping_pong_scenario( + # 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', + # client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + # use_unconstrained_client=True) + + # TODO: scenario reports QPS of 0.0 + #yield _ping_pong_scenario( + # 'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', + # client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + # use_unconstrained_client=True) + + yield _ping_pong_scenario( + 'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1) + + yield _ping_pong_scenario( + 'ruby_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1) def __str__(self): return 'ruby' @@ -638,41 +435,59 @@ class JavaLanguage: return 400 def scenarios(self): - # TODO(jtattermusch): add more scenarios for secure in [True, False]: - if secure: - secstr = 'secure' - secargs = SECURE_SECARGS - else: - secstr = 'insecure' - secargs = None - - yield { - 'name': 'java_protobuf_unary_ping_pong_%s' % secstr, - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': 'SYNC_CLIENT', - 'security_params': secargs, - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': 'UNARY', - 'load_params': { - 'closed_loop': {} - }, - 'payload_config': EMPTY_PROTO_PAYLOAD, - 'histogram_params': HISTOGRAM_PARAMS, - }, - 'server_config': { - 'server_type': 'SYNC_SERVER', - 'security_params': secargs, - 'core_limit': 0, - 'async_server_threads': 1, - }, - 'warmup_seconds': JAVA_WARMUP_SECONDS, - 'benchmark_seconds': BENCHMARK_SECONDS - } + secstr = 'secure' if secure else 'insecure' + + yield _ping_pong_scenario( + 'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True, async_server_threads=1, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + + yield _ping_pong_scenario( + 'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + async_server_threads=1, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + + yield _ping_pong_scenario( + 'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + async_server_threads=1, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + + yield _ping_pong_scenario( + 'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + async_server_threads=1, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + + yield _ping_pong_scenario( + 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + use_unconstrained_client=True, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + + yield _ping_pong_scenario( + 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + use_unconstrained_client=True, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + + yield _ping_pong_scenario( + 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_unconstrained_client=True, use_generic_payload=True, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + + yield _ping_pong_scenario( + 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_unconstrained_client=True, use_generic_payload=True, + async_server_threads=1, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + + # TODO(jtattermusch): add scenarios java vs C++ def __str__(self): return 'java' From 541d5d7ae2745a9e3dc55348df3168c0f51d382e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 May 2016 16:08:49 -0700 Subject: [PATCH 104/155] increase qps_worker max lifetime --- tools/run_tests/run_performance_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 5519666e847..8b67d921f22 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -89,7 +89,7 @@ def create_qpsworker_job(language, shortname=None, jobspec = jobset.JobSpec( cmdline=cmdline, shortname=shortname, - timeout_seconds=30*60) + timeout_seconds=2*60*60) return QpsWorkerJob(jobspec, language, host_and_port) From 253a7109ed1c86df0e482e899bc952e36bdc9f42 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 6 May 2016 03:00:51 +0200 Subject: [PATCH 105/155] The release branch is now 0.14.0-pre1. --- Makefile | 2 +- build.yaml | 2 +- package.json | 2 +- src/core/lib/surface/version.c | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 2 +- src/csharp/build_packages.bat | 2 +- src/node/tools/package.json | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.core | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index e77aa2dd16f..635df30c20a 100644 --- a/Makefile +++ b/Makefile @@ -407,7 +407,7 @@ E = @echo Q = @ endif -VERSION = 0.14.0-dev +VERSION = 0.14.0-pre1 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 1a0888bdc3e..7a74e73c726 100644 --- a/build.yaml +++ b/build.yaml @@ -7,7 +7,7 @@ settings: '#3': Use "-preN" suffixes to identify pre-release versions '#4': Per-language overrides are possible with (eg) ruby_version tag here '#5': See the expand_version.py for all the quirks here - version: 0.14.0-dev + version: 0.14.0-pre1 filegroups: - name: census public_headers: diff --git a/package.json b/package.json index 5ed7f363d37..b22cfc1f938 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "0.14.0-dev", + "version": "0.14.0-pre1", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "http://www.grpc.io/", diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c index fe954cbefb6..0f4b1111f43 100644 --- a/src/core/lib/surface/version.c +++ b/src/core/lib/surface/version.c @@ -36,4 +36,4 @@ #include -const char *grpc_version_string(void) { return "0.14.0-dev"; } +const char *grpc_version_string(void) { return "0.14.0-pre1"; } diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index f7a9cb9c1cb..70ce9a18df5 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -53,6 +53,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "0.14.0-dev"; + public const string CurrentVersion = "0.14.0-pre1"; } } diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat index 9a60be26b63..9c7b877feab 100644 --- a/src/csharp/build_packages.bat +++ b/src/csharp/build_packages.bat @@ -1,7 +1,7 @@ @rem Builds gRPC NuGet packages @rem Current package versions -set VERSION=0.14.0-dev +set VERSION=0.14.0-pre1 set PROTOBUF_VERSION=3.0.0-beta2 @rem Packages that depend on prerelease packages (like Google.Protobuf) need to have prerelease suffix as well. diff --git a/src/node/tools/package.json b/src/node/tools/package.json index d98ed0b1fc3..9bca5eab6f2 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "0.14.0-dev", + "version": "0.14.0-pre1", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "http://www.grpc.io/", diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 873b4e2a913..87fadf9a979 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='0.14.0.dev0' +VERSION='0.14.0rc1' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 67c6a5d5a17..5117c85cea0 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -29,5 +29,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '0.14.0.dev' + VERSION = '0.14.0.pre1' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 12ad21b80eb..ec085d2655b 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -29,6 +29,6 @@ module GRPC module Tools - VERSION = '0.14.0.dev' + VERSION = '0.14.0.pre1' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index b8ae8e20b8b..ee522a0bc31 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='0.14.0.dev0' +VERSION='0.14.0rc1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 664ca03d978..f58f7e72817 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-dev +PROJECT_NUMBER = 0.14.0-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 5188ef1e8da..a7a17bac3fc 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-dev +PROJECT_NUMBER = 0.14.0-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 84b5c2a8efb..d1259e7aee3 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-dev +PROJECT_NUMBER = 0.14.0-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 228c1d98d89..6b7a8be8617 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-dev +PROJECT_NUMBER = 0.14.0-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 16caa50aae7d1669550be35e205039f65cc4c363 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 6 May 2016 03:02:51 +0200 Subject: [PATCH 106/155] Master is now 0.15.0-dev. --- Makefile | 2 +- build.yaml | 2 +- composer.json | 2 +- package.json | 2 +- package.xml | 8 ++++---- src/core/lib/surface/version.c | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/build_packages.bat | 2 +- src/node/tools/package.json | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.core | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- 17 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index e77aa2dd16f..a684ea8611a 100644 --- a/Makefile +++ b/Makefile @@ -407,7 +407,7 @@ E = @echo Q = @ endif -VERSION = 0.14.0-dev +VERSION = 0.15.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 1a0888bdc3e..13916830c1e 100644 --- a/build.yaml +++ b/build.yaml @@ -7,7 +7,7 @@ settings: '#3': Use "-preN" suffixes to identify pre-release versions '#4': Per-language overrides are possible with (eg) ruby_version tag here '#5': See the expand_version.py for all the quirks here - version: 0.14.0-dev + version: 0.15.0-dev filegroups: - name: census public_headers: diff --git a/composer.json b/composer.json index 97b1a5cb49b..b77a59e351c 100644 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc", "type": "library", "description": "gRPC library for PHP", - "version": "0.14.0", + "version": "0.15.0", "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", diff --git a/package.json b/package.json index 5ed7f363d37..54a44ca5513 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "0.14.0-dev", + "version": "0.15.0-dev", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "http://www.grpc.io/", diff --git a/package.xml b/package.xml index 716d6ed2891..152d5d61904 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2016-04-19 - 0.14.0 - 0.14.0 + 0.15.0 + 0.15.0 beta @@ -1054,8 +1054,8 @@ Update to wrap gRPC C Core version 0.10.0 - 0.14.0 - 0.14.0 + 0.15.0 + 0.15.0 beta diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c index fe954cbefb6..aca76d2bb79 100644 --- a/src/core/lib/surface/version.c +++ b/src/core/lib/surface/version.c @@ -36,4 +36,4 @@ #include -const char *grpc_version_string(void) { return "0.14.0-dev"; } +const char *grpc_version_string(void) { return "0.15.0-dev"; } diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index f7a9cb9c1cb..e1609341d9a 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -48,11 +48,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "0.14.0.0"; + public const string CurrentAssemblyFileVersion = "0.15.0.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "0.14.0-dev"; + public const string CurrentVersion = "0.15.0-dev"; } } diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat index 9a60be26b63..7520b0f81a7 100644 --- a/src/csharp/build_packages.bat +++ b/src/csharp/build_packages.bat @@ -1,7 +1,7 @@ @rem Builds gRPC NuGet packages @rem Current package versions -set VERSION=0.14.0-dev +set VERSION=0.15.0-dev set PROTOBUF_VERSION=3.0.0-beta2 @rem Packages that depend on prerelease packages (like Google.Protobuf) need to have prerelease suffix as well. diff --git a/src/node/tools/package.json b/src/node/tools/package.json index d98ed0b1fc3..efdfa811243 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "0.14.0-dev", + "version": "0.15.0-dev", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "http://www.grpc.io/", diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 873b4e2a913..0c13104d9da 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='0.14.0.dev0' +VERSION='0.15.0.dev0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 67c6a5d5a17..01c8c5ac8f1 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -29,5 +29,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '0.14.0.dev' + VERSION = '0.15.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 12ad21b80eb..dca7fd7e72c 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -29,6 +29,6 @@ module GRPC module Tools - VERSION = '0.14.0.dev' + VERSION = '0.15.0.dev' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index b8ae8e20b8b..1267d0e45dc 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='0.14.0.dev0' +VERSION='0.15.0.dev0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 664ca03d978..2a319db9796 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-dev +PROJECT_NUMBER = 0.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 5188ef1e8da..5fdfafbf3ed 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-dev +PROJECT_NUMBER = 0.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 84b5c2a8efb..aabca410dab 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-dev +PROJECT_NUMBER = 0.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 228c1d98d89..3ffc6174ed4 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-dev +PROJECT_NUMBER = 0.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From a21c7e9207490d9bcdd7205c8a5857320a3a450d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 May 2016 17:31:52 -0700 Subject: [PATCH 107/155] add support for scenario categories --- test/cpp/qps/gen_build_yaml.py | 10 +++++++--- tools/run_tests/performance/scenario_config.py | 13 ++++++++++++- tools/run_tests/run_performance_tests.py | 6 +++--- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py index 9d6bf2ab73c..6b3329b046b 100755 --- a/test/cpp/qps/gen_build_yaml.py +++ b/test/cpp/qps/gen_build_yaml.py @@ -43,12 +43,16 @@ sys.path.append(run_tests_root) import performance.scenario_config as scenario_config +def _scenario_json_string(scenario_json): + return json.dumps(scenario_config.remove_nonproto_fields(scenario_json)) + print yaml.dump({ 'tests': [ { 'name': 'json_run_localhost', - 'shortname': 'json_run_localhost:%s' % js['name'], - 'args': ['--scenario_json', pipes.quote(json.dumps(js))], + 'shortname': 'json_run_localhost:%s' % scenario_json['name'], + 'args': ['--scenario_json', + pipes.quote(_scenario_json_string(scenario_json))], 'ci_platforms': ['linux', 'mac', 'posix', 'windows'], 'platforms': ['linux', 'mac', 'posix', 'windows'], 'flaky': False, @@ -58,6 +62,6 @@ print yaml.dump({ 'cpu_cost': 1000.0, 'exclude_configs': [] } - for js in scenario_config.CXXLanguage().scenarios() + for scenario_json in scenario_config.CXXLanguage().scenarios() ] }) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 52efe8b86b3..a5b0d59d02b 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -76,6 +76,14 @@ def _get_secargs(is_secure): return None +def remove_nonproto_fields(scenario): + """Remove special-purpose that contains some extra info about the scenario + but don't belong to the ScenarioConfig protobuf message""" + scenario.pop('CATEGORIES', None) + scenario.pop('SERVER_LANGUAGE', None) + return scenario + + def _ping_pong_scenario(name, rpc_type, client_type, server_type, secure=True, @@ -84,7 +92,8 @@ def _ping_pong_scenario(name, rpc_type, server_language=None, server_core_limit=0, async_server_threads=0, - warmup_seconds=WARMUP_SECONDS): + warmup_seconds=WARMUP_SECONDS, + categories=[]): """Creates a basic ping pong scenario.""" scenario = { 'name': name, @@ -135,6 +144,8 @@ def _ping_pong_scenario(name, rpc_type, if server_language: # the SERVER_LANGUAGE field is recognized by run_performance_tests.py scenario['SERVER_LANGUAGE'] = server_language + if categories: + scenario['CATEGORIES'] = categories return scenario diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 8b67d921f22..c8c1a19783a 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -255,9 +255,9 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', if re.search(args.regex, scenario_json['name']): workers = workers_by_lang[str(language)] # 'SERVER_LANGUAGE' is an indicator for this script to pick - # a server in different language. It doesn't belong to the Scenario - # schema, so we also need to remove it. - custom_server_lang = scenario_json.pop('SERVER_LANGUAGE', None) + # a server in different language. + custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None) + scenario_json = scenario_config.remove_nonproto_fields(scenario_json) if custom_server_lang: if not workers_by_lang.get(custom_server_lang, []): print 'Warning: Skipping scenario %s as' % scenario_json['name'] From c45cd2d2ef42aad241c623e7e0e267f6eaf03d17 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 May 2016 17:34:46 -0700 Subject: [PATCH 108/155] regenerate tests.json --- tools/run_tests/tests.json | 100 ++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 24 deletions(-) diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index cf1154426f4..79a9e73117d 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -22942,7 +22942,7 @@ { "args": [ "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -22963,12 +22963,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -22989,12 +22989,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -23015,12 +23015,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure" + "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -23041,12 +23041,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -23067,12 +23067,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -23093,7 +23093,33 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_secure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure" }, { "args": [ @@ -23124,7 +23150,7 @@ { "args": [ "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -23145,12 +23171,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -23171,12 +23197,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -23197,12 +23223,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure" + "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -23223,12 +23249,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -23249,12 +23275,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -23275,7 +23301,33 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_insecure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure" }, { "args": [ From 427699b638b2f17219f6b5d33bd571e42eb287fb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 May 2016 18:10:14 -0700 Subject: [PATCH 109/155] mark some scenarios as smoketest and add a cmdline flag --- .../run_tests/performance/scenario_config.py | 58 +++++++++++++------ tools/run_tests/run_performance_tests.py | 48 ++++++++------- 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index a5b0d59d02b..8b239951498 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -34,6 +34,8 @@ WARMUP_SECONDS=5 JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in. BENCHMARK_SECONDS=30 +SMOKETEST='smoketest' + SECURE_SECARGS = {'use_test_ca': True, 'server_host_override': 'foo.test.google.fr'} @@ -164,12 +166,14 @@ class CXXLanguage: # TODO(ctiller): add 70% load latency test for secure in [True, False]: secstr = 'secure' if secure else 'insecure' + smoketest_categories = [SMOKETEST] if secure else None yield _ping_pong_scenario( 'cpp_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', use_generic_payload=True, server_core_limit=1, async_server_threads=1, - secure=secure) + secure=secure, + categories=smoketest_categories) yield _ping_pong_scenario( 'cpp_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', @@ -181,7 +185,8 @@ class CXXLanguage: 'cpp_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', server_core_limit=1, async_server_threads=1, - secure=secure) + secure=secure, + categories=smoketest_categories) yield _ping_pong_scenario( 'cpp_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY', @@ -194,7 +199,8 @@ class CXXLanguage: client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', server_core_limit=SINGLE_MACHINE_CORES/2, use_unconstrained_client=True, - secure=secure) + secure=secure, + categories=smoketest_categories) yield _ping_pong_scenario( 'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', @@ -208,7 +214,8 @@ class CXXLanguage: client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', use_unconstrained_client=True, use_generic_payload=True, server_core_limit=SINGLE_MACHINE_CORES/2, - secure=secure) + secure=secure, + categories=smoketest_categories) yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', @@ -236,7 +243,8 @@ class CSharpLanguage: yield _ping_pong_scenario( 'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True) + use_generic_payload=True, + categories=[SMOKETEST]) yield _ping_pong_scenario( 'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', @@ -244,7 +252,8 @@ class CSharpLanguage: yield _ping_pong_scenario( 'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + categories=[SMOKETEST]) yield _ping_pong_scenario( 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', @@ -253,7 +262,8 @@ class CSharpLanguage: yield _ping_pong_scenario( 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True) + use_unconstrained_client=True, + categories=[SMOKETEST]) yield _ping_pong_scenario( 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', @@ -263,7 +273,8 @@ class CSharpLanguage: yield _ping_pong_scenario( 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', server_core_limit=1, async_server_threads=1) + server_language='c++', server_core_limit=1, async_server_threads=1, + categories=[SMOKETEST]) yield _ping_pong_scenario( 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', @@ -300,12 +311,14 @@ class NodeLanguage: yield _ping_pong_scenario( 'node_protobuf_unary_ping_pong', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + categories=[SMOKETEST]) yield _ping_pong_scenario( 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True) + use_unconstrained_client=True, + categories=[SMOKETEST]) # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( @@ -344,7 +357,8 @@ class PythonLanguage: yield _ping_pong_scenario( 'python_generic_async_streaming_ping_pong', rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True) + use_generic_payload=True, + categories=[SMOKETEST]) # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( @@ -358,7 +372,8 @@ class PythonLanguage: yield _ping_pong_scenario( 'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER') + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + categories=[SMOKETEST]) # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( @@ -375,7 +390,8 @@ class PythonLanguage: yield _ping_pong_scenario( 'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', server_core_limit=1, async_server_threads=1) + server_language='c++', server_core_limit=1, async_server_threads=1, + categories=[SMOKETEST]) # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( @@ -401,11 +417,13 @@ class RubyLanguage: def scenarios(self): yield _ping_pong_scenario( 'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER') + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + categories=[SMOKETEST]) yield _ping_pong_scenario( 'ruby_protobuf_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER') + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + categories=[SMOKETEST]) # TODO: scenario reports QPS of 0.0 #yield _ping_pong_scenario( @@ -448,12 +466,14 @@ class JavaLanguage: def scenarios(self): for secure in [True, False]: secstr = 'secure' if secure else 'insecure' + smoketest_categories = [SMOKETEST] if secure else None yield _ping_pong_scenario( 'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', use_generic_payload=True, async_server_threads=1, - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, + categories=smoketest_categories) yield _ping_pong_scenario( 'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', @@ -465,7 +485,8 @@ class JavaLanguage: 'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', async_server_threads=1, - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, + categories=smoketest_categories) yield _ping_pong_scenario( 'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY', @@ -477,7 +498,8 @@ class JavaLanguage: 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', use_unconstrained_client=True, - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, + categories=smoketest_categories) yield _ping_pong_scenario( 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index c8c1a19783a..b1f5889e541 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -244,7 +244,7 @@ def start_qpsworkers(languages, worker_hosts): def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', - bq_result_table=None): + category='all', bq_result_table=None): """Create jobspecs for scenarios to run.""" all_workers = [worker for workers in workers_by_lang.values() @@ -253,25 +253,26 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', for language in languages: for scenario_json in language.scenarios(): if re.search(args.regex, scenario_json['name']): - workers = workers_by_lang[str(language)] - # 'SERVER_LANGUAGE' is an indicator for this script to pick - # a server in different language. - custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None) - scenario_json = scenario_config.remove_nonproto_fields(scenario_json) - if custom_server_lang: - if not workers_by_lang.get(custom_server_lang, []): - print 'Warning: Skipping scenario %s as' % scenario_json['name'] - print('SERVER_LANGUAGE is set to %s yet the language has ' - 'not been selected with -l' % custom_server_lang) - continue - for idx in range(0, scenario_json['num_servers']): - # replace first X workers by workers of a different language - workers[idx] = workers_by_lang[custom_server_lang][idx] - scenario = create_scenario_jobspec(scenario_json, - workers, - remote_host=remote_host, - bq_result_table=bq_result_table) - scenarios.append(scenario) + if category in scenario_json.get('CATEGORIES', []) or category == 'all': + workers = workers_by_lang[str(language)] + # 'SERVER_LANGUAGE' is an indicator for this script to pick + # a server in different language. + custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None) + scenario_json = scenario_config.remove_nonproto_fields(scenario_json) + if custom_server_lang: + if not workers_by_lang.get(custom_server_lang, []): + print 'Warning: Skipping scenario %s as' % scenario_json['name'] + print('SERVER_LANGUAGE is set to %s yet the language has ' + 'not been selected with -l' % custom_server_lang) + continue + for idx in range(0, scenario_json['num_servers']): + # replace first X workers by workers of a different language + workers[idx] = workers_by_lang[custom_server_lang][idx] + scenario = create_scenario_jobspec(scenario_json, + workers, + remote_host=remote_host, + bq_result_table=bq_result_table) + scenarios.append(scenario) # the very last scenario requests shutting down the workers. scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host)) @@ -298,7 +299,7 @@ argp = argparse.ArgumentParser(description='Run performance tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(scenario_config.LANGUAGES.keys()), nargs='+', - default=['all'], + required=True, help='Languages to benchmark.') argp.add_argument('--remote_driver_host', default=None, @@ -311,6 +312,10 @@ argp.add_argument('-r', '--regex', default='.*', type=str, help='Regex to select scenarios to run.') argp.add_argument('--bq_result_table', default=None, type=str, help='Bigquery "dataset.table" to upload results to.') +argp.add_argument('--category', + choices=['smoketest','all'], + default='smoketest', + help='Select a category of tests to run. Smoketest runs by default.') args = argp.parse_args() @@ -354,6 +359,7 @@ try: workers_by_lang=worker_addresses, remote_host=args.remote_driver_host, regex=args.regex, + category=args.category, bq_result_table=args.bq_result_table) if not scenarios: raise Exception('No scenarios to run') From 4340b74761949411224fc7c73a9eb63182116ef8 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 5 May 2016 18:14:26 -0700 Subject: [PATCH 110/155] Fix missing file in grpcio-tools sdist --- tools/distrib/python/grpcio_tools/MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/distrib/python/grpcio_tools/MANIFEST.in b/tools/distrib/python/grpcio_tools/MANIFEST.in index e6ab312f09c..f99fb07ca8f 100644 --- a/tools/distrib/python/grpcio_tools/MANIFEST.in +++ b/tools/distrib/python/grpcio_tools/MANIFEST.in @@ -1,3 +1,4 @@ +include grpc_version.py include protoc_deps.py include protoc_lib_deps.py graft grpc From 2b86729b2521b650ca03c0a20dfe15796cfc89c4 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 5 May 2016 18:17:45 -0700 Subject: [PATCH 111/155] Build grpcio-tools sdist --- tools/run_tests/build_artifact_python.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 35c3a48afae..4320f978e30 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -59,7 +59,11 @@ ${SETARCH_CMD} ${PYTHON} setup.py \ ${SETARCH_CMD} ${PYTHON} setup.py \ bdist_wheel -# Build gRPC tools package +# Build gRPC tools package source distribution +${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py \ + sdist + +# Build gRPC tools package binary distribution ${PYTHON} tools/distrib/python/make_grpcio_tools.py CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \ ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel From ab5309c9b12aef6fd874b5f925ea04df4eebde14 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 5 May 2016 18:39:01 -0700 Subject: [PATCH 112/155] Remove precompiled Python extension hack --- setup.py | 42 +++++------- src/python/grpcio/precompiled.py | 114 ------------------------------- 2 files changed, 18 insertions(+), 138 deletions(-) delete mode 100644 src/python/grpcio/precompiled.py diff --git a/setup.py b/setup.py index 5cd26124f6d..f96824fa883 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,6 @@ sys.path.insert(0, os.path.abspath(PYTHON_STEM)) # Break import-style to ensure we can actually find our in-repo dependencies. import commands -import precompiled import grpc_core_dependencies import grpc_version @@ -173,7 +172,6 @@ COMMAND_CLASS = { 'build_project_metadata': commands.BuildProjectMetadata, 'build_py': commands.BuildPy, 'build_ext': commands.BuildExt, - 'build_tagged_ext': precompiled.BuildTaggedExt, 'gather': commands.Gather, 'run_interop': commands.RunInterop, 'test_lite': commands.TestLite @@ -229,25 +227,21 @@ else: PACKAGES = setuptools.find_packages( PYTHON_STEM, exclude=['tests', 'tests.*']) -setup_arguments = { - 'name': 'grpcio', - 'version': grpc_version.VERSION, - 'license': LICENSE, - 'ext_modules': CYTHON_EXTENSION_MODULES, - 'packages': list(PACKAGES), - 'package_dir': PACKAGE_DIRECTORIES, - # TODO(atash): Figure out why auditwheel doesn't like namespace packages. - #'namespace_packages': ['grpc'], - 'package_data': PACKAGE_DATA, - 'install_requires': INSTALL_REQUIRES, - 'setup_requires': SETUP_REQUIRES, - 'cmdclass': COMMAND_CLASS, - 'tests_require': TESTS_REQUIRE, - 'test_suite': TEST_SUITE, - 'test_loader': TEST_LOADER, - 'test_runner': TEST_RUNNER, -} - -precompiled.update_setup_arguments(setup_arguments) - -setuptools.setup(**setup_arguments) +setuptools.setup( + name='grpcio', + version=grpc_version.VERSION, + license=LICENSE, + ext_modules=CYTHON_EXTENSION_MODULES, + packages=list(PACKAGES), + package_dir=PACKAGE_DIRECTORIES, + # TODO(atash): Figure out why auditwheel doesn't like namespace packages. + #namespace_packages=['grpc'], + package_data=PACKAGE_DATA, + install_requires=INSTALL_REQUIRES, + setup_requires=SETUP_REQUIRES, + cmdclass=COMMAND_CLASS, + tests_require=TESTS_REQUIRE, + test_suite=TEST_SUITE, + test_loader=TEST_LOADER, + test_runner=TEST_RUNNER, +) diff --git a/src/python/grpcio/precompiled.py b/src/python/grpcio/precompiled.py deleted file mode 100644 index b6aa7fc90ee..00000000000 --- a/src/python/grpcio/precompiled.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import platform -import shutil -import sys -import sysconfig - -import setuptools - -import commands -import grpc_version - -try: - from urllib2 import urlopen -except ImportError: - from urllib.request import urlopen - -PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) -BINARIES_REPOSITORY = os.environ.get( - 'GRPC_PYTHON_BINARIES_REPOSITORY', - 'https://storage.googleapis.com/grpc-precompiled-binaries/python') -USE_PRECOMPILED_BINARIES = bool(int(os.environ.get( - 'GRPC_PYTHON_USE_PRECOMPILED_BINARIES', '1'))) - -def _tagged_ext_name(base): - uname = platform.uname() - tags = ( - grpc_version.VERSION, - 'py{}'.format(sysconfig.get_python_version()), - uname[0], - uname[4], - ) - ucs = 'ucs{}'.format(sysconfig.get_config_var('Py_UNICODE_SIZE')) - return '{base}-{tags}-{ucs}'.format( - base=base, tags='-'.join(tags), ucs=ucs) - - -class BuildTaggedExt(setuptools.Command): - - description = 'build the gRPC tagged extensions' - user_options = [] - - def initialize_options(self): - # distutils requires this override. - pass - - def finalize_options(self): - # distutils requires this override. - pass - - def run(self): - if 'linux' in sys.platform: - self.run_command('build_ext') - try: - os.makedirs('dist/') - except OSError: - pass - shutil.copyfile( - os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so'), - 'dist/{}.so'.format(_tagged_ext_name('cygrpc'))) - else: - sys.stderr.write('nothing to do for build_tagged_ext\n') - - -def update_setup_arguments(setup_arguments): - if not USE_PRECOMPILED_BINARIES: - sys.stderr.write('not using precompiled extension') - return - url = '{}/{}.so'.format(BINARIES_REPOSITORY, _tagged_ext_name('cygrpc')) - target_path = os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so') - try: - extension = urlopen(url).read() - except: - sys.stderr.write( - 'could not download precompiled extension: {}\n'.format(url)) - return - try: - with open(target_path, 'w') as target: - target.write(extension) - setup_arguments['ext_modules'] = [] - except: - sys.stderr.write( - 'could not write precompiled extension to directory: {} -> {}\n' - .format(url, target_path)) - return - setup_arguments['package_data']['grpc._cython'].append('cygrpc.so') From 0bdc7ccab9622cd2e62eec6007bff3391bc664f4 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 5 May 2016 19:03:43 -0700 Subject: [PATCH 113/155] Fix #4592 Adds a command that allows the instructions to be used to install gRPC Python from scratch. --- src/python/grpcio/README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index cb3f6b87fe4..afc4fe6a370 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -48,6 +48,7 @@ package named :code:`python-dev`). $ export REPO_ROOT=grpc # REPO_ROOT can be any directory of your choice $ git clone https://github.com/grpc/grpc.git $REPO_ROOT $ cd $REPO_ROOT + $ git submodule update --init # For the next two commands do `sudo pip install` if you get permission-denied errors $ pip install -rrequirements.txt From f4c70caaf44bdcddc22bda869decee2ca410020c Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 5 May 2016 19:14:05 -0700 Subject: [PATCH 114/155] Move manylinux1 precompiler definitions into port_platform.h --- include/grpc/impl/codegen/port_platform.h | 32 +++++++++++++++++++++++ tools/run_tests/artifact_targets.py | 32 +---------------------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 1229d488edb..d2d9ac78084 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -114,6 +114,38 @@ #define GPR_WIN32_ATOMIC 1 #define GPR_MSVC_TLS 1 #endif +#elif defined(GPR_MANYLINUX1) +// TODO(atash): manylinux1 is just another __linux__ but with ancient +// libraries; it should be integrated with the `__linux__` definitions below. +#define GPR_PLATFORM_STRING "manylinux" +#define GPR_POSIX_CRASH_HANDLER 1 +#define GPR_CPU_LINUX 1 +#define GPR_GCC_ATOMIC 1 +#define GPR_GCC_TLS 1 +#define GPR_LINUX 1 +#define GPR_LINUX_LOG 1 +#define GPR_POSIX_SOCKET 1 +#define GPR_POSIX_WAKEUP_FD 1 +#define GPR_POSIX_SOCKETADDR 1 +#define GPR_POSIX_NO_SPECIAL_WAKEUP_FD 1 +#define GPR_POSIX_SOCKETUTILS 1 +#define GPR_HAVE_UNIX_SOCKET 1 +#define GPR_HAVE_IP_PKTINFO 1 +#define GPR_HAVE_IPV6_RECVPKTINFO 1 +#define GPR_LINUX_ENV 1 +#define GPR_POSIX_FILE 1 +#define GPR_POSIX_TMPFILE 1 +#define GPR_POSIX_STRING 1 +#define GPR_POSIX_SUBPROCESS 1 +#define GPR_POSIX_SYNC 1 +#define GPR_POSIX_TIME 1 +#define GPR_GETPID_IN_UNISTD_H 1 +#define GPR_HAVE_MSG_NOSIGNAL 1 +#ifdef _LP64 +#define GPR_ARCH_64 1 +#else /* _LP64 */ +#define GPR_ARCH_32 1 +#endif /* _LP64 */ #elif defined(ANDROID) || defined(__ANDROID__) #define GPR_PLATFORM_STRING "android" #define GPR_ANDROID 1 diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 477bd468887..bd1269ceb72 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -113,37 +113,7 @@ class PythonArtifact: # defines ourselves. # TODO(atash) get better platform-detection support in core so we don't # need to do this manually... - environ['CFLAGS'] = " ".join([ - '-DGPR_NO_AUTODETECT_PLATFORM', - '-DGPR_PLATFORM_STRING=\\"manylinux\\"', - '-DGPR_POSIX_CRASH_HANDLER=1', - '-DGPR_CPU_LINUX=1', - '-DGPR_GCC_ATOMIC=1', - '-DGPR_GCC_TLS=1', - '-DGPR_LINUX=1', - '-DGPR_LINUX_LOG=1', - #'-DGPR_LINUX_MULTIPOLL_WITH_EPOLL=1', - '-DGPR_POSIX_SOCKET=1', - '-DGPR_POSIX_WAKEUP_FD=1', - '-DGPR_POSIX_SOCKETADDR=1', - #'-DGPR_LINUX_EVENTFD=1', - '-DGPR_POSIX_NO_SPECIAL_WAKEUP_FD=1', - #'-DGPR_LINUX_SOCKETUTILS=1', - '-DGPR_POSIX_SOCKETUTILS=1', - '-DGPR_HAVE_UNIX_SOCKET=1', - '-DGPR_HAVE_IP_PKTINFO=1', - '-DGPR_HAVE_IPV6_RECVPKTINFO=1', - '-DGPR_LINUX_ENV=1', - '-DGPR_POSIX_FILE=1', - '-DGPR_POSIX_TMPFILE=1', - '-DGPR_POSIX_STRING=1', - '-DGPR_POSIX_SUBPROCESS=1', - '-DGPR_POSIX_SYNC=1', - '-DGPR_POSIX_TIME=1', - '-DGPR_GETPID_IN_UNISTD_H=1', - '-DGPR_HAVE_MSG_NOSIGNAL=1', - '-DGPR_ARCH_{arch}=1'.format(arch=('32' if self.arch == 'x86' else '64')), - ]) + environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, 'tools/run_tests/build_artifact_python.sh', From 61ba1630f6ae1fbdd1ded1bf6badb45a69235029 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 5 May 2016 19:43:03 -0700 Subject: [PATCH 115/155] Fix Python grpcio-tools windows build --- tools/run_tests/build_artifact_python.bat | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat index 636ae0d393d..342469bba87 100644 --- a/tools/run_tests/build_artifact_python.bat +++ b/tools/run_tests/build_artifact_python.bat @@ -50,6 +50,8 @@ pip install -rrequirements.txt set GRPC_PYTHON_USE_CUSTOM_BDIST=0 set GRPC_PYTHON_BUILD_WITH_CYTHON=1 +@rem TODO(atash): maybe we could avoid the grpc_c.(32|64).python shim above if +@rem this used the right python build? python setup.py bdist_wheel @rem Build gRPC Python tools @@ -57,12 +59,15 @@ set PATH=C:\msys64\mingw%2\bin;%PATH% set CC=C:\msys64\mingw%2\bin\g++.exe set CFLAGS=-fno-wrapv python tools\distrib\python\make_grpcio_tools.py +@rem The following commands *must* be run with the right version of python +@rem otherwise the build get SNAFU'd (so we use the .exe suffix to invoke the python +@rem we set in the %PATH% variable above). if %2 == 32 ( - python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 + python.exe tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 ) else ( - python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 -DMS_WIN64 + python.exe tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 -DMS_WIN64 ) -python tools\distrib\python\grpcio_tools\setup.py bdist_wheel +python.exe tools\distrib\python\grpcio_tools\setup.py bdist_wheel mkdir artifacts xcopy /Y /I /S dist\* artifacts\ || goto :error From 7e660d7e42d0d5b37e41bbb79c373898746c5442 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 5 May 2016 20:09:33 -0700 Subject: [PATCH 116/155] Add Python grpcio-tools documentation --- tools/distrib/python/grpcio_tools/MANIFEST.in | 1 + tools/distrib/python/grpcio_tools/README.rst | 126 ++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 tools/distrib/python/grpcio_tools/README.rst diff --git a/tools/distrib/python/grpcio_tools/MANIFEST.in b/tools/distrib/python/grpcio_tools/MANIFEST.in index e6ab312f09c..f902095ad51 100644 --- a/tools/distrib/python/grpcio_tools/MANIFEST.in +++ b/tools/distrib/python/grpcio_tools/MANIFEST.in @@ -1,5 +1,6 @@ include protoc_deps.py include protoc_lib_deps.py +include README.rst graft grpc graft grpc_root graft third_party diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst new file mode 100644 index 00000000000..3be564ef5be --- /dev/null +++ b/tools/distrib/python/grpcio_tools/README.rst @@ -0,0 +1,126 @@ +gRPC Python Tools +================= + +Package for gRPC Python tools. + +Installation +------------ + +The gRPC Python tools package is available for Linux, Mac OS X, and Windows +running Python 2.7. + +From PyPI +~~~~~~~~~ + +If you are installing locally... + +:: + + $ pip install grpcio-tools + +Else system wide (on Ubuntu)... + +:: + + $ sudo pip install grpcio-tools + +If you're on Windows make sure that you installed the :code:`pip.exe` component +when you installed Python (if not go back and install it!) then invoke: + +:: + + $ pip.exe install grpcio-tools + +Windows users may need to invoke :code:`pip.exe` from a command line ran as +administrator. + +n.b. On Windows and on Mac OS X one *must* have a recent release of :code:`pip` +to retrieve the proper wheel from PyPI. Be sure to upgrade to the latest +version! + +You might also need to install Cython to handle installation via the source +distribution if gRPC Python's system coverage with wheels does not happen to +include your system. + +From Source +~~~~~~~~~~~ + +Building from source requires that you have the Python headers (usually a +package named :code:`python-dev`) and Cython installed. + +:: + + $ export REPO_ROOT=grpc # REPO_ROOT can be any directory of your choice + $ git clone https://github.com/grpc/grpc.git $REPO_ROOT + $ cd $REPO_ROOT + $ git submodule update --init + + $ cd tools/distrib/python/grpcio_tools + $ python ../make_grpcio_tools.py + + # For the next command do `sudo pip install` if you get permission-denied errors + $ pip install . + +You cannot currently install Python from source on Windows. Things might work +out for you in MSYS2 (follow the Linux instructions), but it isn't officially +supported at the moment. + +Troubleshooting +~~~~~~~~~~~~~~~ + +Help, I ... + +* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install + grpc** + + This is likely because :code:`pip` doesn't own the offending dependency, + which in turn is likely because your operating system's package manager owns + it. You'll need to force the installation of the dependency: + + :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY` + + For example, if you get an error like the following: + + :: + + Traceback (most recent call last): + File "", line 17, in + ... + File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 509, in find + raise VersionConflict(dist, req) + pkg_resources.VersionConflict: (six 1.8.0 (/usr/lib/python2.7/dist-packages), Requirement.parse('six>=1.10')) + + You can fix it by doing: + + :: + + sudo pip install --ignore-installed six + +* **... see compiler errors on some platforms when either installing from source or from the source distribution** + + If you see + + :: + + /tmp/pip-build-U8pSsr/cython/Cython/Plex/Scanners.c:4:20: fatal error: Python.h: No such file or directory + #include "Python.h" + ^ + compilation terminated. + + You can fix it by installing `python-dev` package. i.e + + :: + + sudo apt-get install python-dev + + If you see something similar to: + + :: + + third_party/protobuf/src/google/protobuf/stubs/mathlimits.h:173:31: note: in expansion of macro 'SIGNED_INT_MAX' + static const Type kPosMax = SIGNED_INT_MAX(Type); \\ + ^ + And your toolchain is GCC (at the time of this writing, up through at least + GCC 6.0), this is probably a bug where GCC chokes on constant expressions + when the :code:`-fwrapv` flag is specified. You should consider setting your + environment with :code:`CFLAGS=-fno-wrapv` or using clang (:code:`CC=clang`). From 067cce56d4158031ea40f2f91a2e3839ebe2bbec Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 6 May 2016 09:12:39 -0700 Subject: [PATCH 117/155] Fix asan test failure. --- test/core/bad_client/tests/large_metadata.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/core/bad_client/tests/large_metadata.c b/test/core/bad_client/tests/large_metadata.c index b3521439f52..1a8d2a29870 100644 --- a/test/core/bad_client/tests/large_metadata.c +++ b/test/core/bad_client/tests/large_metadata.c @@ -433,6 +433,7 @@ static void server_verifier_sends_too_much_metadata(grpc_server *server, cq_expect_completion(cqv, tag(102), 0); // Operation fails. cq_verify(cqv); + gpr_free((char *)meta.value); grpc_metadata_array_destroy(&request_metadata_recv); grpc_call_details_destroy(&call_details); grpc_call_destroy(s); @@ -469,6 +470,7 @@ static void client_validator(gpr_slice_buffer *incoming) { *p++ = 11; // Compare actual and expected. GPR_ASSERT(gpr_slice_cmp(last_frame, expected) == 0); + gpr_slice_buffer_destroy(&last_frame_buffer); } int main(int argc, char **argv) { From c3eb64980f9cfd907c30c25853a45124138ae88d Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 5 May 2016 16:07:33 -0700 Subject: [PATCH 118/155] node: fix math server minor bug --- src/node/test/math/math_server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/test/math/math_server.js b/src/node/test/math/math_server.js index fa05ed01654..5e3d1dd8641 100644 --- a/src/node/test/math/math_server.js +++ b/src/node/test/math/math_server.js @@ -68,7 +68,7 @@ function mathDiv(call, cb) { function mathFib(stream) { // Here, call is a standard writable Node object Stream var previous = 0, current = 1; - for (var i = 0; i < stream.request.limit; i++) { + for (var i = 0; i < stream.request.getLimit(); i++) { var response = new math.Num(); response.setNum(current); stream.write(response); From 55643fee6b1c02277837b187d2ea706e86d9c315 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 3 May 2016 15:50:07 -0700 Subject: [PATCH 119/155] php: stress test client --- src/php/tests/interop/interop_client.php | 249 ++++++++++++----------- src/php/tests/interop/metrics_client.php | 17 ++ src/php/tests/interop/stress_client.php | 83 ++++++++ 3 files changed, 228 insertions(+), 121 deletions(-) create mode 100644 src/php/tests/interop/metrics_client.php create mode 100644 src/php/tests/interop/stress_client.php diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php index aebf80f6bf0..965a543b10d 100755 --- a/src/php/tests/interop/interop_client.php +++ b/src/php/tests/interop/interop_client.php @@ -388,141 +388,148 @@ function timeoutOnSleepingServer($stub) 'Call status was not DEADLINE_EXCEEDED'); } -$args = getopt('', ['server_host:', 'server_port:', 'test_case:', - 'use_tls::', 'use_test_ca::', - 'server_host_override:', 'oauth_scope:', - 'default_service_account:', ]); -if (!array_key_exists('server_host', $args)) { - throw new Exception('Missing argument: --server_host is required'); -} -if (!array_key_exists('server_port', $args)) { - throw new Exception('Missing argument: --server_port is required'); -} -if (!array_key_exists('test_case', $args)) { - throw new Exception('Missing argument: --test_case is required'); -} - -if ($args['server_port'] == 443) { - $server_address = $args['server_host']; -} else { - $server_address = $args['server_host'].':'.$args['server_port']; -} +function interop_main($args) +{ + if (!array_key_exists('server_host', $args)) { + throw new Exception('Missing argument: --server_host is required'); + } + if (!array_key_exists('server_port', $args)) { + throw new Exception('Missing argument: --server_port is required'); + } + if (!array_key_exists('test_case', $args)) { + throw new Exception('Missing argument: --test_case is required'); + } -$test_case = $args['test_case']; + if ($args['server_port'] == 443) { + $server_address = $args['server_host']; + } else { + $server_address = $args['server_host'].':'.$args['server_port']; + } -$host_override = 'foo.test.google.fr'; -if (array_key_exists('server_host_override', $args)) { - $host_override = $args['server_host_override']; -} + $test_case = $args['test_case']; -$use_tls = false; -if (array_key_exists('use_tls', $args) && - $args['use_tls'] != 'false') { - $use_tls = true; -} + $host_override = 'foo.test.google.fr'; + if (array_key_exists('server_host_override', $args)) { + $host_override = $args['server_host_override']; + } -$use_test_ca = false; -if (array_key_exists('use_test_ca', $args) && - $args['use_test_ca'] != 'false') { - $use_test_ca = true; -} + $use_tls = false; + if (array_key_exists('use_tls', $args) && + $args['use_tls'] != 'false') { + $use_tls = true; + } -$opts = []; + $use_test_ca = false; + if (array_key_exists('use_test_ca', $args) && + $args['use_test_ca'] != 'false') { + $use_test_ca = true; + } -if ($use_tls) { - if ($use_test_ca) { - $ssl_credentials = Grpc\ChannelCredentials::createSsl( - file_get_contents(dirname(__FILE__).'/../data/ca.pem')); + $opts = []; + + if ($use_tls) { + if ($use_test_ca) { + $ssl_credentials = Grpc\ChannelCredentials::createSsl( + file_get_contents(dirname(__FILE__).'/../data/ca.pem')); + } else { + $ssl_credentials = Grpc\ChannelCredentials::createSsl(); + } + $opts['credentials'] = $ssl_credentials; + $opts['grpc.ssl_target_name_override'] = $host_override; } else { - $ssl_credentials = Grpc\ChannelCredentials::createSsl(); + $opts['credentials'] = Grpc\ChannelCredentials::createInsecure(); } - $opts['credentials'] = $ssl_credentials; - $opts['grpc.ssl_target_name_override'] = $host_override; -} else { - $opts['credentials'] = Grpc\ChannelCredentials::createInsecure(); -} -if (in_array($test_case, ['service_account_creds', - 'compute_engine_creds', 'jwt_token_creds', ])) { - if ($test_case == 'jwt_token_creds') { - $auth_credentials = ApplicationDefaultCredentials::getCredentials(); - } else { + if (in_array($test_case, ['service_account_creds', + 'compute_engine_creds', 'jwt_token_creds', ])) { + if ($test_case == 'jwt_token_creds') { + $auth_credentials = ApplicationDefaultCredentials::getCredentials(); + } else { + $auth_credentials = ApplicationDefaultCredentials::getCredentials( + $args['oauth_scope'] + ); + } + $opts['update_metadata'] = $auth_credentials->getUpdateMetadataFunc(); + } + + if ($test_case == 'oauth2_auth_token') { $auth_credentials = ApplicationDefaultCredentials::getCredentials( $args['oauth_scope'] ); + $token = $auth_credentials->fetchAuthToken(); + $update_metadata = + function ($metadata, + $authUri = null, + ClientInterface $client = null) use ($token) { + $metadata_copy = $metadata; + $metadata_copy[CredentialsLoader::AUTH_METADATA_KEY] = + [sprintf('%s %s', + $token['token_type'], + $token['access_token'])]; + + return $metadata_copy; + }; + $opts['update_metadata'] = $update_metadata; } - $opts['update_metadata'] = $auth_credentials->getUpdateMetadataFunc(); -} -if ($test_case == 'oauth2_auth_token') { - $auth_credentials = ApplicationDefaultCredentials::getCredentials( - $args['oauth_scope'] - ); - $token = $auth_credentials->fetchAuthToken(); - $update_metadata = - function ($metadata, - $authUri = null, - ClientInterface $client = null) use ($token) { - $metadata_copy = $metadata; - $metadata_copy[CredentialsLoader::AUTH_METADATA_KEY] = - [sprintf('%s %s', - $token['token_type'], - $token['access_token'])]; - - return $metadata_copy; - }; - $opts['update_metadata'] = $update_metadata; + $stub = new grpc\testing\TestServiceClient($server_address, $opts); + + echo "Connecting to $server_address\n"; + echo "Running test case $test_case\n"; + + switch ($test_case) { + case 'empty_unary': + emptyUnary($stub); + break; + case 'large_unary': + largeUnary($stub); + break; + case 'client_streaming': + clientStreaming($stub); + break; + case 'server_streaming': + serverStreaming($stub); + break; + case 'ping_pong': + pingPong($stub); + break; + case 'empty_stream': + emptyStream($stub); + break; + case 'cancel_after_begin': + cancelAfterBegin($stub); + break; + case 'cancel_after_first_response': + cancelAfterFirstResponse($stub); + break; + case 'timeout_on_sleeping_server': + timeoutOnSleepingServer($stub); + break; + case 'service_account_creds': + serviceAccountCreds($stub, $args); + break; + case 'compute_engine_creds': + computeEngineCreds($stub, $args); + break; + case 'jwt_token_creds': + jwtTokenCreds($stub, $args); + break; + case 'oauth2_auth_token': + oauth2AuthToken($stub, $args); + break; + case 'per_rpc_creds': + perRpcCreds($stub, $args); + break; + default: + echo "Unsupported test case $test_case\n"; + exit(1); + } } -$stub = new grpc\testing\TestServiceClient($server_address, $opts); - -echo "Connecting to $server_address\n"; -echo "Running test case $test_case\n"; - -switch ($test_case) { - case 'empty_unary': - emptyUnary($stub); - break; - case 'large_unary': - largeUnary($stub); - break; - case 'client_streaming': - clientStreaming($stub); - break; - case 'server_streaming': - serverStreaming($stub); - break; - case 'ping_pong': - pingPong($stub); - break; - case 'empty_stream': - emptyStream($stub); - break; - case 'cancel_after_begin': - cancelAfterBegin($stub); - break; - case 'cancel_after_first_response': - cancelAfterFirstResponse($stub); - break; - case 'timeout_on_sleeping_server': - timeoutOnSleepingServer($stub); - break; - case 'service_account_creds': - serviceAccountCreds($stub, $args); - break; - case 'compute_engine_creds': - computeEngineCreds($stub, $args); - break; - case 'jwt_token_creds': - jwtTokenCreds($stub, $args); - break; - case 'oauth2_auth_token': - oauth2AuthToken($stub, $args); - break; - case 'per_rpc_creds': - perRpcCreds($stub, $args); - break; - default: - echo "Unsupported test case $test_case\n"; - exit(1); +if (isset($_SERVER['PHP_SELF']) && preg_match('/interop_client/', $_SERVER['PHP_SELF'])) { + $args = getopt('', ['server_host:', 'server_port:', 'test_case:', + 'use_tls::', 'use_test_ca::', + 'server_host_override:', 'oauth_scope:', + 'default_service_account:', ]); + interop_main($args); } diff --git a/src/php/tests/interop/metrics_client.php b/src/php/tests/interop/metrics_client.php new file mode 100644 index 00000000000..404e5e191ba --- /dev/null +++ b/src/php/tests/interop/metrics_client.php @@ -0,0 +1,17 @@ + $deadline) { + break; + } + if ($client_connection = socket_accept($socket)) { + // there is an incoming request, respond with qps metrics + $input = socket_read($client_connection, 1024); + $qps = round($count / ($current_time - $start_time)); + socket_write($client_connection, "qps: $qps"); + socket_close($client_connection); + } else { + // do actual work, run one interop test case + $args['test_case'] = + $args['test_cases'][mt_rand(0, $num_test_cases - 1)]; + @interop_main($args); + $count++; + } + } + socket_close($socket); + echo "Number of interop tests run in $args[test_duration_secs] seconds: $count.\n"; +} + +// process command line arguments +$raw_args = getopt('', + ['server_addresses::', + 'test_cases:', + 'metrics_port::', + 'test_duration_secs::', + 'num_channels_per_server::', + 'num_stubs_per_channel::']); + +$args = []; + +if (empty($raw_args['server_addresses'])) { + $args['server_host'] = 'localhost'; + $args['server_port'] = '8080'; +} else { + $parts = explode(':', $raw_args['server_addresses']); + $args['server_host'] = $parts[0]; + $args['server_port'] = (count($parts) == 2) ? $parts[1] : ''; +} + +$args['metrics_port'] = empty($raw_args['metrics_port']) ? + '8081' : $args['metrics_port']; + +$args['test_duration_secs'] = empty($raw_args['test_duration_secs']) || + $raw_args['test_duration_secs'] == -1 ? + false : $raw_args['test_duration_secs']; + +$test_cases = []; +$test_case_strs = explode(',', $raw_args['test_cases']); +foreach ($test_case_strs as $test_case_str) { + $parts = explode(':', $test_case_str); + $test_cases = array_merge($test_cases, array_fill(0, $parts[1], $parts[0])); +} +$args['test_cases'] = $test_cases; + +stress_main($args); From c1bb55eef7343655cd23051d1e1b348c99756b40 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Wed, 4 May 2016 09:53:09 -0700 Subject: [PATCH 120/155] reuse interop client stub --- src/php/tests/interop/interop_client.php | 14 ++++++++++++-- src/php/tests/interop/metrics_client.php | 4 ++-- src/php/tests/interop/stress_client.php | 3 ++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php index 965a543b10d..830df133374 100755 --- a/src/php/tests/interop/interop_client.php +++ b/src/php/tests/interop/interop_client.php @@ -388,7 +388,7 @@ function timeoutOnSleepingServer($stub) 'Call status was not DEADLINE_EXCEEDED'); } -function interop_main($args) +function _makeStub($args) { if (!array_key_exists('server_host', $args)) { throw new Exception('Missing argument: --server_host is required'); @@ -474,7 +474,15 @@ function interop_main($args) $stub = new grpc\testing\TestServiceClient($server_address, $opts); - echo "Connecting to $server_address\n"; + return $stub; +} + +function interop_main($args, $stub = false) { + if (!$stub) { + $stub = _makeStub($args); + } + + $test_case = $args['test_case']; echo "Running test case $test_case\n"; switch ($test_case) { @@ -524,6 +532,8 @@ function interop_main($args) echo "Unsupported test case $test_case\n"; exit(1); } + + return $stub; } if (isset($_SERVER['PHP_SELF']) && preg_match('/interop_client/', $_SERVER['PHP_SELF'])) { diff --git a/src/php/tests/interop/metrics_client.php b/src/php/tests/interop/metrics_client.php index 404e5e191ba..7f5ba5b3ccd 100644 --- a/src/php/tests/interop/metrics_client.php +++ b/src/php/tests/interop/metrics_client.php @@ -1,7 +1,7 @@ Date: Thu, 5 May 2016 16:38:15 -0700 Subject: [PATCH 121/155] fix copyright --- src/php/tests/interop/interop_client.php | 2 +- src/php/tests/interop/metrics_client.php | 32 ++++++++++++++++++++++++ src/php/tests/interop/stress_client.php | 32 ++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php index 830df133374..565bfce74f7 100755 --- a/src/php/tests/interop/interop_client.php +++ b/src/php/tests/interop/interop_client.php @@ -1,7 +1,7 @@ Date: Fri, 6 May 2016 11:43:22 -0700 Subject: [PATCH 122/155] Make Ruby library terminate on Ctrl+C on client and server --- src/ruby/ext/grpc/rb_grpc.c | 2 + src/ruby/ext/grpc/rb_signal.c | 66 +++++++++++++++++++++++ src/ruby/ext/grpc/rb_signal.h | 39 ++++++++++++++ src/ruby/lib/grpc.rb | 3 ++ src/ruby/lib/grpc/generic/active_call.rb | 6 +++ src/ruby/lib/grpc/generic/rpc_server.rb | 57 +++----------------- src/ruby/lib/grpc/signals.rb | 69 ++++++++++++++++++++++++ 7 files changed, 191 insertions(+), 51 deletions(-) create mode 100644 src/ruby/ext/grpc/rb_signal.c create mode 100644 src/ruby/ext/grpc/rb_signal.h create mode 100644 src/ruby/lib/grpc/signals.rb diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c index acb47b00558..9145c52ef8d 100644 --- a/src/ruby/ext/grpc/rb_grpc.c +++ b/src/ruby/ext/grpc/rb_grpc.c @@ -50,6 +50,7 @@ #include "rb_loader.h" #include "rb_server.h" #include "rb_server_credentials.h" +#include "rb_signal.h" static VALUE grpc_rb_cTimeVal = Qnil; @@ -332,6 +333,7 @@ void Init_grpc_c() { Init_grpc_channel_credentials(); Init_grpc_server(); Init_grpc_server_credentials(); + Init_grpc_signals(); Init_grpc_status_codes(); Init_grpc_time_consts(); } diff --git a/src/ruby/ext/grpc/rb_signal.c b/src/ruby/ext/grpc/rb_signal.c new file mode 100644 index 00000000000..dd60b0f251f --- /dev/null +++ b/src/ruby/ext/grpc/rb_signal.c @@ -0,0 +1,66 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include + +#include "rb_grpc.h" + +static void (*old_sigint_handler)(int); +static void (*old_sigterm_handler)(int); + +static volatile bool signal_received = false; + +static void handle_signal(int signum) { + signal_received = true; + if (signum == SIGINT) { + old_sigint_handler(signum); + } else if (signum == SIGTERM) { + old_sigterm_handler(signum); + } +} + +static VALUE grpc_rb_signal_received(VALUE self) { + (void)self; + return signal_received ? Qtrue : Qfalse; +} + +void Init_grpc_signals() { + old_sigint_handler = signal(SIGINT, handle_signal); + old_sigterm_handler = signal(SIGTERM, handle_signal); + rb_define_singleton_method(grpc_rb_mGrpcCore, "signal_received?", + grpc_rb_signal_received, 0); +} diff --git a/src/ruby/ext/grpc/rb_signal.h b/src/ruby/ext/grpc/rb_signal.h new file mode 100644 index 00000000000..07e49c0a8be --- /dev/null +++ b/src/ruby/ext/grpc/rb_signal.h @@ -0,0 +1,39 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_RB_SIGNAL_H_ +#define GRPC_RB_SIGNAL_H_ + +void Init_grpc_signals(); + +#endif /* GRPC_RB_SIGNAL_H_ */ diff --git a/src/ruby/lib/grpc.rb b/src/ruby/lib/grpc.rb index 79fa705b1c9..7c9aae30e95 100644 --- a/src/ruby/lib/grpc.rb +++ b/src/ruby/lib/grpc.rb @@ -33,6 +33,7 @@ require_relative 'grpc/errors' require_relative 'grpc/grpc' require_relative 'grpc/logconfig' require_relative 'grpc/notifier' +require_relative 'grpc/signals' require_relative 'grpc/version' require_relative 'grpc/core/time_consts' require_relative 'grpc/generic/active_call' @@ -47,3 +48,5 @@ begin ensure file.close end + +GRPC::Signals.wait_for_signals diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb index ecf3cc32935..fd20a861448 100644 --- a/src/ruby/lib/grpc/generic/active_call.rb +++ b/src/ruby/lib/grpc/generic/active_call.rb @@ -28,7 +28,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. require 'forwardable' +require 'weakref' require_relative 'bidi_call' +require_relative '../signals' class Struct # BatchResult is the struct returned by calls to call#start_batch. @@ -121,6 +123,10 @@ module GRPC @unmarshal = unmarshal @metadata_tag = metadata_tag @op_notifier = nil + weak_self = WeakRef.new(self) + remove_handler = GRPC::Signals.register_handler(&weak_self + .method(:cancel)) + ObjectSpace.define_finalizer(self, remove_handler) end # output_metadata are provides access to hash that can be used to diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index a0f4071adc8..81082d13bf5 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -28,46 +28,13 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. require_relative '../grpc' +require_relative '../signals' require_relative 'active_call' require_relative 'service' require 'thread' -# A global that contains signals the gRPC servers should respond to. -$grpc_signals = [] - # GRPC contains the General RPC module. module GRPC - # Handles the signals in $grpc_signals. - # - # @return false if the server should exit, true if not. - def handle_signals - loop do - sig = $grpc_signals.shift - case sig - when 'INT' - return false - when 'TERM' - return false - when nil - return true - end - end - true - end - module_function :handle_signals - - # Sets up a signal handler that adds signals to the signal handling global. - # - # Signal handlers should do as little as humanly possible. - # Here, they just add themselves to $grpc_signals - # - # RpcServer (and later other parts of gRPC) monitors the signals - # $grpc_signals in its own non-signal context. - def trap_signals - %w(INT TERM).each { |sig| trap(sig) { $grpc_signals << sig } } - end - module_function :trap_signals - # Pool is a simple thread pool. class Pool # Default keep alive period is 1s @@ -328,23 +295,6 @@ module GRPC end end - # Runs the server in its own thread, then waits for signal INT or TERM on - # the current thread to terminate it. - def run_till_terminated - GRPC.trap_signals - t = Thread.new do - run - end - t.abort_on_exception = true - wait_till_running - until running_state == :stopped - sleep SIGNAL_CHECK_PERIOD - break unless GRPC.handle_signals - end - stop - t.join - end - # handle registration of classes # # service is either a class that includes GRPC::GenericService and whose @@ -403,9 +353,14 @@ module GRPC transition_running_state(:running) @run_cond.broadcast end + remove_signal_handler = GRPC::Signals.register_handler { stop } loop_handle_server_calls + # Remove signal handler when server stops + remove_signal_handler.call end + alias_method :run_till_terminated, :run + # Sends RESOURCE_EXHAUSTED if there are too many unprocessed jobs def available?(an_rpc) jobs_count, max = @pool.jobs_waiting, @max_waiting_requests diff --git a/src/ruby/lib/grpc/signals.rb b/src/ruby/lib/grpc/signals.rb new file mode 100644 index 00000000000..7d2ec25dcb3 --- /dev/null +++ b/src/ruby/lib/grpc/signals.rb @@ -0,0 +1,69 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +require 'thread' +require_relative 'grpc' + +# GRPC contains the General RPC module. +module GRPC + # Signals contains gRPC functions related to signal handling + module Signals + @interpreter_exiting = false + @signal_handlers = [] + @handlers_mutex = Mutex.new + + def register_handler(&handler) + @handlers_mutex.synchronize do + @signal_handlers.push(handler) + handler.call if @exit_signal_received + end + # Returns a function to remove the handler + lambda do + @handlers_mutex.synchronize { @signal_handlers.delete(handler) } + end + end + module_function :register_handler + + def wait_for_signals + t = Thread.new do + sleep 1 until GRPC::Core.signal_received? || @interpreter_exiting + unless @interpreter_exiting + @handlers_mutex.synchronize do + @signal_handlers.each(&:call) + end + end + end + at_exit do + @interpreter_exiting = true + t.join + end + end + module_function :wait_for_signals + end +end From 6f0fe0f32728dfa781e49451a755d73df5fa6a88 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 6 May 2016 12:16:28 -0700 Subject: [PATCH 123/155] fix python distribtests with docker on overlay --- test/distrib/python/run_distrib_test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/distrib/python/run_distrib_test.sh b/test/distrib/python/run_distrib_test.sh index 6196e540c82..a0324a92b1b 100755 --- a/test/distrib/python/run_distrib_test.sh +++ b/test/distrib/python/run_distrib_test.sh @@ -48,7 +48,10 @@ which $PYTHON || PYTHON=python which $PIP || PIP=pip # TODO(jtattermusch): this shouldn't be required -${PIP} install --upgrade six pip +# TODO(jtattermusch): run the command twice to workaround docker-on-overlay +# issue https://github.com/docker/docker/issues/12327 +# (first attempt will fail when using docker with overlayFS) +${PIP} install --upgrade six pip || ${PIP} install --upgrade six pip # At least one of the bdist packages has to succeed (whichever one matches the # test machine, anyway). From 4e2f7727a9d39b58d6e3d083a7ed9a5aa6bb5fcc Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 6 May 2016 12:28:58 -0700 Subject: [PATCH 124/155] rename grpc.protoc.compiler to grpc.tools.protoc --- test/distrib/python/run_distrib_test.sh | 2 +- .../python/grpcio_tools/grpc/{protoc => tools}/__init__.py | 0 .../python/grpcio_tools/grpc/{protoc => tools}/main.cc | 2 +- .../python/grpcio_tools/grpc/{protoc => tools}/main.h | 0 .../grpc/{protoc/compiler.py => tools/protoc.py} | 2 +- .../grpcio_tools/grpc/{protoc => tools}/protoc_compiler.pyx | 2 +- tools/distrib/python/grpcio_tools/setup.py | 6 +++--- 7 files changed, 7 insertions(+), 7 deletions(-) rename tools/distrib/python/grpcio_tools/grpc/{protoc => tools}/__init__.py (100%) rename tools/distrib/python/grpcio_tools/grpc/{protoc => tools}/main.cc (98%) rename tools/distrib/python/grpcio_tools/grpc/{protoc => tools}/main.h (100%) rename tools/distrib/python/grpcio_tools/grpc/{protoc/compiler.py => tools/protoc.py} (97%) rename tools/distrib/python/grpcio_tools/grpc/{protoc => tools}/protoc_compiler.pyx (97%) diff --git a/test/distrib/python/run_distrib_test.sh b/test/distrib/python/run_distrib_test.sh index a0324a92b1b..8a983bc2488 100755 --- a/test/distrib/python/run_distrib_test.sh +++ b/test/distrib/python/run_distrib_test.sh @@ -61,6 +61,6 @@ done # TODO(jtattermusch): add a .proto file to the distribtest, generate python # code from it and then use the generated code from distribtest.py -$PYTHON -m grpc.protoc.compiler +$PYTHON -m grpc.tools.protoc $PYTHON distribtest.py diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/__init__.py b/tools/distrib/python/grpcio_tools/grpc/tools/__init__.py similarity index 100% rename from tools/distrib/python/grpcio_tools/grpc/protoc/__init__.py rename to tools/distrib/python/grpcio_tools/grpc/tools/__init__.py diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc b/tools/distrib/python/grpcio_tools/grpc/tools/main.cc similarity index 98% rename from tools/distrib/python/grpcio_tools/grpc/protoc/main.cc rename to tools/distrib/python/grpcio_tools/grpc/tools/main.cc index c9936a3a6b4..81675b4e6fc 100644 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/main.cc +++ b/tools/distrib/python/grpcio_tools/grpc/tools/main.cc @@ -32,7 +32,7 @@ #include "src/compiler/python_generator.h" -#include "grpc/protoc/main.h" +#include "grpc/tools/main.h" int protoc_main(int argc, char* argv[]) { google::protobuf::compiler::CommandLineInterface cli; diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/main.h b/tools/distrib/python/grpcio_tools/grpc/tools/main.h similarity index 100% rename from tools/distrib/python/grpcio_tools/grpc/protoc/main.h rename to tools/distrib/python/grpcio_tools/grpc/tools/main.h diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/compiler.py b/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py similarity index 97% rename from tools/distrib/python/grpcio_tools/grpc/protoc/compiler.py rename to tools/distrib/python/grpcio_tools/grpc/tools/protoc.py index caafc544b2b..b4dd0ecae2e 100644 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/compiler.py +++ b/tools/distrib/python/grpcio_tools/grpc/tools/protoc.py @@ -31,7 +31,7 @@ import sys -from grpc.protoc import protoc_compiler +from grpc.tools import protoc_compiler if __name__ == '__main__': diff --git a/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx b/tools/distrib/python/grpcio_tools/grpc/tools/protoc_compiler.pyx similarity index 97% rename from tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx rename to tools/distrib/python/grpcio_tools/grpc/tools/protoc_compiler.pyx index af15f3db30e..a6530127c04 100644 --- a/tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx +++ b/tools/distrib/python/grpcio_tools/grpc/tools/protoc_compiler.pyx @@ -29,7 +29,7 @@ from libc cimport stdlib -cdef extern from "grpc/protoc/main.h": +cdef extern from "grpc/tools/main.h": int protoc_main(int argc, char *argv[]) def run_main(list args not None): diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 0281c01796d..1a77c430fa7 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -45,13 +45,13 @@ import grpc_version def protoc_ext_module(): plugin_sources = [ - 'grpc/protoc/main.cc', + 'grpc/tools/main.cc', 'grpc_root/src/compiler/python_generator.cc'] + [ os.path.join('third_party/protobuf/src', cc_file) for cc_file in protoc_lib_deps.CC_FILES] plugin_ext = extension.Extension( - name='grpc.protoc.protoc_compiler', - sources=['grpc/protoc/protoc_compiler.pyx'] + plugin_sources, + name='grpc.tools.protoc_compiler', + sources=['grpc/tools/protoc_compiler.pyx'] + plugin_sources, include_dirs=[ '.', 'grpc_root', From 873e419cf73fa747a897cd8e06c8ffad6661efb0 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 6 May 2016 12:44:46 -0700 Subject: [PATCH 125/155] Added explanation about handling signals in C --- src/ruby/ext/grpc/rb_signal.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ruby/ext/grpc/rb_signal.c b/src/ruby/ext/grpc/rb_signal.c index dd60b0f251f..a9e512374ba 100644 --- a/src/ruby/ext/grpc/rb_signal.c +++ b/src/ruby/ext/grpc/rb_signal.c @@ -44,6 +44,10 @@ static void (*old_sigterm_handler)(int); static volatile bool signal_received = false; +/* This has to be handled at the C level instead of Ruby, because Ruby signal + * handlers are constrained to run in the main interpreter thread. If that main + * thread is blocked on grpc_completion_queue_pluck, the signal handlers will + * never run */ static void handle_signal(int signum) { signal_received = true; if (signum == SIGINT) { From 41fd37bbe38cb338b4d7350653658146b9012ae0 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 6 May 2016 13:43:26 -0700 Subject: [PATCH 126/155] Poll for signals more frequently, for greater responsiveness --- src/ruby/lib/grpc/signals.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/lib/grpc/signals.rb b/src/ruby/lib/grpc/signals.rb index 7d2ec25dcb3..2ab85c8bb16 100644 --- a/src/ruby/lib/grpc/signals.rb +++ b/src/ruby/lib/grpc/signals.rb @@ -52,7 +52,7 @@ module GRPC def wait_for_signals t = Thread.new do - sleep 1 until GRPC::Core.signal_received? || @interpreter_exiting + sleep 0.1 until GRPC::Core.signal_received? || @interpreter_exiting unless @interpreter_exiting @handlers_mutex.synchronize do @signal_handlers.each(&:call) From cf239e73093e0c9afe457b2b4b0fe45eae514b6f Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 6 May 2016 14:48:21 -0700 Subject: [PATCH 127/155] Add a default handler to Ruby services that returns UNIMPLEMENTED --- src/ruby/lib/grpc/generic/rpc_server.rb | 6 +-- src/ruby/lib/grpc/generic/service.rb | 16 ++---- src/ruby/spec/generic/rpc_server_spec.rb | 23 ++++++-- src/ruby/spec/generic/service_spec.rb | 69 ------------------------ 4 files changed, 24 insertions(+), 90 deletions(-) diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index a0f4071adc8..60a89572511 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -503,10 +503,8 @@ module GRPC unless cls.include?(GenericService) fail "#{cls} must 'include GenericService'" end - if cls.rpc_descs.size.zero? - fail "#{cls} should specify some rpc descriptions" - end - cls.assert_rpc_descs_have_methods + fail "#{cls} should specify some rpc descriptions" if + cls.rpc_descs.size.zero? end # This should be called while holding @run_mutex diff --git a/src/ruby/lib/grpc/generic/service.rb b/src/ruby/lib/grpc/generic/service.rb index 8e940b5b13e..0a166e823e4 100644 --- a/src/ruby/lib/grpc/generic/service.rb +++ b/src/ruby/lib/grpc/generic/service.rb @@ -110,6 +110,9 @@ module GRPC rpc_descs[name] = RpcDesc.new(name, input, output, marshal_class_method, unmarshal_class_method) + define_method(name) do + fail GRPC::BadStatus, GRPC::Core::StatusCodes::UNIMPLEMENTED + end end def inherited(subclass) @@ -199,19 +202,6 @@ module GRPC end end end - - # Asserts that the appropriate methods are defined for each added rpc - # spec. Is intended to aid verifying that server classes are correctly - # implemented. - def assert_rpc_descs_have_methods - rpc_descs.each_pair do |m, spec| - mth_name = GenericService.underscore(m.to_s).to_sym - unless instance_methods.include?(mth_name) - fail "#{self} does not provide instance method '#{mth_name}'" - end - spec.assert_arity_matches(instance_method(mth_name)) - end - end end def self.included(o) diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index e688057cb17..2a42736237d 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -308,10 +308,6 @@ describe GRPC::RpcServer do expect { @srv.handle(EmptyService) }.to raise_error end - it 'raises if the service does not define its rpc methods' do - expect { @srv.handle(NoRpcImplementation) }.to raise_error - end - it 'raises if a handler method is already registered' do @srv.handle(EchoService) expect { r.handle(EchoService) }.to raise_error @@ -349,6 +345,25 @@ describe GRPC::RpcServer do t.join end + it 'should return UNIMPLEMENTED on unimplemented methods', server: true do + @srv.handle(NoRpcImplementation) + t = Thread.new { @srv.run } + @srv.wait_till_running + req = EchoMsg.new + blk = proc do + cq = GRPC::Core::CompletionQueue.new + stub = GRPC::ClientStub.new(@host, cq, :this_channel_is_insecure, + **client_opts) + stub.request_response('/an_rpc', req, marshal, unmarshal) + end + expect(&blk).to raise_error do |error| + expect(error).to be_a(GRPC::BadStatus) + expect(error.code).to be(GRPC::Core::StatusCodes::UNIMPLEMENTED) + end + @srv.stop + t.join + end + it 'should handle multiple sequential requests', server: true do @srv.handle(EchoService) t = Thread.new { @srv.run } diff --git a/src/ruby/spec/generic/service_spec.rb b/src/ruby/spec/generic/service_spec.rb index 5e7b6c7abaf..76034e4f747 100644 --- a/src/ruby/spec/generic/service_spec.rb +++ b/src/ruby/spec/generic/service_spec.rb @@ -273,73 +273,4 @@ describe GenericService do end end end - - describe '#assert_rpc_descs_have_methods' do - it 'fails if there is no instance method for an rpc descriptor' do - c1 = Class.new do - include GenericService - rpc :AnRpc, GoodMsg, GoodMsg - end - expect { c1.assert_rpc_descs_have_methods }.to raise_error - - c2 = Class.new do - include GenericService - rpc :AnRpc, GoodMsg, GoodMsg - rpc :AnotherRpc, GoodMsg, GoodMsg - - def an_rpc - end - end - expect { c2.assert_rpc_descs_have_methods }.to raise_error - end - - it 'passes if there are corresponding methods for each descriptor' do - c = Class.new do - include GenericService - rpc :AnRpc, GoodMsg, GoodMsg - rpc :AServerStreamer, GoodMsg, stream(GoodMsg) - rpc :AClientStreamer, stream(GoodMsg), GoodMsg - rpc :ABidiStreamer, stream(GoodMsg), stream(GoodMsg) - - def an_rpc(_req, _call) - end - - def a_server_streamer(_req, _call) - end - - def a_client_streamer(_call) - end - - def a_bidi_streamer(_call) - end - end - expect { c.assert_rpc_descs_have_methods }.to_not raise_error - end - - it 'passes for subclasses of that include GenericService' do - base = Class.new do - include GenericService - rpc :AnRpc, GoodMsg, GoodMsg - - def an_rpc(_req, _call) - end - end - c = Class.new(base) - expect { c.assert_rpc_descs_have_methods }.to_not raise_error - expect(c.include?(GenericService)).to be(true) - end - - it 'passes if subclasses define the rpc methods' do - base = Class.new do - include GenericService - rpc :AnRpc, GoodMsg, GoodMsg - end - c = Class.new(base) do - def an_rpc(_req, _call) - end - end - expect { c.assert_rpc_descs_have_methods }.to_not raise_error - expect(c.include?(GenericService)).to be(true) - end - end end From ed4c3f17dd4fce0fa0cf6c310295e4ae5dff729b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 6 May 2016 15:39:08 -0700 Subject: [PATCH 128/155] disable some C# scenarios --- .../run_tests/performance/scenario_config.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 8b239951498..746945b617d 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -265,16 +265,18 @@ class CSharpLanguage: use_unconstrained_client=True, categories=[SMOKETEST]) - yield _ping_pong_scenario( - 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True) + # TODO(jtattermusch): scenario works locally but fails on jenkins + #yield _ping_pong_scenario( + # 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + # use_unconstrained_client=True) - yield _ping_pong_scenario( - 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', server_core_limit=1, async_server_threads=1, - categories=[SMOKETEST]) + # TODO(jtattermusch): scenario works locally but fails on jenkins + #yield _ping_pong_scenario( + # 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + # client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + # server_language='c++', server_core_limit=1, async_server_threads=1, + # categories=[SMOKETEST]) yield _ping_pong_scenario( 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', From 111c95df790324bac3174b17a620c8d0e439adc2 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Thu, 28 Apr 2016 16:45:30 -0700 Subject: [PATCH 129/155] Move podspec to version 0.13 --- gRPC.podspec | 2 +- templates/gRPC.podspec.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gRPC.podspec b/gRPC.podspec index 018306ca643..05ead228de8 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '0.12.0' + version = '0.13.0' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' diff --git a/templates/gRPC.podspec.template b/templates/gRPC.podspec.template index a9948a41df4..316208b47bc 100644 --- a/templates/gRPC.podspec.template +++ b/templates/gRPC.podspec.template @@ -54,7 +54,7 @@ %> Pod::Spec.new do |s| s.name = 'gRPC' - version = '0.12.0' + version = '0.13.0' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' From f58b4f41ff671cd8f38ca524cfedfe454ff72aeb Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 29 Apr 2016 03:07:23 -0700 Subject: [PATCH 130/155] Have Cocoapods read from submodules too, for nanopb --- gRPC.podspec | 3 ++- templates/gRPC.podspec.template | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gRPC.podspec b/gRPC.podspec index 05ead228de8..20db5ccd285 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -44,7 +44,8 @@ Pod::Spec.new do |s| s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } s.source = { :git => 'https://github.com/grpc/grpc.git', - :tag => "release-#{version.gsub(/\./, '_')}-objectivec-#{version}" } + :tag => "release-#{version.gsub(/\./, '_')}-objectivec-#{version}", + :submodules => true } s.ios.deployment_target = '7.1' diff --git a/templates/gRPC.podspec.template b/templates/gRPC.podspec.template index 316208b47bc..d822d6c64e7 100644 --- a/templates/gRPC.podspec.template +++ b/templates/gRPC.podspec.template @@ -62,7 +62,8 @@ s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } s.source = { :git => 'https://github.com/grpc/grpc.git', - :tag => "release-#{version.gsub(/\./, '_')}-objectivec-#{version}" } + :tag => "release-#{version.gsub(/\./, '_')}-objectivec-#{version}", + :submodules => true } s.ios.deployment_target = '7.1' From 4c2056683437bfcd3eed5b76e99a75e8a40e42b4 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 29 Apr 2016 03:08:10 -0700 Subject: [PATCH 131/155] Depend on latest BoringSSL --- gRPC.podspec | 2 +- templates/gRPC.podspec.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gRPC.podspec b/gRPC.podspec index 20db5ccd285..68d0fe480db 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -732,7 +732,7 @@ Pod::Spec.new do |s| ss.requires_arc = false ss.libraries = 'z' - ss.dependency 'BoringSSL', '~> 2.0' + ss.dependency 'BoringSSL', '~> 3.0' # ss.compiler_flags = '-GCC_WARN_INHIBIT_ALL_WARNINGS', '-w' end diff --git a/templates/gRPC.podspec.template b/templates/gRPC.podspec.template index d822d6c64e7..45b923c43e5 100644 --- a/templates/gRPC.podspec.template +++ b/templates/gRPC.podspec.template @@ -98,7 +98,7 @@ ss.requires_arc = false ss.libraries = 'z' - ss.dependency 'BoringSSL', '~> 2.0' + ss.dependency 'BoringSSL', '~> 3.0' # ss.compiler_flags = '-GCC_WARN_INHIBIT_ALL_WARNINGS', '-w' end From f742afccea93fdd4d274175a74805f36b524b951 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 29 Apr 2016 03:33:37 -0700 Subject: [PATCH 132/155] Update BoringSSL podspec version. --- src/objective-c/BoringSSL.podspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/BoringSSL.podspec b/src/objective-c/BoringSSL.podspec index 4a6df910a75..7d1de807169 100644 --- a/src/objective-c/BoringSSL.podspec +++ b/src/objective-c/BoringSSL.podspec @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL' - s.version = '2.0' + s.version = '3.0' s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google’s needs.' # Adapted from the homepage: s.description = <<-DESC @@ -67,7 +67,7 @@ Pod::Spec.new do |s| s.authors = 'Adam Langley', 'David Benjamin', 'Matt Braithwaite' s.source = { :git => 'https://boringssl.googlesource.com/boringssl', - :tag => 'version_for_cocoapods_2.0' } + :tag => 'version_for_cocoapods_3.0' } s.source_files = 'ssl/*.{h,c}', 'ssl/**/*.{h,c}', From f4639d485892341e66ef24521600fb20496c10cf Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 6 May 2016 16:41:36 -0700 Subject: [PATCH 133/155] Move podspec to version 0.14 --- gRPC.podspec | 2 +- templates/gRPC.podspec.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gRPC.podspec b/gRPC.podspec index 68d0fe480db..cfe7e57fb9b 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '0.13.0' + version = '0.14.0' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' diff --git a/templates/gRPC.podspec.template b/templates/gRPC.podspec.template index 45b923c43e5..979cb1ef8e8 100644 --- a/templates/gRPC.podspec.template +++ b/templates/gRPC.podspec.template @@ -54,7 +54,7 @@ %> Pod::Spec.new do |s| s.name = 'gRPC' - version = '0.13.0' + version = '0.14.0' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' From b4e82f7897c8a840249f39f17bc9afb986206588 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Fri, 6 May 2016 12:58:30 -0700 Subject: [PATCH 134/155] Fix windows linkage in Python grpcio-tools --- tools/distrib/python/grpcio_tools/README.rst | 4 ++- tools/distrib/python/grpcio_tools/setup.py | 14 +++++++++- tools/run_tests/build_artifact_python.bat | 29 +++++++++++++------- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst index 3be564ef5be..10d2fe8c309 100644 --- a/tools/distrib/python/grpcio_tools/README.rst +++ b/tools/distrib/python/grpcio_tools/README.rst @@ -46,7 +46,9 @@ From Source ~~~~~~~~~~~ Building from source requires that you have the Python headers (usually a -package named :code:`python-dev`) and Cython installed. +package named :code:`python-dev`) and Cython installed. It further requires a +GCC-like compiler to go smoothly; you can probably get it to work without +GCC-like stuff, but you may end up having a bad time. :: diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 0281c01796d..98f03c8d644 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -30,6 +30,7 @@ from distutils import extension import os import os.path +import shlex import sys import setuptools @@ -40,6 +41,16 @@ from setuptools.command import build_ext os.chdir(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.abspath('.')) +# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are +# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. +# We use these environment variables to thus get around that without locking +# ourselves in w.r.t. the multitude of operating systems this ought to build on. +# By default we assume a GCC-like compiler. +EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', + '-frtti -std=c++11')) +EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', + '-lpthread')) + import protoc_lib_deps import grpc_version @@ -60,7 +71,8 @@ def protoc_ext_module(): ], language='c++', define_macros=[('HAVE_PTHREAD', 1)], - extra_compile_args=['-lpthread', '-frtti', '-std=c++11'], + extra_compile_args=EXTRA_COMPILE_ARGS, + extra_link_args=EXTRA_LINK_ARGS, ) return plugin_ext diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat index 342469bba87..fea0275426f 100644 --- a/tools/run_tests/build_artifact_python.bat +++ b/tools/run_tests/build_artifact_python.bat @@ -41,7 +41,7 @@ copy /Y vsprojects\Release\grpc_dll.dll src\python\grpcio\grpc\_cython\_windows\ copy /Y vsprojects\x64\Release\grpc_dll.dll src\python\grpcio\grpc\_cython\_windows\grpc_c.64.python || goto :error -set PATH=C:\%1;C:\%1\scripts;%PATH% +set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;%PATH% pip install --upgrade six pip install --upgrade setuptools @@ -55,19 +55,28 @@ set GRPC_PYTHON_BUILD_WITH_CYTHON=1 python setup.py bdist_wheel @rem Build gRPC Python tools -set PATH=C:\msys64\mingw%2\bin;%PATH% -set CC=C:\msys64\mingw%2\bin\g++.exe -set CFLAGS=-fno-wrapv +@rem +@rem Because this is windows and *everything seems to hate Windows* we have to +@rem set all of these flags ourselves because Python won't help us (see the +@rem setup.py of the grpcio_tools project). +set GRPC_PYTHON_CFLAGS=-fno-wrapv -frtti -std=c++11 +@rem Further confusing things, MSYS2's mingw64 tries to dynamically link +@rem libgcc, libstdc++, and winpthreads. We have to override this or our +@rem extensions end up linking to MSYS2 DLLs, which the normal Python on +@rem Windows user won't have... and ON TOP OF THIS, there's MinGW's GCC default +@rem behavior of linking msvcrt.dll as the C runtime library, which we need to +@rem override so that Python's distutils doesn't link us against multiple C +@rem runtimes. +python -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])" > temp.txt +set /p PYTHON_MSVCR= Date: Sun, 8 May 2016 18:47:40 -0700 Subject: [PATCH 135/155] comment out the right scenario --- .../run_tests/performance/scenario_config.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 746945b617d..d3937096231 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -259,11 +259,12 @@ class CSharpLanguage: 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') - yield _ping_pong_scenario( - 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True, - categories=[SMOKETEST]) + # TODO(jtattermusch): scenario works locally but fails on jenkins + #yield _ping_pong_scenario( + # 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + # use_unconstrained_client=True, + # categories=[SMOKETEST]) # TODO(jtattermusch): scenario works locally but fails on jenkins #yield _ping_pong_scenario( @@ -271,12 +272,11 @@ class CSharpLanguage: # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', # use_unconstrained_client=True) - # TODO(jtattermusch): scenario works locally but fails on jenkins - #yield _ping_pong_scenario( - # 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - # client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - # server_language='c++', server_core_limit=1, async_server_threads=1, - # categories=[SMOKETEST]) + yield _ping_pong_scenario( + 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1, + categories=[SMOKETEST]) yield _ping_pong_scenario( 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', From 42342cbebb55ee168252accb0433b1bb6cc8e4a7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 9 May 2016 08:12:35 -0700 Subject: [PATCH 136/155] Revert "cronet wrapper code" --- BUILD | 80 --- Makefile | 7 - binding.gyp | 3 - build.yaml | 59 -- config.m4 | 5 - gRPC.podspec | 76 --- grpc.def | 1 - grpc.gemspec | 40 -- include/grpc/grpc_cronet.h | 51 -- package.xml | 40 -- .../client/secure/cronet_channel_create.c | 69 -- .../cronet/transport/cronet_api_dummy.c | 85 --- .../cronet/transport/cronet_transport.c | 640 ------------------ .../grpcio/grpc/_cython/imports.generated.c | 2 - .../grpcio/grpc/_cython/imports.generated.h | 4 - src/python/grpcio/grpc_core_dependencies.py | 3 - src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 - src/ruby/ext/grpc/rb_grpc_imports.generated.h | 4 - .../core/surface/public_headers_must_be_c89.c | 1 - .../objective_c/Cronet/cronet_c_for_grpc.h | 202 ------ tools/doxygen/Doxyfile.core | 1 - tools/doxygen/Doxyfile.core.internal | 40 -- tools/run_tests/sources_and_headers.json | 120 +--- vsprojects/vcxproj/grpc/grpc.vcxproj | 43 -- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 144 ---- 25 files changed, 1 insertion(+), 1721 deletions(-) delete mode 100644 include/grpc/grpc_cronet.h delete mode 100644 src/core/ext/transport/cronet/client/secure/cronet_channel_create.c delete mode 100644 src/core/ext/transport/cronet/transport/cronet_api_dummy.c delete mode 100644 src/core/ext/transport/cronet/transport/cronet_transport.c delete mode 100644 third_party/objective_c/Cronet/cronet_c_for_grpc.h diff --git a/BUILD b/BUILD index fae3596eec6..1da16504388 100644 --- a/BUILD +++ b/BUILD @@ -285,42 +285,6 @@ cc_library( "src/core/ext/client_config/subchannel_call_holder.h", "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", - "include/grpc/byte_buffer.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/status.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/sync.h", - "include/grpc/support/time.h", - "include/grpc/support/useful.h", - "src/core/lib/support/string.h", - "third_party/objective_c/Cronet/cronet_c_for_grpc.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/ext/census/aggregation.h", @@ -475,9 +439,6 @@ cc_library( "src/core/ext/client_config/uri_parser.c", "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c", "src/core/ext/transport/chttp2/client/insecure/channel_create.c", - "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", - "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", - "src/core/ext/transport/cronet/transport/cronet_transport.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/ext/lb_policy/pick_first/pick_first.c", @@ -522,7 +483,6 @@ cc_library( "include/grpc/impl/codegen/sync_posix.h", "include/grpc/impl/codegen/sync_win32.h", "include/grpc/impl/codegen/time.h", - "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "include/grpc/census.h", @@ -1500,9 +1460,6 @@ objc_library( "src/core/ext/client_config/uri_parser.c", "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c", "src/core/ext/transport/chttp2/client/insecure/channel_create.c", - "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", - "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", - "src/core/ext/transport/cronet/transport/cronet_transport.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/ext/lb_policy/pick_first/pick_first.c", @@ -1547,7 +1504,6 @@ objc_library( "include/grpc/impl/codegen/sync_posix.h", "include/grpc/impl/codegen/sync_win32.h", "include/grpc/impl/codegen/time.h", - "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "include/grpc/census.h", @@ -1675,42 +1631,6 @@ objc_library( "src/core/ext/client_config/subchannel_call_holder.h", "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", - "include/grpc/byte_buffer.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/status.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/sync.h", - "include/grpc/support/time.h", - "include/grpc/support/useful.h", - "src/core/lib/support/string.h", - "third_party/objective_c/Cronet/cronet_c_for_grpc.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/ext/census/aggregation.h", diff --git a/Makefile b/Makefile index a684ea8611a..ffaf7701447 100644 --- a/Makefile +++ b/Makefile @@ -2623,9 +2623,6 @@ LIBGRPC_SRC = \ src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ - src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ - src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ - src/core/ext/transport/cronet/transport/cronet_transport.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ @@ -2673,7 +2670,6 @@ PUBLIC_HEADERS_C += \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_win32.h \ include/grpc/impl/codegen/time.h \ - include/grpc/grpc_cronet.h \ include/grpc/grpc_security.h \ include/grpc/grpc_security_constants.h \ include/grpc/census.h \ @@ -14321,9 +14317,6 @@ ifneq ($(OPENSSL_DEP),) # otherwise parallel compilation will fail if a source is compiled first. src/core/ext/transport/chttp2/client/secure/secure_channel_create.c: $(OPENSSL_DEP) src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c: $(OPENSSL_DEP) -src/core/ext/transport/cronet/client/secure/cronet_channel_create.c: $(OPENSSL_DEP) -src/core/ext/transport/cronet/transport/cronet_api_dummy.c: $(OPENSSL_DEP) -src/core/ext/transport/cronet/transport/cronet_transport.c: $(OPENSSL_DEP) src/core/lib/http/httpcli_security_connector.c: $(OPENSSL_DEP) src/core/lib/security/b64.c: $(OPENSSL_DEP) src/core/lib/security/client_auth_filter.c: $(OPENSSL_DEP) diff --git a/binding.gyp b/binding.gyp index 12a745ffb01..4314ab7243d 100644 --- a/binding.gyp +++ b/binding.gyp @@ -709,9 +709,6 @@ 'src/core/ext/client_config/uri_parser.c', 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', - 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', - 'src/core/ext/transport/cronet/transport/cronet_api_dummy.c', - 'src/core/ext/transport/cronet/transport/cronet_transport.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/build.yaml b/build.yaml index 13916830c1e..5e47c084555 100644 --- a/build.yaml +++ b/build.yaml @@ -400,7 +400,6 @@ filegroups: - grpc_client_config - name: grpc_secure public_headers: - - include/grpc/grpc_cronet.h - include/grpc/grpc_security.h - include/grpc/grpc_security_constants.h headers: @@ -548,63 +547,6 @@ filegroups: - grpc_transport_chttp2 - grpc_base - grpc_secure -- name: grpc_transport_cronet_client_secure - headers: - - include/grpc/byte_buffer.h - - include/grpc/grpc.h - - include/grpc/impl/codegen/alloc.h - - include/grpc/impl/codegen/atm.h - - include/grpc/impl/codegen/atm_gcc_atomic.h - - include/grpc/impl/codegen/atm_gcc_sync.h - - include/grpc/impl/codegen/atm_win32.h - - include/grpc/impl/codegen/byte_buffer.h - - include/grpc/impl/codegen/compression_types.h - - include/grpc/impl/codegen/connectivity_state.h - - include/grpc/impl/codegen/grpc_types.h - - include/grpc/impl/codegen/log.h - - include/grpc/impl/codegen/port_platform.h - - include/grpc/impl/codegen/propagation_bits.h - - include/grpc/impl/codegen/slice.h - - include/grpc/impl/codegen/slice_buffer.h - - include/grpc/impl/codegen/status.h - - include/grpc/impl/codegen/sync.h - - include/grpc/impl/codegen/sync_generic.h - - include/grpc/impl/codegen/sync_posix.h - - include/grpc/impl/codegen/sync_win32.h - - include/grpc/impl/codegen/time.h - - include/grpc/status.h - - include/grpc/support/alloc.h - - include/grpc/support/atm.h - - include/grpc/support/host_port.h - - include/grpc/support/log.h - - include/grpc/support/port_platform.h - - include/grpc/support/slice.h - - include/grpc/support/slice_buffer.h - - include/grpc/support/string_util.h - - include/grpc/support/sync.h - - include/grpc/support/time.h - - include/grpc/support/useful.h - - src/core/ext/transport/chttp2/transport/incoming_metadata.h - - src/core/lib/channel/channel_stack.h - - src/core/lib/channel/context.h - - src/core/lib/debug/trace.h - - src/core/lib/iomgr/closure.h - - src/core/lib/iomgr/exec_ctx.h - - src/core/lib/iomgr/pollset.h - - src/core/lib/iomgr/pollset_set.h - - src/core/lib/support/string.h - - src/core/lib/surface/channel.h - - src/core/lib/surface/channel_stack_type.h - - src/core/lib/transport/byte_stream.h - - src/core/lib/transport/metadata.h - - src/core/lib/transport/metadata_batch.h - - src/core/lib/transport/transport.h - - src/core/lib/transport/transport_impl.h - - third_party/objective_c/Cronet/cronet_c_for_grpc.h - src: - - src/core/ext/transport/cronet/client/secure/cronet_channel_create.c - - src/core/ext/transport/cronet/transport/cronet_api_dummy.c - - src/core/ext/transport/cronet/transport/cronet_transport.c - name: nanopb headers: - third_party/nanopb/pb.h @@ -792,7 +734,6 @@ libs: - grpc_transport_chttp2_client_secure - grpc_transport_chttp2_server_insecure - grpc_transport_chttp2_client_insecure - - grpc_transport_cronet_client_secure - grpc_lb_policy_grpclb - grpc_lb_policy_pick_first - grpc_lb_policy_round_robin diff --git a/config.m4 b/config.m4 index 5259e679ba9..74f9ad242ab 100644 --- a/config.m4 +++ b/config.m4 @@ -228,9 +228,6 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ - src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ - src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ - src/core/ext/transport/cronet/transport/cronet_transport.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ @@ -569,8 +566,6 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/server/insecure) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/server/secure) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/transport) - PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/cronet/client/secure) - PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/cronet/transport) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/channel) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/compression) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/debug) diff --git a/gRPC.podspec b/gRPC.podspec index 018306ca643..569f89bf7c0 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -287,42 +287,6 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_call_holder.h', 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', - 'include/grpc/byte_buffer.h', - 'include/grpc/grpc.h', - 'include/grpc/impl/codegen/alloc.h', - 'include/grpc/impl/codegen/atm.h', - 'include/grpc/impl/codegen/atm_gcc_atomic.h', - 'include/grpc/impl/codegen/atm_gcc_sync.h', - 'include/grpc/impl/codegen/atm_win32.h', - 'include/grpc/impl/codegen/byte_buffer.h', - 'include/grpc/impl/codegen/compression_types.h', - 'include/grpc/impl/codegen/connectivity_state.h', - 'include/grpc/impl/codegen/grpc_types.h', - 'include/grpc/impl/codegen/log.h', - 'include/grpc/impl/codegen/port_platform.h', - 'include/grpc/impl/codegen/propagation_bits.h', - 'include/grpc/impl/codegen/slice.h', - 'include/grpc/impl/codegen/slice_buffer.h', - 'include/grpc/impl/codegen/status.h', - 'include/grpc/impl/codegen/sync.h', - 'include/grpc/impl/codegen/sync_generic.h', - 'include/grpc/impl/codegen/sync_posix.h', - 'include/grpc/impl/codegen/sync_win32.h', - 'include/grpc/impl/codegen/time.h', - 'include/grpc/status.h', - 'include/grpc/support/alloc.h', - 'include/grpc/support/atm.h', - 'include/grpc/support/host_port.h', - 'include/grpc/support/log.h', - 'include/grpc/support/port_platform.h', - 'include/grpc/support/slice.h', - 'include/grpc/support/slice_buffer.h', - 'include/grpc/support/string_util.h', - 'include/grpc/support/sync.h', - 'include/grpc/support/time.h', - 'include/grpc/support/useful.h', - 'src/core/lib/support/string.h', - 'third_party/objective_c/Cronet/cronet_c_for_grpc.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', 'third_party/nanopb/pb.h', @@ -361,7 +325,6 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_posix.h', 'include/grpc/impl/codegen/sync_win32.h', 'include/grpc/impl/codegen/time.h', - 'include/grpc/grpc_cronet.h', 'include/grpc/grpc_security.h', 'include/grpc/grpc_security_constants.h', 'include/grpc/census.h', @@ -511,9 +474,6 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/uri_parser.c', 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', - 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', - 'src/core/ext/transport/cronet/transport/cronet_api_dummy.c', - 'src/core/ext/transport/cronet/transport/cronet_transport.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', @@ -670,42 +630,6 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_call_holder.h', 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', - 'include/grpc/byte_buffer.h', - 'include/grpc/grpc.h', - 'include/grpc/impl/codegen/alloc.h', - 'include/grpc/impl/codegen/atm.h', - 'include/grpc/impl/codegen/atm_gcc_atomic.h', - 'include/grpc/impl/codegen/atm_gcc_sync.h', - 'include/grpc/impl/codegen/atm_win32.h', - 'include/grpc/impl/codegen/byte_buffer.h', - 'include/grpc/impl/codegen/compression_types.h', - 'include/grpc/impl/codegen/connectivity_state.h', - 'include/grpc/impl/codegen/grpc_types.h', - 'include/grpc/impl/codegen/log.h', - 'include/grpc/impl/codegen/port_platform.h', - 'include/grpc/impl/codegen/propagation_bits.h', - 'include/grpc/impl/codegen/slice.h', - 'include/grpc/impl/codegen/slice_buffer.h', - 'include/grpc/impl/codegen/status.h', - 'include/grpc/impl/codegen/sync.h', - 'include/grpc/impl/codegen/sync_generic.h', - 'include/grpc/impl/codegen/sync_posix.h', - 'include/grpc/impl/codegen/sync_win32.h', - 'include/grpc/impl/codegen/time.h', - 'include/grpc/status.h', - 'include/grpc/support/alloc.h', - 'include/grpc/support/atm.h', - 'include/grpc/support/host_port.h', - 'include/grpc/support/log.h', - 'include/grpc/support/port_platform.h', - 'include/grpc/support/slice.h', - 'include/grpc/support/slice_buffer.h', - 'include/grpc/support/string_util.h', - 'include/grpc/support/sync.h', - 'include/grpc/support/time.h', - 'include/grpc/support/useful.h', - 'src/core/lib/support/string.h', - 'third_party/objective_c/Cronet/cronet_c_for_grpc.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', 'third_party/nanopb/pb.h', diff --git a/grpc.def b/grpc.def index 09a94a6cd0c..61948ed1b89 100644 --- a/grpc.def +++ b/grpc.def @@ -87,7 +87,6 @@ EXPORTS grpc_header_nonbin_value_is_legal grpc_is_binary_header grpc_call_error_to_string - grpc_cronet_secure_channel_create grpc_auth_property_iterator_next grpc_auth_context_property_iterator grpc_auth_context_peer_identity diff --git a/grpc.gemspec b/grpc.gemspec index ace28715dcd..475fc990ad4 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -169,7 +169,6 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/impl/codegen/sync_posix.h ) s.files += %w( include/grpc/impl/codegen/sync_win32.h ) s.files += %w( include/grpc/impl/codegen/time.h ) - s.files += %w( include/grpc/grpc_cronet.h ) s.files += %w( include/grpc/grpc_security.h ) s.files += %w( include/grpc/grpc_security_constants.h ) s.files += %w( include/grpc/census.h ) @@ -297,42 +296,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/client_config/subchannel_call_holder.h ) s.files += %w( src/core/ext/client_config/subchannel_index.h ) s.files += %w( src/core/ext/client_config/uri_parser.h ) - s.files += %w( include/grpc/byte_buffer.h ) - s.files += %w( include/grpc/grpc.h ) - s.files += %w( include/grpc/impl/codegen/alloc.h ) - s.files += %w( include/grpc/impl/codegen/atm.h ) - s.files += %w( include/grpc/impl/codegen/atm_gcc_atomic.h ) - s.files += %w( include/grpc/impl/codegen/atm_gcc_sync.h ) - s.files += %w( include/grpc/impl/codegen/atm_win32.h ) - s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) - s.files += %w( include/grpc/impl/codegen/compression_types.h ) - s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) - s.files += %w( include/grpc/impl/codegen/grpc_types.h ) - s.files += %w( include/grpc/impl/codegen/log.h ) - s.files += %w( include/grpc/impl/codegen/port_platform.h ) - s.files += %w( include/grpc/impl/codegen/propagation_bits.h ) - s.files += %w( include/grpc/impl/codegen/slice.h ) - s.files += %w( include/grpc/impl/codegen/slice_buffer.h ) - s.files += %w( include/grpc/impl/codegen/status.h ) - s.files += %w( include/grpc/impl/codegen/sync.h ) - s.files += %w( include/grpc/impl/codegen/sync_generic.h ) - s.files += %w( include/grpc/impl/codegen/sync_posix.h ) - s.files += %w( include/grpc/impl/codegen/sync_win32.h ) - s.files += %w( include/grpc/impl/codegen/time.h ) - s.files += %w( include/grpc/status.h ) - s.files += %w( include/grpc/support/alloc.h ) - s.files += %w( include/grpc/support/atm.h ) - s.files += %w( include/grpc/support/host_port.h ) - s.files += %w( include/grpc/support/log.h ) - s.files += %w( include/grpc/support/port_platform.h ) - s.files += %w( include/grpc/support/slice.h ) - s.files += %w( include/grpc/support/slice_buffer.h ) - s.files += %w( include/grpc/support/string_util.h ) - s.files += %w( include/grpc/support/sync.h ) - s.files += %w( include/grpc/support/time.h ) - s.files += %w( include/grpc/support/useful.h ) - s.files += %w( src/core/lib/support/string.h ) - s.files += %w( third_party/objective_c/Cronet/cronet_c_for_grpc.h ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.h ) s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h ) s.files += %w( third_party/nanopb/pb.h ) @@ -491,9 +454,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/client_config/uri_parser.c ) s.files += %w( src/core/ext/transport/chttp2/server/insecure/server_chttp2.c ) s.files += %w( src/core/ext/transport/chttp2/client/insecure/channel_create.c ) - s.files += %w( src/core/ext/transport/cronet/client/secure/cronet_channel_create.c ) - s.files += %w( src/core/ext/transport/cronet/transport/cronet_api_dummy.c ) - s.files += %w( src/core/ext/transport/cronet/transport/cronet_transport.c ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.c ) s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c ) s.files += %w( third_party/nanopb/pb_common.c ) diff --git a/include/grpc/grpc_cronet.h b/include/grpc/grpc_cronet.h deleted file mode 100644 index 295e0f55e80..00000000000 --- a/include/grpc/grpc_cronet.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef GRPC_GRPC_CRONET_H -#define GRPC_GRPC_CRONET_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( - void *engine, const char *target, const grpc_channel_args *args, - void *reserved); - -#ifdef __cplusplus -} -#endif - -#endif /* GRPC_GRPC_CRONET_H */ diff --git a/package.xml b/package.xml index 152d5d61904..feb27175a9b 100644 --- a/package.xml +++ b/package.xml @@ -176,7 +176,6 @@ - @@ -304,42 +303,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -498,9 +461,6 @@ - - - diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c deleted file mode 100644 index df1acddcc08..00000000000 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c +++ /dev/null @@ -1,69 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include - -#include -#include - -#include -#include - -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/transport/transport_impl.h" - -// Cronet transport object -typedef struct cronet_transport { - grpc_transport base; // must be first element in this structure - void *engine; - char *host; -} cronet_transport; - -extern grpc_transport_vtable grpc_cronet_vtable; - -GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( - void *engine, const char *target, const grpc_channel_args *args, - void *reserved) { - cronet_transport *ct = gpr_malloc(sizeof(cronet_transport)); - ct->base.vtable = &grpc_cronet_vtable; - ct->engine = engine; - ct->host = gpr_malloc(strlen(target) + 1); - strcpy(ct->host, target); - gpr_log(GPR_DEBUG, - "grpc_create_cronet_transport: cronet_engine = %p, target=%s", engine, - ct->host); - - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - return grpc_channel_create(&exec_ctx, target, args, - GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); -} diff --git a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c deleted file mode 100644 index 687026c9fde..00000000000 --- a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c +++ /dev/null @@ -1,85 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* This file has empty implementation of all the functions exposed by the cronet -library, so we can build it in all environments */ - -#include - -#include - -#include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" - -#ifdef GRPC_COMPILE_WITH_CRONET -/* link with the real CRONET library in the build system */ -#else -/* Dummy implementation of cronet API just to test for build-ability */ -cronet_bidirectional_stream* cronet_bidirectional_stream_create( - cronet_engine* engine, void* annotation, - cronet_bidirectional_stream_callback* callback) { - GPR_ASSERT(0); - return NULL; -} - -int cronet_bidirectional_stream_destroy(cronet_bidirectional_stream* stream) { - GPR_ASSERT(0); - return 0; -} - -int cronet_bidirectional_stream_start( - cronet_bidirectional_stream* stream, const char* url, int priority, - const char* method, const cronet_bidirectional_stream_header_array* headers, - bool end_of_stream) { - GPR_ASSERT(0); - return 0; -} - -int cronet_bidirectional_stream_read(cronet_bidirectional_stream* stream, - char* buffer, int capacity) { - GPR_ASSERT(0); - return 0; -} - -int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream, - const char* buffer, int count, - bool end_of_stream) { - GPR_ASSERT(0); - return 0; -} - -int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream) { - GPR_ASSERT(0); - return 0; -} - -#endif /* GRPC_COMPILE_WITH_CRONET */ diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c deleted file mode 100644 index 5bb085195c6..00000000000 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ /dev/null @@ -1,640 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "src/core/ext/transport/chttp2/transport/incoming_metadata.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/support/string.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/transport/metadata_batch.h" -#include "src/core/lib/transport/transport_impl.h" -#include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" - -#define GRPC_HEADER_SIZE_IN_BYTES 5 - -// Global flag that gets set with GRPC_TRACE env variable -int grpc_cronet_trace = 1; - -// Cronet transport object -struct grpc_cronet_transport { - grpc_transport base; /* must be first element in this structure */ - cronet_engine *engine; - char *host; -}; - -typedef struct grpc_cronet_transport grpc_cronet_transport; - -enum send_state { - CRONET_SEND_IDLE = 0, - CRONET_REQ_STARTED, - CRONET_SEND_HEADER, - CRONET_WRITE, - CRONET_WRITE_COMPLETED, -}; - -enum recv_state { - CRONET_RECV_IDLE = 0, - CRONET_RECV_READ_LENGTH, - CRONET_RECV_READ_DATA, - CRONET_RECV_CLOSED, -}; - -static const char *recv_state_name[] = { - "CRONET_RECV_IDLE", "CRONET_RECV_READ_LENGTH", "CRONET_RECV_READ_DATA,", - "CRONET_RECV_CLOSED"}; - -// Enum that identifies calling function. -enum e_caller { - PERFORM_STREAM_OP, - ON_READ_COMPLETE, - ON_RESPONSE_HEADERS_RECEIVED, - ON_RESPONSE_TRAILERS_RECEIVED -}; - -enum callback_id { - CB_SEND_INITIAL_METADATA = 0, - CB_SEND_MESSAGE, - CB_SEND_TRAILING_METADATA, - CB_RECV_MESSAGE, - CB_RECV_INITIAL_METADATA, - CB_RECV_TRAILING_METADATA, - CB_NUM_CALLBACKS -}; - -struct stream_obj { - // we store received bytes here as they trickle in. - gpr_slice_buffer write_slice_buffer; - cronet_bidirectional_stream *cbs; - gpr_slice slice; - gpr_slice_buffer read_slice_buffer; - struct grpc_slice_buffer_stream sbs; - char *read_buffer; - int remaining_read_bytes; - int total_read_bytes; - - char *write_buffer; - size_t write_buffer_size; - - // Hold the URL - char *url; - - bool response_headers_received; - bool read_requested; - bool response_trailers_received; - bool read_closed; - - // Recv message stuff - grpc_byte_buffer **recv_message; - // Initial metadata stuff - grpc_metadata_batch *recv_initial_metadata; - // Trailing metadata stuff - grpc_metadata_batch *recv_trailing_metadata; - grpc_chttp2_incoming_metadata_buffer imb; - - // This mutex protects receive state machine execution - gpr_mu recv_mu; - // we can queue up up to 2 callbacks for each OP - grpc_closure *callback_list[CB_NUM_CALLBACKS][2]; - - // storage for header - cronet_bidirectional_stream_header *headers; - uint32_t num_headers; - cronet_bidirectional_stream_header_array header_array; - // state tracking - enum recv_state cronet_recv_state; - enum send_state cronet_send_state; -}; - -typedef struct stream_obj stream_obj; - -static void next_send_step(stream_obj *s); -static void next_recv_step(stream_obj *s, enum e_caller caller); - -static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_pollset *pollset) {} - -static void enqueue_callbacks(grpc_closure *callback_list[]) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - if (callback_list[0]) { - grpc_exec_ctx_enqueue(&exec_ctx, callback_list[0], true, NULL); - callback_list[0] = NULL; - } - if (callback_list[1]) { - grpc_exec_ctx_enqueue(&exec_ctx, callback_list[1], true, NULL); - callback_list[1] = NULL; - } - grpc_exec_ctx_finish(&exec_ctx); -} - -static void on_canceled(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_canceled %p", stream); - } -} - -static void on_failed(cronet_bidirectional_stream *stream, int net_error) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_failed %p, error = %d", stream, net_error); - } -} - -static void on_succeeded(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_succeeded %p", stream); - } -} - -static void on_response_trailers_received( - cronet_bidirectional_stream *stream, - const cronet_bidirectional_stream_header_array *trailers) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_trailers_received"); - } - stream_obj *s = (stream_obj *)stream->annotation; - - memset(&s->imb, 0, sizeof(s->imb)); - grpc_chttp2_incoming_metadata_buffer_init(&s->imb); - unsigned int i = 0; - for (i = 0; i < trailers->count; i++) { - grpc_chttp2_incoming_metadata_buffer_add( - &s->imb, grpc_mdelem_from_metadata_strings( - grpc_mdstr_from_string(trailers->headers[i].key), - grpc_mdstr_from_string(trailers->headers[i].value))); - } - s->response_trailers_received = true; - next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); -} - -static void on_write_completed(cronet_bidirectional_stream *stream, - const char *data) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_write_completed"); - } - stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_SEND_MESSAGE]); - s->cronet_send_state = CRONET_WRITE_COMPLETED; - next_send_step(s); -} - -static void process_recv_message(stream_obj *s, const uint8_t *recv_data) { - gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes); - uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); - memcpy(dst_p, recv_data, (size_t)s->total_read_bytes); - gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice); - grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0); - *s->recv_message = (grpc_byte_buffer *)&s->sbs; -} - -static int parse_grpc_header(const uint8_t *data) { - const uint8_t *p = data + 1; - int length = 0; - length |= ((uint8_t)*p++) << 24; - length |= ((uint8_t)*p++) << 16; - length |= ((uint8_t)*p++) << 8; - length |= ((uint8_t)*p++); - return length; -} - -static void on_read_completed(cronet_bidirectional_stream *stream, char *data, - int count) { - stream_obj *s = (stream_obj *)stream->annotation; - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_read_completed count=%d, total=%d, remaining=%d", - count, s->total_read_bytes, s->remaining_read_bytes); - } - if (count > 0) { - GPR_ASSERT(s->recv_message); - s->remaining_read_bytes -= count; - next_recv_step(s, ON_READ_COMPLETE); - } else { - s->read_closed = true; - next_recv_step(s, ON_READ_COMPLETE); - } -} - -static void on_response_headers_received( - cronet_bidirectional_stream *stream, - const cronet_bidirectional_stream_header_array *headers, - const char *negotiated_protocol) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_headers_received"); - } - stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_RECV_INITIAL_METADATA]); - s->response_headers_received = true; - next_recv_step(s, ON_RESPONSE_HEADERS_RECEIVED); -} - -static void on_request_headers_sent(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_request_headers_sent"); - } - stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_SEND_INITIAL_METADATA]); - s->cronet_send_state = CRONET_SEND_HEADER; - next_send_step(s); -} - -// Callback function pointers (invoked by cronet in response to events) -static cronet_bidirectional_stream_callback callbacks = { - on_request_headers_sent, - on_response_headers_received, - on_read_completed, - on_write_completed, - on_response_trailers_received, - on_succeeded, - on_failed, - on_canceled}; - -static void invoke_closing_callback(stream_obj *s) { - grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, - s->recv_trailing_metadata); - if (s->callback_list[CB_RECV_TRAILING_METADATA]) { - enqueue_callbacks(s->callback_list[CB_RECV_TRAILING_METADATA]); - } -} - -static void set_recv_state(stream_obj *s, enum recv_state state) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "next_state = %s", recv_state_name[state]); - } - s->cronet_recv_state = state; -} - -// This is invoked from perform_stream_op, and all on_xxxx callbacks. -static void next_recv_step(stream_obj *s, enum e_caller caller) { - gpr_mu_lock(&s->recv_mu); - switch (s->cronet_recv_state) { - case CRONET_RECV_IDLE: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE"); - } - if (caller == PERFORM_STREAM_OP || - caller == ON_RESPONSE_HEADERS_RECEIVED) { - if (s->read_closed && s->response_trailers_received) { - invoke_closing_callback(s); - set_recv_state(s, CRONET_RECV_CLOSED); - } else if (s->response_headers_received == true && - s->read_requested == true) { - set_recv_state(s, CRONET_RECV_READ_LENGTH); - s->total_read_bytes = s->remaining_read_bytes = - GRPC_HEADER_SIZE_IN_BYTES; - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); - } - cronet_bidirectional_stream_read(s->cbs, s->read_buffer, - s->remaining_read_bytes); - } - } - break; - case CRONET_RECV_READ_LENGTH: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_LENGTH"); - } - if (caller == ON_READ_COMPLETE) { - if (s->read_closed) { - invoke_closing_callback(s); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); - set_recv_state(s, CRONET_RECV_CLOSED); - } else { - GPR_ASSERT(s->remaining_read_bytes == 0); - set_recv_state(s, CRONET_RECV_READ_DATA); - s->total_read_bytes = s->remaining_read_bytes = - parse_grpc_header((const uint8_t *)s->read_buffer); - s->read_buffer = - gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes); - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); - } - cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer, - s->remaining_read_bytes); - } - } - break; - case CRONET_RECV_READ_DATA: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_DATA"); - } - if (caller == ON_READ_COMPLETE) { - if (s->remaining_read_bytes > 0) { - int offset = s->total_read_bytes - s->remaining_read_bytes; - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); - } - cronet_bidirectional_stream_read( - s->cbs, (char *)s->read_buffer + offset, s->remaining_read_bytes); - } else { - gpr_slice_buffer_init(&s->read_slice_buffer); - uint8_t *p = (uint8_t *)s->read_buffer; - process_recv_message(s, p); - set_recv_state(s, CRONET_RECV_IDLE); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); - } - } - break; - case CRONET_RECV_CLOSED: - break; - default: - GPR_ASSERT(0); // Should not reach here - break; - } - gpr_mu_unlock(&s->recv_mu); -} - -// This function takes the data from s->write_slice_buffer and assembles into -// a contiguous byte stream with 5 byte gRPC header prepended. -static void create_grpc_frame(stream_obj *s) { - gpr_slice slice = gpr_slice_buffer_take_first(&s->write_slice_buffer); - uint8_t *raw_data = GPR_SLICE_START_PTR(slice); - size_t length = GPR_SLICE_LENGTH(slice); - s->write_buffer_size = length + GRPC_HEADER_SIZE_IN_BYTES; - s->write_buffer = gpr_realloc(s->write_buffer, s->write_buffer_size); - uint8_t *p = (uint8_t *)s->write_buffer; - // Append 5 byte header - *p++ = 0; - *p++ = (uint8_t)(length >> 24); - *p++ = (uint8_t)(length >> 16); - *p++ = (uint8_t)(length >> 8); - *p++ = (uint8_t)(length); - // append actual data - memcpy(p, raw_data, length); -} - -static void do_write(stream_obj *s) { - gpr_slice_buffer *sb = &s->write_slice_buffer; - GPR_ASSERT(sb->count <= 1); - if (sb->count > 0) { - create_grpc_frame(s); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); - } - cronet_bidirectional_stream_write(s->cbs, s->write_buffer, - (int)s->write_buffer_size, false); - } -} - -// -static void next_send_step(stream_obj *s) { - switch (s->cronet_send_state) { - case CRONET_SEND_IDLE: - GPR_ASSERT( - s->cbs); // cronet_bidirectional_stream is not initialized yet. - s->cronet_send_state = CRONET_REQ_STARTED; - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", s->url); - } - cronet_bidirectional_stream_start(s->cbs, s->url, 0, "POST", - &s->header_array, false); - // we no longer need the memory that was allocated earlier. - gpr_free(s->header_array.headers); - break; - case CRONET_SEND_HEADER: - do_write(s); - s->cronet_send_state = CRONET_WRITE; - break; - case CRONET_WRITE_COMPLETED: - do_write(s); - break; - default: - GPR_ASSERT(0); - break; - } -} - -static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, - const char *host, - stream_obj *s) { - grpc_linked_mdelem *curr = head; - // Walk the linked list and get number of header fields - uint32_t num_headers_available = 0; - while (curr != NULL) { - curr = curr->next; - num_headers_available++; - } - // Allocate enough memory - s->headers = (cronet_bidirectional_stream_header *)gpr_malloc( - sizeof(cronet_bidirectional_stream_header) * num_headers_available); - - // Walk the linked list again, this time copying the header fields. - // s->num_headers - // can be less than num_headers_available, as some headers are not used for - // cronet - curr = head; - s->num_headers = 0; - while (s->num_headers < num_headers_available) { - grpc_mdelem *mdelem = curr->md; - curr = curr->next; - const char *key = grpc_mdstr_as_c_string(mdelem->key); - const char *value = grpc_mdstr_as_c_string(mdelem->value); - if (strcmp(key, ":scheme") == 0 || strcmp(key, ":method") == 0 || - strcmp(key, ":authority") == 0) { - // Cronet populates these fields on its own. - continue; - } - if (strcmp(key, ":path") == 0) { - // Create URL by appending :path value to the hostname - gpr_asprintf(&s->url, "https://%s%s", host, value); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "extracted URL = %s", s->url); - } - continue; - } - s->headers[s->num_headers].key = key; - s->headers[s->num_headers].value = value; - s->num_headers++; - if (curr == NULL) { - break; - } - } -} - -static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_transport_stream_op *op) { - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - GPR_ASSERT(ct->engine); - stream_obj *s = (stream_obj *)gs; - if (op->recv_trailing_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - recv_trailing_metadata: on_complete=%p", - op->on_complete); - } - s->recv_trailing_metadata = op->recv_trailing_metadata; - GPR_ASSERT(!s->callback_list[CB_RECV_TRAILING_METADATA][0]); - s->callback_list[CB_RECV_TRAILING_METADATA][0] = op->on_complete; - } - if (op->recv_message) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - recv_message: on_complete=%p", - op->on_complete); - } - s->recv_message = (grpc_byte_buffer **)op->recv_message; - GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][0]); - GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][1]); - s->callback_list[CB_RECV_MESSAGE][0] = op->recv_message_ready; - s->callback_list[CB_RECV_MESSAGE][1] = op->on_complete; - s->read_requested = true; - next_recv_step(s, PERFORM_STREAM_OP); - } - if (op->recv_initial_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - recv_initial_metadata:=%p", - op->on_complete); - } - s->recv_initial_metadata = op->recv_initial_metadata; - GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][0]); - GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][1]); - s->callback_list[CB_RECV_INITIAL_METADATA][0] = - op->recv_initial_metadata_ready; - s->callback_list[CB_RECV_INITIAL_METADATA][1] = op->on_complete; - } - if (op->send_initial_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - send_initial_metadata: on_complete=%p", - op->on_complete); - } - s->num_headers = 0; - convert_metadata_to_cronet_headers(op->send_initial_metadata->list.head, - ct->host, s); - s->header_array.count = s->num_headers; - s->header_array.capacity = s->num_headers; - s->header_array.headers = s->headers; - GPR_ASSERT(!s->callback_list[CB_SEND_INITIAL_METADATA][0]); - s->callback_list[CB_SEND_INITIAL_METADATA][0] = op->on_complete; - } - if (op->send_message) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - send_message: on_complete=%p", - op->on_complete); - } - grpc_byte_stream_next(exec_ctx, op->send_message, &s->slice, - op->send_message->length, NULL); - // Check that compression flag is not ON. We don't support compression yet. - // TODO (makdharma): add compression support - GPR_ASSERT(op->send_message->flags == 0); - gpr_slice_buffer_add(&s->write_slice_buffer, s->slice); - if (s->cbs == NULL) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); - } - s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); - GPR_ASSERT(s->cbs); - s->read_closed = false; - s->response_trailers_received = false; - s->response_headers_received = false; - s->cronet_send_state = CRONET_SEND_IDLE; - s->cronet_recv_state = CRONET_RECV_IDLE; - } - GPR_ASSERT(!s->callback_list[CB_SEND_MESSAGE][0]); - s->callback_list[CB_SEND_MESSAGE][0] = op->on_complete; - next_send_step(s); - } - if (op->send_trailing_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - send_trailing_metadata: on_complete=%p", - op->on_complete); - } - GPR_ASSERT(!s->callback_list[CB_SEND_TRAILING_METADATA][0]); - s->callback_list[CB_SEND_TRAILING_METADATA][0] = op->on_complete; - if (s->cbs) { - // Send an "empty" write to the far end to signal that we're done. - // This will induce the server to send down trailers. - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); - } - cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); - } else { - // We never created a stream. This was probably an empty request. - invoke_closing_callback(s); - } - } -} - -static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_stream_refcount *refcount, - const void *server_data) { - stream_obj *s = (stream_obj *)gs; - memset(s->callback_list, 0, sizeof(s->callback_list)); - s->cbs = NULL; - gpr_mu_init(&s->recv_mu); - s->read_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); - s->write_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); - gpr_slice_buffer_init(&s->write_slice_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_transport - init_stream"); - } - return 0; -} - -static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, void *and_free_memory) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Destroy stream"); - } - stream_obj *s = (stream_obj *)gs; - s->cbs = NULL; - gpr_free(s->read_buffer); - gpr_free(s->write_buffer); - gpr_free(s->url); - gpr_mu_destroy(&s->recv_mu); - if (and_free_memory) { - gpr_free(and_free_memory); - } -} - -static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - gpr_free(ct->host); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Destroy transport"); - } -} - -const grpc_transport_vtable grpc_cronet_vtable = { - sizeof(stream_obj), "cronet_http", init_stream, - set_pollset_do_nothing, perform_stream_op, NULL, - destroy_stream, destroy_transport, NULL}; diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c index 09551472b5f..f0a40dbb35c 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.c +++ b/src/python/grpcio/grpc/_cython/imports.generated.c @@ -125,7 +125,6 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; -grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -396,7 +395,6 @@ void pygrpc_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); - grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h index 54c8aaad13e..d5e810b7cf4 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.h +++ b/src/python/grpcio/grpc/_cython/imports.generated.h @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -326,9 +325,6 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import -typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); -extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; -#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 5314329c2cb..dab62530aac 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -222,9 +222,6 @@ CORE_SOURCE_FILES = [ 'src/core/ext/client_config/uri_parser.c', 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', - 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', - 'src/core/ext/transport/cronet/transport/cronet_api_dummy.c', - 'src/core/ext/transport/cronet/transport/cronet_transport.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index cebbe8c40fe..bc43f9d36b5 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -125,7 +125,6 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; -grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -392,7 +391,6 @@ void grpc_rb_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); - grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index d7ea6c574c8..b67361ca25b 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -326,9 +325,6 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import -typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); -extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; -#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index fd6ff2c26f5..3eeb55d033d 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include diff --git a/third_party/objective_c/Cronet/cronet_c_for_grpc.h b/third_party/objective_c/Cronet/cronet_c_for_grpc.h deleted file mode 100644 index 15a511aebd0..00000000000 --- a/third_party/objective_c/Cronet/cronet_c_for_grpc.h +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ -#define COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -/* Cronet Engine API. */ - -/* Opaque object representing Cronet Engine. Created and configured outside - * of this API to facilitate sharing with other components */ -typedef struct cronet_engine { void* obj; } cronet_engine; - -void cronet_engine_add_quic_hint(cronet_engine* engine, - const char* host, - int port, - int alternate_port); - -/* Cronet Bidirectional Stream API */ - -/* Opaque object representing Cronet Bidirectional Stream. */ -typedef struct cronet_bidirectional_stream { - void* obj; - void* annotation; -} cronet_bidirectional_stream; - -/* A single request or response header element. */ -typedef struct cronet_bidirectional_stream_header { - const char* key; - const char* value; -} cronet_bidirectional_stream_header; - -/* Array of request or response headers or trailers. */ -typedef struct cronet_bidirectional_stream_header_array { - size_t count; - size_t capacity; - cronet_bidirectional_stream_header* headers; -} cronet_bidirectional_stream_header_array; - -/* Set of callbacks used to receive callbacks from bidirectional stream. */ -typedef struct cronet_bidirectional_stream_callback { - /* Invoked when request headers are sent. Indicates that stream has initiated - * the request. Consumer may call cronet_bidirectional_stream_write() to start - * writing data. - */ - void (*on_request_headers_sent)(cronet_bidirectional_stream* stream); - - /* Invoked when initial response headers are received. - * Consumer must call cronet_bidirectional_stream_read() to start reading. - * Consumer may call cronet_bidirectional_stream_write() to start writing or - * close the stream. Contents of |headers| is valid for duration of the call. - */ - void (*on_response_headers_received)( - cronet_bidirectional_stream* stream, - const cronet_bidirectional_stream_header_array* headers, - const char* negotiated_protocol); - - /* Invoked when data is read into the buffer passed to - * cronet_bidirectional_stream_read(). Only part of the buffer may be - * populated. To continue reading, call cronet_bidirectional_stream_read(). - * It may be invoked after on_response_trailers_received()}, if there was - * pending read data before trailers were received. - * - * If count is 0, it means the remote side has signaled that it will send no - * more data; future calls to cronet_bidirectional_stream_read() will result - * in the on_data_read() callback or on_succeded() callback if - * cronet_bidirectional_stream_write() was invoked with end_of_stream set to - * true. - */ - void (*on_read_completed)(cronet_bidirectional_stream* stream, - char* data, - int count); - - /** - * Invoked when all data passed to cronet_bidirectional_stream_write() is - * sent. - * To continue writing, call cronet_bidirectional_stream_write(). - */ - void (*on_write_completed)(cronet_bidirectional_stream* stream, - const char* data); - - /* Invoked when trailers are received before closing the stream. Only invoked - * when server sends trailers, which it may not. May be invoked while there is - * read data remaining in local buffer. Contents of |trailers| is valid for - * duration of the call. - */ - void (*on_response_trailers_received)( - cronet_bidirectional_stream* stream, - const cronet_bidirectional_stream_header_array* trailers); - - /** - * Invoked when there is no data to be read or written and the stream is - * closed successfully remotely and locally. Once invoked, no further callback - * methods will be invoked. - */ - void (*on_succeded)(cronet_bidirectional_stream* stream); - - /** - * Invoked if the stream failed for any reason after - * cronet_bidirectional_stream_start(). HTTP/2 error codes are - * mapped to chrome net error codes. Once invoked, no further callback methods - * will be invoked. - */ - void (*on_failed)(cronet_bidirectional_stream* stream, int net_error); - - /** - * Invoked if the stream was canceled via - * cronet_bidirectional_stream_cancel(). Once invoked, no further callback - * methods will be invoked. - */ - void (*on_canceled)(cronet_bidirectional_stream* stream); -} cronet_bidirectional_stream_callback; - -/* Create a new stream object that uses |engine| and |callback|. All stream - * tasks are performed asynchronously on the |engine| network thread. |callback| - * methods are invoked synchronously on the |engine| network thread, but must - * not run tasks on the current thread to prevent blocking networking operations - * and causing exceptions during shutdown. The |annotation| is stored in - * bidirectional stream for arbitrary use by application. - * - * Returned |cronet_bidirectional_stream*| is owned by the caller, and must be - * destroyed using |cronet_bidirectional_stream_destroy|. - * - * Both |calback| and |engine| must remain valid until stream is destroyed. - */ -cronet_bidirectional_stream* cronet_bidirectional_stream_create( - cronet_engine* engine, - void* annotation, - cronet_bidirectional_stream_callback* callback); - -/* TBD: The following methods return int. Should it be a custom type? */ - -/* Destroy stream object. Destroy could be called from any thread, including - * network thread, but is posted, so |stream| is valid until calling task is - * complete. - */ -int cronet_bidirectional_stream_destroy(cronet_bidirectional_stream* stream); - -/* Start the stream by sending request to |url| using |method| and |headers|. If - * |end_of_stream| is true, then no data is expected to be written. - */ -int cronet_bidirectional_stream_start( - cronet_bidirectional_stream* stream, - const char* url, - int priority, - const char* method, - const cronet_bidirectional_stream_header_array* headers, - bool end_of_stream); - -/* Read response data into |buffer| of |capacity| length. Must only be called at - * most once in response to each invocation of the - * on_response_headers_received() and on_read_completed() methods of the - * cronet_bidirectional_stream_callback. - * Each call will result in an invocation of one of the callback's - * on_read_completed method if data is read, its on_succeeded() method if - * the stream is closed, or its on_failed() method if there's an error. - */ -int cronet_bidirectional_stream_read(cronet_bidirectional_stream* stream, - char* buffer, - int capacity); - -/* Read response data into |buffer| of |capacity| length. Must only be called at - * most once in response to each invocation of the - * on_response_headers_received() and on_read_completed() methods of the - * cronet_bidirectional_stream_callback. - * Each call will result in an invocation of one of the callback's - * on_read_completed method if data is read, its on_succeeded() method if - * the stream is closed, or its on_failed() method if there's an error. - */ -int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream, - const char* buffer, - int count, - bool end_of_stream); - -/* Cancels the stream. Can be called at any time after - * cronet_bidirectional_stream_start(). The on_canceled() method of - * cronet_bidirectional_stream_callback will be invoked when cancelation - * is complete and no further callback methods will be invoked. If the - * stream has completed or has not started, calling - * cronet_bidirectional_stream_cancel() has no effect and on_canceled() will not - * be invoked. At most one callback method may be invoked after - * cronet_bidirectional_stream_cancel() has completed. - */ -int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream); - -/* Returns true if the |stream| was successfully started and is now done - * (succeeded, canceled, or failed). - * Returns false if the |stream| stream is not yet started or is in progress. - */ -bool cronet_bidirectional_stream_is_done(cronet_bidirectional_stream* stream); - -#ifdef __cplusplus -} -#endif - -#endif // COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index aabca410dab..eed84252cc2 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -786,7 +786,6 @@ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_win32.h \ include/grpc/impl/codegen/time.h \ -include/grpc/grpc_cronet.h \ include/grpc/grpc_security.h \ include/grpc/grpc_security_constants.h \ include/grpc/census.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 3ffc6174ed4..1fcc1fa140b 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -786,7 +786,6 @@ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_win32.h \ include/grpc/impl/codegen/time.h \ -include/grpc/grpc_cronet.h \ include/grpc/grpc_security.h \ include/grpc/grpc_security_constants.h \ include/grpc/census.h \ @@ -914,42 +913,6 @@ src/core/ext/client_config/subchannel.h \ src/core/ext/client_config/subchannel_call_holder.h \ src/core/ext/client_config/subchannel_index.h \ src/core/ext/client_config/uri_parser.h \ -include/grpc/byte_buffer.h \ -include/grpc/grpc.h \ -include/grpc/impl/codegen/alloc.h \ -include/grpc/impl/codegen/atm.h \ -include/grpc/impl/codegen/atm_gcc_atomic.h \ -include/grpc/impl/codegen/atm_gcc_sync.h \ -include/grpc/impl/codegen/atm_win32.h \ -include/grpc/impl/codegen/byte_buffer.h \ -include/grpc/impl/codegen/compression_types.h \ -include/grpc/impl/codegen/connectivity_state.h \ -include/grpc/impl/codegen/grpc_types.h \ -include/grpc/impl/codegen/log.h \ -include/grpc/impl/codegen/port_platform.h \ -include/grpc/impl/codegen/propagation_bits.h \ -include/grpc/impl/codegen/slice.h \ -include/grpc/impl/codegen/slice_buffer.h \ -include/grpc/impl/codegen/status.h \ -include/grpc/impl/codegen/sync.h \ -include/grpc/impl/codegen/sync_generic.h \ -include/grpc/impl/codegen/sync_posix.h \ -include/grpc/impl/codegen/sync_win32.h \ -include/grpc/impl/codegen/time.h \ -include/grpc/status.h \ -include/grpc/support/alloc.h \ -include/grpc/support/atm.h \ -include/grpc/support/host_port.h \ -include/grpc/support/log.h \ -include/grpc/support/port_platform.h \ -include/grpc/support/slice.h \ -include/grpc/support/slice_buffer.h \ -include/grpc/support/string_util.h \ -include/grpc/support/sync.h \ -include/grpc/support/time.h \ -include/grpc/support/useful.h \ -src/core/lib/support/string.h \ -third_party/objective_c/Cronet/cronet_c_for_grpc.h \ src/core/ext/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h \ third_party/nanopb/pb.h \ @@ -1108,9 +1071,6 @@ src/core/ext/client_config/subchannel_index.c \ src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ -src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ -src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ -src/core/ext/transport/cronet/transport/cronet_transport.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 8c67d2f8440..3b3a49a5b26 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4140,8 +4140,7 @@ "grpc_transport_chttp2_client_insecure", "grpc_transport_chttp2_client_secure", "grpc_transport_chttp2_server_insecure", - "grpc_transport_chttp2_server_secure", - "grpc_transport_cronet_client_secure" + "grpc_transport_chttp2_server_secure" ], "headers": [], "language": "c", @@ -6015,7 +6014,6 @@ "tsi" ], "headers": [ - "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "src/core/lib/security/auth_filters.h", @@ -6031,7 +6029,6 @@ "language": "c", "name": "grpc_secure", "src": [ - "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "src/core/lib/http/httpcli_security_connector.c", @@ -6267,121 +6264,6 @@ "third_party": false, "type": "filegroup" }, - { - "deps": [], - "headers": [ - "include/grpc/byte_buffer.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/status.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/sync.h", - "include/grpc/support/time.h", - "include/grpc/support/useful.h", - "src/core/ext/transport/chttp2/transport/incoming_metadata.h", - "src/core/lib/channel/channel_stack.h", - "src/core/lib/channel/context.h", - "src/core/lib/debug/trace.h", - "src/core/lib/iomgr/closure.h", - "src/core/lib/iomgr/exec_ctx.h", - "src/core/lib/iomgr/pollset.h", - "src/core/lib/iomgr/pollset_set.h", - "src/core/lib/support/string.h", - "src/core/lib/surface/channel.h", - "src/core/lib/surface/channel_stack_type.h", - "src/core/lib/transport/byte_stream.h", - "src/core/lib/transport/metadata.h", - "src/core/lib/transport/metadata_batch.h", - "src/core/lib/transport/transport.h", - "src/core/lib/transport/transport_impl.h", - "third_party/objective_c/Cronet/cronet_c_for_grpc.h" - ], - "language": "c", - "name": "grpc_transport_cronet_client_secure", - "src": [ - "include/grpc/byte_buffer.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/status.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/sync.h", - "include/grpc/support/time.h", - "include/grpc/support/useful.h", - "src/core/ext/transport/chttp2/transport/incoming_metadata.h", - "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", - "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", - "src/core/ext/transport/cronet/transport/cronet_transport.c", - "src/core/lib/channel/channel_stack.h", - "src/core/lib/channel/context.h", - "src/core/lib/debug/trace.h", - "src/core/lib/iomgr/closure.h", - "src/core/lib/iomgr/exec_ctx.h", - "src/core/lib/iomgr/pollset.h", - "src/core/lib/iomgr/pollset_set.h", - "src/core/lib/support/string.h", - "src/core/lib/surface/channel.h", - "src/core/lib/surface/channel_stack_type.h", - "src/core/lib/transport/byte_stream.h", - "src/core/lib/transport/metadata.h", - "src/core/lib/transport/metadata_batch.h", - "src/core/lib/transport/transport.h", - "src/core/lib/transport/transport_impl.h" - ], - "third_party": false, - "type": "filegroup" - }, { "deps": [], "headers": [ diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 7f0e5a83394..03f4eaa5bec 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -293,7 +293,6 @@ - @@ -423,42 +422,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -765,12 +728,6 @@ - - - - - - diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 6d1d69a913d..4617e3de0d7 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -439,15 +439,6 @@ src\core\ext\transport\chttp2\client\insecure - - src\core\ext\transport\cronet\client\secure - - - src\core\ext\transport\cronet\transport - - - src\core\ext\transport\cronet\transport - src\core\ext\lb_policy\grpclb @@ -585,9 +576,6 @@ include\grpc\impl\codegen - - include\grpc - include\grpc @@ -971,114 +959,6 @@ src\core\ext\client_config - - include\grpc - - - include\grpc - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - src\core\lib\support - - - third_party\objective_c\Cronet - src\core\ext\lb_policy\grpclb @@ -1130,9 +1010,6 @@ {def748f5-ed2a-a9bb-40d9-c31d00f0e13b} - - {31de82ea-dc6c-73fb-a640-979b8a7b240c} - {d538af37-07b2-062b-fa2a-d9f882cb2737} @@ -1214,18 +1091,6 @@ {6f34254e-e69f-c9b4-156d-5024bade5408} - - {1e9c85e9-5522-7ef8-0017-7e19990a6194} - - - {d0530883-75d9-b5f7-d594-26735a70ac7b} - - - {4fa6fe90-b7a8-5c8f-d629-db1e68d89eed} - - - {31518af8-5860-6d0d-ff78-4059fce29ec2} - {5b2ded3f-84a5-f6b4-2060-286c7d1dc945} @@ -1250,9 +1115,6 @@ {c4661d64-349f-01c1-1ba8-0602f9047595} - - {27f30339-d694-40f5-db07-4b89b9aeea73} - {a21971fb-304f-da08-b1b2-7bd8df8ac373} @@ -1271,12 +1133,6 @@ {93d6596d-330c-1d27-6f84-3c840e57869e} - - {3a56a516-857e-d2aa-95cc-11685baf4e8c} - - - {a165c6e3-0776-6f40-7351-d7865668e220} - From 0ddf46f10e3d114704bb2beac4f30718957581c3 Mon Sep 17 00:00:00 2001 From: Kirill Katsnelson Date: Tue, 19 Apr 2016 18:49:31 -0700 Subject: [PATCH 137/155] Fully qualify System.Threading.Tasks.Task --- src/compiler/csharp_generator.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/csharp_generator.cc b/src/compiler/csharp_generator.cc index ac0fee1ec4d..29c359c5395 100644 --- a/src/compiler/csharp_generator.cc +++ b/src/compiler/csharp_generator.cc @@ -214,10 +214,10 @@ std::string GetMethodReturnTypeServer(const MethodDescriptor *method) { switch (GetMethodType(method)) { case METHODTYPE_NO_STREAMING: case METHODTYPE_CLIENT_STREAMING: - return "Task<" + GetClassName(method->output_type()) + ">"; + return "global::System.Threading.Tasks.Task<" + GetClassName(method->output_type()) + ">"; case METHODTYPE_SERVER_STREAMING: case METHODTYPE_BIDI_STREAMING: - return "Task"; + return "global::System.Threading.Tasks.Task"; } GOOGLE_LOG(FATAL)<< "Can't get here."; return ""; From d83f791b11287d2ae960c94611a593afae7612d2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 9 May 2016 10:28:10 -0700 Subject: [PATCH 138/155] regenerate C# code --- src/csharp/Grpc.Examples/MathGrpc.cs | 16 ++++----- src/csharp/Grpc.HealthCheck/HealthGrpc.cs | 4 +-- .../Grpc.IntegrationTesting/MetricsGrpc.cs | 8 ++--- .../Grpc.IntegrationTesting/ServicesGrpc.cs | 24 ++++++------- .../Grpc.IntegrationTesting/TestGrpc.cs | 36 +++++++++---------- 5 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/csharp/Grpc.Examples/MathGrpc.cs b/src/csharp/Grpc.Examples/MathGrpc.cs index 2d3034d28b5..d700a18778b 100644 --- a/src/csharp/Grpc.Examples/MathGrpc.cs +++ b/src/csharp/Grpc.Examples/MathGrpc.cs @@ -151,25 +151,25 @@ namespace Math { /// Div divides args.dividend by args.divisor and returns the quotient and /// remainder. /// - Task Div(global::Math.DivArgs request, ServerCallContext context); + global::System.Threading.Tasks.Task Div(global::Math.DivArgs request, ServerCallContext context); /// /// DivMany accepts an arbitrary number of division args from the client stream /// and sends back the results in the reply stream. The stream continues until /// the client closes its end; the server does the same after sending all the /// replies. The stream ends immediately if either end aborts. /// - Task DivMany(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); + global::System.Threading.Tasks.Task DivMany(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); /// /// Fib generates numbers in the Fibonacci sequence. If args.limit > 0, Fib /// generates up to limit numbers; otherwise it continues until the call is /// canceled. Unlike Fib above, Fib has no final FibReply. /// - Task Fib(global::Math.FibArgs request, IServerStreamWriter responseStream, ServerCallContext context); + global::System.Threading.Tasks.Task Fib(global::Math.FibArgs request, IServerStreamWriter responseStream, ServerCallContext context); /// /// Sum sums a stream of numbers, returning the final result once the stream /// is closed. /// - Task Sum(IAsyncStreamReader requestStream, ServerCallContext context); + global::System.Threading.Tasks.Task Sum(IAsyncStreamReader requestStream, ServerCallContext context); } /// Base class for server-side implementations of Math @@ -179,7 +179,7 @@ namespace Math { /// Div divides args.dividend by args.divisor and returns the quotient and /// remainder. /// - public virtual Task Div(global::Math.DivArgs request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task Div(global::Math.DivArgs request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -190,7 +190,7 @@ namespace Math { /// the client closes its end; the server does the same after sending all the /// replies. The stream ends immediately if either end aborts. /// - public virtual Task DivMany(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task DivMany(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -200,7 +200,7 @@ namespace Math { /// generates up to limit numbers; otherwise it continues until the call is /// canceled. Unlike Fib above, Fib has no final FibReply. /// - public virtual Task Fib(global::Math.FibArgs request, IServerStreamWriter responseStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task Fib(global::Math.FibArgs request, IServerStreamWriter responseStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -209,7 +209,7 @@ namespace Math { /// Sum sums a stream of numbers, returning the final result once the stream /// is closed. /// - public virtual Task Sum(IAsyncStreamReader requestStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task Sum(IAsyncStreamReader requestStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } diff --git a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs index 967d1170bed..51c6a39b1d8 100644 --- a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs +++ b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs @@ -72,13 +72,13 @@ namespace Grpc.Health.V1 { [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")] public interface IHealth { - Task Check(global::Grpc.Health.V1.HealthCheckRequest request, ServerCallContext context); + global::System.Threading.Tasks.Task Check(global::Grpc.Health.V1.HealthCheckRequest request, ServerCallContext context); } /// Base class for server-side implementations of Health public abstract class HealthBase { - public virtual Task Check(global::Grpc.Health.V1.HealthCheckRequest request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task Check(global::Grpc.Health.V1.HealthCheckRequest request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } diff --git a/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs b/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs index aa4f1c5c3ec..9d31d1c5146 100644 --- a/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs @@ -112,11 +112,11 @@ namespace Grpc.Testing { /// Returns the values of all the gauges that are currently being maintained by /// the service /// - Task GetAllGauges(global::Grpc.Testing.EmptyMessage request, IServerStreamWriter responseStream, ServerCallContext context); + global::System.Threading.Tasks.Task GetAllGauges(global::Grpc.Testing.EmptyMessage request, IServerStreamWriter responseStream, ServerCallContext context); /// /// Returns the value of one gauge /// - Task GetGauge(global::Grpc.Testing.GaugeRequest request, ServerCallContext context); + global::System.Threading.Tasks.Task GetGauge(global::Grpc.Testing.GaugeRequest request, ServerCallContext context); } /// Base class for server-side implementations of MetricsService @@ -126,7 +126,7 @@ namespace Grpc.Testing { /// Returns the values of all the gauges that are currently being maintained by /// the service /// - public virtual Task GetAllGauges(global::Grpc.Testing.EmptyMessage request, IServerStreamWriter responseStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task GetAllGauges(global::Grpc.Testing.EmptyMessage request, IServerStreamWriter responseStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -134,7 +134,7 @@ namespace Grpc.Testing { /// /// Returns the value of one gauge /// - public virtual Task GetGauge(global::Grpc.Testing.GaugeRequest request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task GetGauge(global::Grpc.Testing.GaugeRequest request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } diff --git a/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs b/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs index 42bf5e0b582..f7071ebf6be 100644 --- a/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs @@ -111,12 +111,12 @@ namespace Grpc.Testing { /// One request followed by one response. /// The server returns the client payload as-is. /// - Task UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context); + global::System.Threading.Tasks.Task UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context); /// /// One request followed by one response. /// The server returns the client payload as-is. /// - Task StreamingCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); + global::System.Threading.Tasks.Task StreamingCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); } /// Base class for server-side implementations of BenchmarkService @@ -126,7 +126,7 @@ namespace Grpc.Testing { /// One request followed by one response. /// The server returns the client payload as-is. /// - public virtual Task UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -135,7 +135,7 @@ namespace Grpc.Testing { /// One request followed by one response. /// The server returns the client payload as-is. /// - public virtual Task StreamingCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task StreamingCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -375,7 +375,7 @@ namespace Grpc.Testing { /// and once the shutdown has finished, the OK status is sent to terminate /// this RPC. /// - Task RunServer(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); + global::System.Threading.Tasks.Task RunServer(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); /// /// Start client with specified workload. /// First request sent specifies the ClientConfig followed by ClientStatus @@ -384,15 +384,15 @@ namespace Grpc.Testing { /// and once the shutdown has finished, the OK status is sent to terminate /// this RPC. /// - Task RunClient(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); + global::System.Threading.Tasks.Task RunClient(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); /// /// Just return the core count - unary call /// - Task CoreCount(global::Grpc.Testing.CoreRequest request, ServerCallContext context); + global::System.Threading.Tasks.Task CoreCount(global::Grpc.Testing.CoreRequest request, ServerCallContext context); /// /// Quit this worker /// - Task QuitWorker(global::Grpc.Testing.Void request, ServerCallContext context); + global::System.Threading.Tasks.Task QuitWorker(global::Grpc.Testing.Void request, ServerCallContext context); } /// Base class for server-side implementations of WorkerService @@ -406,7 +406,7 @@ namespace Grpc.Testing { /// and once the shutdown has finished, the OK status is sent to terminate /// this RPC. /// - public virtual Task RunServer(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task RunServer(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -419,7 +419,7 @@ namespace Grpc.Testing { /// and once the shutdown has finished, the OK status is sent to terminate /// this RPC. /// - public virtual Task RunClient(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task RunClient(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -427,7 +427,7 @@ namespace Grpc.Testing { /// /// Just return the core count - unary call /// - public virtual Task CoreCount(global::Grpc.Testing.CoreRequest request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task CoreCount(global::Grpc.Testing.CoreRequest request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -435,7 +435,7 @@ namespace Grpc.Testing { /// /// Quit this worker /// - public virtual Task QuitWorker(global::Grpc.Testing.Void request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task QuitWorker(global::Grpc.Testing.Void request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } diff --git a/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs b/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs index f1878cbb556..cf43a77118c 100644 --- a/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs @@ -196,34 +196,34 @@ namespace Grpc.Testing { /// /// One empty request followed by one empty response. /// - Task EmptyCall(global::Grpc.Testing.Empty request, ServerCallContext context); + global::System.Threading.Tasks.Task EmptyCall(global::Grpc.Testing.Empty request, ServerCallContext context); /// /// One request followed by one response. /// - Task UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context); + global::System.Threading.Tasks.Task UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context); /// /// One request followed by a sequence of responses (streamed download). /// The server returns the payload with client desired type and sizes. /// - Task StreamingOutputCall(global::Grpc.Testing.StreamingOutputCallRequest request, IServerStreamWriter responseStream, ServerCallContext context); + global::System.Threading.Tasks.Task StreamingOutputCall(global::Grpc.Testing.StreamingOutputCallRequest request, IServerStreamWriter responseStream, ServerCallContext context); /// /// A sequence of requests followed by one response (streamed upload). /// The server returns the aggregated size of client payload as the result. /// - Task StreamingInputCall(IAsyncStreamReader requestStream, ServerCallContext context); + global::System.Threading.Tasks.Task StreamingInputCall(IAsyncStreamReader requestStream, ServerCallContext context); /// /// A sequence of requests with each request served by the server immediately. /// As one request could lead to multiple responses, this interface /// demonstrates the idea of full duplexing. /// - Task FullDuplexCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); + global::System.Threading.Tasks.Task FullDuplexCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); /// /// A sequence of requests followed by a sequence of responses. /// The server buffers all the client requests and then serves them in order. A /// stream of responses are returned to the client when the server starts with /// first request. /// - Task HalfDuplexCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); + global::System.Threading.Tasks.Task HalfDuplexCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); } /// Base class for server-side implementations of TestService @@ -232,7 +232,7 @@ namespace Grpc.Testing { /// /// One empty request followed by one empty response. /// - public virtual Task EmptyCall(global::Grpc.Testing.Empty request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task EmptyCall(global::Grpc.Testing.Empty request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -240,7 +240,7 @@ namespace Grpc.Testing { /// /// One request followed by one response. /// - public virtual Task UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task UnaryCall(global::Grpc.Testing.SimpleRequest request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -249,7 +249,7 @@ namespace Grpc.Testing { /// One request followed by a sequence of responses (streamed download). /// The server returns the payload with client desired type and sizes. /// - public virtual Task StreamingOutputCall(global::Grpc.Testing.StreamingOutputCallRequest request, IServerStreamWriter responseStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task StreamingOutputCall(global::Grpc.Testing.StreamingOutputCallRequest request, IServerStreamWriter responseStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -258,7 +258,7 @@ namespace Grpc.Testing { /// A sequence of requests followed by one response (streamed upload). /// The server returns the aggregated size of client payload as the result. /// - public virtual Task StreamingInputCall(IAsyncStreamReader requestStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task StreamingInputCall(IAsyncStreamReader requestStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -268,7 +268,7 @@ namespace Grpc.Testing { /// As one request could lead to multiple responses, this interface /// demonstrates the idea of full duplexing. /// - public virtual Task FullDuplexCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task FullDuplexCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -279,7 +279,7 @@ namespace Grpc.Testing { /// stream of responses are returned to the client when the server starts with /// first request. /// - public virtual Task HalfDuplexCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task HalfDuplexCall(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -525,7 +525,7 @@ namespace Grpc.Testing { /// /// A call that no server should implement /// - Task UnimplementedCall(global::Grpc.Testing.Empty request, ServerCallContext context); + global::System.Threading.Tasks.Task UnimplementedCall(global::Grpc.Testing.Empty request, ServerCallContext context); } /// Base class for server-side implementations of UnimplementedService @@ -534,7 +534,7 @@ namespace Grpc.Testing { /// /// A call that no server should implement /// - public virtual Task UnimplementedCall(global::Grpc.Testing.Empty request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task UnimplementedCall(global::Grpc.Testing.Empty request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } @@ -669,19 +669,19 @@ namespace Grpc.Testing { [System.Obsolete("Service implementations should inherit from the generated abstract base class instead.")] public interface IReconnectService { - Task Start(global::Grpc.Testing.ReconnectParams request, ServerCallContext context); - Task Stop(global::Grpc.Testing.Empty request, ServerCallContext context); + global::System.Threading.Tasks.Task Start(global::Grpc.Testing.ReconnectParams request, ServerCallContext context); + global::System.Threading.Tasks.Task Stop(global::Grpc.Testing.Empty request, ServerCallContext context); } /// Base class for server-side implementations of ReconnectService public abstract class ReconnectServiceBase { - public virtual Task Start(global::Grpc.Testing.ReconnectParams request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task Start(global::Grpc.Testing.ReconnectParams request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } - public virtual Task Stop(global::Grpc.Testing.Empty request, ServerCallContext context) + public virtual global::System.Threading.Tasks.Task Stop(global::Grpc.Testing.Empty request, ServerCallContext context) { throw new RpcException(new Status(StatusCode.Unimplemented, "")); } From 0545d7fa6900b66dfd37de7a33fe5fdc655a378d Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Mon, 9 May 2016 19:28:17 +0200 Subject: [PATCH 139/155] Revert "Merge pull request #6214 from makdharma/master" This reverts commit 48d833a9d8280216040ef731341502d7d0f157e4, reversing changes made to 66e9d8e8a7717d7ccece629df4454fc546e19daa. --- BUILD | 80 --- Makefile | 7 - binding.gyp | 3 - build.yaml | 59 -- config.m4 | 5 - gRPC.podspec | 76 --- grpc.def | 1 - grpc.gemspec | 40 -- include/grpc/grpc_cronet.h | 51 -- package.xml | 40 -- .../client/secure/cronet_channel_create.c | 69 -- .../cronet/transport/cronet_api_dummy.c | 85 --- .../cronet/transport/cronet_transport.c | 640 ------------------ .../grpcio/grpc/_cython/imports.generated.c | 2 - .../grpcio/grpc/_cython/imports.generated.h | 4 - src/python/grpcio/grpc_core_dependencies.py | 3 - src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 - src/ruby/ext/grpc/rb_grpc_imports.generated.h | 4 - .../core/surface/public_headers_must_be_c89.c | 1 - .../objective_c/Cronet/cronet_c_for_grpc.h | 202 ------ tools/doxygen/Doxyfile.core | 1 - tools/doxygen/Doxyfile.core.internal | 40 -- tools/run_tests/sources_and_headers.json | 120 +--- vsprojects/vcxproj/grpc/grpc.vcxproj | 43 -- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 144 ---- 25 files changed, 1 insertion(+), 1721 deletions(-) delete mode 100644 include/grpc/grpc_cronet.h delete mode 100644 src/core/ext/transport/cronet/client/secure/cronet_channel_create.c delete mode 100644 src/core/ext/transport/cronet/transport/cronet_api_dummy.c delete mode 100644 src/core/ext/transport/cronet/transport/cronet_transport.c delete mode 100644 third_party/objective_c/Cronet/cronet_c_for_grpc.h diff --git a/BUILD b/BUILD index fae3596eec6..1da16504388 100644 --- a/BUILD +++ b/BUILD @@ -285,42 +285,6 @@ cc_library( "src/core/ext/client_config/subchannel_call_holder.h", "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", - "include/grpc/byte_buffer.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/status.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/sync.h", - "include/grpc/support/time.h", - "include/grpc/support/useful.h", - "src/core/lib/support/string.h", - "third_party/objective_c/Cronet/cronet_c_for_grpc.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/ext/census/aggregation.h", @@ -475,9 +439,6 @@ cc_library( "src/core/ext/client_config/uri_parser.c", "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c", "src/core/ext/transport/chttp2/client/insecure/channel_create.c", - "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", - "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", - "src/core/ext/transport/cronet/transport/cronet_transport.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/ext/lb_policy/pick_first/pick_first.c", @@ -522,7 +483,6 @@ cc_library( "include/grpc/impl/codegen/sync_posix.h", "include/grpc/impl/codegen/sync_win32.h", "include/grpc/impl/codegen/time.h", - "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "include/grpc/census.h", @@ -1500,9 +1460,6 @@ objc_library( "src/core/ext/client_config/uri_parser.c", "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c", "src/core/ext/transport/chttp2/client/insecure/channel_create.c", - "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", - "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", - "src/core/ext/transport/cronet/transport/cronet_transport.c", "src/core/ext/lb_policy/grpclb/load_balancer_api.c", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/ext/lb_policy/pick_first/pick_first.c", @@ -1547,7 +1504,6 @@ objc_library( "include/grpc/impl/codegen/sync_posix.h", "include/grpc/impl/codegen/sync_win32.h", "include/grpc/impl/codegen/time.h", - "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "include/grpc/census.h", @@ -1675,42 +1631,6 @@ objc_library( "src/core/ext/client_config/subchannel_call_holder.h", "src/core/ext/client_config/subchannel_index.h", "src/core/ext/client_config/uri_parser.h", - "include/grpc/byte_buffer.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/status.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/sync.h", - "include/grpc/support/time.h", - "include/grpc/support/useful.h", - "src/core/lib/support/string.h", - "third_party/objective_c/Cronet/cronet_c_for_grpc.h", "src/core/ext/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/ext/census/aggregation.h", diff --git a/Makefile b/Makefile index 635df30c20a..45003ad8b57 100644 --- a/Makefile +++ b/Makefile @@ -2623,9 +2623,6 @@ LIBGRPC_SRC = \ src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ - src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ - src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ - src/core/ext/transport/cronet/transport/cronet_transport.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ @@ -2673,7 +2670,6 @@ PUBLIC_HEADERS_C += \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_win32.h \ include/grpc/impl/codegen/time.h \ - include/grpc/grpc_cronet.h \ include/grpc/grpc_security.h \ include/grpc/grpc_security_constants.h \ include/grpc/census.h \ @@ -14321,9 +14317,6 @@ ifneq ($(OPENSSL_DEP),) # otherwise parallel compilation will fail if a source is compiled first. src/core/ext/transport/chttp2/client/secure/secure_channel_create.c: $(OPENSSL_DEP) src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c: $(OPENSSL_DEP) -src/core/ext/transport/cronet/client/secure/cronet_channel_create.c: $(OPENSSL_DEP) -src/core/ext/transport/cronet/transport/cronet_api_dummy.c: $(OPENSSL_DEP) -src/core/ext/transport/cronet/transport/cronet_transport.c: $(OPENSSL_DEP) src/core/lib/http/httpcli_security_connector.c: $(OPENSSL_DEP) src/core/lib/security/b64.c: $(OPENSSL_DEP) src/core/lib/security/client_auth_filter.c: $(OPENSSL_DEP) diff --git a/binding.gyp b/binding.gyp index 12a745ffb01..4314ab7243d 100644 --- a/binding.gyp +++ b/binding.gyp @@ -709,9 +709,6 @@ 'src/core/ext/client_config/uri_parser.c', 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', - 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', - 'src/core/ext/transport/cronet/transport/cronet_api_dummy.c', - 'src/core/ext/transport/cronet/transport/cronet_transport.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/build.yaml b/build.yaml index 7a74e73c726..a21dc688adf 100644 --- a/build.yaml +++ b/build.yaml @@ -400,7 +400,6 @@ filegroups: - grpc_client_config - name: grpc_secure public_headers: - - include/grpc/grpc_cronet.h - include/grpc/grpc_security.h - include/grpc/grpc_security_constants.h headers: @@ -548,63 +547,6 @@ filegroups: - grpc_transport_chttp2 - grpc_base - grpc_secure -- name: grpc_transport_cronet_client_secure - headers: - - include/grpc/byte_buffer.h - - include/grpc/grpc.h - - include/grpc/impl/codegen/alloc.h - - include/grpc/impl/codegen/atm.h - - include/grpc/impl/codegen/atm_gcc_atomic.h - - include/grpc/impl/codegen/atm_gcc_sync.h - - include/grpc/impl/codegen/atm_win32.h - - include/grpc/impl/codegen/byte_buffer.h - - include/grpc/impl/codegen/compression_types.h - - include/grpc/impl/codegen/connectivity_state.h - - include/grpc/impl/codegen/grpc_types.h - - include/grpc/impl/codegen/log.h - - include/grpc/impl/codegen/port_platform.h - - include/grpc/impl/codegen/propagation_bits.h - - include/grpc/impl/codegen/slice.h - - include/grpc/impl/codegen/slice_buffer.h - - include/grpc/impl/codegen/status.h - - include/grpc/impl/codegen/sync.h - - include/grpc/impl/codegen/sync_generic.h - - include/grpc/impl/codegen/sync_posix.h - - include/grpc/impl/codegen/sync_win32.h - - include/grpc/impl/codegen/time.h - - include/grpc/status.h - - include/grpc/support/alloc.h - - include/grpc/support/atm.h - - include/grpc/support/host_port.h - - include/grpc/support/log.h - - include/grpc/support/port_platform.h - - include/grpc/support/slice.h - - include/grpc/support/slice_buffer.h - - include/grpc/support/string_util.h - - include/grpc/support/sync.h - - include/grpc/support/time.h - - include/grpc/support/useful.h - - src/core/ext/transport/chttp2/transport/incoming_metadata.h - - src/core/lib/channel/channel_stack.h - - src/core/lib/channel/context.h - - src/core/lib/debug/trace.h - - src/core/lib/iomgr/closure.h - - src/core/lib/iomgr/exec_ctx.h - - src/core/lib/iomgr/pollset.h - - src/core/lib/iomgr/pollset_set.h - - src/core/lib/support/string.h - - src/core/lib/surface/channel.h - - src/core/lib/surface/channel_stack_type.h - - src/core/lib/transport/byte_stream.h - - src/core/lib/transport/metadata.h - - src/core/lib/transport/metadata_batch.h - - src/core/lib/transport/transport.h - - src/core/lib/transport/transport_impl.h - - third_party/objective_c/Cronet/cronet_c_for_grpc.h - src: - - src/core/ext/transport/cronet/client/secure/cronet_channel_create.c - - src/core/ext/transport/cronet/transport/cronet_api_dummy.c - - src/core/ext/transport/cronet/transport/cronet_transport.c - name: nanopb headers: - third_party/nanopb/pb.h @@ -792,7 +734,6 @@ libs: - grpc_transport_chttp2_client_secure - grpc_transport_chttp2_server_insecure - grpc_transport_chttp2_client_insecure - - grpc_transport_cronet_client_secure - grpc_lb_policy_grpclb - grpc_lb_policy_pick_first - grpc_lb_policy_round_robin diff --git a/config.m4 b/config.m4 index 5259e679ba9..74f9ad242ab 100644 --- a/config.m4 +++ b/config.m4 @@ -228,9 +228,6 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ - src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ - src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ - src/core/ext/transport/cronet/transport/cronet_transport.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ @@ -569,8 +566,6 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/server/insecure) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/server/secure) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/transport) - PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/cronet/client/secure) - PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/cronet/transport) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/channel) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/compression) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/debug) diff --git a/gRPC.podspec b/gRPC.podspec index 018306ca643..569f89bf7c0 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -287,42 +287,6 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_call_holder.h', 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', - 'include/grpc/byte_buffer.h', - 'include/grpc/grpc.h', - 'include/grpc/impl/codegen/alloc.h', - 'include/grpc/impl/codegen/atm.h', - 'include/grpc/impl/codegen/atm_gcc_atomic.h', - 'include/grpc/impl/codegen/atm_gcc_sync.h', - 'include/grpc/impl/codegen/atm_win32.h', - 'include/grpc/impl/codegen/byte_buffer.h', - 'include/grpc/impl/codegen/compression_types.h', - 'include/grpc/impl/codegen/connectivity_state.h', - 'include/grpc/impl/codegen/grpc_types.h', - 'include/grpc/impl/codegen/log.h', - 'include/grpc/impl/codegen/port_platform.h', - 'include/grpc/impl/codegen/propagation_bits.h', - 'include/grpc/impl/codegen/slice.h', - 'include/grpc/impl/codegen/slice_buffer.h', - 'include/grpc/impl/codegen/status.h', - 'include/grpc/impl/codegen/sync.h', - 'include/grpc/impl/codegen/sync_generic.h', - 'include/grpc/impl/codegen/sync_posix.h', - 'include/grpc/impl/codegen/sync_win32.h', - 'include/grpc/impl/codegen/time.h', - 'include/grpc/status.h', - 'include/grpc/support/alloc.h', - 'include/grpc/support/atm.h', - 'include/grpc/support/host_port.h', - 'include/grpc/support/log.h', - 'include/grpc/support/port_platform.h', - 'include/grpc/support/slice.h', - 'include/grpc/support/slice_buffer.h', - 'include/grpc/support/string_util.h', - 'include/grpc/support/sync.h', - 'include/grpc/support/time.h', - 'include/grpc/support/useful.h', - 'src/core/lib/support/string.h', - 'third_party/objective_c/Cronet/cronet_c_for_grpc.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', 'third_party/nanopb/pb.h', @@ -361,7 +325,6 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_posix.h', 'include/grpc/impl/codegen/sync_win32.h', 'include/grpc/impl/codegen/time.h', - 'include/grpc/grpc_cronet.h', 'include/grpc/grpc_security.h', 'include/grpc/grpc_security_constants.h', 'include/grpc/census.h', @@ -511,9 +474,6 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/uri_parser.c', 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', - 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', - 'src/core/ext/transport/cronet/transport/cronet_api_dummy.c', - 'src/core/ext/transport/cronet/transport/cronet_transport.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', @@ -670,42 +630,6 @@ Pod::Spec.new do |s| 'src/core/ext/client_config/subchannel_call_holder.h', 'src/core/ext/client_config/subchannel_index.h', 'src/core/ext/client_config/uri_parser.h', - 'include/grpc/byte_buffer.h', - 'include/grpc/grpc.h', - 'include/grpc/impl/codegen/alloc.h', - 'include/grpc/impl/codegen/atm.h', - 'include/grpc/impl/codegen/atm_gcc_atomic.h', - 'include/grpc/impl/codegen/atm_gcc_sync.h', - 'include/grpc/impl/codegen/atm_win32.h', - 'include/grpc/impl/codegen/byte_buffer.h', - 'include/grpc/impl/codegen/compression_types.h', - 'include/grpc/impl/codegen/connectivity_state.h', - 'include/grpc/impl/codegen/grpc_types.h', - 'include/grpc/impl/codegen/log.h', - 'include/grpc/impl/codegen/port_platform.h', - 'include/grpc/impl/codegen/propagation_bits.h', - 'include/grpc/impl/codegen/slice.h', - 'include/grpc/impl/codegen/slice_buffer.h', - 'include/grpc/impl/codegen/status.h', - 'include/grpc/impl/codegen/sync.h', - 'include/grpc/impl/codegen/sync_generic.h', - 'include/grpc/impl/codegen/sync_posix.h', - 'include/grpc/impl/codegen/sync_win32.h', - 'include/grpc/impl/codegen/time.h', - 'include/grpc/status.h', - 'include/grpc/support/alloc.h', - 'include/grpc/support/atm.h', - 'include/grpc/support/host_port.h', - 'include/grpc/support/log.h', - 'include/grpc/support/port_platform.h', - 'include/grpc/support/slice.h', - 'include/grpc/support/slice_buffer.h', - 'include/grpc/support/string_util.h', - 'include/grpc/support/sync.h', - 'include/grpc/support/time.h', - 'include/grpc/support/useful.h', - 'src/core/lib/support/string.h', - 'third_party/objective_c/Cronet/cronet_c_for_grpc.h', 'src/core/ext/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h', 'third_party/nanopb/pb.h', diff --git a/grpc.def b/grpc.def index 09a94a6cd0c..61948ed1b89 100644 --- a/grpc.def +++ b/grpc.def @@ -87,7 +87,6 @@ EXPORTS grpc_header_nonbin_value_is_legal grpc_is_binary_header grpc_call_error_to_string - grpc_cronet_secure_channel_create grpc_auth_property_iterator_next grpc_auth_context_property_iterator grpc_auth_context_peer_identity diff --git a/grpc.gemspec b/grpc.gemspec index ace28715dcd..475fc990ad4 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -169,7 +169,6 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/impl/codegen/sync_posix.h ) s.files += %w( include/grpc/impl/codegen/sync_win32.h ) s.files += %w( include/grpc/impl/codegen/time.h ) - s.files += %w( include/grpc/grpc_cronet.h ) s.files += %w( include/grpc/grpc_security.h ) s.files += %w( include/grpc/grpc_security_constants.h ) s.files += %w( include/grpc/census.h ) @@ -297,42 +296,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/client_config/subchannel_call_holder.h ) s.files += %w( src/core/ext/client_config/subchannel_index.h ) s.files += %w( src/core/ext/client_config/uri_parser.h ) - s.files += %w( include/grpc/byte_buffer.h ) - s.files += %w( include/grpc/grpc.h ) - s.files += %w( include/grpc/impl/codegen/alloc.h ) - s.files += %w( include/grpc/impl/codegen/atm.h ) - s.files += %w( include/grpc/impl/codegen/atm_gcc_atomic.h ) - s.files += %w( include/grpc/impl/codegen/atm_gcc_sync.h ) - s.files += %w( include/grpc/impl/codegen/atm_win32.h ) - s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) - s.files += %w( include/grpc/impl/codegen/compression_types.h ) - s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) - s.files += %w( include/grpc/impl/codegen/grpc_types.h ) - s.files += %w( include/grpc/impl/codegen/log.h ) - s.files += %w( include/grpc/impl/codegen/port_platform.h ) - s.files += %w( include/grpc/impl/codegen/propagation_bits.h ) - s.files += %w( include/grpc/impl/codegen/slice.h ) - s.files += %w( include/grpc/impl/codegen/slice_buffer.h ) - s.files += %w( include/grpc/impl/codegen/status.h ) - s.files += %w( include/grpc/impl/codegen/sync.h ) - s.files += %w( include/grpc/impl/codegen/sync_generic.h ) - s.files += %w( include/grpc/impl/codegen/sync_posix.h ) - s.files += %w( include/grpc/impl/codegen/sync_win32.h ) - s.files += %w( include/grpc/impl/codegen/time.h ) - s.files += %w( include/grpc/status.h ) - s.files += %w( include/grpc/support/alloc.h ) - s.files += %w( include/grpc/support/atm.h ) - s.files += %w( include/grpc/support/host_port.h ) - s.files += %w( include/grpc/support/log.h ) - s.files += %w( include/grpc/support/port_platform.h ) - s.files += %w( include/grpc/support/slice.h ) - s.files += %w( include/grpc/support/slice_buffer.h ) - s.files += %w( include/grpc/support/string_util.h ) - s.files += %w( include/grpc/support/sync.h ) - s.files += %w( include/grpc/support/time.h ) - s.files += %w( include/grpc/support/useful.h ) - s.files += %w( src/core/lib/support/string.h ) - s.files += %w( third_party/objective_c/Cronet/cronet_c_for_grpc.h ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.h ) s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h ) s.files += %w( third_party/nanopb/pb.h ) @@ -491,9 +454,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/client_config/uri_parser.c ) s.files += %w( src/core/ext/transport/chttp2/server/insecure/server_chttp2.c ) s.files += %w( src/core/ext/transport/chttp2/client/insecure/channel_create.c ) - s.files += %w( src/core/ext/transport/cronet/client/secure/cronet_channel_create.c ) - s.files += %w( src/core/ext/transport/cronet/transport/cronet_api_dummy.c ) - s.files += %w( src/core/ext/transport/cronet/transport/cronet_transport.c ) s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.c ) s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c ) s.files += %w( third_party/nanopb/pb_common.c ) diff --git a/include/grpc/grpc_cronet.h b/include/grpc/grpc_cronet.h deleted file mode 100644 index 295e0f55e80..00000000000 --- a/include/grpc/grpc_cronet.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef GRPC_GRPC_CRONET_H -#define GRPC_GRPC_CRONET_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( - void *engine, const char *target, const grpc_channel_args *args, - void *reserved); - -#ifdef __cplusplus -} -#endif - -#endif /* GRPC_GRPC_CRONET_H */ diff --git a/package.xml b/package.xml index 716d6ed2891..ab6dee06318 100644 --- a/package.xml +++ b/package.xml @@ -176,7 +176,6 @@ - @@ -304,42 +303,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -498,9 +461,6 @@ - - - diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c deleted file mode 100644 index df1acddcc08..00000000000 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.c +++ /dev/null @@ -1,69 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include - -#include -#include - -#include -#include - -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/transport/transport_impl.h" - -// Cronet transport object -typedef struct cronet_transport { - grpc_transport base; // must be first element in this structure - void *engine; - char *host; -} cronet_transport; - -extern grpc_transport_vtable grpc_cronet_vtable; - -GRPCAPI grpc_channel *grpc_cronet_secure_channel_create( - void *engine, const char *target, const grpc_channel_args *args, - void *reserved) { - cronet_transport *ct = gpr_malloc(sizeof(cronet_transport)); - ct->base.vtable = &grpc_cronet_vtable; - ct->engine = engine; - ct->host = gpr_malloc(strlen(target) + 1); - strcpy(ct->host, target); - gpr_log(GPR_DEBUG, - "grpc_create_cronet_transport: cronet_engine = %p, target=%s", engine, - ct->host); - - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - return grpc_channel_create(&exec_ctx, target, args, - GRPC_CLIENT_DIRECT_CHANNEL, (grpc_transport *)ct); -} diff --git a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c b/src/core/ext/transport/cronet/transport/cronet_api_dummy.c deleted file mode 100644 index 687026c9fde..00000000000 --- a/src/core/ext/transport/cronet/transport/cronet_api_dummy.c +++ /dev/null @@ -1,85 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* This file has empty implementation of all the functions exposed by the cronet -library, so we can build it in all environments */ - -#include - -#include - -#include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" - -#ifdef GRPC_COMPILE_WITH_CRONET -/* link with the real CRONET library in the build system */ -#else -/* Dummy implementation of cronet API just to test for build-ability */ -cronet_bidirectional_stream* cronet_bidirectional_stream_create( - cronet_engine* engine, void* annotation, - cronet_bidirectional_stream_callback* callback) { - GPR_ASSERT(0); - return NULL; -} - -int cronet_bidirectional_stream_destroy(cronet_bidirectional_stream* stream) { - GPR_ASSERT(0); - return 0; -} - -int cronet_bidirectional_stream_start( - cronet_bidirectional_stream* stream, const char* url, int priority, - const char* method, const cronet_bidirectional_stream_header_array* headers, - bool end_of_stream) { - GPR_ASSERT(0); - return 0; -} - -int cronet_bidirectional_stream_read(cronet_bidirectional_stream* stream, - char* buffer, int capacity) { - GPR_ASSERT(0); - return 0; -} - -int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream, - const char* buffer, int count, - bool end_of_stream) { - GPR_ASSERT(0); - return 0; -} - -int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream) { - GPR_ASSERT(0); - return 0; -} - -#endif /* GRPC_COMPILE_WITH_CRONET */ diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c deleted file mode 100644 index 5bb085195c6..00000000000 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ /dev/null @@ -1,640 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "src/core/ext/transport/chttp2/transport/incoming_metadata.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/support/string.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/transport/metadata_batch.h" -#include "src/core/lib/transport/transport_impl.h" -#include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" - -#define GRPC_HEADER_SIZE_IN_BYTES 5 - -// Global flag that gets set with GRPC_TRACE env variable -int grpc_cronet_trace = 1; - -// Cronet transport object -struct grpc_cronet_transport { - grpc_transport base; /* must be first element in this structure */ - cronet_engine *engine; - char *host; -}; - -typedef struct grpc_cronet_transport grpc_cronet_transport; - -enum send_state { - CRONET_SEND_IDLE = 0, - CRONET_REQ_STARTED, - CRONET_SEND_HEADER, - CRONET_WRITE, - CRONET_WRITE_COMPLETED, -}; - -enum recv_state { - CRONET_RECV_IDLE = 0, - CRONET_RECV_READ_LENGTH, - CRONET_RECV_READ_DATA, - CRONET_RECV_CLOSED, -}; - -static const char *recv_state_name[] = { - "CRONET_RECV_IDLE", "CRONET_RECV_READ_LENGTH", "CRONET_RECV_READ_DATA,", - "CRONET_RECV_CLOSED"}; - -// Enum that identifies calling function. -enum e_caller { - PERFORM_STREAM_OP, - ON_READ_COMPLETE, - ON_RESPONSE_HEADERS_RECEIVED, - ON_RESPONSE_TRAILERS_RECEIVED -}; - -enum callback_id { - CB_SEND_INITIAL_METADATA = 0, - CB_SEND_MESSAGE, - CB_SEND_TRAILING_METADATA, - CB_RECV_MESSAGE, - CB_RECV_INITIAL_METADATA, - CB_RECV_TRAILING_METADATA, - CB_NUM_CALLBACKS -}; - -struct stream_obj { - // we store received bytes here as they trickle in. - gpr_slice_buffer write_slice_buffer; - cronet_bidirectional_stream *cbs; - gpr_slice slice; - gpr_slice_buffer read_slice_buffer; - struct grpc_slice_buffer_stream sbs; - char *read_buffer; - int remaining_read_bytes; - int total_read_bytes; - - char *write_buffer; - size_t write_buffer_size; - - // Hold the URL - char *url; - - bool response_headers_received; - bool read_requested; - bool response_trailers_received; - bool read_closed; - - // Recv message stuff - grpc_byte_buffer **recv_message; - // Initial metadata stuff - grpc_metadata_batch *recv_initial_metadata; - // Trailing metadata stuff - grpc_metadata_batch *recv_trailing_metadata; - grpc_chttp2_incoming_metadata_buffer imb; - - // This mutex protects receive state machine execution - gpr_mu recv_mu; - // we can queue up up to 2 callbacks for each OP - grpc_closure *callback_list[CB_NUM_CALLBACKS][2]; - - // storage for header - cronet_bidirectional_stream_header *headers; - uint32_t num_headers; - cronet_bidirectional_stream_header_array header_array; - // state tracking - enum recv_state cronet_recv_state; - enum send_state cronet_send_state; -}; - -typedef struct stream_obj stream_obj; - -static void next_send_step(stream_obj *s); -static void next_recv_step(stream_obj *s, enum e_caller caller); - -static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_pollset *pollset) {} - -static void enqueue_callbacks(grpc_closure *callback_list[]) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - if (callback_list[0]) { - grpc_exec_ctx_enqueue(&exec_ctx, callback_list[0], true, NULL); - callback_list[0] = NULL; - } - if (callback_list[1]) { - grpc_exec_ctx_enqueue(&exec_ctx, callback_list[1], true, NULL); - callback_list[1] = NULL; - } - grpc_exec_ctx_finish(&exec_ctx); -} - -static void on_canceled(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_canceled %p", stream); - } -} - -static void on_failed(cronet_bidirectional_stream *stream, int net_error) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_failed %p, error = %d", stream, net_error); - } -} - -static void on_succeeded(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_succeeded %p", stream); - } -} - -static void on_response_trailers_received( - cronet_bidirectional_stream *stream, - const cronet_bidirectional_stream_header_array *trailers) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_trailers_received"); - } - stream_obj *s = (stream_obj *)stream->annotation; - - memset(&s->imb, 0, sizeof(s->imb)); - grpc_chttp2_incoming_metadata_buffer_init(&s->imb); - unsigned int i = 0; - for (i = 0; i < trailers->count; i++) { - grpc_chttp2_incoming_metadata_buffer_add( - &s->imb, grpc_mdelem_from_metadata_strings( - grpc_mdstr_from_string(trailers->headers[i].key), - grpc_mdstr_from_string(trailers->headers[i].value))); - } - s->response_trailers_received = true; - next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); -} - -static void on_write_completed(cronet_bidirectional_stream *stream, - const char *data) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_write_completed"); - } - stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_SEND_MESSAGE]); - s->cronet_send_state = CRONET_WRITE_COMPLETED; - next_send_step(s); -} - -static void process_recv_message(stream_obj *s, const uint8_t *recv_data) { - gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes); - uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); - memcpy(dst_p, recv_data, (size_t)s->total_read_bytes); - gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice); - grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0); - *s->recv_message = (grpc_byte_buffer *)&s->sbs; -} - -static int parse_grpc_header(const uint8_t *data) { - const uint8_t *p = data + 1; - int length = 0; - length |= ((uint8_t)*p++) << 24; - length |= ((uint8_t)*p++) << 16; - length |= ((uint8_t)*p++) << 8; - length |= ((uint8_t)*p++); - return length; -} - -static void on_read_completed(cronet_bidirectional_stream *stream, char *data, - int count) { - stream_obj *s = (stream_obj *)stream->annotation; - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_read_completed count=%d, total=%d, remaining=%d", - count, s->total_read_bytes, s->remaining_read_bytes); - } - if (count > 0) { - GPR_ASSERT(s->recv_message); - s->remaining_read_bytes -= count; - next_recv_step(s, ON_READ_COMPLETE); - } else { - s->read_closed = true; - next_recv_step(s, ON_READ_COMPLETE); - } -} - -static void on_response_headers_received( - cronet_bidirectional_stream *stream, - const cronet_bidirectional_stream_header_array *headers, - const char *negotiated_protocol) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_headers_received"); - } - stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_RECV_INITIAL_METADATA]); - s->response_headers_received = true; - next_recv_step(s, ON_RESPONSE_HEADERS_RECEIVED); -} - -static void on_request_headers_sent(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_request_headers_sent"); - } - stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_SEND_INITIAL_METADATA]); - s->cronet_send_state = CRONET_SEND_HEADER; - next_send_step(s); -} - -// Callback function pointers (invoked by cronet in response to events) -static cronet_bidirectional_stream_callback callbacks = { - on_request_headers_sent, - on_response_headers_received, - on_read_completed, - on_write_completed, - on_response_trailers_received, - on_succeeded, - on_failed, - on_canceled}; - -static void invoke_closing_callback(stream_obj *s) { - grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, - s->recv_trailing_metadata); - if (s->callback_list[CB_RECV_TRAILING_METADATA]) { - enqueue_callbacks(s->callback_list[CB_RECV_TRAILING_METADATA]); - } -} - -static void set_recv_state(stream_obj *s, enum recv_state state) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "next_state = %s", recv_state_name[state]); - } - s->cronet_recv_state = state; -} - -// This is invoked from perform_stream_op, and all on_xxxx callbacks. -static void next_recv_step(stream_obj *s, enum e_caller caller) { - gpr_mu_lock(&s->recv_mu); - switch (s->cronet_recv_state) { - case CRONET_RECV_IDLE: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE"); - } - if (caller == PERFORM_STREAM_OP || - caller == ON_RESPONSE_HEADERS_RECEIVED) { - if (s->read_closed && s->response_trailers_received) { - invoke_closing_callback(s); - set_recv_state(s, CRONET_RECV_CLOSED); - } else if (s->response_headers_received == true && - s->read_requested == true) { - set_recv_state(s, CRONET_RECV_READ_LENGTH); - s->total_read_bytes = s->remaining_read_bytes = - GRPC_HEADER_SIZE_IN_BYTES; - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); - } - cronet_bidirectional_stream_read(s->cbs, s->read_buffer, - s->remaining_read_bytes); - } - } - break; - case CRONET_RECV_READ_LENGTH: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_LENGTH"); - } - if (caller == ON_READ_COMPLETE) { - if (s->read_closed) { - invoke_closing_callback(s); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); - set_recv_state(s, CRONET_RECV_CLOSED); - } else { - GPR_ASSERT(s->remaining_read_bytes == 0); - set_recv_state(s, CRONET_RECV_READ_DATA); - s->total_read_bytes = s->remaining_read_bytes = - parse_grpc_header((const uint8_t *)s->read_buffer); - s->read_buffer = - gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes); - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); - } - cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer, - s->remaining_read_bytes); - } - } - break; - case CRONET_RECV_READ_DATA: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_DATA"); - } - if (caller == ON_READ_COMPLETE) { - if (s->remaining_read_bytes > 0) { - int offset = s->total_read_bytes - s->remaining_read_bytes; - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); - } - cronet_bidirectional_stream_read( - s->cbs, (char *)s->read_buffer + offset, s->remaining_read_bytes); - } else { - gpr_slice_buffer_init(&s->read_slice_buffer); - uint8_t *p = (uint8_t *)s->read_buffer; - process_recv_message(s, p); - set_recv_state(s, CRONET_RECV_IDLE); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); - } - } - break; - case CRONET_RECV_CLOSED: - break; - default: - GPR_ASSERT(0); // Should not reach here - break; - } - gpr_mu_unlock(&s->recv_mu); -} - -// This function takes the data from s->write_slice_buffer and assembles into -// a contiguous byte stream with 5 byte gRPC header prepended. -static void create_grpc_frame(stream_obj *s) { - gpr_slice slice = gpr_slice_buffer_take_first(&s->write_slice_buffer); - uint8_t *raw_data = GPR_SLICE_START_PTR(slice); - size_t length = GPR_SLICE_LENGTH(slice); - s->write_buffer_size = length + GRPC_HEADER_SIZE_IN_BYTES; - s->write_buffer = gpr_realloc(s->write_buffer, s->write_buffer_size); - uint8_t *p = (uint8_t *)s->write_buffer; - // Append 5 byte header - *p++ = 0; - *p++ = (uint8_t)(length >> 24); - *p++ = (uint8_t)(length >> 16); - *p++ = (uint8_t)(length >> 8); - *p++ = (uint8_t)(length); - // append actual data - memcpy(p, raw_data, length); -} - -static void do_write(stream_obj *s) { - gpr_slice_buffer *sb = &s->write_slice_buffer; - GPR_ASSERT(sb->count <= 1); - if (sb->count > 0) { - create_grpc_frame(s); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); - } - cronet_bidirectional_stream_write(s->cbs, s->write_buffer, - (int)s->write_buffer_size, false); - } -} - -// -static void next_send_step(stream_obj *s) { - switch (s->cronet_send_state) { - case CRONET_SEND_IDLE: - GPR_ASSERT( - s->cbs); // cronet_bidirectional_stream is not initialized yet. - s->cronet_send_state = CRONET_REQ_STARTED; - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", s->url); - } - cronet_bidirectional_stream_start(s->cbs, s->url, 0, "POST", - &s->header_array, false); - // we no longer need the memory that was allocated earlier. - gpr_free(s->header_array.headers); - break; - case CRONET_SEND_HEADER: - do_write(s); - s->cronet_send_state = CRONET_WRITE; - break; - case CRONET_WRITE_COMPLETED: - do_write(s); - break; - default: - GPR_ASSERT(0); - break; - } -} - -static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, - const char *host, - stream_obj *s) { - grpc_linked_mdelem *curr = head; - // Walk the linked list and get number of header fields - uint32_t num_headers_available = 0; - while (curr != NULL) { - curr = curr->next; - num_headers_available++; - } - // Allocate enough memory - s->headers = (cronet_bidirectional_stream_header *)gpr_malloc( - sizeof(cronet_bidirectional_stream_header) * num_headers_available); - - // Walk the linked list again, this time copying the header fields. - // s->num_headers - // can be less than num_headers_available, as some headers are not used for - // cronet - curr = head; - s->num_headers = 0; - while (s->num_headers < num_headers_available) { - grpc_mdelem *mdelem = curr->md; - curr = curr->next; - const char *key = grpc_mdstr_as_c_string(mdelem->key); - const char *value = grpc_mdstr_as_c_string(mdelem->value); - if (strcmp(key, ":scheme") == 0 || strcmp(key, ":method") == 0 || - strcmp(key, ":authority") == 0) { - // Cronet populates these fields on its own. - continue; - } - if (strcmp(key, ":path") == 0) { - // Create URL by appending :path value to the hostname - gpr_asprintf(&s->url, "https://%s%s", host, value); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "extracted URL = %s", s->url); - } - continue; - } - s->headers[s->num_headers].key = key; - s->headers[s->num_headers].value = value; - s->num_headers++; - if (curr == NULL) { - break; - } - } -} - -static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_transport_stream_op *op) { - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - GPR_ASSERT(ct->engine); - stream_obj *s = (stream_obj *)gs; - if (op->recv_trailing_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - recv_trailing_metadata: on_complete=%p", - op->on_complete); - } - s->recv_trailing_metadata = op->recv_trailing_metadata; - GPR_ASSERT(!s->callback_list[CB_RECV_TRAILING_METADATA][0]); - s->callback_list[CB_RECV_TRAILING_METADATA][0] = op->on_complete; - } - if (op->recv_message) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - recv_message: on_complete=%p", - op->on_complete); - } - s->recv_message = (grpc_byte_buffer **)op->recv_message; - GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][0]); - GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][1]); - s->callback_list[CB_RECV_MESSAGE][0] = op->recv_message_ready; - s->callback_list[CB_RECV_MESSAGE][1] = op->on_complete; - s->read_requested = true; - next_recv_step(s, PERFORM_STREAM_OP); - } - if (op->recv_initial_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - recv_initial_metadata:=%p", - op->on_complete); - } - s->recv_initial_metadata = op->recv_initial_metadata; - GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][0]); - GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][1]); - s->callback_list[CB_RECV_INITIAL_METADATA][0] = - op->recv_initial_metadata_ready; - s->callback_list[CB_RECV_INITIAL_METADATA][1] = op->on_complete; - } - if (op->send_initial_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - send_initial_metadata: on_complete=%p", - op->on_complete); - } - s->num_headers = 0; - convert_metadata_to_cronet_headers(op->send_initial_metadata->list.head, - ct->host, s); - s->header_array.count = s->num_headers; - s->header_array.capacity = s->num_headers; - s->header_array.headers = s->headers; - GPR_ASSERT(!s->callback_list[CB_SEND_INITIAL_METADATA][0]); - s->callback_list[CB_SEND_INITIAL_METADATA][0] = op->on_complete; - } - if (op->send_message) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - send_message: on_complete=%p", - op->on_complete); - } - grpc_byte_stream_next(exec_ctx, op->send_message, &s->slice, - op->send_message->length, NULL); - // Check that compression flag is not ON. We don't support compression yet. - // TODO (makdharma): add compression support - GPR_ASSERT(op->send_message->flags == 0); - gpr_slice_buffer_add(&s->write_slice_buffer, s->slice); - if (s->cbs == NULL) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); - } - s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); - GPR_ASSERT(s->cbs); - s->read_closed = false; - s->response_trailers_received = false; - s->response_headers_received = false; - s->cronet_send_state = CRONET_SEND_IDLE; - s->cronet_recv_state = CRONET_RECV_IDLE; - } - GPR_ASSERT(!s->callback_list[CB_SEND_MESSAGE][0]); - s->callback_list[CB_SEND_MESSAGE][0] = op->on_complete; - next_send_step(s); - } - if (op->send_trailing_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - send_trailing_metadata: on_complete=%p", - op->on_complete); - } - GPR_ASSERT(!s->callback_list[CB_SEND_TRAILING_METADATA][0]); - s->callback_list[CB_SEND_TRAILING_METADATA][0] = op->on_complete; - if (s->cbs) { - // Send an "empty" write to the far end to signal that we're done. - // This will induce the server to send down trailers. - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); - } - cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); - } else { - // We never created a stream. This was probably an empty request. - invoke_closing_callback(s); - } - } -} - -static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_stream_refcount *refcount, - const void *server_data) { - stream_obj *s = (stream_obj *)gs; - memset(s->callback_list, 0, sizeof(s->callback_list)); - s->cbs = NULL; - gpr_mu_init(&s->recv_mu); - s->read_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); - s->write_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); - gpr_slice_buffer_init(&s->write_slice_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_transport - init_stream"); - } - return 0; -} - -static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, void *and_free_memory) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Destroy stream"); - } - stream_obj *s = (stream_obj *)gs; - s->cbs = NULL; - gpr_free(s->read_buffer); - gpr_free(s->write_buffer); - gpr_free(s->url); - gpr_mu_destroy(&s->recv_mu); - if (and_free_memory) { - gpr_free(and_free_memory); - } -} - -static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - gpr_free(ct->host); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Destroy transport"); - } -} - -const grpc_transport_vtable grpc_cronet_vtable = { - sizeof(stream_obj), "cronet_http", init_stream, - set_pollset_do_nothing, perform_stream_op, NULL, - destroy_stream, destroy_transport, NULL}; diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c index 09551472b5f..f0a40dbb35c 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.c +++ b/src/python/grpcio/grpc/_cython/imports.generated.c @@ -125,7 +125,6 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; -grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -396,7 +395,6 @@ void pygrpc_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); - grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h index 54c8aaad13e..d5e810b7cf4 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.h +++ b/src/python/grpcio/grpc/_cython/imports.generated.h @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -326,9 +325,6 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import -typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); -extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; -#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 5314329c2cb..dab62530aac 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -222,9 +222,6 @@ CORE_SOURCE_FILES = [ 'src/core/ext/client_config/uri_parser.c', 'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', - 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', - 'src/core/ext/transport/cronet/transport/cronet_api_dummy.c', - 'src/core/ext/transport/cronet/transport/cronet_transport.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index cebbe8c40fe..bc43f9d36b5 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -125,7 +125,6 @@ grpc_header_key_is_legal_type grpc_header_key_is_legal_import; grpc_header_nonbin_value_is_legal_type grpc_header_nonbin_value_is_legal_import; grpc_is_binary_header_type grpc_is_binary_header_import; grpc_call_error_to_string_type grpc_call_error_to_string_import; -grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -392,7 +391,6 @@ void grpc_rb_load_imports(HMODULE library) { grpc_header_nonbin_value_is_legal_import = (grpc_header_nonbin_value_is_legal_type) GetProcAddress(library, "grpc_header_nonbin_value_is_legal"); grpc_is_binary_header_import = (grpc_is_binary_header_type) GetProcAddress(library, "grpc_is_binary_header"); grpc_call_error_to_string_import = (grpc_call_error_to_string_type) GetProcAddress(library, "grpc_call_error_to_string"); - grpc_cronet_secure_channel_create_import = (grpc_cronet_secure_channel_create_type) GetProcAddress(library, "grpc_cronet_secure_channel_create"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index d7ea6c574c8..b67361ca25b 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -326,9 +325,6 @@ extern grpc_is_binary_header_type grpc_is_binary_header_import; typedef const char *(*grpc_call_error_to_string_type)(grpc_call_error error); extern grpc_call_error_to_string_type grpc_call_error_to_string_import; #define grpc_call_error_to_string grpc_call_error_to_string_import -typedef grpc_channel *(*grpc_cronet_secure_channel_create_type)(void *engine, const char *target, const grpc_channel_args *args, void *reserved); -extern grpc_cronet_secure_channel_create_type grpc_cronet_secure_channel_create_import; -#define grpc_cronet_secure_channel_create grpc_cronet_secure_channel_create_import typedef const grpc_auth_property *(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator *it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index fd6ff2c26f5..3eeb55d033d 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include diff --git a/third_party/objective_c/Cronet/cronet_c_for_grpc.h b/third_party/objective_c/Cronet/cronet_c_for_grpc.h deleted file mode 100644 index 15a511aebd0..00000000000 --- a/third_party/objective_c/Cronet/cronet_c_for_grpc.h +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ -#define COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -/* Cronet Engine API. */ - -/* Opaque object representing Cronet Engine. Created and configured outside - * of this API to facilitate sharing with other components */ -typedef struct cronet_engine { void* obj; } cronet_engine; - -void cronet_engine_add_quic_hint(cronet_engine* engine, - const char* host, - int port, - int alternate_port); - -/* Cronet Bidirectional Stream API */ - -/* Opaque object representing Cronet Bidirectional Stream. */ -typedef struct cronet_bidirectional_stream { - void* obj; - void* annotation; -} cronet_bidirectional_stream; - -/* A single request or response header element. */ -typedef struct cronet_bidirectional_stream_header { - const char* key; - const char* value; -} cronet_bidirectional_stream_header; - -/* Array of request or response headers or trailers. */ -typedef struct cronet_bidirectional_stream_header_array { - size_t count; - size_t capacity; - cronet_bidirectional_stream_header* headers; -} cronet_bidirectional_stream_header_array; - -/* Set of callbacks used to receive callbacks from bidirectional stream. */ -typedef struct cronet_bidirectional_stream_callback { - /* Invoked when request headers are sent. Indicates that stream has initiated - * the request. Consumer may call cronet_bidirectional_stream_write() to start - * writing data. - */ - void (*on_request_headers_sent)(cronet_bidirectional_stream* stream); - - /* Invoked when initial response headers are received. - * Consumer must call cronet_bidirectional_stream_read() to start reading. - * Consumer may call cronet_bidirectional_stream_write() to start writing or - * close the stream. Contents of |headers| is valid for duration of the call. - */ - void (*on_response_headers_received)( - cronet_bidirectional_stream* stream, - const cronet_bidirectional_stream_header_array* headers, - const char* negotiated_protocol); - - /* Invoked when data is read into the buffer passed to - * cronet_bidirectional_stream_read(). Only part of the buffer may be - * populated. To continue reading, call cronet_bidirectional_stream_read(). - * It may be invoked after on_response_trailers_received()}, if there was - * pending read data before trailers were received. - * - * If count is 0, it means the remote side has signaled that it will send no - * more data; future calls to cronet_bidirectional_stream_read() will result - * in the on_data_read() callback or on_succeded() callback if - * cronet_bidirectional_stream_write() was invoked with end_of_stream set to - * true. - */ - void (*on_read_completed)(cronet_bidirectional_stream* stream, - char* data, - int count); - - /** - * Invoked when all data passed to cronet_bidirectional_stream_write() is - * sent. - * To continue writing, call cronet_bidirectional_stream_write(). - */ - void (*on_write_completed)(cronet_bidirectional_stream* stream, - const char* data); - - /* Invoked when trailers are received before closing the stream. Only invoked - * when server sends trailers, which it may not. May be invoked while there is - * read data remaining in local buffer. Contents of |trailers| is valid for - * duration of the call. - */ - void (*on_response_trailers_received)( - cronet_bidirectional_stream* stream, - const cronet_bidirectional_stream_header_array* trailers); - - /** - * Invoked when there is no data to be read or written and the stream is - * closed successfully remotely and locally. Once invoked, no further callback - * methods will be invoked. - */ - void (*on_succeded)(cronet_bidirectional_stream* stream); - - /** - * Invoked if the stream failed for any reason after - * cronet_bidirectional_stream_start(). HTTP/2 error codes are - * mapped to chrome net error codes. Once invoked, no further callback methods - * will be invoked. - */ - void (*on_failed)(cronet_bidirectional_stream* stream, int net_error); - - /** - * Invoked if the stream was canceled via - * cronet_bidirectional_stream_cancel(). Once invoked, no further callback - * methods will be invoked. - */ - void (*on_canceled)(cronet_bidirectional_stream* stream); -} cronet_bidirectional_stream_callback; - -/* Create a new stream object that uses |engine| and |callback|. All stream - * tasks are performed asynchronously on the |engine| network thread. |callback| - * methods are invoked synchronously on the |engine| network thread, but must - * not run tasks on the current thread to prevent blocking networking operations - * and causing exceptions during shutdown. The |annotation| is stored in - * bidirectional stream for arbitrary use by application. - * - * Returned |cronet_bidirectional_stream*| is owned by the caller, and must be - * destroyed using |cronet_bidirectional_stream_destroy|. - * - * Both |calback| and |engine| must remain valid until stream is destroyed. - */ -cronet_bidirectional_stream* cronet_bidirectional_stream_create( - cronet_engine* engine, - void* annotation, - cronet_bidirectional_stream_callback* callback); - -/* TBD: The following methods return int. Should it be a custom type? */ - -/* Destroy stream object. Destroy could be called from any thread, including - * network thread, but is posted, so |stream| is valid until calling task is - * complete. - */ -int cronet_bidirectional_stream_destroy(cronet_bidirectional_stream* stream); - -/* Start the stream by sending request to |url| using |method| and |headers|. If - * |end_of_stream| is true, then no data is expected to be written. - */ -int cronet_bidirectional_stream_start( - cronet_bidirectional_stream* stream, - const char* url, - int priority, - const char* method, - const cronet_bidirectional_stream_header_array* headers, - bool end_of_stream); - -/* Read response data into |buffer| of |capacity| length. Must only be called at - * most once in response to each invocation of the - * on_response_headers_received() and on_read_completed() methods of the - * cronet_bidirectional_stream_callback. - * Each call will result in an invocation of one of the callback's - * on_read_completed method if data is read, its on_succeeded() method if - * the stream is closed, or its on_failed() method if there's an error. - */ -int cronet_bidirectional_stream_read(cronet_bidirectional_stream* stream, - char* buffer, - int capacity); - -/* Read response data into |buffer| of |capacity| length. Must only be called at - * most once in response to each invocation of the - * on_response_headers_received() and on_read_completed() methods of the - * cronet_bidirectional_stream_callback. - * Each call will result in an invocation of one of the callback's - * on_read_completed method if data is read, its on_succeeded() method if - * the stream is closed, or its on_failed() method if there's an error. - */ -int cronet_bidirectional_stream_write(cronet_bidirectional_stream* stream, - const char* buffer, - int count, - bool end_of_stream); - -/* Cancels the stream. Can be called at any time after - * cronet_bidirectional_stream_start(). The on_canceled() method of - * cronet_bidirectional_stream_callback will be invoked when cancelation - * is complete and no further callback methods will be invoked. If the - * stream has completed or has not started, calling - * cronet_bidirectional_stream_cancel() has no effect and on_canceled() will not - * be invoked. At most one callback method may be invoked after - * cronet_bidirectional_stream_cancel() has completed. - */ -int cronet_bidirectional_stream_cancel(cronet_bidirectional_stream* stream); - -/* Returns true if the |stream| was successfully started and is now done - * (succeeded, canceled, or failed). - * Returns false if the |stream| stream is not yet started or is in progress. - */ -bool cronet_bidirectional_stream_is_done(cronet_bidirectional_stream* stream); - -#ifdef __cplusplus -} -#endif - -#endif // COMPONENTS_CRONET_IOS_CRONET_C_FOR_GRPC_H_ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index d1259e7aee3..8b13e6e7933 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -786,7 +786,6 @@ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_win32.h \ include/grpc/impl/codegen/time.h \ -include/grpc/grpc_cronet.h \ include/grpc/grpc_security.h \ include/grpc/grpc_security_constants.h \ include/grpc/census.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 6b7a8be8617..b3b34778743 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -786,7 +786,6 @@ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_win32.h \ include/grpc/impl/codegen/time.h \ -include/grpc/grpc_cronet.h \ include/grpc/grpc_security.h \ include/grpc/grpc_security_constants.h \ include/grpc/census.h \ @@ -914,42 +913,6 @@ src/core/ext/client_config/subchannel.h \ src/core/ext/client_config/subchannel_call_holder.h \ src/core/ext/client_config/subchannel_index.h \ src/core/ext/client_config/uri_parser.h \ -include/grpc/byte_buffer.h \ -include/grpc/grpc.h \ -include/grpc/impl/codegen/alloc.h \ -include/grpc/impl/codegen/atm.h \ -include/grpc/impl/codegen/atm_gcc_atomic.h \ -include/grpc/impl/codegen/atm_gcc_sync.h \ -include/grpc/impl/codegen/atm_win32.h \ -include/grpc/impl/codegen/byte_buffer.h \ -include/grpc/impl/codegen/compression_types.h \ -include/grpc/impl/codegen/connectivity_state.h \ -include/grpc/impl/codegen/grpc_types.h \ -include/grpc/impl/codegen/log.h \ -include/grpc/impl/codegen/port_platform.h \ -include/grpc/impl/codegen/propagation_bits.h \ -include/grpc/impl/codegen/slice.h \ -include/grpc/impl/codegen/slice_buffer.h \ -include/grpc/impl/codegen/status.h \ -include/grpc/impl/codegen/sync.h \ -include/grpc/impl/codegen/sync_generic.h \ -include/grpc/impl/codegen/sync_posix.h \ -include/grpc/impl/codegen/sync_win32.h \ -include/grpc/impl/codegen/time.h \ -include/grpc/status.h \ -include/grpc/support/alloc.h \ -include/grpc/support/atm.h \ -include/grpc/support/host_port.h \ -include/grpc/support/log.h \ -include/grpc/support/port_platform.h \ -include/grpc/support/slice.h \ -include/grpc/support/slice_buffer.h \ -include/grpc/support/string_util.h \ -include/grpc/support/sync.h \ -include/grpc/support/time.h \ -include/grpc/support/useful.h \ -src/core/lib/support/string.h \ -third_party/objective_c/Cronet/cronet_c_for_grpc.h \ src/core/ext/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.h \ third_party/nanopb/pb.h \ @@ -1108,9 +1071,6 @@ src/core/ext/client_config/subchannel_index.c \ src/core/ext/client_config/uri_parser.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ -src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ -src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ -src/core/ext/transport/cronet/transport/cronet_transport.c \ src/core/ext/lb_policy/grpclb/load_balancer_api.c \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v0/load_balancer.pb.c \ third_party/nanopb/pb_common.c \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 8c67d2f8440..3b3a49a5b26 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4140,8 +4140,7 @@ "grpc_transport_chttp2_client_insecure", "grpc_transport_chttp2_client_secure", "grpc_transport_chttp2_server_insecure", - "grpc_transport_chttp2_server_secure", - "grpc_transport_cronet_client_secure" + "grpc_transport_chttp2_server_secure" ], "headers": [], "language": "c", @@ -6015,7 +6014,6 @@ "tsi" ], "headers": [ - "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "src/core/lib/security/auth_filters.h", @@ -6031,7 +6029,6 @@ "language": "c", "name": "grpc_secure", "src": [ - "include/grpc/grpc_cronet.h", "include/grpc/grpc_security.h", "include/grpc/grpc_security_constants.h", "src/core/lib/http/httpcli_security_connector.c", @@ -6267,121 +6264,6 @@ "third_party": false, "type": "filegroup" }, - { - "deps": [], - "headers": [ - "include/grpc/byte_buffer.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/status.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/sync.h", - "include/grpc/support/time.h", - "include/grpc/support/useful.h", - "src/core/ext/transport/chttp2/transport/incoming_metadata.h", - "src/core/lib/channel/channel_stack.h", - "src/core/lib/channel/context.h", - "src/core/lib/debug/trace.h", - "src/core/lib/iomgr/closure.h", - "src/core/lib/iomgr/exec_ctx.h", - "src/core/lib/iomgr/pollset.h", - "src/core/lib/iomgr/pollset_set.h", - "src/core/lib/support/string.h", - "src/core/lib/surface/channel.h", - "src/core/lib/surface/channel_stack_type.h", - "src/core/lib/transport/byte_stream.h", - "src/core/lib/transport/metadata.h", - "src/core/lib/transport/metadata_batch.h", - "src/core/lib/transport/transport.h", - "src/core/lib/transport/transport_impl.h", - "third_party/objective_c/Cronet/cronet_c_for_grpc.h" - ], - "language": "c", - "name": "grpc_transport_cronet_client_secure", - "src": [ - "include/grpc/byte_buffer.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/status.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/sync.h", - "include/grpc/support/time.h", - "include/grpc/support/useful.h", - "src/core/ext/transport/chttp2/transport/incoming_metadata.h", - "src/core/ext/transport/cronet/client/secure/cronet_channel_create.c", - "src/core/ext/transport/cronet/transport/cronet_api_dummy.c", - "src/core/ext/transport/cronet/transport/cronet_transport.c", - "src/core/lib/channel/channel_stack.h", - "src/core/lib/channel/context.h", - "src/core/lib/debug/trace.h", - "src/core/lib/iomgr/closure.h", - "src/core/lib/iomgr/exec_ctx.h", - "src/core/lib/iomgr/pollset.h", - "src/core/lib/iomgr/pollset_set.h", - "src/core/lib/support/string.h", - "src/core/lib/surface/channel.h", - "src/core/lib/surface/channel_stack_type.h", - "src/core/lib/transport/byte_stream.h", - "src/core/lib/transport/metadata.h", - "src/core/lib/transport/metadata_batch.h", - "src/core/lib/transport/transport.h", - "src/core/lib/transport/transport_impl.h" - ], - "third_party": false, - "type": "filegroup" - }, { "deps": [], "headers": [ diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 7f0e5a83394..03f4eaa5bec 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -293,7 +293,6 @@ - @@ -423,42 +422,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -765,12 +728,6 @@ - - - - - - diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 6d1d69a913d..4617e3de0d7 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -439,15 +439,6 @@ src\core\ext\transport\chttp2\client\insecure - - src\core\ext\transport\cronet\client\secure - - - src\core\ext\transport\cronet\transport - - - src\core\ext\transport\cronet\transport - src\core\ext\lb_policy\grpclb @@ -585,9 +576,6 @@ include\grpc\impl\codegen - - include\grpc - include\grpc @@ -971,114 +959,6 @@ src\core\ext\client_config - - include\grpc - - - include\grpc - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc\impl\codegen - - - include\grpc - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - include\grpc\support - - - src\core\lib\support - - - third_party\objective_c\Cronet - src\core\ext\lb_policy\grpclb @@ -1130,9 +1010,6 @@ {def748f5-ed2a-a9bb-40d9-c31d00f0e13b} - - {31de82ea-dc6c-73fb-a640-979b8a7b240c} - {d538af37-07b2-062b-fa2a-d9f882cb2737} @@ -1214,18 +1091,6 @@ {6f34254e-e69f-c9b4-156d-5024bade5408} - - {1e9c85e9-5522-7ef8-0017-7e19990a6194} - - - {d0530883-75d9-b5f7-d594-26735a70ac7b} - - - {4fa6fe90-b7a8-5c8f-d629-db1e68d89eed} - - - {31518af8-5860-6d0d-ff78-4059fce29ec2} - {5b2ded3f-84a5-f6b4-2060-286c7d1dc945} @@ -1250,9 +1115,6 @@ {c4661d64-349f-01c1-1ba8-0602f9047595} - - {27f30339-d694-40f5-db07-4b89b9aeea73} - {a21971fb-304f-da08-b1b2-7bd8df8ac373} @@ -1271,12 +1133,6 @@ {93d6596d-330c-1d27-6f84-3c840e57869e} - - {3a56a516-857e-d2aa-95cc-11685baf4e8c} - - - {a165c6e3-0776-6f40-7351-d7865668e220} - From 82318676377aea1d2ad035681524584d1657711b Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 9 May 2016 11:21:20 -0700 Subject: [PATCH 140/155] Add explicit license to grpcio-tools setup.py --- tools/distrib/python/grpcio_tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 4cc1a1eaf88..576f7ae32a5 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -83,7 +83,7 @@ def maybe_cythonize(exts): setuptools.setup( name='grpcio_tools', version=grpc_version.VERSION, - license='', + license='3-clause BSD', ext_modules=maybe_cythonize([ protoc_ext_module(), ]), From 49beb933a7af096c2b775ff220d38d79c5eee46c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 9 May 2016 11:22:04 -0700 Subject: [PATCH 141/155] upgrade linux worker setup scripts --- tools/gce/create_linux_worker.sh | 2 +- tools/gce/linux_worker_init.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh index dff0b1ce5f4..c41e4d299bf 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_worker.sh @@ -43,7 +43,7 @@ gcloud compute instances create $INSTANCE_NAME \ --project="$CLOUD_PROJECT" \ --zone "$ZONE" \ --machine-type n1-standard-8 \ - --image ubuntu-14-04 \ + --image ubuntu-15-10 \ --boot-disk-size 1000 echo 'Created GCE instance, waiting 60 seconds for it to come online.' diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh index ef6a5d175c9..afcf7a52d92 100755 --- a/tools/gce/linux_worker_init.sh +++ b/tools/gce/linux_worker_init.sh @@ -37,7 +37,7 @@ set -ex sudo apt-get update # Install JRE -sudo apt-get install -y openjdk-7-jre +sudo apt-get install -y openjdk-8-jre sudo apt-get install -y unzip lsof # Install Docker From 12fd55587d6bbba788f0601a4e106fc27b4cc8d0 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 9 May 2016 14:00:36 -0700 Subject: [PATCH 142/155] Build Node 6 artifacts --- tools/run_tests/build_artifact_node.bat | 2 +- tools/run_tests/build_artifact_node.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/build_artifact_node.bat b/tools/run_tests/build_artifact_node.bat index 84c63c28a25..3c23ac1daea 100644 --- a/tools/run_tests/build_artifact_node.bat +++ b/tools/run_tests/build_artifact_node.bat @@ -27,7 +27,7 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set node_versions=0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 +set node_versions=0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 6.0.0 set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm diff --git a/tools/run_tests/build_artifact_node.sh b/tools/run_tests/build_artifact_node.sh index ef3476a0381..9d06472aa49 100755 --- a/tools/run_tests/build_artifact_node.sh +++ b/tools/run_tests/build_artifact_node.sh @@ -42,7 +42,7 @@ mkdir -p artifacts npm update -node_versions=( 0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 ) +node_versions=( 0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 6.0.0 ) for version in ${node_versions[@]} do From 019a76e300f0170c4ecc09bdc008734ba75c6e6e Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 9 May 2016 14:57:29 -0700 Subject: [PATCH 143/155] Use local copy of node-pre-gyp on Windows to ensure that it is up to date --- tools/run_tests/build_artifact_node.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/build_artifact_node.bat b/tools/run_tests/build_artifact_node.bat index 3c23ac1daea..4f18985a120 100644 --- a/tools/run_tests/build_artifact_node.bat +++ b/tools/run_tests/build_artifact_node.bat @@ -38,12 +38,12 @@ call npm update || goto :error mkdir artifacts for %%v in (%node_versions%) do ( - call node-pre-gyp configure build --target=%%v --target_arch=%1 + call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1 @rem Try again after removing openssl headers rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\iojs-%%v\include\node\openssl" /S /Q - call node-pre-gyp build package testpackage --target=%%v --target_arch=%1 || goto :error + call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1 || goto :error xcopy /Y /I /S build\stage\* artifacts\ || goto :error ) From 33b3b2a3f0fac6105dca46adfe4d68b79fe00aca Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 9 May 2016 14:59:49 -0700 Subject: [PATCH 144/155] Fixed earlier fix with the right command --- tools/run_tests/build_artifact_node.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/build_artifact_node.bat b/tools/run_tests/build_artifact_node.bat index 4f18985a120..c5bd726db7e 100644 --- a/tools/run_tests/build_artifact_node.bat +++ b/tools/run_tests/build_artifact_node.bat @@ -43,7 +43,7 @@ for %%v in (%node_versions%) do ( @rem Try again after removing openssl headers rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\iojs-%%v\include\node\openssl" /S /Q - call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1 || goto :error + call .\node_modules\.bin\node-pre-gyp.cmd build package testpackage --target=%%v --target_arch=%1 || goto :error xcopy /Y /I /S build\stage\* artifacts\ || goto :error ) From 585ceca2143d4261c28fd244bfbefbe409cc0315 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 4 May 2016 12:54:14 -0700 Subject: [PATCH 145/155] Make namespacing of executables exposed by grpc-tools packages consistent between Node and Ruby --- src/node/tools/package.json | 4 ++-- src/ruby/tools/bin/{protoc.rb => grpc_tools_ruby_protoc.rb} | 0 ...c_grpc_ruby_plugin.rb => grpc_tools_ruby_protoc_plugin.rb} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/ruby/tools/bin/{protoc.rb => grpc_tools_ruby_protoc.rb} (100%) rename src/ruby/tools/bin/{protoc_grpc_ruby_plugin.rb => grpc_tools_ruby_protoc_plugin.rb} (100%) diff --git a/src/node/tools/package.json b/src/node/tools/package.json index 9bca5eab6f2..b8b4c9aea1b 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -16,8 +16,8 @@ } ], "bin": { - "grpc-tools-protoc": "./bin/protoc.js", - "grpc-tools-plugin": "./bin/protoc_plugin.js" + "grpc_tools_node_protoc": "./bin/protoc.js", + "grpc_tools_node_protoc_plugin": "./bin/protoc_plugin.js" }, "scripts": { "install": "./node_modules/.bin/node-pre-gyp install" diff --git a/src/ruby/tools/bin/protoc.rb b/src/ruby/tools/bin/grpc_tools_ruby_protoc.rb similarity index 100% rename from src/ruby/tools/bin/protoc.rb rename to src/ruby/tools/bin/grpc_tools_ruby_protoc.rb diff --git a/src/ruby/tools/bin/protoc_grpc_ruby_plugin.rb b/src/ruby/tools/bin/grpc_tools_ruby_protoc_plugin.rb similarity index 100% rename from src/ruby/tools/bin/protoc_grpc_ruby_plugin.rb rename to src/ruby/tools/bin/grpc_tools_ruby_protoc_plugin.rb From f08d3af2077447f9b3161a529f4b35bb93f06fc7 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 4 May 2016 17:43:07 -0700 Subject: [PATCH 146/155] Updated template file --- templates/src/node/tools/package.json.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/src/node/tools/package.json.template b/templates/src/node/tools/package.json.template index 4f673c48d1c..69ad71a3b83 100644 --- a/templates/src/node/tools/package.json.template +++ b/templates/src/node/tools/package.json.template @@ -18,8 +18,8 @@ } ], "bin": { - "grpc-tools-protoc": "./bin/protoc.js", - "grpc-tools-plugin": "./bin/protoc_plugin.js" + "grpc_tools_node_protoc": "./bin/protoc.js", + "grpc_tools_node_protoc_plugin": "./bin/protoc_plugin.js" }, "scripts": { "install": "./node_modules/.bin/node-pre-gyp install" From 65ca9dcabc464a1ea9ea74b8891c77c27008dacd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 4 May 2016 14:23:48 -0700 Subject: [PATCH 147/155] eliminate a thread switch when invoking server-side handler --- src/csharp/Grpc.Core/Server.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/csharp/Grpc.Core/Server.cs b/src/csharp/Grpc.Core/Server.cs index 5b61b7f060f..9b0895d8554 100644 --- a/src/csharp/Grpc.Core/Server.cs +++ b/src/csharp/Grpc.Core/Server.cs @@ -283,6 +283,8 @@ namespace Grpc.Core /// private void HandleNewServerRpc(bool success, BatchContextSafeHandle ctx) { + Task.Run(AllowOneRpc); + if (success) { ServerRpcNew newRpc = ctx.GetServerRpcNew(this); @@ -290,11 +292,9 @@ namespace Grpc.Core // after server shutdown, the callback returns with null call if (!newRpc.Call.IsInvalid) { - Task.Run(async () => await HandleCallAsync(newRpc)).ConfigureAwait(false); + HandleCallAsync(newRpc); // we don't need to await. } } - - AllowOneRpc(); } /// From 26cc1427e93e5f65a5a1a07edd275fdf713e7309 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 4 May 2016 17:21:17 -0700 Subject: [PATCH 148/155] start server with more than one allowed RPCs --- src/csharp/Grpc.Core/Server.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Server.cs b/src/csharp/Grpc.Core/Server.cs index 9b0895d8554..fea76d557ad 100644 --- a/src/csharp/Grpc.Core/Server.cs +++ b/src/csharp/Grpc.Core/Server.cs @@ -48,6 +48,7 @@ namespace Grpc.Core /// public class Server { + const int InitialAllowRpcTokenCount = 10; static readonly ILogger Logger = GrpcEnvironment.Logger.ForType(); readonly AtomicCounter activeCallCounter = new AtomicCounter(); @@ -129,7 +130,13 @@ namespace Grpc.Core startRequested = true; handle.Start(); - AllowOneRpc(); + + // Starting with more than one AllowOneRpc tokens can significantly increase + // unary RPC throughput. + for (int i = 0; i < InitialAllowRpcTokenCount; i++) + { + AllowOneRpc(); + } } } From e458d8463548ff09acb4bdbe953f0e8baae2187b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 9 May 2016 14:21:52 -0700 Subject: [PATCH 149/155] dont create dedicated threads for async client --- .../Grpc.IntegrationTesting/ClientRunners.cs | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs index b4572756f24..9eaf6bf7ce2 100644 --- a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs +++ b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs @@ -142,8 +142,7 @@ namespace Grpc.IntegrationTesting for (int i = 0; i < outstandingRpcsPerChannel; i++) { var timer = CreateTimer(loadParams, 1.0 / this.channels.Count / outstandingRpcsPerChannel); - var threadBody = GetThreadBody(channel, timer); - this.runnerTasks.Add(Task.Factory.StartNew(threadBody, TaskCreationOptions.LongRunning)); + this.runnerTasks.Add(RunClientAsync(channel, timer)); } } } @@ -269,38 +268,30 @@ namespace Grpc.IntegrationTesting } } - private Action GetThreadBody(Channel channel, IInterarrivalTimer timer) + private Task RunClientAsync(Channel channel, IInterarrivalTimer timer) { if (payloadConfig.PayloadCase == PayloadConfig.PayloadOneofCase.BytebufParams) { GrpcPreconditions.CheckArgument(clientType == ClientType.ASYNC_CLIENT, "Generic client only supports async API"); GrpcPreconditions.CheckArgument(rpcType == RpcType.STREAMING, "Generic client only supports streaming calls"); - return () => - { - RunGenericStreamingAsync(channel, timer).Wait(); - }; + return RunGenericStreamingAsync(channel, timer); } GrpcPreconditions.CheckNotNull(payloadConfig.SimpleParams); if (clientType == ClientType.SYNC_CLIENT) { GrpcPreconditions.CheckArgument(rpcType == RpcType.UNARY, "Sync client can only be used for Unary calls in C#"); - return () => RunUnary(channel, timer); + // create a dedicated thread for the synchronous client + return Task.Factory.StartNew(() => RunUnary(channel, timer), TaskCreationOptions.LongRunning); } else if (clientType == ClientType.ASYNC_CLIENT) { switch (rpcType) { case RpcType.UNARY: - return () => - { - RunUnaryAsync(channel, timer).Wait(); - }; + return RunUnaryAsync(channel, timer); case RpcType.STREAMING: - return () => - { - RunStreamingPingPongAsync(channel, timer).Wait(); - }; + return RunStreamingPingPongAsync(channel, timer); } } throw new ArgumentException("Unsupported configuration."); From 16713b91ccfabd5fd904051df17db86dae37b206 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 9 May 2016 14:34:11 -0700 Subject: [PATCH 150/155] enable previously disabled C# scenarios --- .../run_tests/performance/scenario_config.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index d3937096231..8b239951498 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -259,18 +259,16 @@ class CSharpLanguage: 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') - # TODO(jtattermusch): scenario works locally but fails on jenkins - #yield _ping_pong_scenario( - # 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', - # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - # use_unconstrained_client=True, - # categories=[SMOKETEST]) + yield _ping_pong_scenario( + 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + use_unconstrained_client=True, + categories=[SMOKETEST]) - # TODO(jtattermusch): scenario works locally but fails on jenkins - #yield _ping_pong_scenario( - # 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', - # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - # use_unconstrained_client=True) + yield _ping_pong_scenario( + 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + use_unconstrained_client=True) yield _ping_pong_scenario( 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', From df0872f2a609c9fc448923ad6a466f9b9033a63a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 9 May 2016 15:03:05 -0700 Subject: [PATCH 151/155] expose experimental API to set GrpcThreadPool size. --- src/csharp/Grpc.Core/GrpcEnvironment.cs | 34 +++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs index a5c78cc9d7e..bee0ef1d62e 100644 --- a/src/csharp/Grpc.Core/GrpcEnvironment.cs +++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs @@ -45,11 +45,12 @@ namespace Grpc.Core /// public class GrpcEnvironment { - const int THREAD_POOL_SIZE = 4; + const int MinDefaultThreadPoolSize = 4; static object staticLock = new object(); static GrpcEnvironment instance; static int refCount; + static int? customThreadPoolSize; static ILogger logger = new ConsoleLogger(); @@ -122,6 +123,23 @@ namespace Grpc.Core logger = customLogger; } + /// + /// Sets the number of threads in the gRPC thread pool that polls for internal RPC events. + /// Can be only invoke before the GrpcEnviroment is started and cannot be changed afterwards. + /// Setting thread pool size is an advanced setting and you should only use it if you know what you are doing. + /// Most users should rely on the default value provided by gRPC library. + /// Note: this method is part of an experimental API that can change or be removed without any prior notice. + /// + public static void SetThreadPoolSize(int threadCount) + { + lock (staticLock) + { + GrpcPreconditions.CheckState(instance == null, "Can only be set before GrpcEnvironment is initialized"); + GrpcPreconditions.CheckArgument(threadCount > 0, "threadCount needs to be a positive number"); + customThreadPoolSize = threadCount; + } + } + /// /// Creates gRPC environment. /// @@ -129,7 +147,7 @@ namespace Grpc.Core { GrpcNativeInit(); completionRegistry = new CompletionRegistry(this); - threadPool = new GrpcThreadPool(this, THREAD_POOL_SIZE); + threadPool = new GrpcThreadPool(this, GetThreadPoolSizeOrDefault()); threadPool.Start(); } @@ -200,5 +218,17 @@ namespace Grpc.Core debugStats.CheckOK(); } + + private int GetThreadPoolSizeOrDefault() + { + if (customThreadPoolSize.HasValue) + { + return customThreadPoolSize.Value; + } + // In systems with many cores, use half of the cores for GrpcThreadPool + // and the other half for .NET thread pool. This heuristic definitely needs + // more work, but seems to work reasonably well for a start. + return Math.Max(MinDefaultThreadPoolSize, Environment.ProcessorCount / 2); + } } } From 1e1fa0870f60b1c0d23da17b2aa7db9a3ca7f1ae Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 9 May 2016 15:18:11 -0700 Subject: [PATCH 152/155] dont lock to run server_request_call --- src/csharp/Grpc.Core/Server.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/csharp/Grpc.Core/Server.cs b/src/csharp/Grpc.Core/Server.cs index fea76d557ad..3a337ba8319 100644 --- a/src/csharp/Grpc.Core/Server.cs +++ b/src/csharp/Grpc.Core/Server.cs @@ -66,7 +66,7 @@ namespace Grpc.Core readonly TaskCompletionSource shutdownTcs = new TaskCompletionSource(); bool startRequested; - bool shutdownRequested; + volatile bool shutdownRequested; /// /// Create a new server. @@ -246,12 +246,9 @@ namespace Grpc.Core /// private void AllowOneRpc() { - lock (myLock) + if (!shutdownRequested) { - if (!shutdownRequested) - { - handle.RequestCall(HandleNewServerRpc, environment); - } + handle.RequestCall(HandleNewServerRpc, environment); } } From dae51b0fe5fcc85d2ab698d417d711e39286b380 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 9 May 2016 19:30:04 -0700 Subject: [PATCH 153/155] fix compilation on windows --- src/csharp/Grpc.Core/Server.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Server.cs b/src/csharp/Grpc.Core/Server.cs index 3a337ba8319..d538a4671f6 100644 --- a/src/csharp/Grpc.Core/Server.cs +++ b/src/csharp/Grpc.Core/Server.cs @@ -287,7 +287,7 @@ namespace Grpc.Core /// private void HandleNewServerRpc(bool success, BatchContextSafeHandle ctx) { - Task.Run(AllowOneRpc); + Task.Run(() => AllowOneRpc()); if (success) { From 5ecdda759a9b1c8cb1b10ad08d50bb1be7b9e332 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Tue, 10 May 2016 08:29:53 +0200 Subject: [PATCH 154/155] Processing the 0.14 release. --- Makefile | 2 +- build.yaml | 2 +- package.json | 2 +- src/core/lib/surface/version.c | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 2 +- src/csharp/build_packages.bat | 2 +- src/node/tools/package.json | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.core | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 45003ad8b57..29fbb69b62a 100644 --- a/Makefile +++ b/Makefile @@ -407,7 +407,7 @@ E = @echo Q = @ endif -VERSION = 0.14.0-pre1 +VERSION = 0.14.0 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index a21dc688adf..dac4a5c10af 100644 --- a/build.yaml +++ b/build.yaml @@ -7,7 +7,7 @@ settings: '#3': Use "-preN" suffixes to identify pre-release versions '#4': Per-language overrides are possible with (eg) ruby_version tag here '#5': See the expand_version.py for all the quirks here - version: 0.14.0-pre1 + version: 0.14.0 filegroups: - name: census public_headers: diff --git a/package.json b/package.json index b22cfc1f938..a9107e82cf2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "0.14.0-pre1", + "version": "0.14.0", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "http://www.grpc.io/", diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c index 0f4b1111f43..2a0bc5a47d2 100644 --- a/src/core/lib/surface/version.c +++ b/src/core/lib/surface/version.c @@ -36,4 +36,4 @@ #include -const char *grpc_version_string(void) { return "0.14.0-pre1"; } +const char *grpc_version_string(void) { return "0.14.0"; } diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 70ce9a18df5..a22e287c6aa 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -53,6 +53,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "0.14.0-pre1"; + public const string CurrentVersion = "0.14.0"; } } diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat index 9c7b877feab..68b52e211f5 100644 --- a/src/csharp/build_packages.bat +++ b/src/csharp/build_packages.bat @@ -1,7 +1,7 @@ @rem Builds gRPC NuGet packages @rem Current package versions -set VERSION=0.14.0-pre1 +set VERSION=0.14.0 set PROTOBUF_VERSION=3.0.0-beta2 @rem Packages that depend on prerelease packages (like Google.Protobuf) need to have prerelease suffix as well. diff --git a/src/node/tools/package.json b/src/node/tools/package.json index b8b4c9aea1b..25dadcd4cd9 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "0.14.0-pre1", + "version": "0.14.0", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "http://www.grpc.io/", diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 87fadf9a979..f1d07120c06 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='0.14.0rc1' +VERSION='0.14.0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 5117c85cea0..7a0a81bf000 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -29,5 +29,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '0.14.0.pre1' + VERSION = '0.14.0' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index ec085d2655b..80d109f82cd 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -29,6 +29,6 @@ module GRPC module Tools - VERSION = '0.14.0.pre1' + VERSION = '0.14.0' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index ee522a0bc31..6c37bc3d8bc 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='0.14.0rc1' +VERSION='0.14.0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index f58f7e72817..d08a676a117 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-pre1 +PROJECT_NUMBER = 0.14.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index a7a17bac3fc..38fdbe98586 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-pre1 +PROJECT_NUMBER = 0.14.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 8b13e6e7933..917200ed5a0 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-pre1 +PROJECT_NUMBER = 0.14.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index b3b34778743..b79f3951284 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0-pre1 +PROJECT_NUMBER = 0.14.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 2c75731b689ad2fa569668468dfae684a6be9458 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Tue, 10 May 2016 09:26:34 +0200 Subject: [PATCH 155/155] The release branch is now 0.14.1-pre1. --- Makefile | 2 +- build.yaml | 2 +- composer.json | 2 +- package.json | 2 +- package.xml | 8 ++++---- src/core/lib/surface/version.c | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/build_packages.bat | 2 +- src/node/tools/package.json | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.core | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- 17 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 29fbb69b62a..0547faf5bb3 100644 --- a/Makefile +++ b/Makefile @@ -407,7 +407,7 @@ E = @echo Q = @ endif -VERSION = 0.14.0 +VERSION = 0.14.1-pre1 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index dac4a5c10af..1b7d4e03ab0 100644 --- a/build.yaml +++ b/build.yaml @@ -7,7 +7,7 @@ settings: '#3': Use "-preN" suffixes to identify pre-release versions '#4': Per-language overrides are possible with (eg) ruby_version tag here '#5': See the expand_version.py for all the quirks here - version: 0.14.0 + version: 0.14.1-pre1 filegroups: - name: census public_headers: diff --git a/composer.json b/composer.json index 97b1a5cb49b..0bf0ff4b45a 100644 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc", "type": "library", "description": "gRPC library for PHP", - "version": "0.14.0", + "version": "0.14.1", "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", diff --git a/package.json b/package.json index a9107e82cf2..ae5fb0c6572 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "0.14.0", + "version": "0.14.1-pre1", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "http://www.grpc.io/", diff --git a/package.xml b/package.xml index ab6dee06318..c90d6f6b4b9 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2016-04-19 - 0.14.0 - 0.14.0 + 0.14.1 + 0.14.1 beta @@ -1014,8 +1014,8 @@ Update to wrap gRPC C Core version 0.10.0 - 0.14.0 - 0.14.0 + 0.14.1 + 0.14.1 beta diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c index 2a0bc5a47d2..a7fa0f3565d 100644 --- a/src/core/lib/surface/version.c +++ b/src/core/lib/surface/version.c @@ -36,4 +36,4 @@ #include -const char *grpc_version_string(void) { return "0.14.0"; } +const char *grpc_version_string(void) { return "0.14.1-pre1"; } diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index a22e287c6aa..6c6a08775d4 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -48,11 +48,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "0.14.0.0"; + public const string CurrentAssemblyFileVersion = "0.14.1.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "0.14.0"; + public const string CurrentVersion = "0.14.1-pre1"; } } diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat index 68b52e211f5..cf700d7581b 100644 --- a/src/csharp/build_packages.bat +++ b/src/csharp/build_packages.bat @@ -1,7 +1,7 @@ @rem Builds gRPC NuGet packages @rem Current package versions -set VERSION=0.14.0 +set VERSION=0.14.1-pre1 set PROTOBUF_VERSION=3.0.0-beta2 @rem Packages that depend on prerelease packages (like Google.Protobuf) need to have prerelease suffix as well. diff --git a/src/node/tools/package.json b/src/node/tools/package.json index 25dadcd4cd9..da85b6455b4 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "0.14.0", + "version": "0.14.1-pre1", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "http://www.grpc.io/", diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index f1d07120c06..e688e32c5f1 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='0.14.0' +VERSION='0.14.1rc1' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 7a0a81bf000..d8c2578c70c 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -29,5 +29,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '0.14.0' + VERSION = '0.14.1.pre1' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 80d109f82cd..6dd5c339e2c 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -29,6 +29,6 @@ module GRPC module Tools - VERSION = '0.14.0' + VERSION = '0.14.1.pre1' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 6c37bc3d8bc..cfd27d45c27 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='0.14.0' +VERSION='0.14.1rc1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index d08a676a117..8ea1019f407 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0 +PROJECT_NUMBER = 0.14.1-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 38fdbe98586..cb444ef1370 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0 +PROJECT_NUMBER = 0.14.1-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 917200ed5a0..34002354cf1 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0 +PROJECT_NUMBER = 0.14.1-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index b79f3951284..434f6c92a3b 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.14.0 +PROJECT_NUMBER = 0.14.1-pre1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a