From 5eb6962187977a267d848aff41bdfa7f344ce1e3 Mon Sep 17 00:00:00 2001 From: Carl Mastrangelo Date: Mon, 30 Jul 2018 16:11:01 -0700 Subject: [PATCH 01/99] bump grpc java to 1.14.0 --- tools/interop_matrix/client_matrix.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index fae78909daf..26bd4832032 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -178,6 +178,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.1': None }, + { + 'v1.14.0': None + }, ], 'python': [ { From ab8b628b31c8c4b564c09dacac604025f9870463 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Wed, 1 Aug 2018 12:30:57 +0200 Subject: [PATCH 02/99] Create a --channel_creds_type flag for grpc_cli. This replaces several mutually-exclusive flags for a single selector flag which defines the base channel type. It also allows child classes to selectively override the defaults and provide additional channel types (e.g., LOAS in the mono repo). Fixes issue #16060. --- test/cpp/util/cli_credentials.cc | 96 ++++++++++++++++++++++++++------ test/cpp/util/cli_credentials.h | 14 ++++- test/cpp/util/grpc_tool_test.cc | 11 ++-- 3 files changed, 97 insertions(+), 24 deletions(-) diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index d14dc18f168..a560c314109 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -20,7 +20,9 @@ #include -DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); +DEFINE_bool( + enable_ssl, false, + "Whether to use ssl/tls. Deprecated. Use --channel_creds_type=ssl."); DEFINE_bool(use_auth, false, "Whether to create default google credentials."); DEFINE_string( access_token, "", @@ -29,47 +31,105 @@ DEFINE_string( ssl_target, "", "If not empty, treat the server host name as this for ssl/tls certificate " "validation."); +DEFINE_string( + channel_creds_type, "", + "The channel creds type: insecure, ssl, or alts."); namespace grpc { namespace testing { -std::shared_ptr CliCredentials::GetCredentials() +grpc::string CliCredentials::GetDefaultChannelCredsType() const { + // Compatibility logic for --enable_ssl. + if (FLAGS_enable_ssl) { + fprintf(stderr, "warning: --enable_ssl is deprecated. Use " + "--channel_creds_type=ssl.\n"); + return "ssl"; + } + // Implicit channel for GoogleDefaultCredentials is SSL. + if (FLAGS_access_token.empty() && FLAGS_use_auth) { + return "ssl"; + } + return "insecure"; +} + +std::shared_ptr + CliCredentials::GetChannelCredentials() const { + if (FLAGS_channel_creds_type.compare("insecure") == 0) { + return grpc::InsecureChannelCredentials(); + } else if (FLAGS_channel_creds_type.compare("ssl") == 0) { + return grpc::SslCredentials(grpc::SslCredentialsOptions()); + } else if (FLAGS_channel_creds_type.compare("alts") == 0) { + return grpc::experimental::AltsCredentials( + grpc::experimental::AltsCredentialsOptions()); + } + fprintf(stderr, + "--channel_creds_type=%s invalid; must be insecure, ssl or alts.\n", + FLAGS_channel_creds_type.c_str()); + return std::shared_ptr(); +} + +std::shared_ptr CliCredentials::GetCallCredentials() const { if (!FLAGS_access_token.empty()) { if (FLAGS_use_auth) { fprintf(stderr, "warning: use_auth is ignored when access_token is provided."); } - - return grpc::CompositeChannelCredentials( - grpc::SslCredentials(grpc::SslCredentialsOptions()), - grpc::AccessTokenCredentials(FLAGS_access_token)); + return grpc::AccessTokenCredentials(FLAGS_access_token); } + // TODO(@capstan): Support GoogleDefaultCredentials on other channel types. + return std::shared_ptr(); +} - if (FLAGS_use_auth) { - return grpc::GoogleDefaultCredentials(); +std::shared_ptr CliCredentials::GetCredentials() + const { + if (FLAGS_channel_creds_type.empty()) { + FLAGS_channel_creds_type = GetDefaultChannelCredsType(); + } else if (FLAGS_enable_ssl && FLAGS_channel_creds_type.compare("ssl") != 0) { + fprintf(stderr, "warning: ignoring --enable_ssl because " + "--channel_creds_type already set to %s.\n", + FLAGS_channel_creds_type.c_str()); } - - if (FLAGS_enable_ssl) { - return grpc::SslCredentials(grpc::SslCredentialsOptions()); + std::shared_ptr channel_creds; + if (FLAGS_access_token.empty() && FLAGS_use_auth) { + // Today, GoogleDefaultCredentials implies SSL and service account. + if (FLAGS_channel_creds_type.compare("ssl") != 0) { + fprintf(stderr, + "warning: ignoring --channel_creds_type=%s because --use_auth.", + FLAGS_channel_creds_type.c_str()); + } + channel_creds = grpc::GoogleDefaultCredentials(); + } else { + // Legacy transport upgrade logic for insecure requests. + if (!FLAGS_access_token.empty() && + FLAGS_channel_creds_type.compare("insecure") == 0) { + fprintf(stderr, + "warning: --channel_creds_type=insecure upgraded to ssl because " + "an access token was provided.\n"); + FLAGS_channel_creds_type = "ssl"; + } + channel_creds = GetChannelCredentials(); } - - return grpc::InsecureChannelCredentials(); + // Composite any call-type credentials on top of the base channel. + std::shared_ptr call_creds = GetCallCredentials(); + return (channel_creds == nullptr || call_creds == nullptr) ? channel_creds : + grpc::CompositeChannelCredentials(channel_creds, call_creds); } const grpc::string CliCredentials::GetCredentialUsage() const { - return " --enable_ssl ; Set whether to use tls\n" + return " --enable_ssl ; Set whether to use ssl (deprecated)\n" " --use_auth ; Set whether to create default google" " credentials\n" " --access_token ; Set the access token in metadata," " overrides --use_auth\n" - " --ssl_target ; Set server host for tls validation\n"; + " --ssl_target ; Set server host for ssl validation\n" + " --channel_creds_type ; Set to insecure, ssl, alts\n"; } const grpc::string CliCredentials::GetSslTargetNameOverride() const { - bool use_tls = - FLAGS_enable_ssl || (FLAGS_access_token.empty() && FLAGS_use_auth); - return use_tls ? FLAGS_ssl_target : ""; + bool use_ssl = FLAGS_channel_creds_type.compare("ssl") == 0 || + (FLAGS_access_token.empty() && FLAGS_use_auth); + return use_ssl ? FLAGS_ssl_target : ""; } } // namespace testing diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index 8d662356de8..259bd0ab7a6 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -28,9 +28,21 @@ namespace testing { class CliCredentials { public: virtual ~CliCredentials() {} - virtual std::shared_ptr GetCredentials() const; + std::shared_ptr GetCredentials() const; virtual const grpc::string GetCredentialUsage() const; virtual const grpc::string GetSslTargetNameOverride() const; + protected: + // Returns the appropriate channel_creds_type value for the set of legacy + // flag arguments. + virtual grpc::string GetDefaultChannelCredsType() const; + // Returns the base transport channel credentials. Child classes can override + // to support additional channel_creds_types unknown to this base class. + virtual std::shared_ptr GetChannelCredentials() + const; + // Returns call credentials to composite onto the base transport channel + // credentials. Child classes can override to support additional + // authentication flags unknown to this base class. + virtual std::shared_ptr GetCallCredentials() const; }; } // namespace testing diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 7e7f44551ef..555f02af4b3 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -81,7 +81,7 @@ using grpc::testing::EchoResponse; " peer: \"peer\"\n" \ "}\n\n" -DECLARE_bool(enable_ssl); +DECLARE_string(channel_creds_type); DECLARE_string(ssl_target); namespace grpc { @@ -102,7 +102,8 @@ const int kServerDefaultResponseStreamsToSend = 3; class TestCliCredentials final : public grpc::testing::CliCredentials { public: TestCliCredentials(bool secure = false) : secure_(secure) {} - std::shared_ptr GetCredentials() const override { + std::shared_ptr GetChannelCredentials() const + override { if (!secure_) { return InsecureChannelCredentials(); } @@ -769,12 +770,12 @@ TEST_F(GrpcToolTest, CallCommandWithBadMetadata) { TEST_F(GrpcToolTest, ListCommand_OverrideSslHostName) { const grpc::string server_address = SetUpServer(true); - // Test input "grpc_cli ls localhost: --enable_ssl + // Test input "grpc_cli ls localhost: --channel_creds_type=ssl // --ssl_target=z.test.google.fr" std::stringstream output_stream; const char* argv[] = {"grpc_cli", "ls", server_address.c_str()}; FLAGS_l = false; - FLAGS_enable_ssl = true; + FLAGS_channel_creds_type = "ssl"; FLAGS_ssl_target = "z.test.google.fr"; EXPECT_TRUE( 0 == GrpcToolMainLib( @@ -784,7 +785,7 @@ TEST_F(GrpcToolTest, ListCommand_OverrideSslHostName) { "grpc.testing.EchoTestService\n" "grpc.reflection.v1alpha.ServerReflection\n")); - FLAGS_enable_ssl = false; + FLAGS_channel_creds_type = ""; FLAGS_ssl_target = ""; ShutdownServer(); } From 9ddb23442b62beeecb1f06921e76a850faa6ba40 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Wed, 1 Aug 2018 13:08:03 +0200 Subject: [PATCH 03/99] Clang format. --- test/cpp/util/cli_credentials.cc | 20 +++++++++++--------- test/cpp/util/cli_credentials.h | 1 + test/cpp/util/grpc_tool_test.cc | 4 ++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index a560c314109..d223411121b 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -31,9 +31,8 @@ DEFINE_string( ssl_target, "", "If not empty, treat the server host name as this for ssl/tls certificate " "validation."); -DEFINE_string( - channel_creds_type, "", - "The channel creds type: insecure, ssl, or alts."); +DEFINE_string(channel_creds_type, "", + "The channel creds type: insecure, ssl, or alts."); namespace grpc { namespace testing { @@ -41,7 +40,8 @@ namespace testing { grpc::string CliCredentials::GetDefaultChannelCredsType() const { // Compatibility logic for --enable_ssl. if (FLAGS_enable_ssl) { - fprintf(stderr, "warning: --enable_ssl is deprecated. Use " + fprintf(stderr, + "warning: --enable_ssl is deprecated. Use " "--channel_creds_type=ssl.\n"); return "ssl"; } @@ -53,7 +53,7 @@ grpc::string CliCredentials::GetDefaultChannelCredsType() const { } std::shared_ptr - CliCredentials::GetChannelCredentials() const { +CliCredentials::GetChannelCredentials() const { if (FLAGS_channel_creds_type.compare("insecure") == 0) { return grpc::InsecureChannelCredentials(); } else if (FLAGS_channel_creds_type.compare("ssl") == 0) { @@ -86,7 +86,8 @@ std::shared_ptr CliCredentials::GetCredentials() if (FLAGS_channel_creds_type.empty()) { FLAGS_channel_creds_type = GetDefaultChannelCredsType(); } else if (FLAGS_enable_ssl && FLAGS_channel_creds_type.compare("ssl") != 0) { - fprintf(stderr, "warning: ignoring --enable_ssl because " + fprintf(stderr, + "warning: ignoring --enable_ssl because " "--channel_creds_type already set to %s.\n", FLAGS_channel_creds_type.c_str()); } @@ -112,8 +113,9 @@ std::shared_ptr CliCredentials::GetCredentials() } // Composite any call-type credentials on top of the base channel. std::shared_ptr call_creds = GetCallCredentials(); - return (channel_creds == nullptr || call_creds == nullptr) ? channel_creds : - grpc::CompositeChannelCredentials(channel_creds, call_creds); + return (channel_creds == nullptr || call_creds == nullptr) + ? channel_creds + : grpc::CompositeChannelCredentials(channel_creds, call_creds); } const grpc::string CliCredentials::GetCredentialUsage() const { @@ -128,7 +130,7 @@ const grpc::string CliCredentials::GetCredentialUsage() const { const grpc::string CliCredentials::GetSslTargetNameOverride() const { bool use_ssl = FLAGS_channel_creds_type.compare("ssl") == 0 || - (FLAGS_access_token.empty() && FLAGS_use_auth); + (FLAGS_access_token.empty() && FLAGS_use_auth); return use_ssl ? FLAGS_ssl_target : ""; } diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index 259bd0ab7a6..4636d3ca149 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -31,6 +31,7 @@ class CliCredentials { std::shared_ptr GetCredentials() const; virtual const grpc::string GetCredentialUsage() const; virtual const grpc::string GetSslTargetNameOverride() const; + protected: // Returns the appropriate channel_creds_type value for the set of legacy // flag arguments. diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 555f02af4b3..3aae090e818 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -102,8 +102,8 @@ const int kServerDefaultResponseStreamsToSend = 3; class TestCliCredentials final : public grpc::testing::CliCredentials { public: TestCliCredentials(bool secure = false) : secure_(secure) {} - std::shared_ptr GetChannelCredentials() const - override { + std::shared_ptr GetChannelCredentials() + const override { if (!secure_) { return InsecureChannelCredentials(); } From 5c275b5f227a725db7e58da89a7ae7a1c286cde8 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 1 Aug 2018 10:51:14 -0700 Subject: [PATCH 04/99] Fix refcounting issue --- src/objective-c/GRPCClient/GRPCCall.m | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 9783b064402..387a88dc177 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -220,17 +220,17 @@ static NSString *const kBearerPrefix = @"Bearer "; } - (void)cancel { + if (!self.isWaitingForToken) { + [self cancelCall]; + } else { + self.isWaitingForToken = NO; + } [self maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeCancelled userInfo:@{NSLocalizedDescriptionKey : @"Canceled by app"}]]; - if (!self.isWaitingForToken) { - [self cancelCall]; - } else { - self.isWaitingForToken = NO; - } } - (void)maybeFinishWithError:(NSError *)errorOrNil { @@ -292,6 +292,7 @@ static NSString *const kBearerPrefix = @"Bearer "; // don't want to throw, because the app shouldn't crash for a behavior // that's on the hands of any server to have. Instead we finish and ask // the server to cancel. + [strongSelf cancelCall]; [strongSelf maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeResourceExhausted @@ -300,7 +301,6 @@ static NSString *const kBearerPrefix = @"Bearer "; @"Client does not have enough memory to " @"hold the server response." }]]; - [strongSelf cancelCall]; return; } [strongWriteable enqueueValue:data @@ -530,13 +530,13 @@ static NSString *const kBearerPrefix = @"Bearer "; } - (void)connectivityChanged:(NSNotification *)note { + // Cancel underlying call upon this notification + [self cancelCall]; [self maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeUnavailable userInfo:@{ NSLocalizedDescriptionKey : @"Connectivity lost." }]]; - // Cancel underlying call upon this notification - [self cancelCall]; } @end From d6ef707422c8afba5046f395419a2a524a248472 Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 1 Aug 2018 11:22:32 -0700 Subject: [PATCH 05/99] Add more filter priority levels --- .../client_channel/client_channel_plugin.cc | 2 +- .../client_channel/lb_policy/grpclb/grpclb.cc | 2 +- .../ext/filters/deadline/deadline_filter.cc | 4 +-- .../filters/http/client_authority_filter.cc | 12 ++++----- .../ext/filters/http/http_filters_plugin.cc | 27 ++++++++++--------- .../server_load_reporting_filter.cc | 3 ++- .../ext/filters/max_age/max_age_filter.cc | 2 +- .../message_size/message_size_filter.cc | 6 ++--- src/core/lib/channel/connected_channel.cc | 4 +-- src/core/lib/channel/connected_channel.h | 4 +-- src/core/lib/surface/channel_init.h | 27 ++++++++++++++++++- src/core/lib/surface/init.cc | 26 ++++++++---------- src/core/lib/surface/init_secure.cc | 11 +++++--- src/cpp/common/channel_filter.cc | 9 +++++-- src/cpp/common/channel_filter.h | 7 ++++- src/cpp/ext/filters/census/grpc_plugin.cc | 6 +++-- .../channel/minimal_stack_is_minimal_test.cc | 16 +++++------ .../end2end/tests/filter_call_init_fails.cc | 15 ++++++----- test/core/end2end/tests/filter_latency.cc | 19 +++++++------ test/core/end2end/tests/filter_status_code.cc | 19 +++++++------ test/cpp/common/channel_filter_test.cc | 3 ++- test/cpp/end2end/filter_end2end_test.cc | 3 ++- 22 files changed, 133 insertions(+), 94 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel_plugin.cc b/src/core/ext/filters/client_channel/client_channel_plugin.cc index e0784b7e5c1..71da6486604 100644 --- a/src/core/ext/filters/client_channel/client_channel_plugin.cc +++ b/src/core/ext/filters/client_channel/client_channel_plugin.cc @@ -56,7 +56,7 @@ void grpc_client_channel_init(void) { grpc_register_http_proxy_mapper(); grpc_subchannel_index_init(); grpc_channel_init_register_stage( - GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, append_filter, + GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, append_filter, (void*)&grpc_client_channel_filter); grpc_http_connect_register_handshaker_factory(); } diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 2d1f7774748..af6f3fe296a 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -1880,7 +1880,7 @@ void grpc_lb_policy_grpclb_init() { grpc_core::UniquePtr( grpc_core::New())); grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_client_load_reporting_filter, (void*)&grpc_client_load_reporting_filter); } diff --git a/src/core/ext/filters/deadline/deadline_filter.cc b/src/core/ext/filters/deadline/deadline_filter.cc index d23ad67ad51..3bd30593122 100644 --- a/src/core/ext/filters/deadline/deadline_filter.cc +++ b/src/core/ext/filters/deadline/deadline_filter.cc @@ -379,10 +379,10 @@ static bool maybe_add_deadline_filter(grpc_channel_stack_builder* builder, void grpc_deadline_filter_init(void) { grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, maybe_add_deadline_filter, (void*)&grpc_client_deadline_filter); grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, maybe_add_deadline_filter, (void*)&grpc_server_deadline_filter); } diff --git a/src/core/ext/filters/http/client_authority_filter.cc b/src/core/ext/filters/http/client_authority_filter.cc index ddc939ed121..3c0ae47e8d5 100644 --- a/src/core/ext/filters/http/client_authority_filter.cc +++ b/src/core/ext/filters/http/client_authority_filter.cc @@ -146,12 +146,12 @@ static bool add_client_authority_filter(grpc_channel_stack_builder* builder, } void grpc_client_authority_filter_init(void) { - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX, - add_client_authority_filter, - (void*)&grpc_client_authority_filter); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, - add_client_authority_filter, - (void*)&grpc_client_authority_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + add_client_authority_filter, (void*)&grpc_client_authority_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + add_client_authority_filter, (void*)&grpc_client_authority_filter); } void grpc_client_authority_filter_shutdown(void) {} diff --git a/src/core/ext/filters/http/http_filters_plugin.cc b/src/core/ext/filters/http/http_filters_plugin.cc index f03fa0141df..38757710f39 100644 --- a/src/core/ext/filters/http/http_filters_plugin.cc +++ b/src/core/ext/filters/http/http_filters_plugin.cc @@ -18,6 +18,7 @@ #include +#include #include #include "src/core/ext/filters/http/client/http_client_filter.h" @@ -51,15 +52,15 @@ static bool maybe_add_optional_filter(grpc_channel_stack_builder* builder, bool enable = grpc_channel_arg_get_bool( grpc_channel_args_find(channel_args, filtarg->control_channel_arg), !grpc_channel_args_want_minimal_stack(channel_args)); - return enable ? grpc_channel_stack_builder_prepend_filter( + return enable ? grpc_channel_stack_builder_append_filter( builder, filtarg->filter, nullptr, nullptr) : true; } -static bool maybe_add_required_filter(grpc_channel_stack_builder* builder, - void* arg) { +static bool maybe_append_required_filter(grpc_channel_stack_builder* builder, + void* arg) { return is_building_http_like_transport(builder) - ? grpc_channel_stack_builder_prepend_filter( + ? grpc_channel_stack_builder_append_filter( builder, static_cast(arg), nullptr, nullptr) : true; @@ -67,23 +68,23 @@ static bool maybe_add_required_filter(grpc_channel_stack_builder* builder, void grpc_http_filters_init(void) { grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, maybe_add_optional_filter, &compress_filter); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, maybe_add_optional_filter, &compress_filter); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, maybe_add_optional_filter, &compress_filter); grpc_channel_init_register_stage( - GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_required_filter, (void*)&grpc_http_client_filter); + GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_append_required_filter, (void*)&grpc_http_client_filter); grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_required_filter, (void*)&grpc_http_client_filter); + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_append_required_filter, (void*)&grpc_http_client_filter); grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_required_filter, (void*)&grpc_http_server_filter); + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_append_required_filter, (void*)&grpc_http_server_filter); } void grpc_http_filters_shutdown(void) {} diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc index 6529046a5e7..bb2bddf7cb9 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc +++ b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc @@ -345,7 +345,8 @@ struct ServerLoadReportingFilterStaticRegistrar { if (registered) return; RegisterChannelFilter( - "server_load_reporting", GRPC_SERVER_CHANNEL, INT_MAX, + "server_load_reporting", GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_VERY_LOW, true, MaybeAddServerLoadReportingFilter); // Access measures to ensure they are initialized. Otherwise, we can't // create any valid view before the first RPC. diff --git a/src/core/ext/filters/max_age/max_age_filter.cc b/src/core/ext/filters/max_age/max_age_filter.cc index 1fe8288bd0a..7db30d5b488 100644 --- a/src/core/ext/filters/max_age/max_age_filter.cc +++ b/src/core/ext/filters/max_age/max_age_filter.cc @@ -536,7 +536,7 @@ static bool maybe_add_max_age_filter(grpc_channel_stack_builder* builder, void grpc_max_age_filter_init(void) { grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_max_age_filter, nullptr); } diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index c7fc3f2e627..1bd9cf1426b 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -311,13 +311,13 @@ static bool maybe_add_message_size_filter(grpc_channel_stack_builder* builder, void grpc_message_size_filter_init(void) { grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_message_size_filter, nullptr); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_message_size_filter, nullptr); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_message_size_filter, nullptr); } diff --git a/src/core/lib/channel/connected_channel.cc b/src/core/lib/channel/connected_channel.cc index e2ea334dedf..c78849a29b6 100644 --- a/src/core/lib/channel/connected_channel.cc +++ b/src/core/lib/channel/connected_channel.cc @@ -228,8 +228,8 @@ static void bind_transport(grpc_channel_stack* channel_stack, grpc_transport_stream_size(static_cast(t)); } -bool grpc_add_connected_filter(grpc_channel_stack_builder* builder, - void* arg_must_be_null) { +bool grpc_append_connected_filter(grpc_channel_stack_builder* builder, + void* arg_must_be_null) { GPR_ASSERT(arg_must_be_null == nullptr); grpc_transport* t = grpc_channel_stack_builder_get_transport(builder); GPR_ASSERT(t != nullptr); diff --git a/src/core/lib/channel/connected_channel.h b/src/core/lib/channel/connected_channel.h index faa1c73a21f..280daf040da 100644 --- a/src/core/lib/channel/connected_channel.h +++ b/src/core/lib/channel/connected_channel.h @@ -25,8 +25,8 @@ extern const grpc_channel_filter grpc_connected_filter; -bool grpc_add_connected_filter(grpc_channel_stack_builder* builder, - void* arg_must_be_null); +bool grpc_append_connected_filter(grpc_channel_stack_builder* builder, + void* arg_must_be_null); /* Debug helper to dig the transport stream out of a call element */ grpc_stream* grpc_connected_channel_get_stream(grpc_call_element* elem); diff --git a/src/core/lib/surface/channel_init.h b/src/core/lib/surface/channel_init.h index f01852473ba..cae7cb47542 100644 --- a/src/core/lib/surface/channel_init.h +++ b/src/core/lib/surface/channel_init.h @@ -19,13 +19,38 @@ #ifndef GRPC_CORE_LIB_SURFACE_CHANNEL_INIT_H #define GRPC_CORE_LIB_SURFACE_CHANNEL_INIT_H +#include + #include #include "src/core/lib/channel/channel_stack_builder.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/transport.h" -#define GRPC_CHANNEL_INIT_BUILTIN_PRIORITY 10000 +// Priority for channel registration functions to be used in +// grpc_channel_init_register_stage(). The priority dictates the +// order in which the registration functions run. +// +// When used to register a filter, the filter can either be appended or +// prepended, thus dictating whether the filter goes at the top or bottom of +// the stack. Higher priority functions can get closer to the top or bottom +// of the stack than lower priority functions. +enum { + // Default level. Most of filters should use this level if their location in + // the stack does not matter. + GRPC_CHANNEL_INIT_PRIORITY_LOW = 0, + // For filters that should be added after the group of filters with default + // priority, such as auth filters. + GRPC_CHANNEL_INIT_PRIORITY_MED = 10000, + // For filters that need to be close to top or bottom, such as protocol-level + // filters (client_authority, http-client, http-server) and stats (census). + GRPC_CHANNEL_INIT_PRIORITY_HIGH = 20000, + // For filters that need to be very close to the wire or surface. + GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH = 30000, + // For things that have to happen last, such as connected channel filter or + // surface server filter. Consider as reserved for gRPC internals. + GRPC_CHANNEL_INIT_PRIORITY_MAX = INT_MAX +}; /// This module provides a way for plugins (and the grpc core library itself) /// to register mutators for channel stacks. diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc index 0ad82fed99e..7807b261d40 100644 --- a/src/core/lib/surface/init.cc +++ b/src/core/lib/surface/init.cc @@ -70,11 +70,6 @@ static void do_basic_init(void) { g_initializations = 0; } -static bool append_filter(grpc_channel_stack_builder* builder, void* arg) { - return grpc_channel_stack_builder_append_filter( - builder, static_cast(arg), nullptr, nullptr); -} - static bool prepend_filter(grpc_channel_stack_builder* builder, void* arg) { return grpc_channel_stack_builder_prepend_filter( builder, static_cast(arg), nullptr, nullptr); @@ -82,19 +77,20 @@ static bool prepend_filter(grpc_channel_stack_builder* builder, void* arg) { static void register_builtin_channel_init() { grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - grpc_add_connected_filter, nullptr); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + grpc_append_connected_filter, nullptr); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - grpc_add_connected_filter, nullptr); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + grpc_append_connected_filter, nullptr); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - grpc_add_connected_filter, nullptr); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + grpc_append_connected_filter, nullptr); grpc_channel_init_register_stage(GRPC_CLIENT_LAME_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - append_filter, (void*)&grpc_lame_filter); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, prepend_filter, - (void*)&grpc_server_top_filter); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + prepend_filter, (void*)&grpc_lame_filter); + grpc_channel_init_register_stage( + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, prepend_filter, + (void*)&grpc_server_top_filter); } typedef struct grpc_plugin { diff --git a/src/core/lib/surface/init_secure.cc b/src/core/lib/surface/init_secure.cc index 28c6f7b121f..38feae22f5b 100644 --- a/src/core/lib/surface/init_secure.cc +++ b/src/core/lib/surface/init_secure.cc @@ -67,14 +67,17 @@ static bool maybe_prepend_server_auth_filter( } void grpc_register_security_filters(void) { - // Register the auth client with a priority < INT_MAX to allow the authority + // Register the auth client with a low priority to allow the authority // filter -on which the auth filter depends- to be higher on the channel // stack. - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX - 1, + grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MED, maybe_prepend_client_auth_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX - 1, + grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MED, maybe_prepend_client_auth_filter, nullptr); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MED, maybe_prepend_server_auth_filter, nullptr); } diff --git a/src/cpp/common/channel_filter.cc b/src/cpp/common/channel_filter.cc index 422e7bb65ee..0634b0416fd 100644 --- a/src/cpp/common/channel_filter.cc +++ b/src/cpp/common/channel_filter.cc @@ -78,8 +78,13 @@ bool MaybeAddFilter(grpc_channel_stack_builder* builder, void* arg) { grpc_channel_stack_builder_get_channel_arguments(builder); if (!filter.include_filter(*args)) return true; } - return grpc_channel_stack_builder_prepend_filter(builder, &filter.filter, - nullptr, nullptr); + if (filter.prepend) { + return grpc_channel_stack_builder_prepend_filter(builder, &filter.filter, + nullptr, nullptr); + } else { + return grpc_channel_stack_builder_append_filter(builder, &filter.filter, + nullptr, nullptr); + } } } // namespace diff --git a/src/cpp/common/channel_filter.h b/src/cpp/common/channel_filter.h index 5e569c97e68..359c72737c2 100644 --- a/src/cpp/common/channel_filter.h +++ b/src/cpp/common/channel_filter.h @@ -36,7 +36,8 @@ /// \c ChannelData. Then register the filter using something like this: /// \code{.cpp} /// RegisterChannelFilter( -/// "name-of-filter", GRPC_SERVER_CHANNEL, INT_MAX, nullptr); +/// "name-of-filter", GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_LOW, +/// true, nullptr); /// \endcode namespace grpc { @@ -351,6 +352,7 @@ class ChannelFilter final { struct FilterRecord { grpc_channel_stack_type stack_type; int priority; + bool prepend; std::function include_filter; grpc_channel_filter filter; }; @@ -363,12 +365,14 @@ void ChannelFilterPluginShutdown(); /// Registers a new filter. /// Must be called by only one thread at a time. +/// The \a prepend argument decides whether to prepend or append the filter. /// The \a include_filter argument specifies a function that will be called /// to determine at run-time whether or not to add the filter. If the /// value is nullptr, the filter will be added unconditionally. template void RegisterChannelFilter( const char* name, grpc_channel_stack_type stack_type, int priority, + bool prepend, std::function include_filter) { // If we haven't been called before, initialize channel_filters and // call grpc_register_plugin(). @@ -383,6 +387,7 @@ void RegisterChannelFilter( internal::FilterRecord filter_record = { stack_type, priority, + prepend, include_filter, {FilterType::StartTransportStreamOpBatch, FilterType::StartTransportOp, FilterType::call_data_size, FilterType::InitCallElement, diff --git a/src/cpp/ext/filters/census/grpc_plugin.cc b/src/cpp/ext/filters/census/grpc_plugin.cc index f978ed3bf51..f79e0e0e960 100644 --- a/src/cpp/ext/filters/census/grpc_plugin.cc +++ b/src/cpp/ext/filters/census/grpc_plugin.cc @@ -32,10 +32,12 @@ namespace grpc { void RegisterOpenCensusPlugin() { RegisterChannelFilter( - "opencensus_client", GRPC_CLIENT_CHANNEL, INT_MAX /* priority */, + "opencensus_client", GRPC_CLIENT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, true /* prepend */, nullptr /* condition function */); RegisterChannelFilter( - "opencensus_server", GRPC_SERVER_CHANNEL, INT_MAX /* priority */, + "opencensus_server", GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, true /* prepend */, nullptr /* condition function */); // Access measures to ensure they are initialized. Otherwise, creating a view diff --git a/test/core/channel/minimal_stack_is_minimal_test.cc b/test/core/channel/minimal_stack_is_minimal_test.cc index e5953acedcf..5b651ed39b7 100644 --- a/test/core/channel/minimal_stack_is_minimal_test.cc +++ b/test/core/channel/minimal_stack_is_minimal_test.cc @@ -85,21 +85,21 @@ int main(int argc, char** argv) { // tests with a default stack errors += - CHECK_STACK("unknown", nullptr, GRPC_CLIENT_DIRECT_CHANNEL, "authority", - "message_size", "deadline", "connected", NULL); + CHECK_STACK("unknown", nullptr, GRPC_CLIENT_DIRECT_CHANNEL, "deadline", + "authority", "message_size", "connected", NULL); errors += CHECK_STACK("unknown", nullptr, GRPC_CLIENT_SUBCHANNEL, "authority", "message_size", "connected", NULL); errors += CHECK_STACK("unknown", nullptr, GRPC_SERVER_CHANNEL, "server", - "message_size", "deadline", "connected", NULL); + "deadline", "message_size", "connected", NULL); errors += CHECK_STACK("chttp2", nullptr, GRPC_CLIENT_DIRECT_CHANNEL, - "authority", "message_size", "deadline", "http-client", - "message_compress", "connected", NULL); + "deadline", "authority", "message_size", + "message_compress", "http-client", "connected", NULL); errors += CHECK_STACK("chttp2", nullptr, GRPC_CLIENT_SUBCHANNEL, "authority", - "message_size", "http-client", "message_compress", + "message_size", "message_compress", "http-client", "connected", NULL); errors += CHECK_STACK("chttp2", nullptr, GRPC_SERVER_CHANNEL, "server", - "message_size", "deadline", "http-server", - "message_compress", "connected", NULL); + "deadline", "message_size", "message_compress", + "http-server", "connected", NULL); errors += CHECK_STACK(nullptr, nullptr, GRPC_CLIENT_CHANNEL, "client-channel", NULL); diff --git a/test/core/end2end/tests/filter_call_init_fails.cc b/test/core/end2end/tests/filter_call_init_fails.cc index ab96879fe44..07e14214461 100644 --- a/test/core/end2end/tests/filter_call_init_fails.cc +++ b/test/core/end2end/tests/filter_call_init_fails.cc @@ -438,7 +438,6 @@ static bool maybe_add_server_channel_filter(grpc_channel_stack_builder* builder, // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, &test_filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -457,7 +456,6 @@ static bool maybe_add_client_channel_filter(grpc_channel_stack_builder* builder, // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, &test_filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -476,7 +474,6 @@ static bool maybe_add_client_subchannel_filter( // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, &test_filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -487,13 +484,17 @@ static bool maybe_add_client_subchannel_filter( } static void init_plugin(void) { - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_server_channel_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_client_channel_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_client_subchannel_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_client_channel_filter, nullptr); } diff --git a/test/core/end2end/tests/filter_latency.cc b/test/core/end2end/tests/filter_latency.cc index a89db7b094b..02a4d079277 100644 --- a/test/core/end2end/tests/filter_latency.cc +++ b/test/core/end2end/tests/filter_latency.cc @@ -314,7 +314,6 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -326,15 +325,15 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { static void init_plugin(void) { gpr_mu_init(&g_mu); - grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_server_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, + maybe_add_filter, (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_server_filter); } static void destroy_plugin(void) { gpr_mu_destroy(&g_mu); } diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc index ba3cbfa6d11..6ed1de15c62 100644 --- a/test/core/end2end/tests/filter_status_code.cc +++ b/test/core/end2end/tests/filter_status_code.cc @@ -333,7 +333,6 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { // So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -350,15 +349,15 @@ static void init_plugin(void) { g_client_code_recv = false; g_server_code_recv = false; - grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_server_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, + maybe_add_filter, (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_server_filter); } static void destroy_plugin(void) { diff --git a/test/cpp/common/channel_filter_test.cc b/test/cpp/common/channel_filter_test.cc index 7bdd53f9e7a..9b603ca5b48 100644 --- a/test/cpp/common/channel_filter_test.cc +++ b/test/cpp/common/channel_filter_test.cc @@ -50,7 +50,8 @@ class MyCallData : public CallData { // C-core, we don't accidentally break the C++ filter API. TEST(ChannelFilterTest, RegisterChannelFilter) { grpc::RegisterChannelFilter( - "myfilter", GRPC_CLIENT_CHANNEL, INT_MAX, nullptr); + "myfilter", GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_LOW, true, + nullptr); } // TODO(roth): When we have time, add tests for all methods of the diff --git a/test/cpp/end2end/filter_end2end_test.cc b/test/cpp/end2end/filter_end2end_test.cc index 88f8f380c3a..a8022823b10 100644 --- a/test/cpp/end2end/filter_end2end_test.cc +++ b/test/cpp/end2end/filter_end2end_test.cc @@ -323,7 +323,8 @@ TEST_F(FilterEnd2endTest, SimpleBidiStreaming) { void RegisterFilter() { grpc::RegisterChannelFilter( - "test-filter", GRPC_SERVER_CHANNEL, INT_MAX, nullptr); + "test-filter", GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_LOW, true, + nullptr); } } // namespace From 010ffae0fddce8d5c9741f27314f4dafd19f248a Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 1 Aug 2018 13:33:13 -0700 Subject: [PATCH 06/99] Sanity --- src/core/lib/surface/channel_init.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/channel_init.h b/src/core/lib/surface/channel_init.h index cae7cb47542..7509fd664b9 100644 --- a/src/core/lib/surface/channel_init.h +++ b/src/core/lib/surface/channel_init.h @@ -19,10 +19,10 @@ #ifndef GRPC_CORE_LIB_SURFACE_CHANNEL_INIT_H #define GRPC_CORE_LIB_SURFACE_CHANNEL_INIT_H -#include - #include +#include + #include "src/core/lib/channel/channel_stack_builder.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/transport.h" From e61058c8ba634585e560f7acd8e4a579e0f1e63f Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 1 Aug 2018 13:37:14 -0700 Subject: [PATCH 07/99] Review comments --- src/core/lib/surface/channel_init.h | 2 +- src/core/lib/surface/init_secure.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/channel_init.h b/src/core/lib/surface/channel_init.h index 7509fd664b9..d800c83f160 100644 --- a/src/core/lib/surface/channel_init.h +++ b/src/core/lib/surface/channel_init.h @@ -43,7 +43,7 @@ enum { // priority, such as auth filters. GRPC_CHANNEL_INIT_PRIORITY_MED = 10000, // For filters that need to be close to top or bottom, such as protocol-level - // filters (client_authority, http-client, http-server) and stats (census). + // filters (client_authority, http-client, http-server). GRPC_CHANNEL_INIT_PRIORITY_HIGH = 20000, // For filters that need to be very close to the wire or surface. GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH = 30000, diff --git a/src/core/lib/surface/init_secure.cc b/src/core/lib/surface/init_secure.cc index 38feae22f5b..8058aaa804d 100644 --- a/src/core/lib/surface/init_secure.cc +++ b/src/core/lib/surface/init_secure.cc @@ -67,7 +67,7 @@ static bool maybe_prepend_server_auth_filter( } void grpc_register_security_filters(void) { - // Register the auth client with a low priority to allow the authority + // Register the auth client with a medium priority to allow the authority // filter -on which the auth filter depends- to be higher on the channel // stack. grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, From 4e294d2aa389f1936b919c0249001f421e49e605 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 1 Aug 2018 15:26:02 -0700 Subject: [PATCH 08/99] Make strong reference in notification center callback --- src/objective-c/GRPCClient/GRPCCall.m | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 387a88dc177..3c4e87cf5a9 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -531,12 +531,15 @@ static NSString *const kBearerPrefix = @"Bearer "; - (void)connectivityChanged:(NSNotification *)note { // Cancel underlying call upon this notification - [self cancelCall]; - [self maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain - code:GRPCErrorCodeUnavailable - userInfo:@{ - NSLocalizedDescriptionKey : @"Connectivity lost." - }]]; + __strong GRPCCall *strongSelf = self; + if (strongSelf) { + [self cancelCall]; + [self maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain + code:GRPCErrorCodeUnavailable + userInfo:@{ + NSLocalizedDescriptionKey : @"Connectivity lost." + }]]; + } } @end From 072c0f69480448cb1b343f51977ceff2f2bed172 Mon Sep 17 00:00:00 2001 From: Yihua Zhang Date: Wed, 1 Aug 2018 15:43:17 -0700 Subject: [PATCH 09/99] log error messages returned from Nanny --- src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index 06b999899ac..1df1021bb17 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -462,6 +462,14 @@ void alts_tsi_handshaker_handle_response(alts_tsi_handshaker* handshaker, set_unused_bytes(result, &handshaker->recv_bytes, resp->bytes_consumed); } grpc_status_code code = static_cast(resp->status.code); + if (code != GRPC_STATUS_OK) { + grpc_slice* details = static_cast(resp->status.details.arg); + if (details != nullptr) { + char* error_details = grpc_slice_to_c_string(*details); + gpr_log(GPR_ERROR, "Error from handshaker service:%s", error_details); + gpr_free(error_details); + } + } grpc_gcp_handshaker_resp_destroy(resp); cb(alts_tsi_utils_convert_to_tsi_result(code), user_data, bytes_to_send, bytes_to_send_size, result); From 841eb14ff5015388571b99cdb61b9bbc9644f43f Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 1 Aug 2018 15:57:39 -0700 Subject: [PATCH 10/99] typo fix --- .../ext/filters/load_reporting/server_load_reporting_filter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc index bb2bddf7cb9..0c4ffea27bf 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc +++ b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc @@ -346,7 +346,7 @@ struct ServerLoadReportingFilterStaticRegistrar { RegisterChannelFilter( "server_load_reporting", GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_PRIORITY_VERY_LOW, true, + GRPC_CHANNEL_INIT_PRIORITY_LOW, true, MaybeAddServerLoadReportingFilter); // Access measures to ensure they are initialized. Otherwise, we can't // create any valid view before the first RPC. From bfdfa6ad21c06648fcd76fe2609142a209bd2f7e Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Wed, 1 Aug 2018 16:19:10 -0700 Subject: [PATCH 11/99] Bump version to 1.14.0 --- BUILD | 4 ++-- build.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/BUILD b/BUILD index 2d2722780bf..783590a1343 100644 --- a/BUILD +++ b/BUILD @@ -66,9 +66,9 @@ config_setting( # This should be updated along with build.yaml g_stands_for = "gladiolus" -core_version = "6.0.0-pre2" +core_version = "6.0.0" -version = "1.14.0-pre2" +version = "1.14.0" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build.yaml b/build.yaml index d3bc84ffb94..b9dbdc41865 100644 --- a/build.yaml +++ b/build.yaml @@ -12,9 +12,9 @@ settings: '#08': Use "-preN" suffixes to identify pre-release versions '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here - core_version: 6.0.0-pre2 + core_version: 6.0.0 g_stands_for: gladiolus - version: 1.14.0-pre2 + version: 1.14.0 filegroups: - name: alts_proto headers: From 2794a6cae6631b4b6412d8e2ac0ba41a79d37f5a Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Wed, 1 Aug 2018 16:22:01 -0700 Subject: [PATCH 12/99] regenerate projects --- CMakeLists.txt | 2 +- Makefile | 6 +++--- gRPC-C++.podspec | 4 ++-- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 8 ++++---- src/core/lib/surface/version.cc | 2 +- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 2 +- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 6 +++--- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/objective-c/tests/version.h | 4 ++-- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.core | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- 31 files changed, 40 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b4c0226e178..8ea028b5a77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.14.0-pre2") +set(PACKAGE_VERSION "1.14.0") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 8b6cd4554ed..74c01698516 100644 --- a/Makefile +++ b/Makefile @@ -436,9 +436,9 @@ E = @echo Q = @ endif -CORE_VERSION = 6.0.0-pre2 -CPP_VERSION = 1.14.0-pre2 -CSHARP_VERSION = 1.14.0-pre2 +CORE_VERSION = 6.0.0 +CPP_VERSION = 1.14.0 +CSHARP_VERSION = 1.14.0 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index e38d5b48f00..cbf765abb82 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.14.0-pre2' + # version = '1.14.0' version = '0.0.3' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.14.0-pre2' + grpc_version = '1.14.0' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 1e725abf223..85071fb2114 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.14.0-pre2' + version = '1.14.0' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 412ae7adeee..3c4c6e1310e 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.14.0-pre2' + version = '1.14.0' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 1c05a318dc0..c37efca0add 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.14.0-pre2' + version = '1.14.0' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index d2657b35664..9b155cbf444 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.14.0-pre2' + version = '1.14.0' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index d5e8aaf6916..2a30cffb4d8 100644 --- a/package.xml +++ b/package.xml @@ -13,12 +13,12 @@ 2018-01-19 - 1.14.0RC2 - 1.14.0RC2 + 1.14.0 + 1.14.0 - beta - beta + stable + stable Apache 2.0 diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index ad1a4e0370e..f4b1bb0cd82 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -23,6 +23,6 @@ #include -const char* grpc_version_string(void) { return "6.0.0-pre2"; } +const char* grpc_version_string(void) { return "6.0.0"; } const char* grpc_g_stands_for(void) { return "gladiolus"; } diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 21ceb1cc183..de42c6fea30 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.14.0-pre2"; } +grpc::string Version() { return "1.14.0"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index a1fba5fffe8..92bec0e0674 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.14.0-pre2 + 1.14.0 3.5.1 diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index d888937b10e..35888b03001 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -38,6 +38,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.14.0-pre2"; + public const string CurrentVersion = "1.14.0"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index b1dbd9e69bb..bb0ae713606 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.14.0-pre2 +set VERSION=1.14.0 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 59ee72a05b9..e298ea62a9f 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -45,8 +45,8 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.14.0-pre2" -OutputDirectory ../../artifacts -nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.0-pre2" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.14.0-pre2" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.14.0" -OutputDirectory ../../artifacts +nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.0" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.14.0" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 6ebec95ba2e..bf5b7ae3511 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.14.0-pre2' + v = '1.14.0' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 6f8ee8dab94..59c172fc4ba 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-pre2" +#define GRPC_OBJC_VERSION_STRING @"1.14.0" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 74a952fe838..d0a4b496c7a 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-pre2" -#define GRPC_C_VERSION_STRING @"6.0.0-pre2" +#define GRPC_OBJC_VERSION_STRING @"1.14.0" +#define GRPC_C_VERSION_STRING @"6.0.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 9b916b466ed..22631f126ff 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.14.0RC2" +#define PHP_GRPC_VERSION "1.14.0" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index e474a6b50e4..3d3e2531554 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.14.0rc2""" +__version__ = """1.14.0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 61545652fa0..248cd632752 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.14.0rc2' +VERSION = '1.14.0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 61d27e12b39..1e48f81b791 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.14.0rc2' +VERSION = '1.14.0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 69caf17a54b..ec79577825e 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.14.0rc2' +VERSION = '1.14.0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 21bb29fff2e..50e2aa22199 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.14.0rc2' +VERSION = '1.14.0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 7bdce8175cf..18d34cc1a29 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.14.0rc2' +VERSION = '1.14.0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 9698d31dab4..55ae016bf0d 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.14.0.pre2' + VERSION = '1.14.0' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 88169f3a2b3..3bd5529215d 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.14.0.pre2' + VERSION = '1.14.0' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index cf5a7298665..f1f65265271 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.14.0rc2' +VERSION = '1.14.0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index dc8a38658eb..0a051e892ab 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-pre2 +PROJECT_NUMBER = 1.14.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index addfc2da7c0..8547dbd4c22 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-pre2 +PROJECT_NUMBER = 1.14.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 1759195c01f..cfe3455a752 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-pre2 +PROJECT_NUMBER = 6.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 643ee66402f..453adf14d38 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-pre2 +PROJECT_NUMBER = 6.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 20500213e6a14d801464292122b852b0b6d7b9f5 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 2 Aug 2018 09:51:40 +0200 Subject: [PATCH 13/99] add grpc_unity_package to published packages --- tools/internal_ci/linux/grpc_publish_packages.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index ef943db7e15..14492301cc9 100755 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -65,6 +65,7 @@ UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" CSHARP_PACKAGES=( "$UNZIPPED_CSHARP_PACKAGES"/* + "$INPUT_ARTIFACTS"/grpc_unity_package.[0-9]*.zip ) # Python From 1b36ca622c536057a555aea41e81e614a271c8db Mon Sep 17 00:00:00 2001 From: jzeferino Date: Thu, 2 Aug 2018 14:48:35 +0100 Subject: [PATCH 14/99] Improved Xamarin example doc. Xamarin Studio is no longer preferred for Xamarin development, we should use Visual Studio for Mac. Also added more info on how to use the nuget package source. --- examples/csharp/HelloworldXamarin/README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/csharp/HelloworldXamarin/README.md b/examples/csharp/HelloworldXamarin/README.md index 4394dcc0c43..7bbc8cc5e7a 100644 --- a/examples/csharp/HelloworldXamarin/README.md +++ b/examples/csharp/HelloworldXamarin/README.md @@ -6,23 +6,26 @@ EXPERIMENTAL ONLY Support of the Xamarin platform is currently experimental. The example depends on experimental Grpc.Core nuget package that hasn't been officially released and is only available via the [daily builds](https://packages.grpc.io/) -source. +source. +NOTE: To downlaod the package please manually [download](https://packages.grpc.io/archive/2018/07/a3b54ef90841ec45fe5e28f54245b7944d0904f9-d24c85c7-32ed-4924-b9af-80e7a4aeb34d/index.xml) the .nupkg file into a local directory. Then add a nuget source that points to that directory (That can be done in [Visual Studio](https://docs.microsoft.com/en-us/nuget/tools/package-manager-ui#package-sources) or Visual Studio for Mac via "Configure nuget sources"). After that, nuget will also explore that directory when looking for packages. + + BACKGROUND ------------- -The example project supports Xamarin.Android and Xamarin.iOS +The example project supports `Xamarin.Android` and `Xamarin.iOS`. For this sample, we've already generated the server and client stubs from [helloworld.proto][]. PREREQUISITES ------------- -- The latest version Xamarin Studio or Visual Studio 2017 with Xamarin support installed. +- The latest version Visual Studio for Mac or Visual Studio 2017 with Xamarin support installed. BUILD ------- -- Open the `HelloworldXamarin.sln` in Visual Studio (or Xamarin Studio) +- Open the `HelloworldXamarin.sln` in Visual Studio (or Visual Studio for Mac) - Build the solution (Build -> Build All) Try it! From 848fd9d75f6df10f00e8328ff052c0237b3002ab Mon Sep 17 00:00:00 2001 From: Naresh Date: Thu, 2 Aug 2018 14:46:36 +0000 Subject: [PATCH 15/99] Minimal Bazel BUILD files for grpcio Python Follow-up on the additions in #15992. Pad the grpcio Python package with necessary BUILD files along with 2 simple tests. --- src/python/grpcio/grpc/BUILD.bazel | 82 +++++++++++++++++++ src/python/grpcio/grpc/_cython/BUILD.bazel | 46 +++++++++++ src/python/grpcio/grpc/beta/BUILD.bazel | 58 +++++++++++++ .../grpcio/grpc/experimental/BUILD.bazel | 27 ++++++ src/python/grpcio/grpc/framework/BUILD.bazel | 11 +++ .../grpcio/grpc/framework/common/BUILD.bazel | 27 ++++++ .../grpc/framework/foundation/BUILD.bazel | 61 ++++++++++++++ .../grpc/framework/interfaces/BUILD.bazel | 10 +++ .../framework/interfaces/base/BUILD.bazel | 29 +++++++ .../framework/interfaces/face/BUILD.bazel | 32 ++++++++ .../unit/framework/foundation/BUILD.bazel | 17 ++++ 11 files changed, 400 insertions(+) create mode 100644 src/python/grpcio/grpc/BUILD.bazel create mode 100644 src/python/grpcio/grpc/_cython/BUILD.bazel create mode 100644 src/python/grpcio/grpc/beta/BUILD.bazel create mode 100644 src/python/grpcio/grpc/experimental/BUILD.bazel create mode 100644 src/python/grpcio/grpc/framework/BUILD.bazel create mode 100644 src/python/grpcio/grpc/framework/common/BUILD.bazel create mode 100644 src/python/grpcio/grpc/framework/foundation/BUILD.bazel create mode 100644 src/python/grpcio/grpc/framework/interfaces/BUILD.bazel create mode 100644 src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel create mode 100644 src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel create mode 100644 src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel diff --git a/src/python/grpcio/grpc/BUILD.bazel b/src/python/grpcio/grpc/BUILD.bazel new file mode 100644 index 00000000000..3f214bf3b05 --- /dev/null +++ b/src/python/grpcio/grpc/BUILD.bazel @@ -0,0 +1,82 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "grpcio", + srcs = ["__init__.py"], + deps = [ + ":utilities", + ":auth", + ":plugin_wrapping", + ":channel", + ":interceptor", + ":server", + "//src/python/grpcio/grpc/_cython:cygrpc", + "//src/python/grpcio/grpc/beta", + "//src/python/grpcio/grpc/experimental", + "//src/python/grpcio/grpc/framework", + requirement('enum34'), + requirement('six'), + ], + data = [ + "//:grpc", + ], + main = "__init__.py", + imports = ["../",], +) + +py_library( + name = "auth", + srcs = ["_auth.py"], +) + +py_library( + name = "channel", + srcs = ["_channel.py"], + deps = [ + ":common", + ":grpcio_metadata", + ], +) + +py_library( + name = "common", + srcs = ["_common.py"], +) + +py_library( + name = "grpcio_metadata", + srcs = ["_grpcio_metadata.py"], +) + +py_library( + name = "interceptor", + srcs = ["_interceptor.py"], +) + +py_library( + name = "plugin_wrapping", + srcs = ["_plugin_wrapping.py"], + deps = [ + ":common", + ] +) + +py_library( + name = "server", + srcs = ["_server.py"], + deps = [ + ":common", + ":interceptor", + ], +) + +py_library( + name = "utilities", + srcs = ["_utilities.py"], + deps = [ + ":common", + ], +) + diff --git a/src/python/grpcio/grpc/_cython/BUILD.bazel b/src/python/grpcio/grpc/_cython/BUILD.bazel new file mode 100644 index 00000000000..7124e83dee7 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/BUILD.bazel @@ -0,0 +1,46 @@ +package(default_visibility = ["//visibility:public"]) + +load("//bazel:cython_library.bzl", "pyx_library") + +pyx_library( + name = "cygrpc", + srcs = [ + "__init__.py", + "cygrpc.pxd", + "cygrpc.pyx", + "_cygrpc/grpc_string.pyx.pxi", + "_cygrpc/arguments.pyx.pxi", + "_cygrpc/call.pyx.pxi", + "_cygrpc/channel.pyx.pxi", + "_cygrpc/credentials.pyx.pxi", + "_cygrpc/completion_queue.pyx.pxi", + "_cygrpc/event.pyx.pxi", + "_cygrpc/metadata.pyx.pxi", + "_cygrpc/operation.pyx.pxi", + "_cygrpc/records.pyx.pxi", + "_cygrpc/security.pyx.pxi", + "_cygrpc/server.pyx.pxi", + "_cygrpc/tag.pyx.pxi", + "_cygrpc/time.pyx.pxi", + "_cygrpc/grpc_gevent.pyx.pxi", + "_cygrpc/grpc.pxi", + "_cygrpc/arguments.pxd.pxi", + "_cygrpc/call.pxd.pxi", + "_cygrpc/channel.pxd.pxi", + "_cygrpc/credentials.pxd.pxi", + "_cygrpc/completion_queue.pxd.pxi", + "_cygrpc/event.pxd.pxi", + "_cygrpc/metadata.pxd.pxi", + "_cygrpc/operation.pxd.pxi", + "_cygrpc/records.pxd.pxi", + "_cygrpc/security.pxd.pxi", + "_cygrpc/server.pxd.pxi", + "_cygrpc/tag.pxd.pxi", + "_cygrpc/time.pxd.pxi", + "_cygrpc/grpc_gevent.pxd.pxi", + ], + deps = [ + "//:grpc", + ], +) + diff --git a/src/python/grpcio/grpc/beta/BUILD.bazel b/src/python/grpcio/grpc/beta/BUILD.bazel new file mode 100644 index 00000000000..731be5cb25b --- /dev/null +++ b/src/python/grpcio/grpc/beta/BUILD.bazel @@ -0,0 +1,58 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "beta", + srcs = ["__init__.py",], + deps = [ + ":client_adaptations", + ":metadata", + ":server_adaptations", + ":implementations", + ":interfaces", + ":utilities", + ], +) + +py_library( + name = "client_adaptations", + srcs = ["_client_adaptations.py"], + imports=["../../",] +) + +py_library( + name = "metadata", + srcs = ["_metadata.py"], +) + +py_library( + name = "server_adaptations", + srcs = ["_server_adaptations.py"], + imports=["../../",], +) + +py_library( + name = "implementations", + srcs = ["implementations.py"], + imports=["../../",], +) + +py_library( + name = "interfaces", + srcs = ["interfaces.py"], + deps = [ + requirement("six"), + ], + imports=["../../",], +) + +py_library( + name = "utilities", + srcs = ["utilities.py"], + deps = [ + ":implementations", + ":interfaces", + "//src/python/grpcio/grpc/framework/foundation", + ], +) + diff --git a/src/python/grpcio/grpc/experimental/BUILD.bazel b/src/python/grpcio/grpc/experimental/BUILD.bazel new file mode 100644 index 00000000000..6598d02747b --- /dev/null +++ b/src/python/grpcio/grpc/experimental/BUILD.bazel @@ -0,0 +1,27 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "experimental", + srcs = ["__init__.py",], + deps = [ + ":gevent", + ":session_cache", + ], +) + +py_library( + name = "gevent", + srcs = ["gevent.py"], + deps = [ + "//src/python/grpcio/grpc/_cython:cygrpc", + ], +) + +py_library( + name = "session_cache", + srcs = ["session_cache.py"], + deps = [ + "//src/python/grpcio/grpc/_cython:cygrpc", + ], +) diff --git a/src/python/grpcio/grpc/framework/BUILD.bazel b/src/python/grpcio/grpc/framework/BUILD.bazel new file mode 100644 index 00000000000..55b4f4d2dfe --- /dev/null +++ b/src/python/grpcio/grpc/framework/BUILD.bazel @@ -0,0 +1,11 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "framework", + srcs = ["__init__.py",], + deps = [ + "//src/python/grpcio/grpc/framework/common", + "//src/python/grpcio/grpc/framework/foundation", + "//src/python/grpcio/grpc/framework/interfaces", + ], +) diff --git a/src/python/grpcio/grpc/framework/common/BUILD.bazel b/src/python/grpcio/grpc/framework/common/BUILD.bazel new file mode 100644 index 00000000000..9d9ef682c90 --- /dev/null +++ b/src/python/grpcio/grpc/framework/common/BUILD.bazel @@ -0,0 +1,27 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "common", + srcs = ["__init__.py",], + deps = [ + ":cardinality", + ":style", + ], +) + +py_library( + name = "cardinality", + srcs = ["cardinality.py"], + deps = [ + requirement("enum34"), + ], +) + +py_library( + name = "style", + srcs = ["style.py"], + deps = [ + requirement("enum34"), + ], +) diff --git a/src/python/grpcio/grpc/framework/foundation/BUILD.bazel b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel new file mode 100644 index 00000000000..1287fdd44ed --- /dev/null +++ b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel @@ -0,0 +1,61 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "foundation", + srcs = ["__init__.py",], + deps = [ + ":abandonment", + ":callable_util", + ":future", + ":logging_pool", + ":stream_util", + ":stream", + ], +) + +py_library( + name = "abandonment", + srcs = ["abandonment.py"], +) + +py_library( + name = "callable_util", + srcs = ["callable_util.py"], + deps = [ + requirement("enum34"), + requirement("six"), + ], +) + +py_library( + name = "future", + srcs = ["future.py"], + deps = [ + requirement("six"), + ], +) + +py_library( + name = "logging_pool", + srcs = ["logging_pool.py"], + deps = [ + requirement("futures"), + ], +) + +py_library( + name = "stream_util", + srcs = ["stream_util.py"], + deps = [ + ":stream", + ], +) + +py_library( + name = "stream", + srcs = ["stream.py"], + deps = [ + requirement("six"), + ], +) diff --git a/src/python/grpcio/grpc/framework/interfaces/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/BUILD.bazel new file mode 100644 index 00000000000..b81e196cc3c --- /dev/null +++ b/src/python/grpcio/grpc/framework/interfaces/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "interfaces", + srcs = ["__init__.py",], + deps = [ + "//src/python/grpcio/grpc/framework/interfaces/base", + "//src/python/grpcio/grpc/framework/interfaces/face", + ], +) diff --git a/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel new file mode 100644 index 00000000000..408a66a6310 --- /dev/null +++ b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel @@ -0,0 +1,29 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "base_lib", + srcs = ["__init__.py",], + deps = [ + ":base", + ":utilities", + ], +) + +py_library( + name = "base", + srcs = ["base.py"], + deps = [ + "//src/python/grpcio/grpc/framework/foundation:abandonment", + requirement("enum34"), + requirement("six"), + ], +) + +py_library( + name = "utilities", + srcs = ["utilities.py"], + deps = [ + requirement("enum34"), + ], +) diff --git a/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel new file mode 100644 index 00000000000..e683e7cc426 --- /dev/null +++ b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel @@ -0,0 +1,32 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "face", + srcs = ["__init__.py",], + deps = [ + ":face_lib", + ":utilities", + ], +) + +py_library( + name = "face_lib", + srcs = ["face.py"], + deps = [ + "//src/python/grpcio/grpc/framework/foundation", + "//src/python/grpcio/grpc/framework/common", + requirement("enum34"), + requirement("six"), + ], +) + +py_library( + name = "utilities", + srcs = ["utilities.py"], + deps = [ + "//src/python/grpcio/grpc/framework/common", + "//src/python/grpcio/grpc/framework/foundation:stream", + ":face_lib", + ], +) diff --git a/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel new file mode 100644 index 00000000000..d69186e1fde --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel @@ -0,0 +1,17 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "stream_testing", + srcs = ["stream_testing.py"], +) + +py_test( + name = "logging_pool_test", + srcs = ["_logging_pool_test.py"], + main = "_logging_pool_test.py", + size = "small", + deps = [ + "//src/python/grpcio/grpc:grpcio", + ], +) + From c88fd0656a6760eb19485d1fa0d5e63e4d44368a Mon Sep 17 00:00:00 2001 From: yang-g Date: Thu, 2 Aug 2018 09:12:46 -0700 Subject: [PATCH 16/99] comments --- src/core/lib/surface/channel_init.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/surface/channel_init.h b/src/core/lib/surface/channel_init.h index d800c83f160..6543796b4c8 100644 --- a/src/core/lib/surface/channel_init.h +++ b/src/core/lib/surface/channel_init.h @@ -45,7 +45,8 @@ enum { // For filters that need to be close to top or bottom, such as protocol-level // filters (client_authority, http-client, http-server). GRPC_CHANNEL_INIT_PRIORITY_HIGH = 20000, - // For filters that need to be very close to the wire or surface. + // For filters that need to be very close to the wire or surface, such as + // stats filters (census). GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH = 30000, // For things that have to happen last, such as connected channel filter or // surface server filter. Consider as reserved for gRPC internals. From 63ce61b4eb31e209cc9fbecfad1c1d48c1c5c633 Mon Sep 17 00:00:00 2001 From: jzeferino Date: Thu, 2 Aug 2018 21:37:43 +0100 Subject: [PATCH 17/99] Fixed typo. --- examples/csharp/HelloworldXamarin/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/csharp/HelloworldXamarin/README.md b/examples/csharp/HelloworldXamarin/README.md index 7bbc8cc5e7a..d2f00a4d63a 100644 --- a/examples/csharp/HelloworldXamarin/README.md +++ b/examples/csharp/HelloworldXamarin/README.md @@ -7,7 +7,7 @@ Support of the Xamarin platform is currently experimental. The example depends on experimental Grpc.Core nuget package that hasn't been officially released and is only available via the [daily builds](https://packages.grpc.io/) source. -NOTE: To downlaod the package please manually [download](https://packages.grpc.io/archive/2018/07/a3b54ef90841ec45fe5e28f54245b7944d0904f9-d24c85c7-32ed-4924-b9af-80e7a4aeb34d/index.xml) the .nupkg file into a local directory. Then add a nuget source that points to that directory (That can be done in [Visual Studio](https://docs.microsoft.com/en-us/nuget/tools/package-manager-ui#package-sources) or Visual Studio for Mac via "Configure nuget sources"). After that, nuget will also explore that directory when looking for packages. +NOTE: To download the package please manually [download](https://packages.grpc.io/archive/2018/07/a3b54ef90841ec45fe5e28f54245b7944d0904f9-d24c85c7-32ed-4924-b9af-80e7a4aeb34d/index.xml) the .nupkg file into a local directory. Then add a nuget source that points to that directory (That can be done in [Visual Studio](https://docs.microsoft.com/en-us/nuget/tools/package-manager-ui#package-sources) or Visual Studio for Mac via "Configure nuget sources"). After that, nuget will also explore that directory when looking for packages. From 6956daa41f4a9bda5d1f381cbf0f74bec6636019 Mon Sep 17 00:00:00 2001 From: Jonathan Moss Date: Fri, 3 Aug 2018 15:15:52 +1000 Subject: [PATCH 18/99] adding python version environmental markers in the new style --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 388e629ec2a..b87843f335d 100644 --- a/setup.py +++ b/setup.py @@ -276,11 +276,11 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'six>=1.5.2', + "six>=1.5.2", + "futures>=2.2.0 ; python_version<'3.2'", + "enum34>=1.0.4 ; python_version<'3.4'" ) -if not PY3: - INSTALL_REQUIRES += ('futures>=2.2.0', 'enum34>=1.0.4') SETUP_REQUIRES = INSTALL_REQUIRES + ( 'sphinx>=1.3', From 7c0cd2915ed73948658c184c4f7a162d9babc4b3 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 3 Aug 2018 13:07:27 +0200 Subject: [PATCH 19/99] HelloworldXamarin readme improvements --- examples/csharp/HelloworldXamarin/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/csharp/HelloworldXamarin/README.md b/examples/csharp/HelloworldXamarin/README.md index d2f00a4d63a..e47855de5e1 100644 --- a/examples/csharp/HelloworldXamarin/README.md +++ b/examples/csharp/HelloworldXamarin/README.md @@ -6,10 +6,9 @@ EXPERIMENTAL ONLY Support of the Xamarin platform is currently experimental. The example depends on experimental Grpc.Core nuget package that hasn't been officially released and is only available via the [daily builds](https://packages.grpc.io/) -source. -NOTE: To download the package please manually [download](https://packages.grpc.io/archive/2018/07/a3b54ef90841ec45fe5e28f54245b7944d0904f9-d24c85c7-32ed-4924-b9af-80e7a4aeb34d/index.xml) the .nupkg file into a local directory. Then add a nuget source that points to that directory (That can be done in [Visual Studio](https://docs.microsoft.com/en-us/nuget/tools/package-manager-ui#package-sources) or Visual Studio for Mac via "Configure nuget sources"). After that, nuget will also explore that directory when looking for packages. - +source. +HINT: To download the package, please manually download the latest `.nupkg` packages from "Daily Builds" in [packages.grpc.io](https://packages.grpc.io/) into a local directory. Then add a nuget source that points to that directory (That can be [done in Visual Studio](https://docs.microsoft.com/en-us/nuget/tools/package-manager-ui#package-sources) or Visual Studio for Mac via "Configure nuget sources"). After that, nuget will also explore that directory when looking for packages. BACKGROUND ------------- @@ -20,7 +19,7 @@ For this sample, we've already generated the server and client stubs from [hello PREREQUISITES ------------- -- The latest version Visual Studio for Mac or Visual Studio 2017 with Xamarin support installed. +- The latest version Visual Studio 2017 or Visual Studio for Mac with Xamarin support installed. BUILD ------- @@ -31,7 +30,7 @@ BUILD Try it! ------- -You can deploy the example apps directly through Xamarin Studio IDE. +You can deploy the example apps directly through Visual Studio IDE. Deployments can target both Android and iOS (both support physical device deployment as well as simulator). From 2398365c4ea0d06fd29b6c218bd287835a74aae9 Mon Sep 17 00:00:00 2001 From: Noah Eisen Date: Fri, 3 Aug 2018 08:41:31 -0400 Subject: [PATCH 20/99] Revert "Immediately run write closures for failed stream" --- src/core/ext/transport/chttp2/transport/chttp2_transport.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index bd6fec6fbe2..9ad271753ce 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1208,7 +1208,7 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t, grpc_error_add_child(closure->error_data.error, error); } if (closure->next_data.scratch < CLOSURE_BARRIER_FIRST_REF_BIT) { - if (s->seen_error || (t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) || + if ((t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) || !(closure->next_data.scratch & CLOSURE_BARRIER_MAY_COVER_WRITE)) { GRPC_CLOSURE_RUN(closure, closure->error_data.error); } else { From cddfd4f99f17db3f216f78fa8126efa4626e8a71 Mon Sep 17 00:00:00 2001 From: Hope Casey-Allen Date: Fri, 3 Aug 2018 13:31:11 -0700 Subject: [PATCH 21/99] Revert "Immediately run write closures for failed stream" --- src/core/ext/transport/chttp2/transport/chttp2_transport.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index bd6fec6fbe2..9ad271753ce 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1208,7 +1208,7 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t, grpc_error_add_child(closure->error_data.error, error); } if (closure->next_data.scratch < CLOSURE_BARRIER_FIRST_REF_BIT) { - if (s->seen_error || (t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) || + if ((t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) || !(closure->next_data.scratch & CLOSURE_BARRIER_MAY_COVER_WRITE)) { GRPC_CLOSURE_RUN(closure, closure->error_data.error); } else { From 428c25744cdd867f511f3049dfff0d35dec9b001 Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Fri, 3 Aug 2018 16:13:49 -0700 Subject: [PATCH 22/99] add 1.14.0 to interop matrix --- tools/interop_matrix/client_matrix.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 26bd4832032..dff33c88c02 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -96,6 +96,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], 'go': [ { @@ -222,6 +225,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], 'node': [ { @@ -307,6 +313,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], 'php': [ { @@ -348,6 +357,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], 'csharp': [ { @@ -394,6 +406,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], } From b733c4dc250aede3c470b8459e7bda1efd50efed Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 6 Aug 2018 13:26:02 +0200 Subject: [PATCH 23/99] add instructions how to use Unity --- src/csharp/experimental/README.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/csharp/experimental/README.md b/src/csharp/experimental/README.md index 212791ac381..bd53cbcd357 100644 --- a/src/csharp/experimental/README.md +++ b/src/csharp/experimental/README.md @@ -17,6 +17,19 @@ Xamarin.iOS - supported architectures: arm64 (iPhone 6+) and x86_64 (iPhone simulator) # Unity -gRPC C# currently doesn't support Unity, but some proof-of-concept -work has been done. There is in-progress effort to provide users -with a pre-built gRPC package that can be used in their projects. + +gRPC C# now has experimental support for Unity. Please try using gRPC with +Unity and provide feedback! + +How to test gRPC in a Unity project +1. Create a Unity project that targets .NET 4.x (Edit -> Project Settings -> Editor -> Scripting Runtime Version). gRPC uses APIs that are only available in .NET4.5+ so this is a requirement. +2. Download the latest development build of `grpc_unity_package.VERSION.zip` from + [daily builds](https://packages.grpc.io/) +3. Extract the `.zip` file in the `Assets` directory in your Unity project +4. Unity IDE will pick up all the bundled files and add them to project automatically. + You should be able to use gRPC and Protobuf in your scripts from now on. + +What's currently bundled in the `grpc_unity_package` +- Grpc.Core and its dependencies +- Google.Protobuf +- Precompiled native libraries for Linux, MacOS, Windows, Android and iOS. From 7b5e32a3128a4a1ba53bb3e337cc40fd7dd35b05 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 6 Aug 2018 10:17:05 -0700 Subject: [PATCH 24/99] Refresh pb2 files in examples/python/multiplex --- examples/python/multiplex/helloworld_pb2.py | 145 ++------ .../python/multiplex/helloworld_pb2_grpc.py | 3 +- examples/python/multiplex/route_guide_pb2.py | 310 +++--------------- .../python/multiplex/route_guide_pb2_grpc.py | 3 +- 4 files changed, 77 insertions(+), 384 deletions(-) diff --git a/examples/python/multiplex/helloworld_pb2.py b/examples/python/multiplex/helloworld_pb2.py index 6665b1f6878..e18ab9acc7a 100644 --- a/examples/python/multiplex/helloworld_pb2.py +++ b/examples/python/multiplex/helloworld_pb2.py @@ -21,7 +21,6 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_pb=_b('\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2I\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x42\x36\n\x1bio.grpc.examples.helloworldB\x0fHelloWorldProtoP\x01\xa2\x02\x03HLWb\x06proto3') ) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -89,6 +88,7 @@ _HELLOREPLY = _descriptor.Descriptor( DESCRIPTOR.message_types_by_name['HelloRequest'] = _HELLOREQUEST DESCRIPTOR.message_types_by_name['HelloReply'] = _HELLOREPLY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) HelloRequest = _reflection.GeneratedProtocolMessageType('HelloRequest', (_message.Message,), dict( DESCRIPTOR = _HELLOREQUEST, @@ -107,123 +107,28 @@ _sym_db.RegisterMessage(HelloReply) DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW')) -try: - # THESE ELEMENTS WILL BE DEPRECATED. - # Please use the generated *_pb2_grpc.py files instead. - import grpc - from grpc.framework.common import cardinality - from grpc.framework.interfaces.face import utilities as face_utilities - from grpc.beta import implementations as beta_implementations - from grpc.beta import interfaces as beta_interfaces - - - class GreeterStub(object): - """The greeting service definition. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.SayHello = channel.unary_unary( - '/helloworld.Greeter/SayHello', - request_serializer=HelloRequest.SerializeToString, - response_deserializer=HelloReply.FromString, - ) - - - class GreeterServicer(object): - """The greeting service definition. - """ - - def SayHello(self, request, context): - """Sends a greeting - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - - def add_GreeterServicer_to_server(servicer, server): - rpc_method_handlers = { - 'SayHello': grpc.unary_unary_rpc_method_handler( - servicer.SayHello, - request_deserializer=HelloRequest.FromString, - response_serializer=HelloReply.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'helloworld.Greeter', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - class BetaGreeterServicer(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """The greeting service definition. - """ - def SayHello(self, request, context): - """Sends a greeting - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - - - class BetaGreeterStub(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """The greeting service definition. - """ - def SayHello(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """Sends a greeting - """ - raise NotImplementedError() - SayHello.future = None - - - def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_deserializers = { - ('helloworld.Greeter', 'SayHello'): HelloRequest.FromString, - } - response_serializers = { - ('helloworld.Greeter', 'SayHello'): HelloReply.SerializeToString, - } - method_implementations = { - ('helloworld.Greeter', 'SayHello'): face_utilities.unary_unary_inline(servicer.SayHello), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - - - def beta_create_Greeter_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_serializers = { - ('helloworld.Greeter', 'SayHello'): HelloRequest.SerializeToString, - } - response_deserializers = { - ('helloworld.Greeter', 'SayHello'): HelloReply.FromString, - } - cardinalities = { - 'SayHello': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'helloworld.Greeter', cardinalities, options=stub_options) -except ImportError: - pass + +_GREETER = _descriptor.ServiceDescriptor( + name='Greeter', + full_name='helloworld.Greeter', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=93, + serialized_end=166, + methods=[ + _descriptor.MethodDescriptor( + name='SayHello', + full_name='helloworld.Greeter.SayHello', + index=0, + containing_service=None, + input_type=_HELLOREQUEST, + output_type=_HELLOREPLY, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_GREETER) + +DESCRIPTOR.services_by_name['Greeter'] = _GREETER + # @@protoc_insertion_point(module_scope) diff --git a/examples/python/multiplex/helloworld_pb2_grpc.py b/examples/python/multiplex/helloworld_pb2_grpc.py index 682dc36cd89..18e07d16797 100644 --- a/examples/python/multiplex/helloworld_pb2_grpc.py +++ b/examples/python/multiplex/helloworld_pb2_grpc.py @@ -1,6 +1,5 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities import helloworld_pb2 as helloworld__pb2 diff --git a/examples/python/multiplex/route_guide_pb2.py b/examples/python/multiplex/route_guide_pb2.py index e6775eb8140..fbbc194a580 100644 --- a/examples/python/multiplex/route_guide_pb2.py +++ b/examples/python/multiplex/route_guide_pb2.py @@ -21,7 +21,6 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_pb=_b('\n\x11route_guide.proto\x12\nrouteguide\",\n\x05Point\x12\x10\n\x08latitude\x18\x01 \x01(\x05\x12\x11\n\tlongitude\x18\x02 \x01(\x05\"I\n\tRectangle\x12\x1d\n\x02lo\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x1d\n\x02hi\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"<\n\x07\x46\x65\x61ture\x12\x0c\n\x04name\x18\x01 \x01(\t\x12#\n\x08location\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"A\n\tRouteNote\x12#\n\x08location\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x0f\n\x07message\x18\x02 \x01(\t\"b\n\x0cRouteSummary\x12\x13\n\x0bpoint_count\x18\x01 \x01(\x05\x12\x15\n\rfeature_count\x18\x02 \x01(\x05\x12\x10\n\x08\x64istance\x18\x03 \x01(\x05\x12\x14\n\x0c\x65lapsed_time\x18\x04 \x01(\x05\x32\x85\x02\n\nRouteGuide\x12\x36\n\nGetFeature\x12\x11.routeguide.Point\x1a\x13.routeguide.Feature\"\x00\x12>\n\x0cListFeatures\x12\x15.routeguide.Rectangle\x1a\x13.routeguide.Feature\"\x00\x30\x01\x12>\n\x0bRecordRoute\x12\x11.routeguide.Point\x1a\x18.routeguide.RouteSummary\"\x00(\x01\x12?\n\tRouteChat\x12\x15.routeguide.RouteNote\x1a\x15.routeguide.RouteNote\"\x00(\x01\x30\x01\x42\x36\n\x1bio.grpc.examples.routeguideB\x0fRouteGuideProtoP\x01\xa2\x02\x03RTGb\x06proto3') ) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -238,6 +237,7 @@ DESCRIPTOR.message_types_by_name['Rectangle'] = _RECTANGLE DESCRIPTOR.message_types_by_name['Feature'] = _FEATURE DESCRIPTOR.message_types_by_name['RouteNote'] = _ROUTENOTE DESCRIPTOR.message_types_by_name['RouteSummary'] = _ROUTESUMMARY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) Point = _reflection.GeneratedProtocolMessageType('Point', (_message.Message,), dict( DESCRIPTOR = _POINT, @@ -277,265 +277,55 @@ _sym_db.RegisterMessage(RouteSummary) DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.routeguideB\017RouteGuideProtoP\001\242\002\003RTG')) -try: - # THESE ELEMENTS WILL BE DEPRECATED. - # Please use the generated *_pb2_grpc.py files instead. - import grpc - from grpc.framework.common import cardinality - from grpc.framework.interfaces.face import utilities as face_utilities - from grpc.beta import implementations as beta_implementations - from grpc.beta import interfaces as beta_interfaces - - - class RouteGuideStub(object): - """Interface exported by the server. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.GetFeature = channel.unary_unary( - '/routeguide.RouteGuide/GetFeature', - request_serializer=Point.SerializeToString, - response_deserializer=Feature.FromString, - ) - self.ListFeatures = channel.unary_stream( - '/routeguide.RouteGuide/ListFeatures', - request_serializer=Rectangle.SerializeToString, - response_deserializer=Feature.FromString, - ) - self.RecordRoute = channel.stream_unary( - '/routeguide.RouteGuide/RecordRoute', - request_serializer=Point.SerializeToString, - response_deserializer=RouteSummary.FromString, - ) - self.RouteChat = channel.stream_stream( - '/routeguide.RouteGuide/RouteChat', - request_serializer=RouteNote.SerializeToString, - response_deserializer=RouteNote.FromString, - ) - - - class RouteGuideServicer(object): - """Interface exported by the server. - """ - - def GetFeature(self, request, context): - """A simple RPC. - - Obtains the feature at a given position. - - A feature with an empty name is returned if there's no feature at the given - position. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ListFeatures(self, request, context): - """A server-to-client streaming RPC. - - Obtains the Features available within the given Rectangle. Results are - streamed rather than returned at once (e.g. in a response message with a - repeated field), as the rectangle may cover a large area and contain a - huge number of features. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def RecordRoute(self, request_iterator, context): - """A client-to-server streaming RPC. - - Accepts a stream of Points on a route being traversed, returning a - RouteSummary when traversal is completed. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def RouteChat(self, request_iterator, context): - """A Bidirectional streaming RPC. - - Accepts a stream of RouteNotes sent while a route is being traversed, - while receiving other RouteNotes (e.g. from other users). - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - - def add_RouteGuideServicer_to_server(servicer, server): - rpc_method_handlers = { - 'GetFeature': grpc.unary_unary_rpc_method_handler( - servicer.GetFeature, - request_deserializer=Point.FromString, - response_serializer=Feature.SerializeToString, - ), - 'ListFeatures': grpc.unary_stream_rpc_method_handler( - servicer.ListFeatures, - request_deserializer=Rectangle.FromString, - response_serializer=Feature.SerializeToString, - ), - 'RecordRoute': grpc.stream_unary_rpc_method_handler( - servicer.RecordRoute, - request_deserializer=Point.FromString, - response_serializer=RouteSummary.SerializeToString, - ), - 'RouteChat': grpc.stream_stream_rpc_method_handler( - servicer.RouteChat, - request_deserializer=RouteNote.FromString, - response_serializer=RouteNote.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'routeguide.RouteGuide', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - class BetaRouteGuideServicer(object): - """The Beta API is deprecated for 0.15.0 and later. - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """Interface exported by the server. - """ - def GetFeature(self, request, context): - """A simple RPC. - - Obtains the feature at a given position. - - A feature with an empty name is returned if there's no feature at the given - position. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def ListFeatures(self, request, context): - """A server-to-client streaming RPC. - - Obtains the Features available within the given Rectangle. Results are - streamed rather than returned at once (e.g. in a response message with a - repeated field), as the rectangle may cover a large area and contain a - huge number of features. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def RecordRoute(self, request_iterator, context): - """A client-to-server streaming RPC. - - Accepts a stream of Points on a route being traversed, returning a - RouteSummary when traversal is completed. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def RouteChat(self, request_iterator, context): - """A Bidirectional streaming RPC. - - Accepts a stream of RouteNotes sent while a route is being traversed, - while receiving other RouteNotes (e.g. from other users). - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - - - class BetaRouteGuideStub(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """Interface exported by the server. - """ - def GetFeature(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """A simple RPC. - - Obtains the feature at a given position. - - A feature with an empty name is returned if there's no feature at the given - position. - """ - raise NotImplementedError() - GetFeature.future = None - def ListFeatures(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """A server-to-client streaming RPC. - - Obtains the Features available within the given Rectangle. Results are - streamed rather than returned at once (e.g. in a response message with a - repeated field), as the rectangle may cover a large area and contain a - huge number of features. - """ - raise NotImplementedError() - def RecordRoute(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None): - """A client-to-server streaming RPC. - - Accepts a stream of Points on a route being traversed, returning a - RouteSummary when traversal is completed. - """ - raise NotImplementedError() - RecordRoute.future = None - def RouteChat(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None): - """A Bidirectional streaming RPC. - - Accepts a stream of RouteNotes sent while a route is being traversed, - while receiving other RouteNotes (e.g. from other users). - """ - raise NotImplementedError() - - - def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_deserializers = { - ('routeguide.RouteGuide', 'GetFeature'): Point.FromString, - ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.FromString, - ('routeguide.RouteGuide', 'RecordRoute'): Point.FromString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString, - } - response_serializers = { - ('routeguide.RouteGuide', 'GetFeature'): Feature.SerializeToString, - ('routeguide.RouteGuide', 'ListFeatures'): Feature.SerializeToString, - ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.SerializeToString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString, - } - method_implementations = { - ('routeguide.RouteGuide', 'GetFeature'): face_utilities.unary_unary_inline(servicer.GetFeature), - ('routeguide.RouteGuide', 'ListFeatures'): face_utilities.unary_stream_inline(servicer.ListFeatures), - ('routeguide.RouteGuide', 'RecordRoute'): face_utilities.stream_unary_inline(servicer.RecordRoute), - ('routeguide.RouteGuide', 'RouteChat'): face_utilities.stream_stream_inline(servicer.RouteChat), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - - - def beta_create_RouteGuide_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - """The Beta API is deprecated for 0.15.0 and later. +_ROUTEGUIDE = _descriptor.ServiceDescriptor( + name='RouteGuide', + full_name='routeguide.RouteGuide', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=384, + serialized_end=645, + methods=[ + _descriptor.MethodDescriptor( + name='GetFeature', + full_name='routeguide.RouteGuide.GetFeature', + index=0, + containing_service=None, + input_type=_POINT, + output_type=_FEATURE, + options=None, + ), + _descriptor.MethodDescriptor( + name='ListFeatures', + full_name='routeguide.RouteGuide.ListFeatures', + index=1, + containing_service=None, + input_type=_RECTANGLE, + output_type=_FEATURE, + options=None, + ), + _descriptor.MethodDescriptor( + name='RecordRoute', + full_name='routeguide.RouteGuide.RecordRoute', + index=2, + containing_service=None, + input_type=_POINT, + output_type=_ROUTESUMMARY, + options=None, + ), + _descriptor.MethodDescriptor( + name='RouteChat', + full_name='routeguide.RouteGuide.RouteChat', + index=3, + containing_service=None, + input_type=_ROUTENOTE, + output_type=_ROUTENOTE, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_ROUTEGUIDE) + +DESCRIPTOR.services_by_name['RouteGuide'] = _ROUTEGUIDE - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_serializers = { - ('routeguide.RouteGuide', 'GetFeature'): Point.SerializeToString, - ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.SerializeToString, - ('routeguide.RouteGuide', 'RecordRoute'): Point.SerializeToString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString, - } - response_deserializers = { - ('routeguide.RouteGuide', 'GetFeature'): Feature.FromString, - ('routeguide.RouteGuide', 'ListFeatures'): Feature.FromString, - ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.FromString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString, - } - cardinalities = { - 'GetFeature': cardinality.Cardinality.UNARY_UNARY, - 'ListFeatures': cardinality.Cardinality.UNARY_STREAM, - 'RecordRoute': cardinality.Cardinality.STREAM_UNARY, - 'RouteChat': cardinality.Cardinality.STREAM_STREAM, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'routeguide.RouteGuide', cardinalities, options=stub_options) -except ImportError: - pass # @@protoc_insertion_point(module_scope) diff --git a/examples/python/multiplex/route_guide_pb2_grpc.py b/examples/python/multiplex/route_guide_pb2_grpc.py index 27b24c747db..05c1b793128 100644 --- a/examples/python/multiplex/route_guide_pb2_grpc.py +++ b/examples/python/multiplex/route_guide_pb2_grpc.py @@ -1,6 +1,5 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities import route_guide_pb2 as route__guide__pb2 From 67b8328146b79160e6a37523ea9c10416216faae Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 24 Jul 2018 07:22:25 -0700 Subject: [PATCH 25/99] Add Python3.7 32bit build on Windows --- tools/internal_ci/windows/grpc_build_artifacts.bat | 1 + tools/run_tests/artifacts/artifact_targets.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 4d528e01285..0e02bdb6ecc 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -18,6 +18,7 @@ rename C:\Python27_32bit Python27_32bits rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits +rename C:\Python37_32bit Python37_32bits @rem enter repo root cd /d %~dp0\..\..\.. diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index edde3eae28d..40cbfa00219 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -378,6 +378,7 @@ def targets(): PythonArtifact('windows', 'x86', 'Python34_32bits'), PythonArtifact('windows', 'x86', 'Python35_32bits'), PythonArtifact('windows', 'x86', 'Python36_32bits'), + PythonArtifact('windows', 'x86', 'Python37_32bits'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact('windows', 'x64', 'Python35'), From 8baaac5c14f2f699989e79b0af2f717ad0a2cbb4 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 24 Jul 2018 07:20:40 -0700 Subject: [PATCH 26/99] Revert manual install of Python3.7 on Windows build images --- tools/internal_ci/helper_scripts/prepare_build_windows.bat | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index 0164e4a5617..f987f8a8cb5 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -14,7 +14,7 @@ @rem make sure msys binaries are preferred over cygwin binaries @rem set path to python 2.7 -set PATH=C:\tools\msys64\usr\bin;C:\Python27;C:\Python37;%PATH% +set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% @rem If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests if defined KOKORO_GITHUB_PULL_REQUEST_NUMBER if defined RUN_TESTS_FLAGS ( @@ -34,9 +34,6 @@ netsh interface ip add dnsservers "Local Area Connection 8" 8.8.4.4 index=3 @rem Needed for big_query_utils python -m pip install google-api-python-client -@rem Install Python 3.7 -chocolatey install -y -r python3 --version 3.7 - @rem Disable some unwanted dotnet options set NUGET_XMLDOC_MODE=skip set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true From 8c5f24cf3c00891cf061678ad0e3937d2ea7aece Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 6 Aug 2018 10:59:15 -0700 Subject: [PATCH 27/99] update test --- test/core/end2end/tests/filter_causes_close.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/core/end2end/tests/filter_causes_close.cc b/test/core/end2end/tests/filter_causes_close.cc index a7f4268803f..891c1b8c1fb 100644 --- a/test/core/end2end/tests/filter_causes_close.cc +++ b/test/core/end2end/tests/filter_causes_close.cc @@ -261,8 +261,9 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { } static void init_plugin(void) { - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, 0, maybe_add_filter, - nullptr); + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_add_filter, nullptr); } static void destroy_plugin(void) {} From 5ffb9bf8c4ffcde396a3a649be396a8f70b4e2eb Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 6 Aug 2018 09:38:21 -0700 Subject: [PATCH 28/99] Add server reflection guide for Python --- doc/python/server_reflection.md | 61 +++++++++++++++++++ .../greeter_server_with_reflection.py | 52 ++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 doc/python/server_reflection.md create mode 100644 examples/python/helloworld/greeter_server_with_reflection.py diff --git a/doc/python/server_reflection.md b/doc/python/server_reflection.md new file mode 100644 index 00000000000..5a68e3c3019 --- /dev/null +++ b/doc/python/server_reflection.md @@ -0,0 +1,61 @@ +# gRPC Python Server Reflection + +This document shows how to use gRPC Server Reflection in gRPC Python. +Please see [C++ Server Reflection Tutorial](../server_reflection_tutorial.md) +for general information and more examples how to use server reflection. + +## Enable server reflection in Python servers + +gRPC Python Server Reflection is an add-on library. +To use it, first install the [grpcio-reflection](https://pypi.org/project/grpcio-reflection/) +PyPI package into your project. + +Note that with Python you need to manually register the service +descriptors with the reflection service implementation when creating a server +(this isn't necessary with e.g. C++ or Java) +```python +# add the following import statement to use server reflection +from grpc_reflection.v1alpha import reflection +# ... +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + # the reflection service will be aware of "Greeter" and "ServerReflection" services. + SERVICE_NAMES = ( + helloworld_pb2.DESCRIPTOR.services_by_name['Greeter'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port('[::]:50051') + server.start() +``` + +Please see +[greeter_server_with_reflection.py](https://github.com/grpc/grpc/blob/master/examples/python/helloworld/greeter_server_with_reflection.py) +in the examples directory for the full example, which extends the gRPC [Python +`Greeter` example](https://github.com/grpc/tree/master/examples/python/helloworld) on a +reflection-enabled server. + +After starting the server, you can verify that the server reflection +is working properly by using the [`grpc_cli` command line +tool](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md): + + ```sh + $ grpc_cli ls localhost:50051 + ``` + + output: + ```sh + grpc.reflection.v1alpha.ServerReflection + helloworld.Greeter + ``` + + For more examples and instructions how to use the `grpc_cli` tool, + please refer to the [`grpc_cli` documentation](../command_line_tool.md) + and the [C++ Server Reflection Tutorial](../server_reflection_tutorial.md). + +## Additional Resources + +The [Server Reflection Protocol](../server-reflection.md) provides detailed +information about how the server reflection works and describes the server reflection +protocol in detail. diff --git a/examples/python/helloworld/greeter_server_with_reflection.py b/examples/python/helloworld/greeter_server_with_reflection.py new file mode 100644 index 00000000000..5ba8782dfca --- /dev/null +++ b/examples/python/helloworld/greeter_server_with_reflection.py @@ -0,0 +1,52 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The reflection-enabled version of gRPC helloworld.Greeter server.""" + +from concurrent import futures +import time + +import grpc +from grpc_reflection.v1alpha import reflection + +import helloworld_pb2 +import helloworld_pb2_grpc + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +class Greeter(helloworld_pb2_grpc.GreeterServicer): + + def SayHello(self, request, context): + return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name) + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + SERVICE_NAMES = ( + helloworld_pb2.DESCRIPTOR.services_by_name['Greeter'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port('[::]:50051') + server.start() + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + + +if __name__ == '__main__': + serve() From ed40f7044586990311caa64d7112ed3e0e51ce57 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 6 Aug 2018 13:41:41 -0700 Subject: [PATCH 29/99] Linkify grpc_cli in C# reflection tutorial --- doc/csharp/server_reflection.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/csharp/server_reflection.md b/doc/csharp/server_reflection.md index 97216802694..7827cbb21f0 100644 --- a/doc/csharp/server_reflection.md +++ b/doc/csharp/server_reflection.md @@ -30,7 +30,8 @@ server.Start(); ``` After starting the server, you can verify that the server reflection -is working properly by using the `grpc_cli` command line tool: +is working properly by using the [`grpc_cli` command line +tool](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md): ```sh $ grpc_cli ls localhost:50051 From f7e72560b664cce34bdf3c64b411cd6a153219ad Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 6 Aug 2018 14:46:37 -0700 Subject: [PATCH 30/99] Add experimental API for resetting connection backoff. --- grpc.def | 1 + include/grpc/grpc.h | 5 ++++ include/grpcpp/channel.h | 9 ++++++ .../filters/client_channel/client_channel.cc | 11 +++++++ .../ext/filters/client_channel/lb_policy.h | 5 +++- .../client_channel/lb_policy/grpclb/grpclb.cc | 10 +++++++ .../lb_policy/pick_first/pick_first.cc | 8 +++++ .../lb_policy/round_robin/round_robin.cc | 8 +++++ .../lb_policy/subchannel_list.h | 27 +++++++++++++++++ .../ext/filters/client_channel/resolver.h | 8 +++++ .../resolver/dns/c_ares/dns_resolver_ares.cc | 9 ++++++ .../resolver/dns/native/dns_resolver.cc | 9 ++++++ .../ext/filters/client_channel/subchannel.cc | 18 +++++++++++ .../ext/filters/client_channel/subchannel.h | 7 +++++ src/core/lib/surface/channel.cc | 11 +++++++ src/core/lib/transport/transport.h | 2 ++ src/cpp/client/channel_cc.cc | 8 +++++ src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 ++ src/ruby/ext/grpc/rb_grpc_imports.generated.h | 3 ++ .../core/surface/public_headers_must_be_c89.c | 1 + test/cpp/end2end/client_lb_end2end_test.cc | 30 +++++++++++++++++++ 21 files changed, 191 insertions(+), 1 deletion(-) diff --git a/grpc.def b/grpc.def index 5b98792662c..5e9d86c769c 100644 --- a/grpc.def +++ b/grpc.def @@ -42,6 +42,7 @@ EXPORTS grpc_census_call_get_context grpc_channel_get_target grpc_channel_get_info + grpc_channel_reset_connect_backoff grpc_insecure_channel_create grpc_lame_client_channel_create grpc_channel_destroy diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index f0eb2c0121b..bc6cd340af2 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -269,6 +269,11 @@ GRPCAPI char* grpc_channel_get_target(grpc_channel* channel); GRPCAPI void grpc_channel_get_info(grpc_channel* channel, const grpc_channel_info* channel_info); +/** EXPERIMENTAL. Resets the channel's connect backoff. + TODO(roth): When we see whether this proves useful, either promote + to non-experimental or remove it. */ +GRPCAPI void grpc_channel_reset_connect_backoff(grpc_channel* channel); + /** Create a client channel to 'target'. Additional channel level configuration MAY be provided by grpc_channel_args, though the expectation is that most clients will want to simply pass NULL. The user data in 'args' need only diff --git a/include/grpcpp/channel.h b/include/grpcpp/channel.h index 4b45d5382cd..fed02bf7bc7 100644 --- a/include/grpcpp/channel.h +++ b/include/grpcpp/channel.h @@ -30,6 +30,14 @@ struct grpc_channel; namespace grpc { + +namespace experimental { +/// Resets the channel's connection backoff. +/// TODO(roth): Once we see whether this proves useful, either create a gRFC +/// and change this to be a method of the Channel class, or remove it. +void ChannelResetConnectionBackoff(Channel* channel); +} // namespace experimental + /// Channels represent a connection to an endpoint. Created by \a CreateChannel. class Channel final : public ChannelInterface, public internal::CallHook, @@ -52,6 +60,7 @@ class Channel final : public ChannelInterface, private: template friend class internal::BlockingUnaryCallImpl; + friend void experimental::ChannelResetConnectionBackoff(Channel* channel); friend std::shared_ptr CreateChannelInternal( const grpc::string& host, grpc_channel* c_channel); Channel(const grpc::string& host, grpc_channel* c_channel); diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index fead8feb17b..b06f09d8c7b 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -622,6 +622,17 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) { } GRPC_ERROR_UNREF(op->disconnect_with_error); } + + if (op->reset_connect_backoff) { + if (chand->resolver != nullptr) { + chand->resolver->ResetBackoffLocked(); + chand->resolver->RequestReresolutionLocked(); + } + if (chand->lb_policy != nullptr) { + chand->lb_policy->ResetBackoffLocked(); + } + } + GRPC_CHANNEL_STACK_UNREF(chand->owning_stack, "start_transport_op"); GRPC_CLOSURE_SCHED(op->on_consumed, GRPC_ERROR_NONE); diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 31c08246ae4..3c0a9c11180 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -144,7 +144,10 @@ class LoadBalancingPolicy /// consider whether this method is still needed. virtual void ExitIdleLocked() GRPC_ABSTRACT; - /// populates child_subchannels and child_channels with the uuids of this + /// Resets connection backoff. + virtual void ResetBackoffLocked() GRPC_ABSTRACT; + + /// Populates child_subchannels and child_channels with the uuids of this /// LB policy's referenced children. This is not invoked from the /// client_channel's combiner. The implementation is responsible for /// providing its own synchronization. diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 2d1f7774748..cf029ef4c18 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -134,6 +134,7 @@ class GrpcLb : public LoadBalancingPolicy { grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* child_channels) override; @@ -1214,6 +1215,15 @@ void GrpcLb::ExitIdleLocked() { } } +void GrpcLb::ResetBackoffLocked() { + if (lb_channel_ != nullptr) { + grpc_channel_reset_connect_backoff(lb_channel_); + } + if (rr_policy_ != nullptr) { + rr_policy_->ResetBackoffLocked(); + } +} + bool GrpcLb::PickLocked(PickState* pick, grpc_error** error) { PendingPick* pp = PendingPickCreate(pick); bool pick_done = false; diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 46acbf628b0..2b6a9ba8c53 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -57,6 +57,7 @@ class PickFirst : public LoadBalancingPolicy { grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -259,6 +260,13 @@ void PickFirst::ExitIdleLocked() { } } +void PickFirst::ResetBackoffLocked() { + subchannel_list_->ResetBackoffLocked(); + if (latest_pending_subchannel_list_ != nullptr) { + latest_pending_subchannel_list_->ResetBackoffLocked(); + } +} + bool PickFirst::PickLocked(PickState* pick, grpc_error** error) { // If we have a selected subchannel already, return synchronously. if (selected_ != nullptr) { diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index 9c3a15c67b0..fea84331d80 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -68,6 +68,7 @@ class RoundRobin : public LoadBalancingPolicy { grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -333,6 +334,13 @@ void RoundRobin::ExitIdleLocked() { } } +void RoundRobin::ResetBackoffLocked() { + subchannel_list_->ResetBackoffLocked(); + if (latest_pending_subchannel_list_ != nullptr) { + latest_pending_subchannel_list_->ResetBackoffLocked(); + } +} + bool RoundRobin::DoPickLocked(PickState* pick) { const size_t next_ready_index = subchannel_list_->GetNextReadySubchannelIndexLocked(); diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h index 018ac3bb868..0fa2f04e73b 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -107,6 +107,11 @@ class SubchannelData { // being unreffed. virtual void UnrefSubchannelLocked(const char* reason); + // Resets the connection backoff. + // TODO(roth): This method should go away when we move the backoff + // code out of the subchannel and into the LB policies. + void ResetBackoffLocked(); + // Starts watching the connectivity state of the subchannel. // ProcessConnectivityChangeLocked() will be called when the // connectivity state changes. @@ -206,6 +211,11 @@ class SubchannelList LoadBalancingPolicy* policy() const { return policy_; } TraceFlag* tracer() const { return tracer_; } + // Resets connection backoff of all subchannels. + // TODO(roth): We will probably need to rethink this as part of moving + // the backoff code out of subchannels and into LB policies. + void ResetBackoffLocked(); + // Note: Caller must ensure that this is invoked inside of the combiner. void Orphan() override { ShutdownLocked(); @@ -298,6 +308,14 @@ void SubchannelData:: } } +template +void SubchannelData::ResetBackoffLocked() { + if (subchannel_ != nullptr) { + grpc_subchannel_reset_backoff(subchannel_); + } +} + template void SubchannelData::StartConnectivityWatchLocked() { @@ -544,6 +562,15 @@ void SubchannelList::ShutdownLocked() { } } +template +void SubchannelList::ResetBackoffLocked() { + for (size_t i = 0; i < subchannels_.size(); i++) { + SubchannelDataType* sd = &subchannels_[i]; + sd->ResetBackoffLocked(); + } +} + } // namespace grpc_core #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_SUBCHANNEL_LIST_H */ diff --git a/src/core/ext/filters/client_channel/resolver.h b/src/core/ext/filters/client_channel/resolver.h index c7e37e4468e..48f2e890952 100644 --- a/src/core/ext/filters/client_channel/resolver.h +++ b/src/core/ext/filters/client_channel/resolver.h @@ -94,6 +94,14 @@ class Resolver : public InternallyRefCountedWithTracing { /// throw away unselected subchannels. virtual void RequestReresolutionLocked() GRPC_ABSTRACT; + /// Resets the re-resolution backoff, if any. + /// This needs to be implemented only by pull-based implementations; + /// for push-based implementations, it will be a no-op. + /// TODO(roth): Pull the backoff code out of resolver and into + /// client_channel, so that it can be shared across resolver + /// implementations. At that point, this method can go away. + virtual void ResetBackoffLocked() {} + void Orphan() override { // Invoke ShutdownAndUnrefLocked() inside of the combiner. GRPC_CLOSURE_SCHED( diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index 7050e82121c..f2bb5f3c712 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -66,6 +66,8 @@ class AresDnsResolver : public Resolver { void RequestReresolutionLocked() override; + void ResetBackoffLocked() override; + void ShutdownLocked() override; private: @@ -187,6 +189,13 @@ void AresDnsResolver::RequestReresolutionLocked() { } } +void AresDnsResolver::ResetBackoffLocked() { + if (have_next_resolution_timer_) { + grpc_timer_cancel(&next_resolution_timer_); + } + backoff_.Reset(); +} + void AresDnsResolver::ShutdownLocked() { if (have_next_resolution_timer_) { grpc_timer_cancel(&next_resolution_timer_); diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc index fae4c33a178..282caf215c7 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc @@ -58,6 +58,8 @@ class NativeDnsResolver : public Resolver { void RequestReresolutionLocked() override; + void ResetBackoffLocked() override; + void ShutdownLocked() override; private: @@ -158,6 +160,13 @@ void NativeDnsResolver::RequestReresolutionLocked() { } } +void NativeDnsResolver::ResetBackoffLocked() { + if (have_next_resolution_timer_) { + grpc_timer_cancel(&next_resolution_timer_); + } + backoff_.Reset(); +} + void NativeDnsResolver::ShutdownLocked() { if (have_next_resolution_timer_) { grpc_timer_cancel(&next_resolution_timer_); diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 71ef8c518bb..48c6030c895 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -132,6 +132,8 @@ struct grpc_subchannel { bool have_alarm; /** have we started the backoff loop */ bool backoff_begun; + // reset_backoff() was called while alarm was pending + bool deferred_reset_backoff; /** our alarm */ grpc_timer alarm; @@ -438,6 +440,9 @@ static void on_alarm(void* arg, grpc_error* error) { if (c->disconnected) { error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING("Disconnected", &error, 1); + } else if (c->deferred_reset_backoff) { + c->deferred_reset_backoff = false; + error = GRPC_ERROR_NONE; } else { GRPC_ERROR_REF(error); } @@ -675,6 +680,19 @@ static void on_subchannel_connected(void* arg, grpc_error* error) { grpc_channel_args_destroy(delete_channel_args); } +void grpc_subchannel_reset_backoff(grpc_subchannel* subchannel) { + gpr_mu_lock(&subchannel->mu); + if (subchannel->have_alarm) { + subchannel->deferred_reset_backoff = true; + grpc_timer_cancel(&subchannel->alarm); + } else { + subchannel->backoff_begun = false; + subchannel->backoff->Reset(); + maybe_start_connecting_locked(subchannel); + } + gpr_mu_unlock(&subchannel->mu); +} + /* * grpc_subchannel_call implementation */ diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index 9e53f7d5423..a135035d62a 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -145,6 +145,13 @@ grpc_subchannel_get_connected_subchannel(grpc_subchannel* c); const grpc_subchannel_key* grpc_subchannel_get_key( const grpc_subchannel* subchannel); +// Resets the connection backoff of the subchannel. +// TODO(roth): Move connection backoff out of subchannels and up into LB +// policy code (probably by adding a SubchannelGroup between +// SubchannelList and SubchannelData), at which point this method can +// go away. +void grpc_subchannel_reset_backoff(grpc_subchannel* subchannel); + /** continue processing a transport op */ void grpc_subchannel_call_process_op(grpc_subchannel_call* subchannel_call, grpc_transport_stream_op_batch* op); diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index 7cbd61adef5..82635d3c215 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -281,6 +281,17 @@ void grpc_channel_get_info(grpc_channel* channel, elem->filter->get_channel_info(elem, channel_info); } +void grpc_channel_reset_connect_backoff(grpc_channel* channel) { + grpc_core::ExecCtx exec_ctx; + GRPC_API_TRACE("grpc_channel_reset_connect_backoff(channel=%p)", 1, + (channel)); + grpc_transport_op* op = grpc_make_transport_op(nullptr); + op->reset_connect_backoff = true; + grpc_channel_element* elem = + grpc_channel_stack_element(CHANNEL_STACK_FROM_CHANNEL(channel), 0); + elem->filter->start_transport_op(elem, op); +} + static grpc_call* grpc_channel_create_call_internal( grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask, grpc_completion_queue* cq, grpc_pollset_set* pollset_set_alternative, diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 585b9dfae91..9e784635c69 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -282,6 +282,8 @@ typedef struct grpc_transport_op { /** Called when the ping ack is received */ grpc_closure* on_ack; } send_ping; + // If true, will reset the channel's connection backoff. + bool reset_connect_backoff; /*************************************************************************** * remaining fields are initialized and used at the discretion of the diff --git a/src/cpp/client/channel_cc.cc b/src/cpp/client/channel_cc.cc index 867f31f0255..39b891c2e1f 100644 --- a/src/cpp/client/channel_cc.cc +++ b/src/cpp/client/channel_cc.cc @@ -84,6 +84,14 @@ grpc::string Channel::GetServiceConfigJSON() const { &channel_info.service_config_json); } +namespace experimental { + +void ChannelResetConnectionBackoff(Channel* channel) { + grpc_channel_reset_connect_backoff(channel->c_channel_); +} + +} // namespace experimental + internal::Call Channel::CreateCall(const internal::RpcMethod& method, ClientContext* context, CompletionQueue* cq) { diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 2443532bb8e..38b68462df1 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -65,6 +65,7 @@ grpc_census_call_set_context_type grpc_census_call_set_context_import; grpc_census_call_get_context_type grpc_census_call_get_context_import; grpc_channel_get_target_type grpc_channel_get_target_import; grpc_channel_get_info_type grpc_channel_get_info_import; +grpc_channel_reset_connect_backoff_type grpc_channel_reset_connect_backoff_import; grpc_insecure_channel_create_type grpc_insecure_channel_create_import; grpc_lame_client_channel_create_type grpc_lame_client_channel_create_import; grpc_channel_destroy_type grpc_channel_destroy_import; @@ -315,6 +316,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_census_call_get_context_import = (grpc_census_call_get_context_type) GetProcAddress(library, "grpc_census_call_get_context"); grpc_channel_get_target_import = (grpc_channel_get_target_type) GetProcAddress(library, "grpc_channel_get_target"); grpc_channel_get_info_import = (grpc_channel_get_info_type) GetProcAddress(library, "grpc_channel_get_info"); + grpc_channel_reset_connect_backoff_import = (grpc_channel_reset_connect_backoff_type) GetProcAddress(library, "grpc_channel_reset_connect_backoff"); grpc_insecure_channel_create_import = (grpc_insecure_channel_create_type) GetProcAddress(library, "grpc_insecure_channel_create"); grpc_lame_client_channel_create_import = (grpc_lame_client_channel_create_type) GetProcAddress(library, "grpc_lame_client_channel_create"); grpc_channel_destroy_import = (grpc_channel_destroy_type) GetProcAddress(library, "grpc_channel_destroy"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index b08a1f94f7b..d6add00d12e 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -170,6 +170,9 @@ extern grpc_channel_get_target_type grpc_channel_get_target_import; typedef void(*grpc_channel_get_info_type)(grpc_channel* channel, const grpc_channel_info* channel_info); extern grpc_channel_get_info_type grpc_channel_get_info_import; #define grpc_channel_get_info grpc_channel_get_info_import +typedef void(*grpc_channel_reset_connect_backoff_type)(grpc_channel* channel); +extern grpc_channel_reset_connect_backoff_type grpc_channel_reset_connect_backoff_import; +#define grpc_channel_reset_connect_backoff grpc_channel_reset_connect_backoff_import typedef grpc_channel*(*grpc_insecure_channel_create_type)(const char* target, const grpc_channel_args* args, void* reserved); extern grpc_insecure_channel_create_type grpc_insecure_channel_create_import; #define grpc_insecure_channel_create grpc_insecure_channel_create_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 9f4ad2b4d75..7b3e875cf0a 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -104,6 +104,7 @@ int main(int argc, char **argv) { printf("%lx", (unsigned long) grpc_census_call_get_context); printf("%lx", (unsigned long) grpc_channel_get_target); printf("%lx", (unsigned long) grpc_channel_get_info); + printf("%lx", (unsigned long) grpc_channel_reset_connect_backoff); printf("%lx", (unsigned long) grpc_insecure_channel_create); printf("%lx", (unsigned long) grpc_lame_client_channel_create); printf("%lx", (unsigned long) grpc_channel_destroy); diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index c5a73a24698..7fe0da8aaec 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -408,6 +408,36 @@ TEST_F(ClientLbEnd2endTest, PickFirstBackOffMinReconnect) { gpr_atm_rel_store(&g_connection_delay_ms, 0); } +TEST_F(ClientLbEnd2endTest, PickFirstResetConnectionBackoff) { + ChannelArguments args; + constexpr int kInitialBackOffMs = 1000; + args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs); + const std::vector ports = {grpc_pick_unused_port_or_die()}; + auto channel = BuildChannel("pick_first", args); + auto stub = BuildStub(channel); + SetNextResolution(ports); + // The channel won't become connected (there's no server). + EXPECT_FALSE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + // Bring up a server on the chosen port. + StartServers(1, ports); + const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC); + // Wait for connect, but not long enough. This proves that we're + // being throttled by initial backoff. + EXPECT_FALSE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + // Reset connection backoff. + experimental::ChannelResetConnectionBackoff(channel.get()); + // Wait for connect. Should happen ~immediately. + EXPECT_TRUE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + const gpr_timespec t1 = gpr_now(GPR_CLOCK_MONOTONIC); + const grpc_millis waited_ms = gpr_time_to_millis(gpr_time_sub(t1, t0)); + gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited_ms); + // We should have waited less than kInitialBackOffMs. + EXPECT_LT(waited_ms, kInitialBackOffMs); +} + TEST_F(ClientLbEnd2endTest, PickFirstUpdates) { // Start servers and send one RPC per server. const int kNumServers = 3; From 10d129476a78b4adcad68775b1dca509c7641b28 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Mon, 6 Aug 2018 15:56:19 -0700 Subject: [PATCH 31/99] Use param --- test/cpp/end2end/async_end2end_test.cc | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 3d31c9d8100..c9246f08068 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -1709,7 +1709,7 @@ TEST_P(AsyncEnd2endServerTryCancelTest, ServerBidiStreamingTryCancelAfter) { } std::vector CreateTestScenarios(bool test_secure, - int test_big_limit) { + bool test_message_size_limit) { std::vector scenarios; std::vector credentials_types; std::vector messages; @@ -1731,13 +1731,18 @@ std::vector CreateTestScenarios(bool test_secure, GPR_ASSERT(!credentials_types.empty()); messages.push_back("Hello"); - for (int sz = 1; sz <= test_big_limit; sz *= 32) { - grpc::string big_msg; - for (int i = 0; i < sz * 1024; i++) { - char c = 'a' + (i % 26); - big_msg += c; + if (test_message_size_limit) { + for (size_t k = 1; k < GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH / 1024; + k *= 32) { + grpc::string big_msg; + for (size_t i = 0; i < k * 1024; ++i) { + char c = 'a' + (i % 26); + big_msg += c; + } + messages.push_back(big_msg); } - messages.push_back(big_msg); + messages.push_back( + grpc::string(GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH - 10, 'a')); } // TODO (sreek) Renable tests with health check service after the issue @@ -1758,10 +1763,10 @@ std::vector CreateTestScenarios(bool test_secure, } INSTANTIATE_TEST_CASE_P(AsyncEnd2end, AsyncEnd2endTest, - ::testing::ValuesIn(CreateTestScenarios(true, 1024))); + ::testing::ValuesIn(CreateTestScenarios(true, true))); INSTANTIATE_TEST_CASE_P(AsyncEnd2endServerTryCancel, AsyncEnd2endServerTryCancelTest, - ::testing::ValuesIn(CreateTestScenarios(false, 0))); + ::testing::ValuesIn(CreateTestScenarios(false, false))); } // namespace } // namespace testing From f43103dcec1e678b66a28d92f6cadbbcbb8ec2ac Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Mon, 6 Aug 2018 18:08:53 -0700 Subject: [PATCH 32/99] Add lang check --- .github/mergeable.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/mergeable.yml b/.github/mergeable.yml index 660d8cb4405..30692095c46 100644 --- a/.github/mergeable.yml +++ b/.github/mergeable.yml @@ -1,6 +1,14 @@ mergeable: pull_requests: label: - must_include: - regex: "release notes: yes|release notes: no" - message: "Add release notes yes/no label. For yes, add lang label" + or: + - and: + - must_include: + regex: 'release notes: yes' + message: 'Please include release note: yes' + - must_include: + regex: '^lang\/' + message: 'Please include a language label' + - must_include: + regex: 'release notes: no' + message: 'Please include release note: no' From 29e4947c49f89472cd00ccf51efb7c89ec8950b0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 6 Aug 2018 18:22:31 +0200 Subject: [PATCH 33/99] portable zip for unity package --- src/csharp/build_unitypackage.bat | 4 +++- templates/src/csharp/build_unitypackage.bat.template | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 0dcc38ee96d..9c53114b849 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -68,7 +68,9 @@ copy /Y Grpc.Core\bin\Release\net45\System.Interactive.Async.dll unitypackage\un copy /Y Grpc.HealthCheck\bin\Release\net45\Google.Protobuf.dll unitypackage\unitypackage_skeleton\Plugins\Google.Protobuf\lib\net45\Google.Protobuf.dll || goto :error @rem create a zipfile that will act as a Unity package -powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('unitypackage\unitypackage_skeleton', 'grpc_unity_package.zip');" +cd unitypackage\unitypackage_skeleton +zip -r ..\..\grpc_unity_package.zip Plugins +cd ..\.. copy /Y grpc_unity_package.zip ..\..\artifacts\grpc_unity_package.%VERSION%.zip || goto :error goto :EOF diff --git a/templates/src/csharp/build_unitypackage.bat.template b/templates/src/csharp/build_unitypackage.bat.template index a53e385f9b1..76ec10dbd90 100755 --- a/templates/src/csharp/build_unitypackage.bat.template +++ b/templates/src/csharp/build_unitypackage.bat.template @@ -70,7 +70,9 @@ copy /Y Grpc.HealthCheck\bin\Release\net45\Google.Protobuf.dll unitypackage\unitypackage_skeleton\Plugins\Google.Protobuf\lib\net45\Google.Protobuf.dll || goto :error @rem create a zipfile that will act as a Unity package - powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('unitypackage\unitypackage_skeleton', 'grpc_unity_package.zip');" + cd unitypackage\unitypackage_skeleton + zip -r ..\..\grpc_unity_package.zip Plugins + cd ..\.. copy /Y grpc_unity_package.zip ..\..\artifacts\grpc_unity_package.%VERSION%.zip || goto :error goto :EOF From 29fea1e4dfc21240d7b687f286fb0942e2d25825 Mon Sep 17 00:00:00 2001 From: Jean de Klerk Date: Tue, 7 Aug 2018 11:38:53 +0200 Subject: [PATCH 34/99] docs: add python server reflectlion link --- doc/server-reflection.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/server-reflection.md b/doc/server-reflection.md index c9b476599f1..3716dc5a21d 100644 --- a/doc/server-reflection.md +++ b/doc/server-reflection.md @@ -191,6 +191,6 @@ each language: - [Go](https://github.com/grpc/grpc-go/blob/master/Documentation/server-reflection-tutorial.md#enable-server-reflection) - [C++](https://grpc.io/grpc/cpp/md_doc_server_reflection_tutorial.html) - [C#](https://github.com/grpc/grpc/blob/master/doc/csharp/server_reflection.md) -- Python: (tutorial not yet written) +- [Python](https://github.com/grpc/grpc/blob/master/doc/python/server_reflection.md) - Ruby: not yet implemented [#2567](https://github.com/grpc/grpc/issues/2567) - Node: not yet implemented [#2568](https://github.com/grpc/grpc/issues/2568) From 894745dfe39185620cf644508fd6fec8510b22bd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 1 Jun 2018 10:00:37 +0200 Subject: [PATCH 35/99] update protobuf submodule to latest 3.6.0 --- third_party/protobuf | 2 +- tools/run_tests/sanity/check_submodules.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/protobuf b/third_party/protobuf index b5fbb742af1..ab8edf1dbe2 160000 --- a/third_party/protobuf +++ b/third_party/protobuf @@ -1 +1 @@ -Subproject commit b5fbb742af122b565925987e65c08957739976a7 +Subproject commit ab8edf1dbe2237b4717869eaab11a2998541ad8d diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index fb159bc2d20..d60d1e7c753 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -36,7 +36,7 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) 6599cac0965be8e5a835ab7a5684bbef033d5ad0 third_party/libcxx (heads/release_60) 9245d481eb3e890f708ff2d7dadf2a10c04748ba third_party/libcxxabi (heads/release_60) - b5fbb742af122b565925987e65c08957739976a7 third_party/protobuf (v3.3.1-641-gb5fbb74) + ab8edf1dbe2237b4717869eaab11a2998541ad8d third_party/protobuf (v3.6.0) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) EOF From 0ff1249d4d18d418777219460ca7065827e19b51 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 1 Jun 2018 10:05:51 +0200 Subject: [PATCH 36/99] update python grpcio_tools --- tools/distrib/python/grpcio_tools/protoc_lib_deps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index a0e1419ac12..546b076dd66 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -14,10 +14,10 @@ # limitations under the License. # AUTO-GENERATED BY make_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc', 'google/protobuf/compiler/js/embed.cc'] +CC_FILES=['google/protobuf/compiler/main.cc', 'google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/implicit_weak_message.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] CC_INCLUDE='third_party/protobuf/src' PROTO_INCLUDE='third_party/protobuf/src' -PROTOBUF_SUBMODULE_VERSION="b5fbb742af122b565925987e65c08957739976a7" +PROTOBUF_SUBMODULE_VERSION="ab8edf1dbe2237b4717869eaab11a2998541ad8d" From e8c3bf00bceafae7a2465b94936d4e895ddc5815 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 1 Jun 2018 10:07:26 +0200 Subject: [PATCH 37/99] update grpc_deps.bzl --- bazel/grpc_deps.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 7c349dbcdcf..faac0c18213 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -116,9 +116,9 @@ def grpc_deps(): if "com_google_protobuf" not in native.existing_rules(): native.http_archive( name = "com_google_protobuf", - strip_prefix = "protobuf-b5fbb742af122b565925987e65c08957739976a7", - url = "https://github.com/google/protobuf/archive/b5fbb742af122b565925987e65c08957739976a7.tar.gz", - ) + strip_prefix = "protobuf-ab8edf1dbe2237b4717869eaab11a2998541ad8d", + url = "https://github.com/google/protobuf/archive/ab8edf1dbe2237b4717869eaab11a2998541ad8d.tar.gz", + ) if "com_github_nanopb_nanopb" not in native.existing_rules(): native.new_http_archive( From bc8495da7a6187ba75eab7dbf91f34214866d397 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 2 Jul 2018 17:47:32 +0200 Subject: [PATCH 38/99] upgrade python to protobuf 3.6.0 --- src/python/grpcio_health_checking/setup.py | 2 +- src/python/grpcio_reflection/setup.py | 2 +- src/python/grpcio_testing/setup.py | 2 +- src/python/grpcio_tests/setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 35c09827ba7..db2edae2cea 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -57,7 +57,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index 589d0ff5565..b4087d87b49 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -58,7 +58,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_testing/setup.py b/src/python/grpcio_testing/setup.py index eb480a5464a..6ceb1fc5c9d 100644 --- a/src/python/grpcio_testing/setup.py +++ b/src/python/grpcio_testing/setup.py @@ -29,7 +29,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index 1262e485715..74c7705c970 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -41,7 +41,7 @@ INSTALL_REQUIRES = ( 'grpcio>={version}'.format(version=grpc_version.VERSION), 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), - 'oauth2client>=1.4.7', 'protobuf>=3.5.2.post1', 'six>=1.10', + 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', 'google-auth>=1.0.0', 'requests>=2.14.2') if not PY3: From 5e56966b1be9d6eb825de77fc9711c4c49123b81 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 2 Jul 2018 17:49:01 +0200 Subject: [PATCH 39/99] upgrade C# to protobuf 3.6.0 --- src/csharp/Grpc.Core/Version.csproj.include | 2 +- templates/src/csharp/Grpc.Core/Version.csproj.include.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 1a746acdd4f..6b0731eb401 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -2,6 +2,6 @@ 1.15.0-dev - 3.5.1 + 3.6.0 diff --git a/templates/src/csharp/Grpc.Core/Version.csproj.include.template b/templates/src/csharp/Grpc.Core/Version.csproj.include.template index 398b198dbd1..a950b1615d2 100755 --- a/templates/src/csharp/Grpc.Core/Version.csproj.include.template +++ b/templates/src/csharp/Grpc.Core/Version.csproj.include.template @@ -4,6 +4,6 @@ ${settings.csharp_version} - 3.5.1 + 3.6.0 From ff3b554e47fc2488e4a28cbbbd13ee0a03873381 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 2 Jul 2018 17:56:25 +0200 Subject: [PATCH 40/99] regenerate C# protos --- src/csharp/Grpc.IntegrationTesting/Control.cs | 26 +++++++++---------- .../Grpc.IntegrationTesting/EchoMessages.cs | 8 +++--- .../Grpc.IntegrationTesting/Messages.cs | 26 ++++++++----------- src/csharp/Grpc.IntegrationTesting/Stats.cs | 6 ++--- src/csharp/Grpc.Reflection/Reflection.cs | 2 +- 5 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/Control.cs b/src/csharp/Grpc.IntegrationTesting/Control.cs index f3284a5390f..6e003484513 100644 --- a/src/csharp/Grpc.IntegrationTesting/Control.cs +++ b/src/csharp/Grpc.IntegrationTesting/Control.cs @@ -1039,14 +1039,14 @@ namespace Grpc.Testing { public ClientConfig(ClientConfig other) : this() { serverTargets_ = other.serverTargets_.Clone(); clientType_ = other.clientType_; - SecurityParams = other.securityParams_ != null ? other.SecurityParams.Clone() : null; + securityParams_ = other.securityParams_ != null ? other.securityParams_.Clone() : null; outstandingRpcsPerChannel_ = other.outstandingRpcsPerChannel_; clientChannels_ = other.clientChannels_; asyncClientThreads_ = other.asyncClientThreads_; rpcType_ = other.rpcType_; - LoadParams = other.loadParams_ != null ? other.LoadParams.Clone() : null; - PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; - HistogramParams = other.histogramParams_ != null ? other.HistogramParams.Clone() : null; + loadParams_ = other.loadParams_ != null ? other.loadParams_.Clone() : null; + payloadConfig_ = other.payloadConfig_ != null ? other.payloadConfig_.Clone() : null; + histogramParams_ = other.histogramParams_ != null ? other.histogramParams_.Clone() : null; coreList_ = other.coreList_.Clone(); coreLimit_ = other.coreLimit_; otherClientApi_ = other.otherClientApi_; @@ -1647,7 +1647,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ClientStatus(ClientStatus other) : this() { - Stats = other.stats_ != null ? other.Stats.Clone() : null; + stats_ = other.stats_ != null ? other.stats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -2122,11 +2122,11 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerConfig(ServerConfig other) : this() { serverType_ = other.serverType_; - SecurityParams = other.securityParams_ != null ? other.SecurityParams.Clone() : null; + securityParams_ = other.securityParams_ != null ? other.securityParams_.Clone() : null; port_ = other.port_; asyncServerThreads_ = other.asyncServerThreads_; coreLimit_ = other.coreLimit_; - PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; + payloadConfig_ = other.payloadConfig_ != null ? other.payloadConfig_.Clone() : null; coreList_ = other.coreList_.Clone(); otherServerApi_ = other.otherServerApi_; threadsPerCq_ = other.threadsPerCq_; @@ -2758,7 +2758,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerStatus(ServerStatus other) : this() { - Stats = other.stats_ != null ? other.Stats.Clone() : null; + stats_ = other.stats_ != null ? other.stats_.Clone() : null; port_ = other.port_; cores_ = other.cores_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); @@ -3293,9 +3293,9 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public Scenario(Scenario other) : this() { name_ = other.name_; - ClientConfig = other.clientConfig_ != null ? other.ClientConfig.Clone() : null; + clientConfig_ = other.clientConfig_ != null ? other.clientConfig_.Clone() : null; numClients_ = other.numClients_; - ServerConfig = other.serverConfig_ != null ? other.ServerConfig.Clone() : null; + serverConfig_ = other.serverConfig_ != null ? other.serverConfig_.Clone() : null; numServers_ = other.numServers_; warmupSeconds_ = other.warmupSeconds_; benchmarkSeconds_ = other.benchmarkSeconds_; @@ -4422,12 +4422,12 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ScenarioResult(ScenarioResult other) : this() { - Scenario = other.scenario_ != null ? other.Scenario.Clone() : null; - Latencies = other.latencies_ != null ? other.Latencies.Clone() : null; + scenario_ = other.scenario_ != null ? other.scenario_.Clone() : null; + latencies_ = other.latencies_ != null ? other.latencies_.Clone() : null; clientStats_ = other.clientStats_.Clone(); serverStats_ = other.serverStats_.Clone(); serverCores_ = other.serverCores_.Clone(); - Summary = other.summary_ != null ? other.Summary.Clone() : null; + summary_ = other.summary_ != null ? other.summary_.Clone() : null; clientSuccess_ = other.clientSuccess_.Clone(); serverSuccess_ = other.serverSuccess_.Clone(); requestResults_ = other.requestResults_.Clone(); diff --git a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs index 39c3d76ce82..04bea8c185f 100644 --- a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs +++ b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs @@ -435,10 +435,10 @@ namespace Grpc.Testing { expectedClientIdentity_ = other.expectedClientIdentity_; skipCancelledCheck_ = other.skipCancelledCheck_; expectedTransportSecurityType_ = other.expectedTransportSecurityType_; - DebugInfo = other.debugInfo_ != null ? other.DebugInfo.Clone() : null; + debugInfo_ = other.debugInfo_ != null ? other.debugInfo_.Clone() : null; serverDie_ = other.serverDie_; binaryErrorDetails_ = other.binaryErrorDetails_; - ExpectedError = other.expectedError_ != null ? other.ExpectedError.Clone() : null; + expectedError_ = other.expectedError_ != null ? other.expectedError_.Clone() : null; serverSleepUs_ = other.serverSleepUs_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -968,7 +968,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public EchoRequest(EchoRequest other) : this() { message_ = other.message_; - Param = other.param_ != null ? other.Param.Clone() : null; + param_ = other.param_ != null ? other.param_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1316,7 +1316,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public EchoResponse(EchoResponse other) : this() { message_ = other.message_; - Param = other.param_ != null ? other.Param.Clone() : null; + param_ = other.param_ != null ? other.param_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.IntegrationTesting/Messages.cs b/src/csharp/Grpc.IntegrationTesting/Messages.cs index b5c93babd20..35546f1b671 100644 --- a/src/csharp/Grpc.IntegrationTesting/Messages.cs +++ b/src/csharp/Grpc.IntegrationTesting/Messages.cs @@ -77,7 +77,6 @@ namespace Grpc.Testing { } #region Enums /// - /// DEPRECATED, don't use. To be removed shortly. /// The type of payload that should be returned. /// public enum PayloadType { @@ -269,7 +268,6 @@ namespace Grpc.Testing { public const int TypeFieldNumber = 1; private global::Grpc.Testing.PayloadType type_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// The type of data in body. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -585,12 +583,12 @@ namespace Grpc.Testing { public SimpleRequest(SimpleRequest other) : this() { responseType_ = other.responseType_; responseSize_ = other.responseSize_; - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; fillUsername_ = other.fillUsername_; fillOauthScope_ = other.fillOauthScope_; - ResponseCompressed = other.responseCompressed_ != null ? other.ResponseCompressed.Clone() : null; - ResponseStatus = other.responseStatus_ != null ? other.ResponseStatus.Clone() : null; - ExpectCompressed = other.expectCompressed_ != null ? other.ExpectCompressed.Clone() : null; + responseCompressed_ = other.responseCompressed_ != null ? other.responseCompressed_.Clone() : null; + responseStatus_ = other.responseStatus_ != null ? other.responseStatus_.Clone() : null; + expectCompressed_ = other.expectCompressed_ != null ? other.expectCompressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -603,7 +601,6 @@ namespace Grpc.Testing { public const int ResponseTypeFieldNumber = 1; private global::Grpc.Testing.PayloadType responseType_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// Desired payload type in the response from the server. /// If response_type is RANDOM, server randomly chooses one from other formats. /// @@ -964,7 +961,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public SimpleResponse(SimpleResponse other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; username_ = other.username_; oauthScope_ = other.oauthScope_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); @@ -1168,8 +1165,8 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public StreamingInputCallRequest(StreamingInputCallRequest other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; - ExpectCompressed = other.expectCompressed_ != null ? other.ExpectCompressed.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; + expectCompressed_ = other.expectCompressed_ != null ? other.expectCompressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1486,7 +1483,7 @@ namespace Grpc.Testing { public ResponseParameters(ResponseParameters other) : this() { size_ = other.size_; intervalUs_ = other.intervalUs_; - Compressed = other.compressed_ != null ? other.Compressed.Clone() : null; + compressed_ = other.compressed_ != null ? other.compressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1693,8 +1690,8 @@ namespace Grpc.Testing { public StreamingOutputCallRequest(StreamingOutputCallRequest other) : this() { responseType_ = other.responseType_; responseParameters_ = other.responseParameters_.Clone(); - Payload = other.payload_ != null ? other.Payload.Clone() : null; - ResponseStatus = other.responseStatus_ != null ? other.ResponseStatus.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; + responseStatus_ = other.responseStatus_ != null ? other.responseStatus_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1707,7 +1704,6 @@ namespace Grpc.Testing { public const int ResponseTypeFieldNumber = 1; private global::Grpc.Testing.PayloadType responseType_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// Desired payload type in the response from the server. /// If response_type is RANDOM, the payload from each response in the stream /// might be of different types. This is to simulate a mixed type of payload @@ -1927,7 +1923,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public StreamingOutputCallResponse(StreamingOutputCallResponse other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.IntegrationTesting/Stats.cs b/src/csharp/Grpc.IntegrationTesting/Stats.cs index 8160646215a..af83eef7ba8 100644 --- a/src/csharp/Grpc.IntegrationTesting/Stats.cs +++ b/src/csharp/Grpc.IntegrationTesting/Stats.cs @@ -86,7 +86,7 @@ namespace Grpc.Testing { totalCpuTime_ = other.totalCpuTime_; idleCpuTime_ = other.idleCpuTime_; cqPollCount_ = other.cqPollCount_; - CoreStats = other.coreStats_ != null ? other.CoreStats.Clone() : null; + coreStats_ = other.coreStats_ != null ? other.coreStats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -993,13 +993,13 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ClientStats(ClientStats other) : this() { - Latencies = other.latencies_ != null ? other.Latencies.Clone() : null; + latencies_ = other.latencies_ != null ? other.latencies_.Clone() : null; timeElapsed_ = other.timeElapsed_; timeUser_ = other.timeUser_; timeSystem_ = other.timeSystem_; requestResults_ = other.requestResults_.Clone(); cqPollCount_ = other.cqPollCount_; - CoreStats = other.coreStats_ != null ? other.CoreStats.Clone() : null; + coreStats_ = other.coreStats_ != null ? other.coreStats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.Reflection/Reflection.cs b/src/csharp/Grpc.Reflection/Reflection.cs index 84b2a0a8421..e319be5bff7 100644 --- a/src/csharp/Grpc.Reflection/Reflection.cs +++ b/src/csharp/Grpc.Reflection/Reflection.cs @@ -610,7 +610,7 @@ namespace Grpc.Reflection.V1Alpha { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerReflectionResponse(ServerReflectionResponse other) : this() { validHost_ = other.validHost_; - OriginalRequest = other.originalRequest_ != null ? other.OriginalRequest.Clone() : null; + originalRequest_ = other.originalRequest_ != null ? other.originalRequest_.Clone() : null; switch (other.MessageResponseCase) { case MessageResponseOneofCase.FileDescriptorResponse: FileDescriptorResponse = other.FileDescriptorResponse.Clone(); From 02d4afb94c8f06c7184691f27ed0518f9c359ea9 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 Jul 2018 09:19:52 +0200 Subject: [PATCH 41/99] yapf code --- src/python/grpcio_tests/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index 74c7705c970..a94c0963ec1 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -41,8 +41,8 @@ INSTALL_REQUIRES = ( 'grpcio>={version}'.format(version=grpc_version.VERSION), 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), - 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', - 'google-auth>=1.0.0', 'requests>=2.14.2') + 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', 'google-auth>=1.0.0', + 'requests>=2.14.2') if not PY3: INSTALL_REQUIRES += ('futures>=2.2.0',) From 6ed3c45cbd7e7a27bbe2d4071e711b80366f2f9f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 Jul 2018 09:33:52 +0200 Subject: [PATCH 42/99] android patch no longer needer --- tools/internal_ci/linux/grpc_android.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/internal_ci/linux/grpc_android.sh b/tools/internal_ci/linux/grpc_android.sh index 2220145dacc..42c7f5fb042 100755 --- a/tools/internal_ci/linux/grpc_android.sh +++ b/tools/internal_ci/linux/grpc_android.sh @@ -25,13 +25,6 @@ git submodule update --init # Build protoc and grpc_cpp_plugin. Codegen is not cross-compiled to Android make HAS_SYSTEM_PROTOBUF=false -# TODO(ericgribkoff) Remove when this commit (already in master) is included in -# next protobuf release -cd third_party/protobuf -git fetch -git cherry-pick 7daa320065f3bea2b54bf983337d1724f153422d -m 1 - - # Build and run interop instrumentation tests on Firebase Test Lab cd "${REPO_ROOT}/src/android/test/interop/" From fcf533d916ba0d4e0a7cd8e3aa15982c84e99b4a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 3 Jul 2018 10:06:47 +0200 Subject: [PATCH 43/99] try fix nanopb generation --- tools/codegen/core/gen_nano_proto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/codegen/core/gen_nano_proto.sh b/tools/codegen/core/gen_nano_proto.sh index 5c781761ba1..15495917467 100755 --- a/tools/codegen/core/gen_nano_proto.sh +++ b/tools/codegen/core/gen_nano_proto.sh @@ -68,7 +68,7 @@ popd # this should be the same version as the submodule we compile against # ideally we'd update this as a template to ensure that -pip install protobuf==3.5.2 +pip install protobuf==3.6.0 pushd "$(dirname $INPUT_PROTO)" > /dev/null From d4c1c9b18c92452e0221b081e652cb6f5bfc4236 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 Jul 2018 11:24:39 +0200 Subject: [PATCH 44/99] try fixing protobuf build in win distribtest --- test/distrib/cpp/run_distrib_test_cmake.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/distrib/cpp/run_distrib_test_cmake.bat b/test/distrib/cpp/run_distrib_test_cmake.bat index 8eb3b201b13..7658156be1d 100644 --- a/test/distrib/cpp/run_distrib_test_cmake.bat +++ b/test/distrib/cpp/run_distrib_test_cmake.bat @@ -39,7 +39,7 @@ cd ../../.. cd third_party/protobuf/cmake mkdir build cd build -cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF .. +cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -DZLIB_ROOT=%INSTALL_DIR% -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF .. cmake --build . --config Release --target install || goto :error cd ../../../.. From d1396550bc70323537ae56be9c333da19c4ed556 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 Jul 2018 12:00:30 +0200 Subject: [PATCH 45/99] try fix c++ vs2017 build --- CMakeLists.txt | 2 ++ templates/CMakeLists.txt.template | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e8e65d4b71d..26a4a638605 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,8 @@ if (MSVC) add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) + # TODO(jtattermusch): revisit warnings disabled when upgrading to protobuf3.6.0 + add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) # TODO(jtattermusch): needed to build boringssl with VS2017, revisit later diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index de13d02e2a3..99804d1cb5f 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -149,6 +149,8 @@ add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) + # TODO(jtattermusch): revisit warnings disabled when upgrading to protobuf3.6.0 + add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) # TODO(jtattermusch): needed to build boringssl with VS2017, revisit later From 18aac34e293546836f91103758adaee921a6a504 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 Jul 2018 13:23:43 +0200 Subject: [PATCH 46/99] try fixing cxx distrib test --- test/distrib/cpp/run_distrib_test_routeguide.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/distrib/cpp/run_distrib_test_routeguide.sh b/test/distrib/cpp/run_distrib_test_routeguide.sh index dc69ab83773..a81692d920c 100755 --- a/test/distrib/cpp/run_distrib_test_routeguide.sh +++ b/test/distrib/cpp/run_distrib_test_routeguide.sh @@ -19,7 +19,7 @@ set -ex cd "$(dirname "$0")/../../.." cd third_party/protobuf && ./autogen.sh && \ -./configure && make -j4 && make check && make install && ldconfig +./configure && make -j4 && make install && ldconfig cd ../.. && make -j4 && make install From c42dd8b3c935fa651a62fdd5a828680b8d808ae2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 Jul 2018 14:32:57 +0200 Subject: [PATCH 47/99] attempt to fix protoc build --- .../grpc_artifact_protoc/Dockerfile | 30 ++++++++++++------- .../artifacts/build_artifact_protoc.sh | 4 +-- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/tools/dockerfile/grpc_artifact_protoc/Dockerfile b/tools/dockerfile/grpc_artifact_protoc/Dockerfile index 33e2f2970c3..44fde4f99f0 100644 --- a/tools/dockerfile/grpc_artifact_protoc/Dockerfile +++ b/tools/dockerfile/grpc_artifact_protoc/Dockerfile @@ -30,24 +30,34 @@ RUN yum install -y git \ glibc-devel \ glibc-devel.i686 -# Install GCC 4.7 to support -static-libstdc++ -RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d -RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' -RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" -RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo +# Install GCC 4.8 +RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d +RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' +RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" +RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo # We'll get and "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error caused by # docker issue when using overlay storage driver, but all the stuff we need # will be installed, so for now we just ignore the error. # https://github.com/docker/docker/issues/10180 -RUN yum install -y devtoolset-1.1 \ - devtoolset-1.1-libstdc++-devel \ - devtoolset-1.1-libstdc++-devel.i686 || true +RUN yum install -y devtoolset-2-build \ + devtoolset-2-toolchain \ + devtoolset-2-binutils \ + devtoolset-2-gcc \ + devtoolset-2-gcc-c++ \ + devtoolset-2-libstdc++-devel \ + devtoolset-2-libstdc++-devel.i686 || true + +# Again, ignore the "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error. +RUN yum install -y ca-certificates || true # renew certs to prevent download error for ius-release.rpm + +# TODO(jtattermusch): gRPC makefile uses "which" to detect the availability of gcc +RUN yum install -y which || true # Update Git to version >1.7 to allow cloning submodules with --reference arg. RUN yum remove -y git && yum clean all RUN yum install -y https://centos6.iuscommunity.org/ius-release.rpm && yum clean all RUN yum install -y git2u && yum clean all -# Start in devtoolset environment that uses GCC 4.7 -CMD ["scl", "enable", "devtoolset-1.1", "bash"] +# Start in devtoolset environment that uses GCC 4.8 +CMD ["scl", "enable", "devtoolset-2", "bash"] diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh index b531fc9a5d4..6d433f2dad4 100755 --- a/tools/run_tests/artifacts/build_artifact_protoc.sh +++ b/tools/run_tests/artifacts/build_artifact_protoc.sh @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Use devtoolset environment that has GCC 4.7 before set -ex -source scl_source enable devtoolset-1.1 +# Use devtoolset environment that has GCC 4.8 before set -ex +source scl_source enable devtoolset-2 set -ex From 50b684a6b6b07a93dad34b02fd3a1040e584a882 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 10 Jul 2018 10:59:01 +0200 Subject: [PATCH 48/99] upgrade third_party/protobuf to v3.6.1 --- third_party/protobuf | 2 +- tools/run_tests/sanity/check_submodules.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/protobuf b/third_party/protobuf index ab8edf1dbe2..48cb18e5c41 160000 --- a/third_party/protobuf +++ b/third_party/protobuf @@ -1 +1 @@ -Subproject commit ab8edf1dbe2237b4717869eaab11a2998541ad8d +Subproject commit 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index d60d1e7c753..2c7c140716f 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -36,7 +36,7 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) 6599cac0965be8e5a835ab7a5684bbef033d5ad0 third_party/libcxx (heads/release_60) 9245d481eb3e890f708ff2d7dadf2a10c04748ba third_party/libcxxabi (heads/release_60) - ab8edf1dbe2237b4717869eaab11a2998541ad8d third_party/protobuf (v3.6.0) + 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 third_party/protobuf (v3.6.0.1-37-g48cb18e5) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) EOF From cbb9feea012e8946c19fd2cf2055a62a56d63184 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 2 Aug 2018 08:45:08 +0200 Subject: [PATCH 49/99] upgrade protobuf in grpc_deps.bzl to v3.6.1 --- bazel/grpc_deps.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index faac0c18213..d7519b14532 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -116,8 +116,8 @@ def grpc_deps(): if "com_google_protobuf" not in native.existing_rules(): native.http_archive( name = "com_google_protobuf", - strip_prefix = "protobuf-ab8edf1dbe2237b4717869eaab11a2998541ad8d", - url = "https://github.com/google/protobuf/archive/ab8edf1dbe2237b4717869eaab11a2998541ad8d.tar.gz", + strip_prefix = "protobuf-48cb18e5c419ddd23d9badcfe4e9df7bde1979b2", + url = "https://github.com/google/protobuf/archive/48cb18e5c419ddd23d9badcfe4e9df7bde1979b2.tar.gz", ) if "com_github_nanopb_nanopb" not in native.existing_rules(): From d404e322bff7dbdcb925a56dba0ec7bc3206e09c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 2 Aug 2018 08:46:51 +0200 Subject: [PATCH 50/99] run make_grpcio_tools.py --- tools/distrib/python/grpcio_tools/protoc_lib_deps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index 546b076dd66..7d10db0329c 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -14,10 +14,10 @@ # limitations under the License. # AUTO-GENERATED BY make_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/main.cc', 'google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/implicit_weak_message.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/implicit_weak_message.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] CC_INCLUDE='third_party/protobuf/src' PROTO_INCLUDE='third_party/protobuf/src' -PROTOBUF_SUBMODULE_VERSION="ab8edf1dbe2237b4717869eaab11a2998541ad8d" +PROTOBUF_SUBMODULE_VERSION="48cb18e5c419ddd23d9badcfe4e9df7bde1979b2" From 666dc768355736b3260ec5d118cebaf1a4056d17 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 2 Aug 2018 08:54:34 +0200 Subject: [PATCH 51/99] regenerate C# protos --- .../Grpc.IntegrationTesting/EchoMessages.cs | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs index 04bea8c185f..80a1007e9a5 100644 --- a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs +++ b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs @@ -28,7 +28,7 @@ namespace Grpc.Testing { "DGdycGMudGVzdGluZyIyCglEZWJ1Z0luZm8SFQoNc3RhY2tfZW50cmllcxgB", "IAMoCRIOCgZkZXRhaWwYAiABKAkiUAoLRXJyb3JTdGF0dXMSDAoEY29kZRgB", "IAEoBRIVCg1lcnJvcl9tZXNzYWdlGAIgASgJEhwKFGJpbmFyeV9lcnJvcl9k", - "ZXRhaWxzGAMgASgJIuIDCg1SZXF1ZXN0UGFyYW1zEhUKDWVjaG9fZGVhZGxp", + "ZXRhaWxzGAMgASgJIv8DCg1SZXF1ZXN0UGFyYW1zEhUKDWVjaG9fZGVhZGxp", "bmUYASABKAgSHgoWY2xpZW50X2NhbmNlbF9hZnRlcl91cxgCIAEoBRIeChZz", "ZXJ2ZXJfY2FuY2VsX2FmdGVyX3VzGAMgASgFEhUKDWVjaG9fbWV0YWRhdGEY", "BCABKAgSGgoSY2hlY2tfYXV0aF9jb250ZXh0GAUgASgIEh8KF3Jlc3BvbnNl", @@ -39,18 +39,18 @@ namespace Grpc.Testing { "Zy5EZWJ1Z0luZm8SEgoKc2VydmVyX2RpZRgMIAEoCBIcChRiaW5hcnlfZXJy", "b3JfZGV0YWlscxgNIAEoCRIxCg5leHBlY3RlZF9lcnJvchgOIAEoCzIZLmdy", "cGMudGVzdGluZy5FcnJvclN0YXR1cxIXCg9zZXJ2ZXJfc2xlZXBfdXMYDyAB", - "KAUiSgoLRWNob1JlcXVlc3QSDwoHbWVzc2FnZRgBIAEoCRIqCgVwYXJhbRgC", - "IAEoCzIbLmdycGMudGVzdGluZy5SZXF1ZXN0UGFyYW1zIkYKDlJlc3BvbnNl", - "UGFyYW1zEhgKEHJlcXVlc3RfZGVhZGxpbmUYASABKAMSDAoEaG9zdBgCIAEo", - "CRIMCgRwZWVyGAMgASgJIkwKDEVjaG9SZXNwb25zZRIPCgdtZXNzYWdlGAEg", - "ASgJEisKBXBhcmFtGAIgASgLMhwuZ3JwYy50ZXN0aW5nLlJlc3BvbnNlUGFy", - "YW1zYgZwcm90bzM=")); + "KAUSGwoTYmFja2VuZF9jaGFubmVsX2lkeBgQIAEoBSJKCgtFY2hvUmVxdWVz", + "dBIPCgdtZXNzYWdlGAEgASgJEioKBXBhcmFtGAIgASgLMhsuZ3JwYy50ZXN0", + "aW5nLlJlcXVlc3RQYXJhbXMiRgoOUmVzcG9uc2VQYXJhbXMSGAoQcmVxdWVz", + "dF9kZWFkbGluZRgBIAEoAxIMCgRob3N0GAIgASgJEgwKBHBlZXIYAyABKAki", + "TAoMRWNob1Jlc3BvbnNlEg8KB21lc3NhZ2UYASABKAkSKwoFcGFyYW0YAiAB", + "KAsyHC5ncnBjLnRlc3RpbmcuUmVzcG9uc2VQYXJhbXNiBnByb3RvMw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.DebugInfo), global::Grpc.Testing.DebugInfo.Parser, new[]{ "StackEntries", "Detail" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ErrorStatus), global::Grpc.Testing.ErrorStatus.Parser, new[]{ "Code", "ErrorMessage", "BinaryErrorDetails" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.RequestParams), global::Grpc.Testing.RequestParams.Parser, new[]{ "EchoDeadline", "ClientCancelAfterUs", "ServerCancelAfterUs", "EchoMetadata", "CheckAuthContext", "ResponseMessageLength", "EchoPeer", "ExpectedClientIdentity", "SkipCancelledCheck", "ExpectedTransportSecurityType", "DebugInfo", "ServerDie", "BinaryErrorDetails", "ExpectedError", "ServerSleepUs" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.RequestParams), global::Grpc.Testing.RequestParams.Parser, new[]{ "EchoDeadline", "ClientCancelAfterUs", "ServerCancelAfterUs", "EchoMetadata", "CheckAuthContext", "ResponseMessageLength", "EchoPeer", "ExpectedClientIdentity", "SkipCancelledCheck", "ExpectedTransportSecurityType", "DebugInfo", "ServerDie", "BinaryErrorDetails", "ExpectedError", "ServerSleepUs", "BackendChannelIdx" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoRequest), global::Grpc.Testing.EchoRequest.Parser, new[]{ "Message", "Param" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ResponseParams), global::Grpc.Testing.ResponseParams.Parser, new[]{ "RequestDeadline", "Host", "Peer" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoResponse), global::Grpc.Testing.EchoResponse.Parser, new[]{ "Message", "Param" }, null, null, null) @@ -440,6 +440,7 @@ namespace Grpc.Testing { binaryErrorDetails_ = other.binaryErrorDetails_; expectedError_ = other.expectedError_ != null ? other.expectedError_.Clone() : null; serverSleepUs_ = other.serverSleepUs_; + backendChannelIdx_ = other.backendChannelIdx_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -622,6 +623,20 @@ namespace Grpc.Testing { } } + /// Field number for the "backend_channel_idx" field. + public const int BackendChannelIdxFieldNumber = 16; + private int backendChannelIdx_; + /// + /// which backend to send request to + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int BackendChannelIdx { + get { return backendChannelIdx_; } + set { + backendChannelIdx_ = value; + } + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public override bool Equals(object other) { return Equals(other as RequestParams); @@ -650,6 +665,7 @@ namespace Grpc.Testing { if (BinaryErrorDetails != other.BinaryErrorDetails) return false; if (!object.Equals(ExpectedError, other.ExpectedError)) return false; if (ServerSleepUs != other.ServerSleepUs) return false; + if (BackendChannelIdx != other.BackendChannelIdx) return false; return Equals(_unknownFields, other._unknownFields); } @@ -671,6 +687,7 @@ namespace Grpc.Testing { if (BinaryErrorDetails.Length != 0) hash ^= BinaryErrorDetails.GetHashCode(); if (expectedError_ != null) hash ^= ExpectedError.GetHashCode(); if (ServerSleepUs != 0) hash ^= ServerSleepUs.GetHashCode(); + if (BackendChannelIdx != 0) hash ^= BackendChannelIdx.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -744,6 +761,10 @@ namespace Grpc.Testing { output.WriteRawTag(120); output.WriteInt32(ServerSleepUs); } + if (BackendChannelIdx != 0) { + output.WriteRawTag(128, 1); + output.WriteInt32(BackendChannelIdx); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -797,6 +818,9 @@ namespace Grpc.Testing { if (ServerSleepUs != 0) { size += 1 + pb::CodedOutputStream.ComputeInt32Size(ServerSleepUs); } + if (BackendChannelIdx != 0) { + size += 2 + pb::CodedOutputStream.ComputeInt32Size(BackendChannelIdx); + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -859,6 +883,9 @@ namespace Grpc.Testing { if (other.ServerSleepUs != 0) { ServerSleepUs = other.ServerSleepUs; } + if (other.BackendChannelIdx != 0) { + BackendChannelIdx = other.BackendChannelIdx; + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -936,6 +963,10 @@ namespace Grpc.Testing { ServerSleepUs = input.ReadInt32(); break; } + case 128: { + BackendChannelIdx = input.ReadInt32(); + break; + } } } } From 23c3c477566752521aa147fce513ba80a9fb00cf Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 7 Aug 2018 16:10:24 +0200 Subject: [PATCH 52/99] rephrase comment --- CMakeLists.txt | 2 +- templates/CMakeLists.txt.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 26a4a638605..855b921ada1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,7 +100,7 @@ if (MSVC) add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) - # TODO(jtattermusch): revisit warnings disabled when upgrading to protobuf3.6.0 + # TODO(jtattermusch): revisit warnings that were silenced as part of upgrade to protobuf3.6.0 add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 99804d1cb5f..2a514ed7b68 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -149,7 +149,7 @@ add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) - # TODO(jtattermusch): revisit warnings disabled when upgrading to protobuf3.6.0 + # TODO(jtattermusch): revisit warnings that were silenced as part of upgrade to protobuf3.6.0 add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) From 1cc7904eca29acc61dcd99d225efcd30e1f60ad7 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 7 Aug 2018 16:20:00 +0200 Subject: [PATCH 53/99] better way to disable bazel caching --- tools/internal_ci/linux/grpc_asan_on_foundry.sh | 2 +- tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh | 2 +- tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh | 2 +- tools/internal_ci/linux/grpc_tsan_on_foundry.sh | 2 +- tools/internal_ci/linux/grpc_ubsan_on_foundry.sh | 3 +-- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/internal_ci/linux/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/grpc_asan_on_foundry.sh index 5099fa03016..a6367ad9628 100644 --- a/tools/internal_ci/linux/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_asan_on_foundry.sh @@ -14,6 +14,6 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh index 192d9d1b2d0..51cb66f5b86 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh index 6fb3c77892e..cbba9067ad3 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh index 25531ead2b5..2ba7d469ecf 100644 --- a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index d9b039f1ecd..338b1b6a0d0 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -61,8 +61,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --runs_per_test_detects_flakes \ - --runs_per_test=2 \ + --cache_test_results=no \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying From 3b2b237b3b9c0ffd1bfd7f2074d5680a2cf62b8d Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Tue, 7 Aug 2018 09:22:33 -0700 Subject: [PATCH 54/99] Make `google_default_credentials` be an additional channel type. --- test/cpp/util/cli_credentials.cc | 66 ++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index d223411121b..10ee7917c78 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -23,7 +23,9 @@ DEFINE_bool( enable_ssl, false, "Whether to use ssl/tls. Deprecated. Use --channel_creds_type=ssl."); -DEFINE_bool(use_auth, false, "Whether to create default google credentials."); +DEFINE_bool(use_auth, false, + "Whether to create default google credentials. Deprecated. Use " + "--channel_creds_type=google_default_credentials."); DEFINE_string( access_token, "", "The access token that will be sent to the server to authenticate RPCs."); @@ -31,8 +33,10 @@ DEFINE_string( ssl_target, "", "If not empty, treat the server host name as this for ssl/tls certificate " "validation."); -DEFINE_string(channel_creds_type, "", - "The channel creds type: insecure, ssl, or alts."); +DEFINE_string( + channel_creds_type, "", + "The channel creds type: insecure, ssl, google_default_credentials or " + "alts."); namespace grpc { namespace testing { @@ -45,9 +49,12 @@ grpc::string CliCredentials::GetDefaultChannelCredsType() const { "--channel_creds_type=ssl.\n"); return "ssl"; } - // Implicit channel for GoogleDefaultCredentials is SSL. + // Compatibility logic for --use_auth. if (FLAGS_access_token.empty() && FLAGS_use_auth) { - return "ssl"; + fprintf(stderr, + "warning: --use_auth is deprecated. Use " + "--channel_creds_type=google_default_credentials.\n"); + return "google_default_credentials"; } return "insecure"; } @@ -58,12 +65,16 @@ CliCredentials::GetChannelCredentials() const { return grpc::InsecureChannelCredentials(); } else if (FLAGS_channel_creds_type.compare("ssl") == 0) { return grpc::SslCredentials(grpc::SslCredentialsOptions()); + } else if (FLAGS_channel_creds_type.compare("google_default_credentials") == + 0) { + return grpc::GoogleDefaultCredentials(); } else if (FLAGS_channel_creds_type.compare("alts") == 0) { return grpc::experimental::AltsCredentials( grpc::experimental::AltsCredentialsOptions()); } fprintf(stderr, - "--channel_creds_type=%s invalid; must be insecure, ssl or alts.\n", + "--channel_creds_type=%s invalid; must be insecure, ssl, " + "google_default_credentials or alts.\n", FLAGS_channel_creds_type.c_str()); return std::shared_ptr(); } @@ -77,7 +88,6 @@ std::shared_ptr CliCredentials::GetCallCredentials() } return grpc::AccessTokenCredentials(FLAGS_access_token); } - // TODO(@capstan): Support GoogleDefaultCredentials on other channel types. return std::shared_ptr(); } @@ -90,27 +100,23 @@ std::shared_ptr CliCredentials::GetCredentials() "warning: ignoring --enable_ssl because " "--channel_creds_type already set to %s.\n", FLAGS_channel_creds_type.c_str()); + } else if (FLAGS_use_auth && FLAGS_channel_creds_type.compare( + "google_default_credentials") != 0) { + fprintf(stderr, + "warning: ignoring --use_auth because " + "--channel_creds_type already set to %s.\n", + FLAGS_channel_creds_type.c_str()); } - std::shared_ptr channel_creds; - if (FLAGS_access_token.empty() && FLAGS_use_auth) { - // Today, GoogleDefaultCredentials implies SSL and service account. - if (FLAGS_channel_creds_type.compare("ssl") != 0) { - fprintf(stderr, - "warning: ignoring --channel_creds_type=%s because --use_auth.", - FLAGS_channel_creds_type.c_str()); - } - channel_creds = grpc::GoogleDefaultCredentials(); - } else { - // Legacy transport upgrade logic for insecure requests. - if (!FLAGS_access_token.empty() && - FLAGS_channel_creds_type.compare("insecure") == 0) { - fprintf(stderr, - "warning: --channel_creds_type=insecure upgraded to ssl because " - "an access token was provided.\n"); - FLAGS_channel_creds_type = "ssl"; - } - channel_creds = GetChannelCredentials(); + // Legacy transport upgrade logic for insecure requests. + if (!FLAGS_access_token.empty() && + FLAGS_channel_creds_type.compare("insecure") == 0) { + fprintf(stderr, + "warning: --channel_creds_type=insecure upgraded to ssl because " + "an access token was provided.\n"); + FLAGS_channel_creds_type = "ssl"; } + std::shared_ptr channel_creds = + GetChannelCredentials(); // Composite any call-type credentials on top of the base channel. std::shared_ptr call_creds = GetCallCredentials(); return (channel_creds == nullptr || call_creds == nullptr) @@ -125,12 +131,14 @@ const grpc::string CliCredentials::GetCredentialUsage() const { " --access_token ; Set the access token in metadata," " overrides --use_auth\n" " --ssl_target ; Set server host for ssl validation\n" - " --channel_creds_type ; Set to insecure, ssl, alts\n"; + " --channel_creds_type ; Set to insecure, ssl, alts or\n" + " ; google_default_credentials\n"; } const grpc::string CliCredentials::GetSslTargetNameOverride() const { - bool use_ssl = FLAGS_channel_creds_type.compare("ssl") == 0 || - (FLAGS_access_token.empty() && FLAGS_use_auth); + bool use_ssl = + FLAGS_channel_creds_type.compare("ssl") == 0 || + FLAGS_channel_creds_type.compare("google_default_credentials") == 0; return use_ssl ? FLAGS_ssl_target : ""; } From ae013976b24ce95140d3a9b129f15d4714532e28 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 7 Aug 2018 09:34:51 -0700 Subject: [PATCH 55/99] Surface error_string to ObjC users --- src/objective-c/GRPCClient/private/GRPCWrappedCall.m | 8 ++++++-- src/objective-c/GRPCClient/private/NSError+GRPC.h | 4 +++- src/objective-c/GRPCClient/private/NSError+GRPC.m | 10 ++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index f28e494868e..7781d27ca4f 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -187,6 +187,7 @@ grpc_slice _details; size_t _detailsCapacity; grpc_metadata_array _trailers; + const char *_errorString; } - (instancetype)init { @@ -200,6 +201,7 @@ _op.data.recv_status_on_client.status_details = &_details; grpc_metadata_array_init(&_trailers); _op.data.recv_status_on_client.trailing_metadata = &_trailers; + _op.data.recv_status_on_client.error_string = &_errorString; if (handler) { // Prevent reference cycle with _handler __weak typeof(self) weakSelf = self; @@ -207,8 +209,9 @@ __strong typeof(self) strongSelf = weakSelf; if (strongSelf) { char *details = grpc_slice_to_c_string(strongSelf->_details); - NSError *error = - [NSError grpc_errorFromStatusCode:strongSelf->_statusCode details:details]; + NSError *error = [NSError grpc_errorFromStatusCode:strongSelf->_statusCode + details:details + errorString:strongSelf->_errorString]; NSDictionary *trailers = [NSDictionary grpc_dictionaryFromMetadataArray:strongSelf->_trailers]; handler(error, trailers); @@ -223,6 +226,7 @@ - (void)dealloc { grpc_metadata_array_destroy(&_trailers); grpc_slice_unref(_details); + gpr_free((void *)_errorString); } @end diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.h b/src/objective-c/GRPCClient/private/NSError+GRPC.h index e96b7297c28..a63e76ee4d0 100644 --- a/src/objective-c/GRPCClient/private/NSError+GRPC.h +++ b/src/objective-c/GRPCClient/private/NSError+GRPC.h @@ -24,5 +24,7 @@ * Returns nil if the status code is OK. Otherwise, a NSError whose code is one of |GRPCErrorCode| * and whose domain is |kGRPCErrorDomain|. */ -+ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode details:(char *)details; ++ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode + details:(char *)details + errorString:(const char *)errorString; @end diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.m b/src/objective-c/GRPCClient/private/NSError+GRPC.m index c2e65e4d8a2..199b2ebb6c3 100644 --- a/src/objective-c/GRPCClient/private/NSError+GRPC.m +++ b/src/objective-c/GRPCClient/private/NSError+GRPC.m @@ -23,13 +23,19 @@ NSString *const kGRPCErrorDomain = @"io.grpc"; @implementation NSError (GRPC) -+ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode details:(char *)details { ++ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode + details:(char *)details + errorString:(const char *)errorString { if (statusCode == GRPC_STATUS_OK) { return nil; } NSString *message = [NSString stringWithCString:details encoding:NSUTF8StringEncoding]; + NSString *debugMessage = [NSString stringWithCString:errorString encoding:NSUTF8StringEncoding]; return [NSError errorWithDomain:kGRPCErrorDomain code:statusCode - userInfo:@{NSLocalizedDescriptionKey : message}]; + userInfo:@{ + NSLocalizedDescriptionKey : message, + NSDebugDescriptionErrorKey : debugMessage + }]; } @end From 8a8f33928f906208e9d3ed128e367aa6481b8507 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 7 Aug 2018 10:52:32 -0700 Subject: [PATCH 56/99] clang-format --- src/objective-c/GRPCClient/GRPCCall.m | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 3c4e87cf5a9..8ce88c7db2d 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -230,7 +230,6 @@ static NSString *const kBearerPrefix = @"Bearer "; errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeCancelled userInfo:@{NSLocalizedDescriptionKey : @"Canceled by app"}]]; - } - (void)maybeFinishWithError:(NSError *)errorOrNil { @@ -534,11 +533,12 @@ static NSString *const kBearerPrefix = @"Bearer "; __strong GRPCCall *strongSelf = self; if (strongSelf) { [self cancelCall]; - [self maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain - code:GRPCErrorCodeUnavailable - userInfo:@{ - NSLocalizedDescriptionKey : @"Connectivity lost." - }]]; + [self + maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain + code:GRPCErrorCodeUnavailable + userInfo:@{ + NSLocalizedDescriptionKey : @"Connectivity lost." + }]]; } } From 520bcfb64aed867373b2ffd643cc6a685b1e9cc1 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 7 Aug 2018 11:27:40 -0700 Subject: [PATCH 57/99] Upgrade protobuf podspecs in grpc --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler.podspec | 2 +- .../src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 6ad9166b32e..5e9a9a45131 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.5.0' + s.dependency '!ProtoCompiler', '3.6.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index 12598e616a6..b98339941e5 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.5.0' + v = '3.6.0' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC diff --git a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template index 5c1358f7c37..30b6c5684cc 100644 --- a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template +++ b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template @@ -103,7 +103,7 @@ s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.5.0' + s.dependency '!ProtoCompiler', '3.6.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' From 3b1e34843b6470aa6845e8400b7b7d38f579f8a8 Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Tue, 7 Aug 2018 11:38:53 -0700 Subject: [PATCH 58/99] Update README --- tools/interop_matrix/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index 40c02a1bc7a..db84d9b454c 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -9,7 +9,7 @@ from specific releases/tag, are used to test version compatiblity between gRPC r We have continuous nightly test setup to test gRPC backward compatibility between old clients and latest server. When a gRPC developer creates a new gRPC release, s/he is also responsible to add the just-released gRPC client to the nightly test. The steps are: - Add (or update) an entry in `./client_matrix.py` file to reference the github tag for the release. - Build new client docker image(s). For example, for C and wrapper languages release `v1.9.9`, do - - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --language cxx csharp python ruby php` + - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --upload_images --language cxx csharp python ruby php` - Verify that the new docker image was built successfully and uploaded to GCR. For example, - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos. - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`. From 8617879613133c06782c955b03c0bd3d1f8517e2 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 7 Aug 2018 12:04:35 -0700 Subject: [PATCH 59/99] Add test for returned debug information --- src/objective-c/tests/GRPCClientTests.m | 35 +++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 2a169800a09..6dda346514e 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -591,4 +591,39 @@ static GRPCProtoMethod *kFullDuplexCallMethod; [self testTimeoutBackoffWithTimeout:0.3 Backoff:0.7]; } +- (void)testErrorDebugInformation { + __weak XCTestExpectation *expectation = [self expectationWithDescription:@"RPC unauthorized."]; + + RMTSimpleRequest *request = [RMTSimpleRequest message]; + request.fillUsername = YES; + request.fillOauthScope = YES; + GRXWriter *requestsWriter = [GRXWriter writerWithValue:[request data]]; + + GRPCCall *call = [[GRPCCall alloc] initWithHost:kRemoteSSLHost + path:kUnaryCallMethod.HTTPPath + requestsWriter:requestsWriter]; + + call.oauth2AccessToken = @"bogusToken"; + + id responsesWriteable = + [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + XCTFail(@"Received unexpected response: %@", value); + } + completionHandler:^(NSError *errorOrNil) { + XCTAssertNotNil(errorOrNil, @"Finished without error!"); + NSDictionary *userInfo = errorOrNil.userInfo; + NSString *debugInformation = userInfo[NSDebugDescriptionErrorKey]; + XCTAssertNotNil(debugInformation); + XCTAssertNotEqual([debugInformation length], 0); + NSString *challengeHeader = call.oauth2ChallengeHeader; + XCTAssertGreaterThan(challengeHeader.length, 0, @"No challenge in response headers %@", + call.responseHeaders); + [expectation fulfill]; + }]; + + [call startWithWriteable:responsesWriteable]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; +} + @end From bcd237d6aac31380ce3be7b8f9aa918ee89e6441 Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Tue, 7 Aug 2018 12:34:48 -0700 Subject: [PATCH 60/99] Bump version to 1.14.1 --- BUILD | 2 +- build.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD b/BUILD index 783590a1343..50193d99221 100644 --- a/BUILD +++ b/BUILD @@ -68,7 +68,7 @@ g_stands_for = "gladiolus" core_version = "6.0.0" -version = "1.14.0" +version = "1.14.1" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build.yaml b/build.yaml index b9dbdc41865..fa06279e7af 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 6.0.0 g_stands_for: gladiolus - version: 1.14.0 + version: 1.14.1 filegroups: - name: alts_proto headers: From 7d41ba6370acafc76c5ec36b100630448d4e6da3 Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Tue, 7 Aug 2018 12:37:28 -0700 Subject: [PATCH 61/99] Regenerate projects --- CMakeLists.txt | 2 +- Makefile | 4 ++-- gRPC-C++.podspec | 4 ++-- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 6 +++--- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 29 files changed, 35 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ea028b5a77..a59fd818e3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.14.0") +set(PACKAGE_VERSION "1.14.1") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 74c01698516..d508ec5db2c 100644 --- a/Makefile +++ b/Makefile @@ -437,8 +437,8 @@ Q = @ endif CORE_VERSION = 6.0.0 -CPP_VERSION = 1.14.0 -CSHARP_VERSION = 1.14.0 +CPP_VERSION = 1.14.1 +CSHARP_VERSION = 1.14.1 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index cbf765abb82..1f4eea3e87e 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.14.0' + # version = '1.14.1' version = '0.0.3' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.14.0' + grpc_version = '1.14.1' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 85071fb2114..fec599ddded 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.14.0' + version = '1.14.1' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 3c4c6e1310e..02f81a3bf45 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.14.0' + version = '1.14.1' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index c37efca0add..3d78e9b9cb3 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.14.0' + version = '1.14.1' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 9b155cbf444..4adeddd1676 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.14.0' + version = '1.14.1' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index 2a30cffb4d8..643e4adb62a 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2018-01-19 - 1.14.0 - 1.14.0 + 1.14.1 + 1.14.1 stable diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index de42c6fea30..82bfac21ec8 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.14.0"; } +grpc::string Version() { return "1.14.1"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 92bec0e0674..04409e09551 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.14.0 + 1.14.1 3.5.1 diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 35888b03001..7cd98ce34b0 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "1.14.0.0"; + public const string CurrentAssemblyFileVersion = "1.14.1.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.14.0"; + public const string CurrentVersion = "1.14.1"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index bb0ae713606..54a1097cd83 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.14.0 +set VERSION=1.14.1 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index e298ea62a9f..eabd219a4a0 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -45,8 +45,8 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.14.0" -OutputDirectory ../../artifacts -nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.0" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.14.0" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.14.1" -OutputDirectory ../../artifacts +nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.1" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.14.1" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index bf5b7ae3511..d1e9792fa53 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.14.0' + v = '1.14.1' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 59c172fc4ba..8b1626fe094 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0" +#define GRPC_OBJC_VERSION_STRING @"1.14.1" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index d0a4b496c7a..a8080512eed 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0" +#define GRPC_OBJC_VERSION_STRING @"1.14.1" #define GRPC_C_VERSION_STRING @"6.0.0" diff --git a/src/php/composer.json b/src/php/composer.json index 94918417d9f..21f09feb812 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.14.0", + "version": "1.14.1", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 22631f126ff..b37dcb56824 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.14.0" +#define PHP_GRPC_VERSION "1.14.1" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 3d3e2531554..40b79b34cc6 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.14.0""" +__version__ = """1.14.1""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 248cd632752..74df810d090 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.14.0' +VERSION = '1.14.1' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 1e48f81b791..6da128ff279 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.14.0' +VERSION = '1.14.1' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index ec79577825e..69b69ec3e05 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.14.0' +VERSION = '1.14.1' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 50e2aa22199..8f3036dde57 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.14.0' +VERSION = '1.14.1' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 18d34cc1a29..60374c84521 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.14.0' +VERSION = '1.14.1' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 55ae016bf0d..473724a8d4e 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.14.0' + VERSION = '1.14.1' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 3bd5529215d..387512ff897 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.14.0' + VERSION = '1.14.1' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index f1f65265271..a710233084b 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.14.0' +VERSION = '1.14.1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 0a051e892ab..c80ec3b79a7 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0 +PROJECT_NUMBER = 1.14.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 8547dbd4c22..af8012d8ffc 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0 +PROJECT_NUMBER = 1.14.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From e1a0f235e1e2328aa5ebc50fbd0847c998db632f Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Tue, 7 Aug 2018 15:27:40 -0700 Subject: [PATCH 62/99] Shorten flag value to gdc. --- test/cpp/util/cli_credentials.cc | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index 10ee7917c78..acf4ef8ef10 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -25,7 +25,7 @@ DEFINE_bool( "Whether to use ssl/tls. Deprecated. Use --channel_creds_type=ssl."); DEFINE_bool(use_auth, false, "Whether to create default google credentials. Deprecated. Use " - "--channel_creds_type=google_default_credentials."); + "--channel_creds_type=gdc."); DEFINE_string( access_token, "", "The access token that will be sent to the server to authenticate RPCs."); @@ -35,8 +35,8 @@ DEFINE_string( "validation."); DEFINE_string( channel_creds_type, "", - "The channel creds type: insecure, ssl, google_default_credentials or " - "alts."); + "The channel creds type: insecure, ssl, gdc (Google Default Credentials) " + "or alts."); namespace grpc { namespace testing { @@ -53,8 +53,8 @@ grpc::string CliCredentials::GetDefaultChannelCredsType() const { if (FLAGS_access_token.empty() && FLAGS_use_auth) { fprintf(stderr, "warning: --use_auth is deprecated. Use " - "--channel_creds_type=google_default_credentials.\n"); - return "google_default_credentials"; + "--channel_creds_type=gdc.\n"); + return "gdc"; } return "insecure"; } @@ -65,16 +65,15 @@ CliCredentials::GetChannelCredentials() const { return grpc::InsecureChannelCredentials(); } else if (FLAGS_channel_creds_type.compare("ssl") == 0) { return grpc::SslCredentials(grpc::SslCredentialsOptions()); - } else if (FLAGS_channel_creds_type.compare("google_default_credentials") == - 0) { + } else if (FLAGS_channel_creds_type.compare("gdc") == 0) { return grpc::GoogleDefaultCredentials(); } else if (FLAGS_channel_creds_type.compare("alts") == 0) { return grpc::experimental::AltsCredentials( grpc::experimental::AltsCredentialsOptions()); } fprintf(stderr, - "--channel_creds_type=%s invalid; must be insecure, ssl, " - "google_default_credentials or alts.\n", + "--channel_creds_type=%s invalid; must be insecure, ssl, gdc or " + "alts.\n", FLAGS_channel_creds_type.c_str()); return std::shared_ptr(); } @@ -100,8 +99,7 @@ std::shared_ptr CliCredentials::GetCredentials() "warning: ignoring --enable_ssl because " "--channel_creds_type already set to %s.\n", FLAGS_channel_creds_type.c_str()); - } else if (FLAGS_use_auth && FLAGS_channel_creds_type.compare( - "google_default_credentials") != 0) { + } else if (FLAGS_use_auth && FLAGS_channel_creds_type.compare("gdc") != 0) { fprintf(stderr, "warning: ignoring --use_auth because " "--channel_creds_type already set to %s.\n", @@ -131,14 +129,12 @@ const grpc::string CliCredentials::GetCredentialUsage() const { " --access_token ; Set the access token in metadata," " overrides --use_auth\n" " --ssl_target ; Set server host for ssl validation\n" - " --channel_creds_type ; Set to insecure, ssl, alts or\n" - " ; google_default_credentials\n"; + " --channel_creds_type ; Set to insecure, ssl, gdc, or alts\n"; } const grpc::string CliCredentials::GetSslTargetNameOverride() const { - bool use_ssl = - FLAGS_channel_creds_type.compare("ssl") == 0 || - FLAGS_channel_creds_type.compare("google_default_credentials") == 0; + bool use_ssl = FLAGS_channel_creds_type.compare("ssl") == 0 || + FLAGS_channel_creds_type.compare("gdc") == 0; return use_ssl ? FLAGS_ssl_target : ""; } From 949096934d77b93aa3a115695289ff49b1efcd22 Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Tue, 7 Aug 2018 15:37:08 -0700 Subject: [PATCH 63/99] Revert #15983 - gRPC channels blocking indefinitely and not respecting deadlines on network disconnect --- .../chttp2/transport/chttp2_transport.cc | 15 ++------------- .../ext/transport/chttp2/transport/internal.h | 13 +++---------- .../transport/chttp2/transport/stream_lists.cc | 17 ----------------- 3 files changed, 5 insertions(+), 40 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 9ad271753ce..bc6fa0d0eb0 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -813,11 +813,7 @@ static void set_write_state(grpc_chttp2_transport* t, write_state_name(st), reason)); t->write_state = st; if (st == GRPC_CHTTP2_WRITE_STATE_IDLE) { - grpc_chttp2_stream* s; - while (grpc_chttp2_list_pop_waiting_for_write_stream(t, &s)) { - GRPC_CLOSURE_LIST_SCHED(&s->run_after_write); - GRPC_CHTTP2_STREAM_UNREF(s, "chttp2:write_closure_sched"); - } + GRPC_CLOSURE_LIST_SCHED(&t->run_after_write); if (t->close_transport_on_writes_finished != nullptr) { grpc_error* err = t->close_transport_on_writes_finished; t->close_transport_on_writes_finished = nullptr; @@ -1212,10 +1208,7 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t, !(closure->next_data.scratch & CLOSURE_BARRIER_MAY_COVER_WRITE)) { GRPC_CLOSURE_RUN(closure, closure->error_data.error); } else { - if (grpc_chttp2_list_add_waiting_for_write_stream(t, s)) { - GRPC_CHTTP2_STREAM_REF(s, "chttp2:pending_write_closure"); - } - grpc_closure_list_append(&s->run_after_write, closure, + grpc_closure_list_append(&t->run_after_write, closure, closure->error_data.error); } } @@ -2016,10 +2009,6 @@ static void remove_stream(grpc_chttp2_transport* t, uint32_t id, void grpc_chttp2_cancel_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s, grpc_error* due_to_error) { - GRPC_CLOSURE_LIST_SCHED(&s->run_after_write); - if (grpc_chttp2_list_remove_waiting_for_write_stream(t, s)) { - GRPC_CHTTP2_STREAM_UNREF(s, "chttp2:pending_write_closure"); - } if (!t->is_client && !s->sent_trailing_metadata && grpc_error_has_clear_grpc_status(due_to_error)) { close_from_api(t, s, due_to_error); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 4f1a08d98b5..ca6e7159787 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -54,8 +54,6 @@ typedef enum { /** streams that are waiting to start because there are too many concurrent streams on the connection */ GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY, - /** streams with closures waiting to be run on a write **/ - GRPC_CHTTP2_LIST_WAITING_FOR_WRITE, STREAM_LIST_COUNT /* must be last */ } grpc_chttp2_stream_list_id; @@ -433,6 +431,9 @@ struct grpc_chttp2_transport { */ grpc_error* close_transport_on_writes_finished; + /* a list of closures to run after writes are finished */ + grpc_closure_list run_after_write; + /* buffer pool state */ /** have we scheduled a benign cleanup? */ bool benign_reclaimer_registered; @@ -583,7 +584,6 @@ struct grpc_chttp2_stream { grpc_slice_buffer flow_controlled_buffer; - grpc_closure_list run_after_write; grpc_chttp2_write_cb* on_flow_controlled_cbs; grpc_chttp2_write_cb* on_write_finished_cbs; grpc_chttp2_write_cb* finish_after_write; @@ -686,13 +686,6 @@ bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport* t, bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s); -bool grpc_chttp2_list_add_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream* s); -bool grpc_chttp2_list_pop_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream** s); -bool grpc_chttp2_list_remove_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream* s); - /********* Flow Control ***************/ // Takes in a flow control action and performs all the needed operations. diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.cc b/src/core/ext/transport/chttp2/transport/stream_lists.cc index 50bfe36a863..6626170a7e4 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.cc +++ b/src/core/ext/transport/chttp2/transport/stream_lists.cc @@ -35,8 +35,6 @@ static const char* stream_list_id_string(grpc_chttp2_stream_list_id id) { return "stalled_by_stream"; case GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY: return "waiting_for_concurrency"; - case GRPC_CHTTP2_LIST_WAITING_FOR_WRITE: - return "waiting_for_write"; case STREAM_LIST_COUNT: GPR_UNREACHABLE_CODE(return "unknown"); } @@ -216,18 +214,3 @@ bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } - -bool grpc_chttp2_list_add_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream* s) { - return stream_list_add(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE); -} - -bool grpc_chttp2_list_pop_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream** s) { - return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE); -} - -bool grpc_chttp2_list_remove_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream* s) { - return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE); -} From 07f190d2d571d295952e8449550b9d908c6ad718 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 7 Aug 2018 17:03:25 -0700 Subject: [PATCH 64/99] Fix binary_size.py --- tools/profiling/ios_bin/binary_size.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/ios_bin/binary_size.py b/tools/profiling/ios_bin/binary_size.py index 91c43f44243..d4d134fef37 100755 --- a/tools/profiling/ios_bin/binary_size.py +++ b/tools/profiling/ios_bin/binary_size.py @@ -75,6 +75,7 @@ def get_size(where, frameworks): def build(where, frameworks): + subprocess.check_call(['make', 'clean']) shutil.rmtree( 'src/objective-c/examples/Sample/Build/Build-%s' % where, ignore_errors=True) From d562ad11accb84c06c15e8e1b9177ac353f1c01c Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 7 Aug 2018 17:19:55 -0700 Subject: [PATCH 65/99] clang-format --- src/objective-c/tests/GRPCClientTests.m | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 6dda346514e..a0de8ba8990 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -606,20 +606,20 @@ static GRPCProtoMethod *kFullDuplexCallMethod; call.oauth2AccessToken = @"bogusToken"; id responsesWriteable = - [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { - XCTFail(@"Received unexpected response: %@", value); - } - completionHandler:^(NSError *errorOrNil) { - XCTAssertNotNil(errorOrNil, @"Finished without error!"); - NSDictionary *userInfo = errorOrNil.userInfo; - NSString *debugInformation = userInfo[NSDebugDescriptionErrorKey]; - XCTAssertNotNil(debugInformation); - XCTAssertNotEqual([debugInformation length], 0); - NSString *challengeHeader = call.oauth2ChallengeHeader; - XCTAssertGreaterThan(challengeHeader.length, 0, @"No challenge in response headers %@", - call.responseHeaders); - [expectation fulfill]; - }]; + [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + XCTFail(@"Received unexpected response: %@", value); + } + completionHandler:^(NSError *errorOrNil) { + XCTAssertNotNil(errorOrNil, @"Finished without error!"); + NSDictionary *userInfo = errorOrNil.userInfo; + NSString *debugInformation = userInfo[NSDebugDescriptionErrorKey]; + XCTAssertNotNil(debugInformation); + XCTAssertNotEqual([debugInformation length], 0); + NSString *challengeHeader = call.oauth2ChallengeHeader; + XCTAssertGreaterThan(challengeHeader.length, 0, @"No challenge in response headers %@", + call.responseHeaders); + [expectation fulfill]; + }]; [call startWithWriteable:responsesWriteable]; From 15b1f6b292db74ec169df5dddb387df823c707ea Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 8 Aug 2018 11:45:05 +0200 Subject: [PATCH 66/99] deduplicate GUIDs for unity .meta files --- .../runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta | 2 +- .../runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta | 2 +- .../Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta index ccaf0c20ec2..6f93617280d 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: e5beceb1c8fb2403ab3dea319dcd9a2e +guid: e1f44cc7ecd4244448817ccae6de42a3 PluginImporter: externalObjects: {} serializedVersion: 2 diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta index 1560e207fbd..82578436b83 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: e5beceb1c8fb2403ab3dea319dcd9a2e +guid: 04fe0e4dcf310416b991e57c99e5d55f PluginImporter: externalObjects: {} serializedVersion: 2 diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta index 4e216b8b425..3ebdd010e88 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: e5beceb1c8fb2403ab3dea319dcd9a2e +guid: 245e3d515096b414fbcdd1fd4160161a PluginImporter: externalObjects: {} serializedVersion: 2 From 310a5b9dcd0cd38d4f463e799343dc941fde7b1a Mon Sep 17 00:00:00 2001 From: Nicolas Noble Date: Wed, 8 Aug 2018 05:57:31 -0700 Subject: [PATCH 67/99] Create ssl-performance.md First draft --- doc/ssl-performance.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 doc/ssl-performance.md diff --git a/doc/ssl-performance.md b/doc/ssl-performance.md new file mode 100644 index 00000000000..5af7afdfe49 --- /dev/null +++ b/doc/ssl-performance.md @@ -0,0 +1,30 @@ +# SSL in gRPC and performance + +The SSL requirement of gRPC isn't necessarily making it easy to integrate. The HTTP/2 protocol requires ALPN support, which is a fairly new handshake protocol only supported by recent implementations. + +As a result, we've tried hard to provide a smooth experience to our users when compiling and distributing gRPC, but this may come at performance costs due to this. More specifically, we will sometime build the SSL library by disabling assembly code, which can impact performances by an order of magnitude when processing encrypted streams. + +Build system | Condition | Platform | Assembly code +---|---|---|-- +Makefile | with OpenSSL 1.0.2 development files | all | :heavy_check_mark: +Makefile | all other cases | all | :x: +Bazel | | Linux | :heavy_check_mark: +Bazel | | MacOS | :heavy_check_mark: +Bazel | | Windows | :x: +CMake | | Windows | :x: +CMake | | all others | :heavy_check_mark: + + +In addition, we are shipping packages for language implementations. These packages are source packages, but also have pre-built binaries being distributed. Building packages from source may give a different result in some cases. + +Language | From source | Platform | Assembly code +---|---|---|--- +Node.JS | n/a | Linux | :heavy_check_mark: +Node.JS | n/a | MacOS | :heavy_check_mark: +Node.JS | n/a | Windows | :x: +Electron | n/a | all | :heavy_check_mark: +Ruby | Yes | all | Same as the `Makefile` case from above +Ruby | No | all | :x: +PHP | Yes | all | Same as the `Makefile` case from above +PHP | No | all | :x: +Python | n/a | all | :x: From ddc7fd845b6649124d801a9f01e09a27c863bffb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 6 Aug 2018 13:46:53 +0200 Subject: [PATCH 68/99] make Grpc.Core nuget work for Xamarin.Forms --- src/csharp/Grpc.Core/Grpc.Core.csproj | 10 +++++----- .../Grpc.Core/build/MonoAndroid/Grpc.Core.targets | 9 +++------ .../Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets | 4 ++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index fc32271063f..481400125ae 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -47,23 +47,23 @@ true - runtimes/monoandroid/armeabi-v7a/libgrpc_csharp_ext.so + native/android/armeabi-v7a/libgrpc_csharp_ext.so true - runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so + native/android/arm64-v8a/libgrpc_csharp_ext.so true - runtimes/monoandroid/x86/libgrpc_csharp_ext.so + native/android/x86/libgrpc_csharp_ext.so true - runtimes/ios/native/libgrpc_csharp_ext.a + native/ios/universal/libgrpc_csharp_ext.a true - runtimes/ios/native/libgrpc.a + native/ios/universal/libgrpc.a true diff --git a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets b/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets index d75e5a2f2f9..250d3bd0cd6 100644 --- a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets +++ b/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets @@ -1,25 +1,22 @@ - - <_GrpcCoreNugetNativePath Condition="'$(_GrpcCoreNugetNativePath)' == ''">$(MSBuildThisFileDirectory)..\..\ - - + Always arm64-v8a - + Always armeabi-v7a - + Always x86 diff --git a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets b/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets index 658158f6ea9..dda1cdd1e84 100644 --- a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets +++ b/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets @@ -2,11 +2,11 @@ - + Static True - + Static True From cc131600d54601ab5d611a846605d0c4ca922c14 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 8 Aug 2018 17:51:21 +0200 Subject: [PATCH 69/99] use recommended names for MonoAndroid and Xamarin.iOS --- src/csharp/Grpc.Core/Grpc.Core.csproj | 8 ++++---- .../{MonoAndroid => MonoAndroid10}/Grpc.Core.targets | 0 .../{Xamarin.iOS => Xamarin.iOS10}/Grpc.Core.targets | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename src/csharp/Grpc.Core/build/{MonoAndroid => MonoAndroid10}/Grpc.Core.targets (100%) rename src/csharp/Grpc.Core/build/{Xamarin.iOS => Xamarin.iOS10}/Grpc.Core.targets (100%) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 481400125ae..dc5683c9753 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -70,12 +70,12 @@ build/net45/ true - - build/MonoAndroid/ + + build/MonoAndroid10/ true - - build/Xamarin.iOS/ + + build/Xamarin.iOS10/ true diff --git a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets b/src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets similarity index 100% rename from src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets rename to src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets diff --git a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets b/src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets similarity index 100% rename from src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets rename to src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets From b60c1f10cd36e18fda9fd53248c7f4cf46e493e3 Mon Sep 17 00:00:00 2001 From: Nicolas Noble Date: Wed, 8 Aug 2018 08:54:52 -0700 Subject: [PATCH 70/99] Ruby always disables assembly. --- doc/ssl-performance.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/ssl-performance.md b/doc/ssl-performance.md index 5af7afdfe49..f0d2e0a0748 100644 --- a/doc/ssl-performance.md +++ b/doc/ssl-performance.md @@ -23,7 +23,6 @@ Node.JS | n/a | Linux | :heavy_check_mark: Node.JS | n/a | MacOS | :heavy_check_mark: Node.JS | n/a | Windows | :x: Electron | n/a | all | :heavy_check_mark: -Ruby | Yes | all | Same as the `Makefile` case from above Ruby | No | all | :x: PHP | Yes | all | Same as the `Makefile` case from above PHP | No | all | :x: From 7faa3fecad62580fd0a644bb814e1b488ef6248b Mon Sep 17 00:00:00 2001 From: Nicolas Noble Date: Wed, 8 Aug 2018 08:55:19 -0700 Subject: [PATCH 71/99] Update assembly code heading a bit. --- doc/ssl-performance.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ssl-performance.md b/doc/ssl-performance.md index f0d2e0a0748..2604e26a808 100644 --- a/doc/ssl-performance.md +++ b/doc/ssl-performance.md @@ -4,7 +4,7 @@ The SSL requirement of gRPC isn't necessarily making it easy to integrate. The H As a result, we've tried hard to provide a smooth experience to our users when compiling and distributing gRPC, but this may come at performance costs due to this. More specifically, we will sometime build the SSL library by disabling assembly code, which can impact performances by an order of magnitude when processing encrypted streams. -Build system | Condition | Platform | Assembly code +Build system | Condition | Platform | Uses assembly code ---|---|---|-- Makefile | with OpenSSL 1.0.2 development files | all | :heavy_check_mark: Makefile | all other cases | all | :x: @@ -17,7 +17,7 @@ CMake | | all others | :heavy_check_mark: In addition, we are shipping packages for language implementations. These packages are source packages, but also have pre-built binaries being distributed. Building packages from source may give a different result in some cases. -Language | From source | Platform | Assembly code +Language | From source | Platform | Uses assembly code ---|---|---|--- Node.JS | n/a | Linux | :heavy_check_mark: Node.JS | n/a | MacOS | :heavy_check_mark: From a49a2a824d2bb0948dd3723d82a0c456d8a5b3e2 Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Wed, 8 Aug 2018 09:17:44 -0700 Subject: [PATCH 72/99] update roots.pem --- etc/roots.pem | 203 ++++++++++---------------------------------------- 1 file changed, 40 insertions(+), 163 deletions(-) diff --git a/etc/roots.pem b/etc/roots.pem index 5dbd1ae6edb..c22dfe69f87 100644 --- a/etc/roots.pem +++ b/etc/roots.pem @@ -3734,169 +3734,6 @@ lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR -----END CERTIFICATE----- -# Issuer: CN=Certplus Root CA G1 O=Certplus -# Subject: CN=Certplus Root CA G1 O=Certplus -# Label: "Certplus Root CA G1" -# Serial: 1491911565779898356709731176965615564637713 -# MD5 Fingerprint: 7f:09:9c:f7:d9:b9:5c:69:69:56:d5:37:3e:14:0d:42 -# SHA1 Fingerprint: 22:fd:d0:b7:fd:a2:4e:0d:ac:49:2c:a0:ac:a6:7b:6a:1f:e3:f7:66 -# SHA256 Fingerprint: 15:2a:40:2b:fc:df:2c:d5:48:05:4d:22:75:b3:9c:7f:ca:3e:c0:97:80:78:b0:f0:ea:76:e5:61:a6:c7:43:3e ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUA -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBa -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB -ANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHNr49a -iZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt -6kuJPKNxQv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP -0FG7Yn2ksYyy/yARujVjBYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f -6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTvLRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDE -EW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2z4QTd28n6v+WZxcIbekN -1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc4nBvCGrc -h2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCT -mehd4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV -4EJQeIQEQWGw9CEjjy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPO -WftwenMGE9nTdDckQQoRb5fc5+R+ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1Ud -DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSowcCbkahDFXxd -Bie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHYlwuBsTANBgkq -hkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh -66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7 -/SMNkPX0XtPGYX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BS -S7CTKtQ+FjPlnsZlFT5kOwQ/2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j -2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F6ALEUz65noe8zDUa3qHpimOHZR4R -Kttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilXCNQ314cnrUlZp5Gr -RHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWetUNy -6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEV -V/xuZDDCVRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5 -g4VCXA9DO2pJNdWY9BW/+mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl -++O/QmueD6i9a5jc2NvLi6Td11n0bt3+qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo= ------END CERTIFICATE----- - -# Issuer: CN=Certplus Root CA G2 O=Certplus -# Subject: CN=Certplus Root CA G2 O=Certplus -# Label: "Certplus Root CA G2" -# Serial: 1492087096131536844209563509228951875861589 -# MD5 Fingerprint: a7:ee:c4:78:2d:1b:ee:2d:b9:29:ce:d6:a7:96:32:31 -# SHA1 Fingerprint: 4f:65:8e:1f:e9:06:d8:28:02:e9:54:47:41:c9:54:25:5d:69:cc:1a -# SHA256 Fingerprint: 6c:c0:50:41:e6:44:5e:74:69:6c:4c:fb:c9:f8:0f:54:3b:7e:ab:bb:44:b4:ce:6f:78:7c:6a:99:71:c4:2f:17 ------BEGIN CERTIFICATE----- -MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABM0PW1aC3/BFGtat -93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uNAm8x -Ik0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0P -AQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwj -FNiPwyCrKGBZMB8GA1UdIwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqG -SM49BAMDA2gAMGUCMHD+sAvZ94OX7PNVHdTcswYO/jOYnYs5kGuUIe22113WTNch -p+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjlvPl5adytRSv3tjFzzAal -U5ORGpOucGpnutee5WEaXw== ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G1 O=OpenTrust -# Subject: CN=OpenTrust Root CA G1 O=OpenTrust -# Label: "OpenTrust Root CA G1" -# Serial: 1492036577811947013770400127034825178844775 -# MD5 Fingerprint: 76:00:cc:81:29:cd:55:5e:88:6a:7a:2e:f7:4d:39:da -# SHA1 Fingerprint: 79:91:e8:34:f7:e2:ee:dd:08:95:01:52:e9:55:2d:14:e9:58:d5:7e -# SHA256 Fingerprint: 56:c7:71:28:d9:8c:18:d9:1b:4c:fd:ff:bc:25:ee:91:03:d4:75:8e:a2:ab:ad:82:6a:90:f3:45:7d:46:0e:b4 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcxMB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7faYp6b -wiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX -/uMftk87ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR0 -77F9jAHiOH3BX2pfJLKOYheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGP -uY4zbGneWK2gDqdkVBFpRGZPTBKnjix9xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLx -p2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO9z0M+Yo0FMT7MzUj8czx -Kselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq3ywgsNw2 -TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+W -G+Oin6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPw -vFEVVJSmdz7QdFG9URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYY -EQRVzXR7z2FwefR7LFxckvzluFqrTJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUl0YhVyE1 -2jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/PxN3DlCPaTKbYw -DQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E -PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kf -gLMtMrpkZ2CvuVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbS -FXJfLkur1J1juONI5f6ELlgKn0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0 -V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLhX4SPgPL0DTatdrOjteFkdjpY3H1P -XlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80nR14SohWZ25g/4/I -i+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcmGS3t -TAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L91 -09S5zvE/bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/Ky -Pu1svf0OnWZzsD2097+o4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJ -AwSQiumPv+i2tCqjI40cHLI5kqiPAlxAOXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj -1oxx ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G2 O=OpenTrust -# Subject: CN=OpenTrust Root CA G2 O=OpenTrust -# Label: "OpenTrust Root CA G2" -# Serial: 1492012448042702096986875987676935573415441 -# MD5 Fingerprint: 57:24:b6:59:24:6b:ae:c8:fe:1c:0c:20:f2:c0:4e:eb -# SHA1 Fingerprint: 79:5f:88:60:c5:ab:7c:3d:92:e6:cb:f4:8d:e1:45:cd:11:ef:60:0b -# SHA256 Fingerprint: 27:99:58:29:fe:6a:75:15:c1:bf:e8:48:f9:c4:76:1d:b1:6c:22:59:29:25:7b:f4:0d:08:94:f2:9e:a8:ba:f2 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcyMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+Ntmh -/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78e -CbY2albz4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/6 -1UWY0jUJ9gNDlP7ZvyCVeYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fE -FY8ElggGQgT4hNYdvJGmQr5J1WqIP7wtUdGejeBSzFfdNTVY27SPJIjki9/ca1TS -gSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz3GIZ38i1MH/1PCZ1Eb3X -G7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj3CzMpSZy -YhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaH -vGOz9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4 -t/bQWVyJ98LVtZR00dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/ -gh7PU3+06yzbXfZqfUAkBXKJOAGTy3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUajn6QiL3 -5okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59M4PLuG53hq8w -DQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz -Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0 -nXGEL8pZ0keImUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qT -RmTFAHneIWv2V6CG1wZy7HBGS4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpT -wm+bREx50B1ws9efAvSyB7DH5fitIw6mVskpEndI2S9G/Tvw/HRwkqWOOAgfZDC2 -t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ6e18CL13zSdkzJTa -TkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97krgCf2 -o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU -3jg9CcCoSmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eA -iN1nE28daCSLT7d0geX0YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14f -WKGVyasvc0rQLW6aWQ9VGHgtPFGml4vmu7JwqkwR3v98KzfUetF3NI/n+UL3PIEM -S1IK ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G3 O=OpenTrust -# Subject: CN=OpenTrust Root CA G3 O=OpenTrust -# Label: "OpenTrust Root CA G3" -# Serial: 1492104908271485653071219941864171170455615 -# MD5 Fingerprint: 21:37:b4:17:16:92:7b:67:46:70:a9:96:d7:a8:13:24 -# SHA1 Fingerprint: 6e:26:64:f3:56:bf:34:55:bf:d1:93:3f:7c:01:de:d8:13:da:8a:a6 -# SHA256 Fingerprint: b7:c3:62:31:70:6e:81:07:8c:36:7c:b8:96:19:8f:1e:32:08:dd:92:69:49:dd:8f:57:09:a4:10:f7:5b:62:92 ------BEGIN CERTIFICATE----- -MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAx -CzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5U -cnVzdCBSb290IENBIEczMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFow -QDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwUT3Bl -blRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARK7liuTcpm -3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5Bta1d -oYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4G -A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5 -DMlv4VBN0BBY3JWIbTAfBgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAK -BggqhkjOPQQDAwNpADBmAjEAj6jcnboMBBf6Fek9LykBl7+BFjNAk2z8+e2AcG+q -j9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta3U1fJAuwACEl74+nBCZx -4nxp5V2a+EEfOzmTk51V6s2N8fvB ------END CERTIFICATE----- - # Issuer: CN=ISRG Root X1 O=Internet Security Research Group # Subject: CN=ISRG Root X1 O=Internet Security Research Group # Label: "ISRG Root X1" @@ -4440,3 +4277,43 @@ MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== -----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- From fd1771cfa096fe5f8adca3f096a42f901728a674 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Wed, 8 Aug 2018 10:08:46 -0700 Subject: [PATCH 73/99] Revert "adding python version environmental markers in the new style" --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index b87843f335d..388e629ec2a 100644 --- a/setup.py +++ b/setup.py @@ -276,11 +276,11 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - "six>=1.5.2", - "futures>=2.2.0 ; python_version<'3.2'", - "enum34>=1.0.4 ; python_version<'3.4'" + 'six>=1.5.2', ) +if not PY3: + INSTALL_REQUIRES += ('futures>=2.2.0', 'enum34>=1.0.4') SETUP_REQUIRES = INSTALL_REQUIRES + ( 'sphinx>=1.3', From b9decfb35d95aeee9245c4d2b197bb5b8bcb2133 Mon Sep 17 00:00:00 2001 From: Junhao Li Date: Wed, 8 Aug 2018 14:31:52 -0700 Subject: [PATCH 74/99] include cmath to fix gcc8 compilation error This fix resolves the following compilation error on GCC8.1.1: ERROR: /home/junhao/tools/grpc/BUILD:1350:1: C++ compilation of rule '//:grpcpp_server_load_reporting' failed (Exit 1) src/cpp/server/load_reporter/util.cc:31:12: error: no member named 'isnormal' in namespace 'std' if (std::isnormal(cost_value)) { ~~~~~^ 1 error generated. --- src/cpp/server/load_reporter/util.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cpp/server/load_reporter/util.cc b/src/cpp/server/load_reporter/util.cc index a2f2f11e707..0f8fa764ad0 100644 --- a/src/cpp/server/load_reporter/util.cc +++ b/src/cpp/server/load_reporter/util.cc @@ -21,6 +21,7 @@ #include #include +#include namespace grpc { namespace load_reporter { From db1a5962e0e81fb6aa1fefbb7a4e3038f5c32ccc Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 8 Aug 2018 15:17:19 -0700 Subject: [PATCH 75/99] Explictly Flush exec_ctx after resetting call_combiner_set_notify_on_cancel to avoid the need to take refs on the stack for cancellation closures on exec_ctx --- src/core/lib/iomgr/call_combiner.h | 5 ++++- src/core/lib/security/transport/client_auth_filter.cc | 4 ---- src/core/lib/security/transport/server_auth_filter.cc | 2 -- src/core/lib/surface/call.cc | 5 ++++- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/core/lib/iomgr/call_combiner.h b/src/core/lib/iomgr/call_combiner.h index 641fa180820..6f7ddd40435 100644 --- a/src/core/lib/iomgr/call_combiner.h +++ b/src/core/lib/iomgr/call_combiner.h @@ -102,7 +102,10 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner, /// If \a closure is NULL, then no closure will be invoked on /// cancellation; this effectively unregisters the previously set closure. /// However, most filters will not need to explicitly unregister their -/// callbacks, as this is done automatically when the call is destroyed. +/// callbacks, as this is done automatically when the call is destroyed. Filters +/// that schedule the cancellation closure on ExecCtx do not need to take a ref +/// on the call stack to guarantee closure liveness. This is done by explicitly +/// flushing ExecCtx after the unregistration during call destruction. void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner, grpc_closure* closure); diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc index 9b5c6f34906..0f125e7c26d 100644 --- a/src/core/lib/security/transport/client_auth_filter.cc +++ b/src/core/lib/security/transport/client_auth_filter.cc @@ -167,7 +167,6 @@ static void cancel_get_request_metadata(void* arg, grpc_error* error) { grpc_call_credentials_cancel_get_request_metadata( calld->creds, &calld->md_array, GRPC_ERROR_REF(error)); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_get_request_metadata"); } static void send_security_metadata(grpc_call_element* elem, @@ -222,7 +221,6 @@ static void send_security_metadata(grpc_call_element* elem, GRPC_ERROR_UNREF(error); } else { // Async return; register cancellation closure with call combiner. - GRPC_CALL_STACK_REF(calld->owning_call, "cancel_get_request_metadata"); grpc_call_combiner_set_notify_on_cancel( calld->call_combiner, GRPC_CLOSURE_INIT(&calld->get_request_metadata_cancel_closure, @@ -265,7 +263,6 @@ static void cancel_check_call_host(void* arg, grpc_error* error) { chand->security_connector, &calld->async_result_closure, GRPC_ERROR_REF(error)); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_check_call_host"); } static void auth_start_transport_stream_op_batch( @@ -318,7 +315,6 @@ static void auth_start_transport_stream_op_batch( GRPC_ERROR_UNREF(error); } else { // Async return; register cancellation closure with call combiner. - GRPC_CALL_STACK_REF(calld->owning_call, "cancel_check_call_host"); grpc_call_combiner_set_notify_on_cancel( calld->call_combiner, GRPC_CLOSURE_INIT(&calld->check_call_host_cancel_closure, diff --git a/src/core/lib/security/transport/server_auth_filter.cc b/src/core/lib/security/transport/server_auth_filter.cc index 2dbefdf1317..19cbb03b63e 100644 --- a/src/core/lib/security/transport/server_auth_filter.cc +++ b/src/core/lib/security/transport/server_auth_filter.cc @@ -156,7 +156,6 @@ static void cancel_call(void* arg, grpc_error* error) { on_md_processing_done_inner(elem, nullptr, 0, nullptr, 0, GRPC_ERROR_REF(error)); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_call"); } static void recv_initial_metadata_ready(void* arg, grpc_error* error) { @@ -168,7 +167,6 @@ static void recv_initial_metadata_ready(void* arg, grpc_error* error) { if (chand->creds != nullptr && chand->creds->processor.process != nullptr) { // We're calling out to the application, so we need to make sure // to drop the call combiner early if we get cancelled. - GRPC_CALL_STACK_REF(calld->owning_call, "cancel_call"); GRPC_CLOSURE_INIT(&calld->cancel_closure, cancel_call, elem, grpc_schedule_on_exec_ctx); grpc_call_combiner_set_notify_on_cancel(calld->call_combiner, diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index dbad5ded4d4..52053e686b9 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -613,8 +613,11 @@ void grpc_call_unref(grpc_call* c) { // Unset the call combiner cancellation closure. This has the // effect of scheduling the previously set cancellation closure, if // any, so that it can release any internal references it may be - // holding to the call stack. + // holding to the call stack. Also flush the closures on exec_ctx so that + // filters that schedule cancel notification closures on exec_ctx do not + // need to take a ref of the call stack to guarantee closure liveness. grpc_call_combiner_set_notify_on_cancel(&c->call_combiner, nullptr); + grpc_core::ExecCtx::Get()->Flush(); } GRPC_CALL_INTERNAL_UNREF(c, "destroy"); } From 5cd8b1eb811e79ad68bf91a0296507c153053ecf Mon Sep 17 00:00:00 2001 From: Alex Polcyn Date: Sat, 16 Jun 2018 04:08:55 +0000 Subject: [PATCH 76/99] Enable c-ares queries on Windows --- CMakeLists.txt | 12 - .../dns/c_ares/grpc_ares_ev_driver.cc | 17 +- .../resolver/dns/c_ares/grpc_ares_ev_driver.h | 22 +- .../dns/c_ares/grpc_ares_ev_driver_posix.cc | 18 +- .../dns/c_ares/grpc_ares_ev_driver_windows.cc | 508 +++++++++++++++++- .../resolver/dns/c_ares/grpc_ares_wrapper.cc | 2 + .../resolver/dns/c_ares/grpc_ares_wrapper.h | 7 + src/core/lib/iomgr/iocp_windows.cc | 13 +- src/core/lib/iomgr/socket_windows.cc | 4 + src/core/lib/iomgr/socket_windows.h | 2 + src/core/lib/iomgr/tcp_windows.cc | 4 +- src/core/lib/iomgr/tcp_windows.h | 2 + .../resolver_component_tests_defs.include | 19 +- test/cpp/naming/cancel_ares_query_test.cc | 55 +- test/cpp/naming/gen_build_yaml.py | 4 +- .../manual_run_resolver_component_test.py | 36 ++ test/cpp/naming/resolver_component_test.cc | 72 ++- .../naming/resolver_component_tests_runner.py | 31 +- .../naming/resolver_test_record_groups.yaml | 8 + tools/run_tests/generated/tests.json | 6 +- 20 files changed, 771 insertions(+), 71 deletions(-) create mode 100644 test/cpp/naming/manual_run_resolver_component_test.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 855b921ada1..f242ee92bbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -662,12 +662,8 @@ add_dependencies(buildtests_cxx transport_security_common_api_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx writes_per_rpc_test) endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_test_unsecure) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_test) -endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker_unsecure) endif() @@ -676,9 +672,7 @@ add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker) endif() add_dependencies(buildtests_cxx address_sorting_test_unsecure) add_dependencies(buildtests_cxx address_sorting_test) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx cancel_ares_query_test) -endif() add_custom_target(buildtests DEPENDS buildtests_c buildtests_cxx) @@ -16213,7 +16207,6 @@ target_link_libraries(inproc_nosec_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(resolver_component_test_unsecure test/cpp/naming/resolver_component_test.cc @@ -16253,10 +16246,8 @@ target_link_libraries(resolver_component_test_unsecure ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(resolver_component_test test/cpp/naming/resolver_component_test.cc @@ -16296,7 +16287,6 @@ target_link_libraries(resolver_component_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -16467,7 +16457,6 @@ target_link_libraries(address_sorting_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(cancel_ares_query_test test/cpp/naming/cancel_ares_query_test.cc @@ -16507,7 +16496,6 @@ target_link_libraries(cancel_ares_query_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc index 0068d0d5f4b..fdbd07ebf51 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc @@ -74,6 +74,8 @@ struct grpc_ares_ev_driver { bool shutting_down; /** request object that's using this ev driver */ grpc_ares_request* request; + /** Owned by the ev_driver. Creates new GrpcPolledFd's */ + grpc_core::UniquePtr polled_fd_factory; }; static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver); @@ -93,7 +95,7 @@ static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver* ev_driver) { GRPC_COMBINER_UNREF(ev_driver->combiner, "free ares event driver"); ares_destroy(ev_driver->channel); grpc_ares_complete_request_locked(ev_driver->request); - gpr_free(ev_driver); + grpc_core::Delete(ev_driver); } } @@ -118,13 +120,11 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver, grpc_pollset_set* pollset_set, grpc_combiner* combiner, grpc_ares_request* request) { - *ev_driver = static_cast( - gpr_malloc(sizeof(grpc_ares_ev_driver))); + *ev_driver = grpc_core::New(); ares_options opts; memset(&opts, 0, sizeof(opts)); opts.flags |= ARES_FLAG_STAYOPEN; int status = ares_init_options(&(*ev_driver)->channel, &opts, ARES_OPT_FLAGS); - grpc_core::ConfigureAresChannelLocked(&(*ev_driver)->channel); gpr_log(GPR_DEBUG, "grpc_ares_ev_driver_create_locked"); if (status != ARES_SUCCESS) { char* err_msg; @@ -142,6 +142,10 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver, (*ev_driver)->working = false; (*ev_driver)->shutting_down = false; (*ev_driver)->request = request; + (*ev_driver)->polled_fd_factory = + grpc_core::NewGrpcPolledFdFactory((*ev_driver)->combiner); + (*ev_driver) + ->polled_fd_factory->ConfigureAresChannelLocked((*ev_driver)->channel); return GRPC_ERROR_NONE; } @@ -245,8 +249,9 @@ static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver) { // Create a new fd_node if sock[i] is not in the fd_node list. if (fdn == nullptr) { fdn = static_cast(gpr_malloc(sizeof(fd_node))); - fdn->grpc_polled_fd = grpc_core::NewGrpcPolledFdLocked( - socks[i], ev_driver->pollset_set); + fdn->grpc_polled_fd = + ev_driver->polled_fd_factory->NewGrpcPolledFdLocked( + socks[i], ev_driver->pollset_set, ev_driver->combiner); gpr_log(GPR_DEBUG, "new fd: %s", fdn->grpc_polled_fd->GetName()); fdn->ev_driver = ev_driver; fdn->readable_registered = false; diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h index 2c9db710116..671c537fe72 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h @@ -81,10 +81,24 @@ class GrpcPolledFd { GRPC_ABSTRACT_BASE_CLASS }; -/* Creates a new wrapped fd for the current platform */ -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set); -void ConfigureAresChannelLocked(ares_channel* channel); +/* A GrpcPolledFdFactory is 1-to-1 with and owned by the + * ares event driver. It knows how to create GrpcPolledFd's + * for the current platform, and the ares driver uses it for all of + * its fd's. */ +class GrpcPolledFdFactory { + public: + virtual ~GrpcPolledFdFactory() {} + /* Creates a new wrapped fd for the current platform */ + virtual GrpcPolledFd* NewGrpcPolledFdLocked( + ares_socket_t as, grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) GRPC_ABSTRACT; + /* Optionally configures the ares channel after creation */ + virtual void ConfigureAresChannelLocked(ares_channel channel) GRPC_ABSTRACT; + + GRPC_ABSTRACT_BASE_CLASS +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner); } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc index fffe9eda8e5..aa58e1aaf50 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc @@ -86,12 +86,20 @@ class GrpcPolledFdPosix : public GrpcPolledFd { grpc_pollset_set* driver_pollset_set_; }; -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set) { - return grpc_core::New(as, driver_pollset_set); -} +class GrpcPolledFdFactoryPosix : public GrpcPolledFdFactory { + public: + GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, + grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) override { + return New(as, driver_pollset_set); + } -void ConfigureAresChannelLocked(ares_channel* channel) {} + void ConfigureAresChannelLocked(ares_channel channel) override {} +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner) { + return UniquePtr(New()); +} } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc index 5d65ae3ab37..02121aa0ab4 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc @@ -21,38 +21,516 @@ #if GRPC_ARES == 1 && defined(GPR_WINDOWS) #include + +#include +#include +#include +#include +#include #include +#include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/socket_windows.h" +#include "src/core/lib/iomgr/tcp_windows.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" + +/* TODO(apolcyn): remove this hack after fixing upstream. + * Our grpc/c-ares code on Windows uses the ares_set_socket_functions API, + * which uses "struct iovec" type, which on Windows is defined inside of + * a c-ares header that is not public. + * See https://github.com/c-ares/c-ares/issues/206. */ +struct iovec { + void* iov_base; + size_t iov_len; +}; namespace grpc_core { -/* TODO: fill in the body of GrpcPolledFdWindows to enable c-ares on Windows. - This dummy implementation only allows grpc to compile on windows with - GRPC_ARES=1. */ +/* c-ares creates its own sockets and is meant to read them when readable and + * write them when writeable. To fit this socket usage model into the grpc + * windows poller (which gives notifications when attempted reads and writes are + * actually fulfilled rather than possible), this GrpcPolledFdWindows class + * takes advantage of the ares_set_socket_functions API and acts as a virtual + * socket. It holds its own read and write buffers which are written to and read + * from c-ares and are used with the grpc windows poller, and it, e.g., + * manufactures virtual socket error codes when it e.g. needs to tell the c-ares + * library to wait for an async read. */ class GrpcPolledFdWindows : public GrpcPolledFd { public: - GrpcPolledFdWindows() { abort(); } - ~GrpcPolledFdWindows() { abort(); } + enum WriteState { + WRITE_IDLE, + WRITE_REQUESTED, + WRITE_PENDING, + WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY, + }; + + GrpcPolledFdWindows(ares_socket_t as, grpc_combiner* combiner) + : read_buf_(grpc_empty_slice()), + write_buf_(grpc_empty_slice()), + write_state_(WRITE_IDLE), + gotten_into_driver_list_(false) { + gpr_asprintf(&name_, "c-ares socket: %" PRIdPTR, as); + winsocket_ = grpc_winsocket_create(as, name_); + combiner_ = GRPC_COMBINER_REF(combiner, name_); + GRPC_CLOSURE_INIT(&outer_read_closure_, + &GrpcPolledFdWindows::OnIocpReadable, this, + grpc_combiner_scheduler(combiner_)); + GRPC_CLOSURE_INIT(&outer_write_closure_, + &GrpcPolledFdWindows::OnIocpWriteable, this, + grpc_combiner_scheduler(combiner_)); + } + + ~GrpcPolledFdWindows() { + GRPC_COMBINER_UNREF(combiner_, name_); + grpc_slice_unref_internal(read_buf_); + grpc_slice_unref_internal(write_buf_); + GPR_ASSERT(read_closure_ == nullptr); + GPR_ASSERT(write_closure_ == nullptr); + grpc_winsocket_destroy(winsocket_); + gpr_free(name_); + } + + void ScheduleAndNullReadClosure(grpc_error* error) { + GRPC_CLOSURE_SCHED(read_closure_, error); + read_closure_ = nullptr; + } + + void ScheduleAndNullWriteClosure(grpc_error* error) { + GRPC_CLOSURE_SCHED(write_closure_, error); + write_closure_ = nullptr; + } + void RegisterForOnReadableLocked(grpc_closure* read_closure) override { - abort(); + GPR_ASSERT(read_closure_ == nullptr); + read_closure_ = read_closure; + GPR_ASSERT(GRPC_SLICE_LENGTH(read_buf_) == 0); + grpc_slice_unref_internal(read_buf_); + read_buf_ = GRPC_SLICE_MALLOC(4192); + WSABUF buffer; + buffer.buf = (char*)GRPC_SLICE_START_PTR(read_buf_); + buffer.len = GRPC_SLICE_LENGTH(read_buf_); + memset(&winsocket_->read_info.overlapped, 0, sizeof(OVERLAPPED)); + recv_from_source_addr_len_ = sizeof(recv_from_source_addr_); + DWORD flags = 0; + if (WSARecvFrom(grpc_winsocket_wrapped_socket(winsocket_), &buffer, 1, + nullptr, &flags, (sockaddr*)recv_from_source_addr_, + &recv_from_source_addr_len_, + &winsocket_->read_info.overlapped, nullptr)) { + char* msg = gpr_format_message(WSAGetLastError()); + grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + GRPC_CARES_TRACE_LOG( + "RegisterForOnReadableLocked: WSARecvFrom error:|%s|. fd:|%s|", msg, + GetName()); + gpr_free(msg); + if (WSAGetLastError() != WSA_IO_PENDING) { + ScheduleAndNullReadClosure(error); + return; + } + } + grpc_socket_notify_on_read(winsocket_, &outer_read_closure_); } + void RegisterForOnWriteableLocked(grpc_closure* write_closure) override { + GRPC_CARES_TRACE_LOG( + "RegisterForOnWriteableLocked. fd:|%s|. Current write state: %d", + GetName(), write_state_); + GPR_ASSERT(write_closure_ == nullptr); + write_closure_ = write_closure; + switch (write_state_) { + case WRITE_IDLE: + ScheduleAndNullWriteClosure(GRPC_ERROR_NONE); + break; + case WRITE_REQUESTED: + write_state_ = WRITE_PENDING; + SendWriteBuf(nullptr, &winsocket_->write_info.overlapped); + grpc_socket_notify_on_write(winsocket_, &outer_write_closure_); + break; + case WRITE_PENDING: + case WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY: + abort(); + } + } + + bool IsFdStillReadableLocked() override { + return GRPC_SLICE_LENGTH(read_buf_) > 0; + } + + void ShutdownLocked(grpc_error* error) override { + grpc_winsocket_shutdown(winsocket_); + } + + ares_socket_t GetWrappedAresSocketLocked() override { + return grpc_winsocket_wrapped_socket(winsocket_); + } + + const char* GetName() override { return name_; } + + ares_ssize_t RecvFrom(void* data, ares_socket_t data_len, int flags, + struct sockaddr* from, ares_socklen_t* from_len) { + GRPC_CARES_TRACE_LOG( + "RecvFrom called on fd:|%s|. Current read buf length:|%d|", GetName(), + GRPC_SLICE_LENGTH(read_buf_)); + if (GRPC_SLICE_LENGTH(read_buf_) == 0) { + WSASetLastError(WSAEWOULDBLOCK); + return -1; + } + ares_ssize_t bytes_read = 0; + for (size_t i = 0; i < GRPC_SLICE_LENGTH(read_buf_) && i < data_len; i++) { + ((char*)data)[i] = GRPC_SLICE_START_PTR(read_buf_)[i]; + bytes_read++; + } + read_buf_ = grpc_slice_sub_no_ref(read_buf_, bytes_read, + GRPC_SLICE_LENGTH(read_buf_)); + /* c-ares overloads this recv_from virtual socket function to receive + * data on both UDP and TCP sockets, and from is nullptr for TCP. */ + if (from != nullptr) { + GPR_ASSERT(*from_len <= recv_from_source_addr_len_); + memcpy(from, &recv_from_source_addr_, recv_from_source_addr_len_); + *from_len = recv_from_source_addr_len_; + } + return bytes_read; + } + + grpc_slice FlattenIovec(const struct iovec* iov, int iov_count) { + int total = 0; + for (int i = 0; i < iov_count; i++) { + total += iov[i].iov_len; + } + grpc_slice out = GRPC_SLICE_MALLOC(total); + size_t cur = 0; + for (int i = 0; i < iov_count; i++) { + for (int k = 0; k < iov[i].iov_len; k++) { + GRPC_SLICE_START_PTR(out)[cur++] = ((char*)iov[i].iov_base)[k]; + } + } + return out; + } + + int SendWriteBuf(LPDWORD bytes_sent_ptr, LPWSAOVERLAPPED overlapped) { + WSABUF buf; + buf.len = GRPC_SLICE_LENGTH(write_buf_); + buf.buf = (char*)GRPC_SLICE_START_PTR(write_buf_); + DWORD flags = 0; + int out = WSASend(grpc_winsocket_wrapped_socket(winsocket_), &buf, 1, + bytes_sent_ptr, flags, overlapped, nullptr); + GRPC_CARES_TRACE_LOG( + "WSASend: name:%s. buf len:%d. bytes sent: %d. overlapped %p. return " + "val: %d", + GetName(), buf.len, *bytes_sent_ptr, overlapped, out); + return out; + } + + ares_ssize_t TrySendWriteBufSyncNonBlocking() { + GPR_ASSERT(write_state_ == WRITE_IDLE); + ares_ssize_t total_sent; + DWORD bytes_sent = 0; + if (SendWriteBuf(&bytes_sent, nullptr) != 0) { + char* msg = gpr_format_message(WSAGetLastError()); + GRPC_CARES_TRACE_LOG( + "TrySendWriteBufSyncNonBlocking: SendWriteBuf error:|%s|. fd:|%s|", + msg, GetName()); + gpr_free(msg); + if (WSAGetLastError() == WSA_IO_PENDING) { + WSASetLastError(WSAEWOULDBLOCK); + write_state_ = WRITE_REQUESTED; + } + } + write_buf_ = grpc_slice_sub_no_ref(write_buf_, bytes_sent, + GRPC_SLICE_LENGTH(write_buf_)); + return bytes_sent; + } + + ares_ssize_t SendV(const struct iovec* iov, int iov_count) { + GRPC_CARES_TRACE_LOG("SendV called on fd:|%s|. Current write state: %d", + GetName(), write_state_); + switch (write_state_) { + case WRITE_IDLE: + GPR_ASSERT(GRPC_SLICE_LENGTH(write_buf_) == 0); + grpc_slice_unref_internal(write_buf_); + write_buf_ = FlattenIovec(iov, iov_count); + return TrySendWriteBufSyncNonBlocking(); + case WRITE_REQUESTED: + case WRITE_PENDING: + WSASetLastError(WSAEWOULDBLOCK); + return -1; + case WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY: + grpc_slice currently_attempted = FlattenIovec(iov, iov_count); + GPR_ASSERT(GRPC_SLICE_LENGTH(currently_attempted) >= + GRPC_SLICE_LENGTH(write_buf_)); + ares_ssize_t total_sent = 0; + for (size_t i = 0; i < GRPC_SLICE_LENGTH(write_buf_); i++) { + GPR_ASSERT(GRPC_SLICE_START_PTR(currently_attempted)[i] == + GRPC_SLICE_START_PTR(write_buf_)[i]); + total_sent++; + } + grpc_slice_unref_internal(write_buf_); + write_buf_ = + grpc_slice_sub_no_ref(currently_attempted, total_sent, + GRPC_SLICE_LENGTH(currently_attempted)); + write_state_ = WRITE_IDLE; + total_sent += TrySendWriteBufSyncNonBlocking(); + return total_sent; + } abort(); } - bool IsFdStillReadableLocked() override { abort(); } - void ShutdownLocked(grpc_error* error) override { abort(); } - ares_socket_t GetWrappedAresSocketLocked() override { abort(); } - const char* GetName() override { abort(); } + + int Connect(const struct sockaddr* target, ares_socklen_t target_len) { + SOCKET s = grpc_winsocket_wrapped_socket(winsocket_); + GRPC_CARES_TRACE_LOG("Connect: fd:|%s|", GetName()); + int out = + WSAConnect(s, target, target_len, nullptr, nullptr, nullptr, nullptr); + if (out != 0) { + char* msg = gpr_format_message(WSAGetLastError()); + GRPC_CARES_TRACE_LOG("Connect error code:|%d|, msg:|%s|. fd:|%s|", + WSAGetLastError(), msg, GetName()); + gpr_free(msg); + // c-ares expects a posix-style connect API + out = -1; + } + return out; + } + + static void OnIocpReadable(void* arg, grpc_error* error) { + GrpcPolledFdWindows* polled_fd = static_cast(arg); + polled_fd->OnIocpReadableInner(error); + } + + void OnIocpReadableInner(grpc_error* error) { + if (error == GRPC_ERROR_NONE) { + if (winsocket_->read_info.wsa_error != 0) { + /* WSAEMSGSIZE would be due to receiving more data + * than our read buffer's fixed capacity. Assume that + * the connection is TCP and read the leftovers + * in subsequent c-ares reads. */ + if (winsocket_->read_info.wsa_error != WSAEMSGSIZE) { + GRPC_ERROR_UNREF(error); + char* msg = gpr_format_message(winsocket_->read_info.wsa_error); + GRPC_CARES_TRACE_LOG( + "OnIocpReadableInner. winsocket error:|%s|. fd:|%s|", msg, + GetName()); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + gpr_free(msg); + } + } + } + if (error == GRPC_ERROR_NONE) { + read_buf_ = grpc_slice_sub_no_ref(read_buf_, 0, + winsocket_->read_info.bytes_transfered); + } else { + grpc_slice_unref_internal(read_buf_); + read_buf_ = grpc_empty_slice(); + } + GRPC_CARES_TRACE_LOG( + "OnIocpReadable finishing. read buf length now:|%d|. :fd:|%s|", + GRPC_SLICE_LENGTH(read_buf_), GetName()); + ScheduleAndNullReadClosure(error); + } + + static void OnIocpWriteable(void* arg, grpc_error* error) { + GrpcPolledFdWindows* polled_fd = static_cast(arg); + polled_fd->OnIocpWriteableInner(error); + } + + void OnIocpWriteableInner(grpc_error* error) { + GRPC_CARES_TRACE_LOG("OnIocpWriteableInner. fd:|%s|", GetName()); + if (error == GRPC_ERROR_NONE) { + if (winsocket_->write_info.wsa_error != 0) { + char* msg = gpr_format_message(winsocket_->write_info.wsa_error); + GRPC_CARES_TRACE_LOG( + "OnIocpWriteableInner. winsocket error:|%s|. fd:|%s|", msg, + GetName()); + GRPC_ERROR_UNREF(error); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + gpr_free(msg); + } + } + GPR_ASSERT(write_state_ == WRITE_PENDING); + if (error == GRPC_ERROR_NONE) { + write_state_ = WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY; + write_buf_ = grpc_slice_sub_no_ref( + write_buf_, 0, winsocket_->write_info.bytes_transfered); + } else { + grpc_slice_unref_internal(write_buf_); + write_buf_ = grpc_empty_slice(); + } + ScheduleAndNullWriteClosure(error); + } + + bool gotten_into_driver_list() const { return gotten_into_driver_list_; } + void set_gotten_into_driver_list() { gotten_into_driver_list_ = true; } + + grpc_combiner* combiner_; + char recv_from_source_addr_[200]; + ares_socklen_t recv_from_source_addr_len_; + grpc_slice read_buf_; + grpc_slice write_buf_; + grpc_closure* read_closure_ = nullptr; + grpc_closure* write_closure_ = nullptr; + grpc_closure outer_read_closure_; + grpc_closure outer_write_closure_; + grpc_winsocket* winsocket_; + WriteState write_state_; + char* name_ = nullptr; + bool gotten_into_driver_list_; }; -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set) { - return nullptr; -} +struct SockToPolledFdEntry { + SockToPolledFdEntry(SOCKET s, GrpcPolledFdWindows* fd) + : socket(s), polled_fd(fd) {} + SOCKET socket; + GrpcPolledFdWindows* polled_fd; + SockToPolledFdEntry* next = nullptr; +}; + +/* A SockToPolledFdMap can make ares_socket_t types (SOCKET's on windows) + * to GrpcPolledFdWindow's, and is used to find the appropriate + * GrpcPolledFdWindows to handle a virtual socket call when c-ares makes that + * socket call on the ares_socket_t type. Instances are owned by and one-to-one + * with a GrpcPolledFdWindows factory and event driver */ +class SockToPolledFdMap { + public: + SockToPolledFdMap(grpc_combiner* combiner) { + combiner_ = GRPC_COMBINER_REF(combiner, "sock to polled fd map"); + } + + ~SockToPolledFdMap() { + GPR_ASSERT(head_ == nullptr); + GRPC_COMBINER_UNREF(combiner_, "sock to polled fd map"); + } + + void AddNewSocket(SOCKET s, GrpcPolledFdWindows* polled_fd) { + SockToPolledFdEntry* new_node = New(s, polled_fd); + new_node->next = head_; + head_ = new_node; + } + + GrpcPolledFdWindows* LookupPolledFd(SOCKET s) { + for (SockToPolledFdEntry* node = head_; node != nullptr; + node = node->next) { + if (node->socket == s) { + GPR_ASSERT(node->polled_fd != nullptr); + return node->polled_fd; + } + } + abort(); + } + + void RemoveEntry(SOCKET s) { + GPR_ASSERT(head_ != nullptr); + SockToPolledFdEntry** prev = &head_; + for (SockToPolledFdEntry* node = head_; node != nullptr; + node = node->next) { + if (node->socket == s) { + *prev = node->next; + Delete(node); + return; + } + prev = &node->next; + } + abort(); + } + + /* These virtual socket functions are called from within the c-ares + * library. These methods generally dispatch those socket calls to the + * appropriate methods. The virtual "socket" and "close" methods are + * special and instead create/add and remove/destroy GrpcPolledFdWindows + * objects. + */ + static ares_socket_t Socket(int af, int type, int protocol, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + SOCKET s = WSASocket(af, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + if (s == INVALID_SOCKET) { + return s; + } + grpc_tcp_set_non_block(s); + GrpcPolledFdWindows* polled_fd = + New(s, map->combiner_); + map->AddNewSocket(s, polled_fd); + return s; + } + + static int Connect(ares_socket_t as, const struct sockaddr* target, + ares_socklen_t target_len, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->Connect(target, target_len); + } + + static ares_ssize_t SendV(ares_socket_t as, const struct iovec* iov, + int iovec_count, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->SendV(iov, iovec_count); + } + + static ares_ssize_t RecvFrom(ares_socket_t as, void* data, size_t data_len, + int flags, struct sockaddr* from, + ares_socklen_t* from_len, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->RecvFrom(data, data_len, flags, from, from_len); + } + + static int CloseSocket(SOCKET s, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(s); + map->RemoveEntry(s); + // If a gRPC polled fd has not made it in to the driver's list yet, then + // the driver has not and will never see this socket. + if (!polled_fd->gotten_into_driver_list()) { + polled_fd->ShutdownLocked(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Shut down c-ares fd before without it ever having made it into the " + "driver's list")); + return 0; + } + return 0; + } + + private: + SockToPolledFdEntry* head_ = nullptr; + grpc_combiner* combiner_; +}; + +const struct ares_socket_functions custom_ares_sock_funcs = { + &SockToPolledFdMap::Socket /* socket */, + &SockToPolledFdMap::CloseSocket /* close */, + &SockToPolledFdMap::Connect /* connect */, + &SockToPolledFdMap::RecvFrom /* recvfrom */, + &SockToPolledFdMap::SendV /* sendv */, +}; + +class GrpcPolledFdFactoryWindows : public GrpcPolledFdFactory { + public: + GrpcPolledFdFactoryWindows(grpc_combiner* combiner) + : sock_to_polled_fd_map_(combiner) {} + + GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, + grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) override { + GrpcPolledFdWindows* polled_fd = sock_to_polled_fd_map_.LookupPolledFd(as); + // Set a flag so that the virtual socket "close" method knows it + // doesn't need to call ShutdownLocked, since now the driver will. + polled_fd->set_gotten_into_driver_list(); + return polled_fd; + } -void ConfigureAresChannelLocked(ares_channel* channel) { abort(); } + void ConfigureAresChannelLocked(ares_channel channel) override { + ares_set_socket_functions(channel, &custom_ares_sock_funcs, + &sock_to_polled_fd_map_); + } + + private: + SockToPolledFdMap sock_to_polled_fd_map_; +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner) { + return UniquePtr( + New(combiner)); +} } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc index b3d6437e9a6..485998f5e44 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc @@ -49,6 +49,8 @@ static gpr_mu g_init_mu; grpc_core::TraceFlag grpc_trace_cares_address_sorting(false, "cares_address_sorting"); +grpc_core::TraceFlag grpc_trace_cares_resolver(false, "cares_resolver"); + struct grpc_ares_request { /** indicates the DNS server to use, if specified */ struct ares_addr_port_node dns_server_addr; diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h index 17eaa7ccf06..ca5779e1d79 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h @@ -28,6 +28,13 @@ extern grpc_core::TraceFlag grpc_trace_cares_address_sorting; +extern grpc_core::TraceFlag grpc_trace_cares_resolver; + +#define GRPC_CARES_TRACE_LOG(format, ...) \ + if (grpc_trace_cares_resolver.enabled()) { \ + gpr_log(GPR_DEBUG, "(c-ares resolver) " format, __VA_ARGS__); \ + } + typedef struct grpc_ares_request grpc_ares_request; /* Asynchronously resolve \a name. Use \a default_port if a port isn't diff --git a/src/core/lib/iomgr/iocp_windows.cc b/src/core/lib/iomgr/iocp_windows.cc index ce772310362..ad325fe2156 100644 --- a/src/core/lib/iomgr/iocp_windows.cc +++ b/src/core/lib/iomgr/iocp_windows.cc @@ -89,10 +89,15 @@ grpc_iocp_work_status grpc_iocp_work(grpc_millis deadline) { } else { abort(); } - success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes, - FALSE, &flags); - info->bytes_transfered = bytes; - info->wsa_error = success ? 0 : WSAGetLastError(); + if (socket->shutdown_called) { + info->bytes_transfered = 0; + info->wsa_error = WSA_OPERATION_ABORTED; + } else { + success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes, + FALSE, &flags); + info->bytes_transfered = bytes; + info->wsa_error = success ? 0 : WSAGetLastError(); + } GPR_ASSERT(overlapped == &info->overlapped); grpc_socket_become_ready(socket, info); return GRPC_IOCP_WORK_WORK; diff --git a/src/core/lib/iomgr/socket_windows.cc b/src/core/lib/iomgr/socket_windows.cc index 4ad31cb35d1..999c6646ad4 100644 --- a/src/core/lib/iomgr/socket_windows.cc +++ b/src/core/lib/iomgr/socket_windows.cc @@ -52,6 +52,10 @@ grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name) { return r; } +SOCKET grpc_winsocket_wrapped_socket(grpc_winsocket* socket) { + return socket->socket; +} + /* Schedule a shutdown of the socket operations. Will call the pending operations to abort them. We need to do that this way because of the various callsites of that function, which happens to be in various diff --git a/src/core/lib/iomgr/socket_windows.h b/src/core/lib/iomgr/socket_windows.h index b09b9da5628..46d7d583560 100644 --- a/src/core/lib/iomgr/socket_windows.h +++ b/src/core/lib/iomgr/socket_windows.h @@ -92,6 +92,8 @@ typedef struct grpc_winsocket { it will be responsible for closing it. */ grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name); +SOCKET grpc_winsocket_wrapped_socket(grpc_winsocket* socket); + /* Initiate an asynchronous shutdown of the socket. Will call off any pending operation to cancel them. */ void grpc_winsocket_shutdown(grpc_winsocket* socket); diff --git a/src/core/lib/iomgr/tcp_windows.cc b/src/core/lib/iomgr/tcp_windows.cc index 5d316d477b7..b3cb442f18f 100644 --- a/src/core/lib/iomgr/tcp_windows.cc +++ b/src/core/lib/iomgr/tcp_windows.cc @@ -53,7 +53,7 @@ extern grpc_core::TraceFlag grpc_tcp_trace; -static grpc_error* set_non_block(SOCKET sock) { +grpc_error* grpc_tcp_set_non_block(SOCKET sock) { int status; uint32_t param = 1; DWORD ret; @@ -90,7 +90,7 @@ static grpc_error* enable_loopback_fast_path(SOCKET sock) { grpc_error* grpc_tcp_prepare_socket(SOCKET sock) { grpc_error* err; - err = set_non_block(sock); + err = grpc_tcp_set_non_block(sock); if (err != GRPC_ERROR_NONE) return err; err = set_dualstack(sock); if (err != GRPC_ERROR_NONE) return err; diff --git a/src/core/lib/iomgr/tcp_windows.h b/src/core/lib/iomgr/tcp_windows.h index 161a545a2a5..04ef8102b65 100644 --- a/src/core/lib/iomgr/tcp_windows.h +++ b/src/core/lib/iomgr/tcp_windows.h @@ -46,6 +46,8 @@ grpc_endpoint* grpc_tcp_create(grpc_winsocket* socket, grpc_error* grpc_tcp_prepare_socket(SOCKET sock); +grpc_error* grpc_tcp_set_non_block(SOCKET sock); + #endif #endif /* GRPC_CORE_LIB_IOMGR_TCP_WINDOWS_H */ diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include index bc981dc83e1..b34845e01a3 100644 --- a/templates/test/cpp/naming/resolver_component_tests_defs.include +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -22,6 +22,7 @@ import tempfile import os import time import signal +import platform argp = argparse.ArgumentParser(description='Run c-ares resolver tests') @@ -43,6 +44,11 @@ args = argp.parse_args() def test_runner_log(msg): sys.stderr.write('\n%s: %s\n' % (__file__, msg)) +def python_args(arg_list): + if platform.system() == 'Windows': + return [sys.executable] + arg_list + return arg_list + cur_resolver = os.environ.get('GRPC_DNS_RESOLVER') if cur_resolver and cur_resolver != 'ares': test_runner_log(('WARNING: cur resolver set to %s. This set of tests ' @@ -50,26 +56,27 @@ if cur_resolver and cur_resolver != 'ares': test_runner_log('Exit 1 without running tests.') sys.exit(1) os.environ.update({'GRPC_DNS_RESOLVER': 'ares'}) +os.environ.update({'GRPC_TRACE': 'cares_resolver'}) def wait_until_dns_server_is_up(args, dns_server_subprocess, dns_server_subprocess_output): for i in range(0, 30): test_runner_log('Health check: attempt to connect to DNS server over TCP.') - tcp_connect_subprocess = subprocess.Popen([ + tcp_connect_subprocess = subprocess.Popen(python_args([ args.tcp_connect_bin_path, '--server_host', '127.0.0.1', '--server_port', str(args.dns_server_port), - '--timeout', str(1)]) + '--timeout', str(1)])) tcp_connect_subprocess.communicate() if tcp_connect_subprocess.returncode == 0: test_runner_log(('Health check: attempt to make an A-record ' 'query to DNS server.')) - dns_resolver_subprocess = subprocess.Popen([ + dns_resolver_subprocess = subprocess.Popen(python_args([ args.dns_resolver_bin_path, '--qname', 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp', '--server_host', '127.0.0.1', - '--server_port', str(args.dns_server_port)], + '--server_port', str(args.dns_server_port)]), stdout=subprocess.PIPE) dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() if dns_resolver_subprocess.returncode == 0: @@ -91,10 +98,10 @@ def wait_until_dns_server_is_up(args, dns_server_subprocess_output = tempfile.mktemp() with open(dns_server_subprocess_output, 'w') as l: - dns_server_subprocess = subprocess.Popen([ + dns_server_subprocess = subprocess.Popen(python_args([ args.dns_server_bin_path, '--port', str(args.dns_server_port), - '--records_config_path', args.records_config_path], + '--records_config_path', args.records_config_path]), stdin=subprocess.PIPE, stdout=l, stderr=l) diff --git a/test/cpp/naming/cancel_ares_query_test.cc b/test/cpp/naming/cancel_ares_query_test.cc index 0d59bf6fb60..dec7c171dc0 100644 --- a/test/cpp/naming/cancel_ares_query_test.cc +++ b/test/cpp/naming/cancel_ares_query_test.cc @@ -45,11 +45,14 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" -// TODO: pull in different headers when enabling this -// test on windows. Also set BAD_SOCKET_RETURN_VAL -// to INVALID_SOCKET on windows. +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/sockaddr_windows.h" +#include "src/core/lib/iomgr/socket_windows.h" +#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET +#else #include "src/core/lib/iomgr/sockaddr_posix.h" #define BAD_SOCKET_RETURN_VAL -1 +#endif namespace { @@ -91,7 +94,13 @@ class FakeNonResponsiveDNSServer { abort(); } } - ~FakeNonResponsiveDNSServer() { close(socket_); } + ~FakeNonResponsiveDNSServer() { +#ifdef GPR_WINDOWS + closesocket(socket_); +#else + close(socket_); +#endif + } private: int socket_; @@ -193,6 +202,38 @@ TEST(CancelDuringAresQuery, TestCancelActiveDNSQuery) { TestCancelActiveDNSQuery(&args); } +#ifdef GPR_WINDOWS + +void MaybePollArbitraryPollsetTwice() { + grpc_pollset* pollset = (grpc_pollset*)gpr_zalloc(grpc_pollset_size()); + gpr_mu* mu; + grpc_pollset_init(pollset, &mu); + grpc_pollset_worker* worker = nullptr; + // Make a zero timeout poll + gpr_mu_lock(mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(pollset, &worker, grpc_core::ExecCtx::Get()->Now())); + gpr_mu_unlock(mu); + grpc_core::ExecCtx::Get()->Flush(); + // Make a second zero-timeout poll (in case the first one + // short-circuited by picking up a previous "kick") + gpr_mu_lock(mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(pollset, &worker, grpc_core::ExecCtx::Get()->Now())); + gpr_mu_unlock(mu); + grpc_core::ExecCtx::Get()->Flush(); + grpc_pollset_destroy(pollset); + gpr_free(pollset); +} + +#else + +void MaybePollArbitraryPollsetTwice() {} + +#endif + TEST(CancelDuringAresQuery, TestFdsAreDeletedFromPollsetSet) { grpc_core::ExecCtx exec_ctx; ArgsStruct args; @@ -209,6 +250,12 @@ TEST(CancelDuringAresQuery, TestFdsAreDeletedFromPollsetSet) { // this test. This test only cares about what happens to fd's that c-ares // opens. TestCancelActiveDNSQuery(&args); + // This test relies on the assumption that cancelling a c-ares query + // will flush out all callbacks on the current exec ctx, which is true + // on posix platforms but not on Windows, because fd shutdown on Windows + // requires a trip through the polling loop to schedule the callback. + // So we need to do extra polling work on Windows to free things up. + MaybePollArbitraryPollsetTwice(); EXPECT_EQ(grpc_iomgr_count_objects_for_testing(), 0u); grpc_pollset_set_destroy(fake_other_pollset_set); } diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py index 5dad2ea7af6..1c9d0676b89 100755 --- a/test/cpp/naming/gen_build_yaml.py +++ b/test/cpp/naming/gen_build_yaml.py @@ -68,7 +68,7 @@ def main(): 'gtest': False, 'run': False, 'src': ['test/cpp/naming/resolver_component_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util' + unsecure_build_config_suffix, 'grpc_test_util' + unsecure_build_config_suffix, @@ -129,7 +129,7 @@ def main(): 'gtest': True, 'run': True, 'src': ['test/cpp/naming/cancel_ares_query_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util', 'grpc_test_util', diff --git a/test/cpp/naming/manual_run_resolver_component_test.py b/test/cpp/naming/manual_run_resolver_component_test.py new file mode 100644 index 00000000000..fb2157741a0 --- /dev/null +++ b/test/cpp/naming/manual_run_resolver_component_test.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +# The c-ares test suite doesn't get ran regularly on Windows, but +# this script provides a way to run a lot of the tests manually. +_MSBUILD_CONFIG = os.environ['CONFIG'] +os.chdir(os.path.join('..', '..', os.getcwd())) +# This port is arbitrary, but it needs to be available. +_DNS_SERVER_PORT = 15353 + +subprocess.call([ + sys.executable, + 'test\\cpp\\naming\\resolver_component_tests_runner.py', + '--test_bin_path', 'cmake\\build\\%s\\resolver_component_test.exe' % _MSBUILD_CONFIG, + '--dns_server_bin_path', 'test\\cpp\\naming\\utils\\dns_server.py', + '--records_config_path', 'test\\cpp\\naming\\resolver_test_record_groups.yaml', + '--dns_server_port', str(_DNS_SERVER_PORT), + '--dns_resolver_bin_path', 'test\\cpp\\naming\\utils\\dns_resolver.py', + '--tcp_connect_bin_path', 'test\\cpp\\naming\\utils\\tcp_connect.py', +]) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc index 6ac548120ca..3dc6e7178cc 100644 --- a/test/cpp/naming/resolver_component_test.cc +++ b/test/cpp/naming/resolver_component_test.cc @@ -16,6 +16,8 @@ * */ +#include + #include #include #include @@ -55,8 +57,15 @@ // TODO: pull in different headers when enabling this // test on windows. Also set BAD_SOCKET_RETURN_VAL // to INVALID_SOCKET on windows. +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/sockaddr_windows.h" +#include "src/core/lib/iomgr/socket_windows.h" +#include "src/core/lib/iomgr/tcp_windows.h" +#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET +#else #include "src/core/lib/iomgr/sockaddr_posix.h" #define BAD_SOCKET_RETURN_VAL -1 +#endif using grpc::SubProcess; using std::vector; @@ -241,6 +250,62 @@ void CheckLBPolicyResultLocked(grpc_channel_args* channel_args, } } +#ifdef GPR_WINDOWS +void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { + sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_port = htons(dummy_port); + ((char*)&addr.sin6_addr)[15] = 1; + for (;;) { + if (gpr_event_get(done_ev)) { + return; + } + std::vector sockets; + for (size_t i = 0; i < 50; i++) { + SOCKET s = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, + WSA_FLAG_OVERLAPPED); + ASSERT_TRUE(s != BAD_SOCKET_RETURN_VAL) + << "Failed to create TCP ipv6 socket"; + gpr_log(GPR_DEBUG, "Opened socket: %d", s); + char val = 1; + ASSERT_TRUE(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) != + SOCKET_ERROR) + << "Failed to set socketopt reuseaddr. WSA error: " + + std::to_string(WSAGetLastError()); + ASSERT_TRUE(grpc_tcp_set_non_block(s) == GRPC_ERROR_NONE) + << "Failed to set socket non-blocking"; + ASSERT_TRUE(bind(s, (const sockaddr*)&addr, sizeof(addr)) != SOCKET_ERROR) + << "Failed to bind socket " + std::to_string(s) + + " to [::1]:" + std::to_string(dummy_port) + + ". WSA error: " + std::to_string(WSAGetLastError()); + ASSERT_TRUE(listen(s, 1) != SOCKET_ERROR) + << "Failed to listen on socket " + std::to_string(s) + + ". WSA error: " + std::to_string(WSAGetLastError()); + sockets.push_back(s); + } + // Do a non-blocking accept followed by a close on all of those sockets. + // Do this in a separate loop to try to induce a time window to hit races. + for (size_t i = 0; i < sockets.size(); i++) { + gpr_log(GPR_DEBUG, "non-blocking accept then close on %d", sockets[i]); + ASSERT_TRUE(accept(sockets[i], nullptr, nullptr) == INVALID_SOCKET) + << "Accept on dummy socket unexpectedly accepted actual connection."; + ASSERT_TRUE(WSAGetLastError() == WSAEWOULDBLOCK) + << "OpenAndCloseSocketsStressLoop accept on socket " + + std::to_string(sockets[i]) + + " failed in " + "an unexpected way. " + "WSA error: " + + std::to_string(WSAGetLastError()) + + ". Socket use-after-close bugs are likely."; + ASSERT_TRUE(closesocket(sockets[i]) != SOCKET_ERROR) + << "Failed to close socket: " + std::to_string(sockets[i]) + + ". WSA error: " + std::to_string(WSAGetLastError()); + } + } + return; +} +#else void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { // The goal of this loop is to catch socket // "use after close" bugs within the c-ares resolver by acting @@ -311,6 +376,7 @@ void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { } } } +#endif void CheckResolverResultLocked(void* argsp, grpc_error* err) { EXPECT_EQ(err, GRPC_ERROR_NONE); @@ -372,9 +438,9 @@ void RunResolvesRelevantRecordsTest(void (*OnDoneLocked)(void* arg, args.expected_lb_policy = FLAGS_expected_lb_policy; // maybe build the address with an authority char* whole_uri = nullptr; - GPR_ASSERT(asprintf(&whole_uri, "dns://%s/%s", - FLAGS_local_dns_server_address.c_str(), - FLAGS_target_name.c_str())); + GPR_ASSERT(gpr_asprintf(&whole_uri, "dns://%s/%s", + FLAGS_local_dns_server_address.c_str(), + FLAGS_target_name.c_str())); // create resolver and resolve grpc_core::OrphanablePtr resolver = grpc_core::ResolverRegistry::CreateResolver(whole_uri, nullptr, diff --git a/test/cpp/naming/resolver_component_tests_runner.py b/test/cpp/naming/resolver_component_tests_runner.py index 69386ebeb03..1873eec35bd 100755 --- a/test/cpp/naming/resolver_component_tests_runner.py +++ b/test/cpp/naming/resolver_component_tests_runner.py @@ -22,6 +22,7 @@ import tempfile import os import time import signal +import platform argp = argparse.ArgumentParser(description='Run c-ares resolver tests') @@ -43,6 +44,11 @@ args = argp.parse_args() def test_runner_log(msg): sys.stderr.write('\n%s: %s\n' % (__file__, msg)) +def python_args(arg_list): + if platform.system() == 'Windows': + return [sys.executable] + arg_list + return arg_list + cur_resolver = os.environ.get('GRPC_DNS_RESOLVER') if cur_resolver and cur_resolver != 'ares': test_runner_log(('WARNING: cur resolver set to %s. This set of tests ' @@ -50,26 +56,27 @@ if cur_resolver and cur_resolver != 'ares': test_runner_log('Exit 1 without running tests.') sys.exit(1) os.environ.update({'GRPC_DNS_RESOLVER': 'ares'}) +os.environ.update({'GRPC_TRACE': 'cares_resolver'}) def wait_until_dns_server_is_up(args, dns_server_subprocess, dns_server_subprocess_output): for i in range(0, 30): test_runner_log('Health check: attempt to connect to DNS server over TCP.') - tcp_connect_subprocess = subprocess.Popen([ + tcp_connect_subprocess = subprocess.Popen(python_args([ args.tcp_connect_bin_path, '--server_host', '127.0.0.1', '--server_port', str(args.dns_server_port), - '--timeout', str(1)]) + '--timeout', str(1)])) tcp_connect_subprocess.communicate() if tcp_connect_subprocess.returncode == 0: test_runner_log(('Health check: attempt to make an A-record ' 'query to DNS server.')) - dns_resolver_subprocess = subprocess.Popen([ + dns_resolver_subprocess = subprocess.Popen(python_args([ args.dns_resolver_bin_path, '--qname', 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp', '--server_host', '127.0.0.1', - '--server_port', str(args.dns_server_port)], + '--server_port', str(args.dns_server_port)]), stdout=subprocess.PIPE) dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() if dns_resolver_subprocess.returncode == 0: @@ -91,10 +98,10 @@ def wait_until_dns_server_is_up(args, dns_server_subprocess_output = tempfile.mktemp() with open(dns_server_subprocess_output, 'w') as l: - dns_server_subprocess = subprocess.Popen([ + dns_server_subprocess = subprocess.Popen(python_args([ args.dns_server_bin_path, '--port', str(args.dns_server_port), - '--records_config_path', args.records_config_path], + '--records_config_path', args.records_config_path]), stdin=subprocess.PIPE, stdout=l, stderr=l) @@ -112,6 +119,18 @@ wait_until_dns_server_is_up(args, dns_server_subprocess_output) num_test_failures = 0 +test_runner_log('Run test with target: %s' % 'no-srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.') +current_test_subprocess = subprocess.Popen([ + args.test_bin_path, + '--target_name', 'no-srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.', + '--expected_addrs', '5.5.5.5:443,False', + '--expected_chosen_service_config', '', + '--expected_lb_policy', '', + '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port]) +current_test_subprocess.communicate() +if current_test_subprocess.returncode != 0: + num_test_failures += 1 + test_runner_log('Run test with target: %s' % 'srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.') current_test_subprocess = subprocess.Popen([ args.test_bin_path, diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml index 6c4f89d09b8..3c51a00c7b1 100644 --- a/test/cpp/naming/resolver_test_record_groups.yaml +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -1,5 +1,13 @@ resolver_tests_common_zone_name: resolver-tests-version-4.grpctestingexp. resolver_component_tests: +- expected_addrs: + - {address: '5.5.5.5:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: no-srv-ipv4-single-target + records: + no-srv-ipv4-single-target: + - {TTL: '2100', data: 5.5.5.5, type: A} - expected_addrs: - {address: '1.2.3.4:1234', is_balancer: true} expected_chosen_service_config: null diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 5815f82fefb..cf3b54e0440 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -5784,7 +5784,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5796,7 +5797,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, From 5cb99c959231f1c8a949b1e5e75a2d7cde7d9648 Mon Sep 17 00:00:00 2001 From: Junhao Li Date: Thu, 9 Aug 2018 00:51:17 -0400 Subject: [PATCH 77/99] fix format --- src/cpp/server/load_reporter/util.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpp/server/load_reporter/util.cc b/src/cpp/server/load_reporter/util.cc index 0f8fa764ad0..89bdf57049c 100644 --- a/src/cpp/server/load_reporter/util.cc +++ b/src/cpp/server/load_reporter/util.cc @@ -20,9 +20,10 @@ #include -#include #include +#include + namespace grpc { namespace load_reporter { namespace experimental { From 5a664d7260c0de1f80ebe0813f322f237ff2b610 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 9 Aug 2018 09:02:32 -0700 Subject: [PATCH 78/99] check if channel is closed before starting core ops --- .../grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index 893df8eac67..aa187e88a62 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -309,13 +309,18 @@ cdef SegregatedCall _segregated_call( _ChannelState state, int flags, method, host, object deadline, object metadata, CallCredentials credentials, operationses_and_user_tags): cdef _CallState call_state = _CallState() - cdef grpc_completion_queue *c_completion_queue = ( - grpc_completion_queue_create_for_next(NULL)) cdef SegregatedCall segregated_call + cdef grpc_completion_queue *c_completion_queue def on_success(started_tags): state.segregated_call_states.add(call_state) + with state.condition: + if state.open: + c_completion_queue = (grpc_completion_queue_create_for_next(NULL)) + else: + raise ValueError('Cannot invoke RPC on closed channel!') + try: _call( state, call_state, c_completion_queue, on_success, flags, method, host, @@ -443,8 +448,11 @@ cdef class Channel: def check_connectivity_state(self, bint try_to_connect): with self._state.condition: - return grpc_channel_check_connectivity_state( - self._state.c_channel, try_to_connect) + if self._state.open: + return grpc_channel_check_connectivity_state( + self._state.c_channel, try_to_connect) + else: + raise ValueError('Cannot invoke RPC on closed channel!') def watch_connectivity_state( self, grpc_connectivity_state last_observed_state, object deadline): From bcd747d42d11c92c3d37aef8d8b45a043ff5d79c Mon Sep 17 00:00:00 2001 From: tdbhacks Date: Fri, 20 Jul 2018 16:41:44 -0700 Subject: [PATCH 79/99] Added system roots feature to load roots from OS trust store Added a flag-guarded feature that allows gRPC to load TLS/SSL roots from the OS trust store. This is the Linux-specific implementation of such feature. --- BUILD | 4 + CMakeLists.txt | 43 +++++ Makefile | 54 ++++++ build.yaml | 15 ++ config.m4 | 2 + config.w32 | 2 + gRPC-C++.podspec | 2 + gRPC-Core.podspec | 6 + grpc.gemspec | 4 + grpc.gyp | 2 + package.xml | 4 + .../security_connector/load_system_roots.h | 29 +++ .../load_system_roots_fallback.cc | 32 ++++ .../load_system_roots_linux.cc | 165 ++++++++++++++++++ .../load_system_roots_linux.h | 44 +++++ .../security_connector/security_connector.cc | 18 +- src/python/grpcio/grpc_core_dependencies.py | 2 + test/core/security/BUILD | 25 ++- test/core/security/etc/BUILD | 22 +++ test/core/security/etc/README | 2 + test/core/security/etc/bundle.pem | 63 +++++++ test/core/security/etc/test_roots/cert1.pem | 21 +++ test/core/security/etc/test_roots/cert2.pem | 21 +++ test/core/security/etc/test_roots/cert3.pem | 21 +++ test/core/security/linux_system_roots_test.cc | 104 +++++++++++ test/core/security/security_connector_test.cc | 12 +- tools/doxygen/Doxyfile.core.internal | 4 + .../generated/sources_and_headers.json | 23 +++ tools/run_tests/generated/tests.json | 24 +++ 29 files changed, 760 insertions(+), 10 deletions(-) create mode 100644 src/core/lib/security/security_connector/load_system_roots.h create mode 100644 src/core/lib/security/security_connector/load_system_roots_fallback.cc create mode 100644 src/core/lib/security/security_connector/load_system_roots_linux.cc create mode 100644 src/core/lib/security/security_connector/load_system_roots_linux.h create mode 100644 test/core/security/etc/BUILD create mode 100644 test/core/security/etc/README create mode 100644 test/core/security/etc/bundle.pem create mode 100644 test/core/security/etc/test_roots/cert1.pem create mode 100644 test/core/security/etc/test_roots/cert2.pem create mode 100644 test/core/security/etc/test_roots/cert3.pem create mode 100644 test/core/security/linux_system_roots_test.cc diff --git a/BUILD b/BUILD index 81390dd1aa1..35cf86288d6 100644 --- a/BUILD +++ b/BUILD @@ -1499,6 +1499,8 @@ grpc_cc_library( "src/core/lib/security/credentials/plugin/plugin_credentials.cc", "src/core/lib/security/credentials/ssl/ssl_credentials.cc", "src/core/lib/security/security_connector/alts_security_connector.cc", + "src/core/lib/security/security_connector/load_system_roots_fallback.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.cc", "src/core/lib/security/security_connector/local_security_connector.cc", "src/core/lib/security/security_connector/security_connector.cc", "src/core/lib/security/transport/client_auth_filter.cc", @@ -1527,6 +1529,8 @@ grpc_cc_library( "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.h", "src/core/lib/security/transport/auth_filters.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index f242ee92bbf..c4526d2af5d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -581,6 +581,7 @@ add_dependencies(buildtests_cxx generic_end2end_test) add_dependencies(buildtests_cxx golden_file_test) add_dependencies(buildtests_cxx grpc_alts_credentials_options_test) add_dependencies(buildtests_cxx grpc_cli) +add_dependencies(buildtests_cxx grpc_linux_system_roots_test) add_dependencies(buildtests_cxx grpc_tool_test) add_dependencies(buildtests_cxx grpclb_api_test) add_dependencies(buildtests_cxx grpclb_end2end_test) @@ -1129,6 +1130,8 @@ add_library(grpc src/core/lib/security/credentials/plugin/plugin_credentials.cc src/core/lib/security/credentials/ssl/ssl_credentials.cc src/core/lib/security/security_connector/alts_security_connector.cc + src/core/lib/security/security_connector/load_system_roots_fallback.cc + src/core/lib/security/security_connector/load_system_roots_linux.cc src/core/lib/security/security_connector/local_security_connector.cc src/core/lib/security/security_connector/security_connector.cc src/core/lib/security/transport/client_auth_filter.cc @@ -1559,6 +1562,8 @@ add_library(grpc_cronet src/core/lib/security/credentials/plugin/plugin_credentials.cc src/core/lib/security/credentials/ssl/ssl_credentials.cc src/core/lib/security/security_connector/alts_security_connector.cc + src/core/lib/security/security_connector/load_system_roots_fallback.cc + src/core/lib/security/security_connector/load_system_roots_linux.cc src/core/lib/security/security_connector/local_security_connector.cc src/core/lib/security/security_connector/security_connector.cc src/core/lib/security/transport/client_auth_filter.cc @@ -12146,6 +12151,44 @@ if (gRPC_INSTALL) endif() endif (gRPC_BUILD_CODEGEN) +if (gRPC_BUILD_TESTS) + +add_executable(grpc_linux_system_roots_test + test/core/security/linux_system_roots_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(grpc_linux_system_roots_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(grpc_linux_system_roots_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_CODEGEN) add_executable(grpc_node_plugin diff --git a/Makefile b/Makefile index 5174ab6719c..9d23de866cd 100644 --- a/Makefile +++ b/Makefile @@ -1170,6 +1170,7 @@ grpc_alts_credentials_options_test: $(BINDIR)/$(CONFIG)/grpc_alts_credentials_op grpc_cli: $(BINDIR)/$(CONFIG)/grpc_cli grpc_cpp_plugin: $(BINDIR)/$(CONFIG)/grpc_cpp_plugin grpc_csharp_plugin: $(BINDIR)/$(CONFIG)/grpc_csharp_plugin +grpc_linux_system_roots_test: $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test grpc_node_plugin: $(BINDIR)/$(CONFIG)/grpc_node_plugin grpc_objective_c_plugin: $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin grpc_php_plugin: $(BINDIR)/$(CONFIG)/grpc_php_plugin @@ -1670,6 +1671,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/grpclb_end2end_test \ @@ -1849,6 +1851,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/grpclb_end2end_test \ @@ -2316,6 +2319,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/golden_file_test || ( echo test golden_file_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_alts_credentials_options_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test || ( echo test grpc_alts_credentials_options_test failed ; exit 1 ) + $(E) "[RUN] Testing grpc_linux_system_roots_test" + $(Q) $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test || ( echo test grpc_linux_system_roots_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_tool_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_tool_test || ( echo test grpc_tool_test failed ; exit 1 ) $(E) "[RUN] Testing grpclb_api_test" @@ -3608,6 +3613,8 @@ LIBGRPC_SRC = \ src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/security_connector/alts_security_connector.cc \ + src/core/lib/security/security_connector/load_system_roots_fallback.cc \ + src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/transport/client_auth_filter.cc \ @@ -4037,6 +4044,8 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/security_connector/alts_security_connector.cc \ + src/core/lib/security/security_connector/load_system_roots_fallback.cc \ + src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/transport/client_auth_filter.cc \ @@ -17907,6 +17916,49 @@ ifneq ($(NO_DEPS),true) endif +GRPC_LINUX_SYSTEM_ROOTS_TEST_SRC = \ + test/core/security/linux_system_roots_test.cc \ + +GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_LINUX_SYSTEM_ROOTS_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+. + +$(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test: $(PROTOBUF_DEP) $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/core/security/linux_system_roots_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_grpc_linux_system_roots_test: $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS:.o=.dep) +endif +endif + + GRPC_NODE_PLUGIN_SRC = \ src/compiler/node_plugin.cc \ @@ -24651,6 +24703,8 @@ src/core/lib/security/credentials/oauth2/oauth2_credentials.cc: $(OPENSSL_DEP) src/core/lib/security/credentials/plugin/plugin_credentials.cc: $(OPENSSL_DEP) src/core/lib/security/credentials/ssl/ssl_credentials.cc: $(OPENSSL_DEP) src/core/lib/security/security_connector/alts_security_connector.cc: $(OPENSSL_DEP) +src/core/lib/security/security_connector/load_system_roots_fallback.cc: $(OPENSSL_DEP) +src/core/lib/security/security_connector/load_system_roots_linux.cc: $(OPENSSL_DEP) src/core/lib/security/security_connector/local_security_connector.cc: $(OPENSSL_DEP) src/core/lib/security/security_connector/security_connector.cc: $(OPENSSL_DEP) src/core/lib/security/transport/client_auth_filter.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 70af96046cb..bd9c4237a11 100644 --- a/build.yaml +++ b/build.yaml @@ -792,6 +792,8 @@ filegroups: - src/core/lib/security/credentials/plugin/plugin_credentials.h - src/core/lib/security/credentials/ssl/ssl_credentials.h - src/core/lib/security/security_connector/alts_security_connector.h + - src/core/lib/security/security_connector/load_system_roots.h + - src/core/lib/security/security_connector/load_system_roots_linux.h - src/core/lib/security/security_connector/local_security_connector.h - src/core/lib/security/security_connector/security_connector.h - src/core/lib/security/transport/auth_filters.h @@ -819,6 +821,8 @@ filegroups: - src/core/lib/security/credentials/plugin/plugin_credentials.cc - src/core/lib/security/credentials/ssl/ssl_credentials.cc - src/core/lib/security/security_connector/alts_security_connector.cc + - src/core/lib/security/security_connector/load_system_roots_fallback.cc + - src/core/lib/security/security_connector/load_system_roots_linux.cc - src/core/lib/security/security_connector/local_security_connector.cc - src/core/lib/security/security_connector/security_connector.cc - src/core/lib/security/transport/client_auth_filter.cc @@ -4698,6 +4702,17 @@ targets: secure: false vs_config_type: Application vs_project_guid: '{3C813052-A49A-4662-B90A-1ADBEC7EE453}' +- name: grpc_linux_system_roots_test + gtest: true + build: test + language: c++ + src: + - test/core/security/linux_system_roots_test.cc + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_node_plugin build: protoc language: c++ diff --git a/config.m4 b/config.m4 index aa40a698a64..a46b076fe92 100644 --- a/config.m4 +++ b/config.m4 @@ -280,6 +280,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/security_connector/alts_security_connector.cc \ + src/core/lib/security/security_connector/load_system_roots_fallback.cc \ + src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/transport/client_auth_filter.cc \ diff --git a/config.w32 b/config.w32 index 5afa4466acd..3aea5fa7f20 100644 --- a/config.w32 +++ b/config.w32 @@ -255,6 +255,8 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\security\\credentials\\plugin\\plugin_credentials.cc " + "src\\core\\lib\\security\\credentials\\ssl\\ssl_credentials.cc " + "src\\core\\lib\\security\\security_connector\\alts_security_connector.cc " + + "src\\core\\lib\\security\\security_connector\\load_system_roots_fallback.cc " + + "src\\core\\lib\\security\\security_connector\\load_system_roots_linux.cc " + "src\\core\\lib\\security\\security_connector\\local_security_connector.cc " + "src\\core\\lib\\security\\security_connector\\security_connector.cc " + "src\\core\\lib\\security\\transport\\client_auth_filter.cc " + diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 1d9237bf627..1d3cedb16be 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -278,6 +278,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/security_connector/alts_security_connector.h', + 'src/core/lib/security/security_connector/load_system_roots.h', + 'src/core/lib/security/security_connector/load_system_roots_linux.h', 'src/core/lib/security/security_connector/local_security_connector.h', 'src/core/lib/security/security_connector/security_connector.h', 'src/core/lib/security/transport/auth_filters.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 5c3649afbde..1998bc8b4c5 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -289,6 +289,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/security_connector/alts_security_connector.h', + 'src/core/lib/security/security_connector/load_system_roots.h', + 'src/core/lib/security/security_connector/load_system_roots_linux.h', 'src/core/lib/security/security_connector/local_security_connector.h', 'src/core/lib/security/security_connector/security_connector.h', 'src/core/lib/security/transport/auth_filters.h', @@ -705,6 +707,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', 'src/core/lib/security/security_connector/alts_security_connector.cc', + 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', + 'src/core/lib/security/security_connector/load_system_roots_linux.cc', 'src/core/lib/security/security_connector/local_security_connector.cc', 'src/core/lib/security/security_connector/security_connector.cc', 'src/core/lib/security/transport/client_auth_filter.cc', @@ -882,6 +886,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/security_connector/alts_security_connector.h', + 'src/core/lib/security/security_connector/load_system_roots.h', + 'src/core/lib/security/security_connector/load_system_roots_linux.h', 'src/core/lib/security/security_connector/local_security_connector.h', 'src/core/lib/security/security_connector/security_connector.h', 'src/core/lib/security/transport/auth_filters.h', diff --git a/grpc.gemspec b/grpc.gemspec index c250316b995..55d53cb71db 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -222,6 +222,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.h ) s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.h ) s.files += %w( src/core/lib/security/security_connector/alts_security_connector.h ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots.h ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots_linux.h ) s.files += %w( src/core/lib/security/security_connector/local_security_connector.h ) s.files += %w( src/core/lib/security/security_connector/security_connector.h ) s.files += %w( src/core/lib/security/transport/auth_filters.h ) @@ -642,6 +644,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.cc ) s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.cc ) s.files += %w( src/core/lib/security/security_connector/alts_security_connector.cc ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots_fallback.cc ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots_linux.cc ) s.files += %w( src/core/lib/security/security_connector/local_security_connector.cc ) s.files += %w( src/core/lib/security/security_connector/security_connector.cc ) s.files += %w( src/core/lib/security/transport/client_auth_filter.cc ) diff --git a/grpc.gyp b/grpc.gyp index 25082fe540a..ba4e8185c61 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -472,6 +472,8 @@ 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', 'src/core/lib/security/security_connector/alts_security_connector.cc', + 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', + 'src/core/lib/security/security_connector/load_system_roots_linux.cc', 'src/core/lib/security/security_connector/local_security_connector.cc', 'src/core/lib/security/security_connector/security_connector.cc', 'src/core/lib/security/transport/client_auth_filter.cc', diff --git a/package.xml b/package.xml index acdc6ffdb38..76bdd5ac8f6 100644 --- a/package.xml +++ b/package.xml @@ -227,6 +227,8 @@ + + @@ -647,6 +649,8 @@ + + diff --git a/src/core/lib/security/security_connector/load_system_roots.h b/src/core/lib/security/security_connector/load_system_roots.h new file mode 100644 index 00000000000..8d4af5b2c6c --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots.h @@ -0,0 +1,29 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H +#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H + +namespace grpc_core { + +// Returns a slice containing roots from the OS trust store +grpc_slice LoadSystemRootCerts(); + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H */ \ No newline at end of file diff --git a/src/core/lib/security/security_connector/load_system_roots_fallback.cc b/src/core/lib/security/security_connector/load_system_roots_fallback.cc new file mode 100644 index 00000000000..73d1245f331 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots_fallback.cc @@ -0,0 +1,32 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include "src/core/lib/security/security_connector/load_system_roots.h" + +#ifndef GPR_LINUX + +namespace grpc_core { + +grpc_slice LoadSystemRootCerts() { return grpc_empty_slice(); } + +} // namespace grpc_core + +#endif /* GPR_LINUX */ diff --git a/src/core/lib/security/security_connector/load_system_roots_linux.cc b/src/core/lib/security/security_connector/load_system_roots_linux.cc new file mode 100644 index 00000000000..924fa8a3e26 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots_linux.cc @@ -0,0 +1,165 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include "src/core/lib/security/security_connector/load_system_roots_linux.h" + +#ifdef GPR_LINUX + +#include "src/core/lib/security/security_connector/load_system_roots.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/inlined_vector.h" +#include "src/core/lib/iomgr/load_file.h" + +namespace grpc_core { +namespace { + +const char* kLinuxCertFiles[] = { + "/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/ca-bundle.pem", "/etc/pki/tls/cacert.pem", + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}; +const char* kLinuxCertDirectories[] = { + "/etc/ssl/certs", "/system/etc/security/cacerts", "/usr/local/share/certs", + "/etc/pki/tls/certs", "/etc/openssl/certs"}; + +grpc_slice GetSystemRootCerts() { + grpc_slice valid_bundle_slice = grpc_empty_slice(); + size_t num_cert_files_ = GPR_ARRAY_SIZE(kLinuxCertFiles); + for (size_t i = 0; i < num_cert_files_; i++) { + grpc_error* error = + grpc_load_file(kLinuxCertFiles[i], 1, &valid_bundle_slice); + if (error == GRPC_ERROR_NONE) { + return valid_bundle_slice; + } + } + return grpc_empty_slice(); +} + +} // namespace + +void GetAbsoluteFilePath(const char* valid_file_dir, + const char* file_entry_name, char* path_buffer) { + if (valid_file_dir != nullptr && file_entry_name != nullptr) { + int path_len = snprintf(path_buffer, MAXPATHLEN, "%s/%s", valid_file_dir, + file_entry_name); + if (path_len == 0) { + gpr_log(GPR_ERROR, "failed to get absolute path for file: %s", + file_entry_name); + } + } +} + +grpc_slice CreateRootCertsBundle(const char* certs_directory) { + grpc_slice bundle_slice = grpc_empty_slice(); + if (certs_directory == nullptr) { + return bundle_slice; + } + DIR* ca_directory = opendir(certs_directory); + if (ca_directory == nullptr) { + return bundle_slice; + } + struct FileData { + char path[MAXPATHLEN]; + off_t size; + }; + InlinedVector roots_filenames; + size_t total_bundle_size = 0; + struct dirent* directory_entry; + while ((directory_entry = readdir(ca_directory)) != nullptr) { + struct stat dir_entry_stat; + const char* file_entry_name = directory_entry->d_name; + FileData file_data; + GetAbsoluteFilePath(certs_directory, file_entry_name, file_data.path); + int stat_return = stat(file_data.path, &dir_entry_stat); + if (stat_return == -1 || !S_ISREG(dir_entry_stat.st_mode)) { + // no subdirectories. + if (stat_return == -1) { + gpr_log(GPR_ERROR, "failed to get status for file: %s", file_data.path); + } + continue; + } + file_data.size = dir_entry_stat.st_size; + total_bundle_size += file_data.size; + roots_filenames.push_back(file_data); + } + closedir(ca_directory); + char* bundle_string = static_cast(gpr_zalloc(total_bundle_size + 1)); + size_t bytes_read = 0; + for (size_t i = 0; i < roots_filenames.size(); i++) { + int file_descriptor = open(roots_filenames[i].path, O_RDONLY); + if (file_descriptor != -1) { + // Read file into bundle. + size_t cert_file_size = roots_filenames[i].size; + int read_ret = + read(file_descriptor, bundle_string + bytes_read, cert_file_size); + if (read_ret != -1) { + bytes_read += read_ret; + } else { + gpr_log(GPR_ERROR, "failed to read file: %s", roots_filenames[i].path); + } + } + } + bundle_slice = grpc_slice_new(bundle_string, bytes_read, gpr_free); + return bundle_slice; +} + +grpc_slice LoadSystemRootCerts() { + grpc_slice result = grpc_empty_slice(); + // Prioritize user-specified custom directory if flag is set. + char* custom_dir = gpr_getenv("GRPC_SYSTEM_SSL_ROOTS_DIR"); + if (custom_dir != nullptr) { + result = CreateRootCertsBundle(custom_dir); + gpr_free(custom_dir); + } + // If the custom directory is empty/invalid/not specified, fallback to + // distribution-specific directory. + if (GRPC_SLICE_IS_EMPTY(result)) { + result = GetSystemRootCerts(); + } + if (GRPC_SLICE_IS_EMPTY(result)) { + for (size_t i = 0; i < GPR_ARRAY_SIZE(kLinuxCertDirectories); i++) { + result = CreateRootCertsBundle(kLinuxCertDirectories[i]); + if (!GRPC_SLICE_IS_EMPTY(result)) { + break; + } + } + } + return result; +} + +} // namespace grpc_core + +#endif /* GPR_LINUX */ diff --git a/src/core/lib/security/security_connector/load_system_roots_linux.h b/src/core/lib/security/security_connector/load_system_roots_linux.h new file mode 100644 index 00000000000..12617df4928 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots_linux.h @@ -0,0 +1,44 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H +#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H + +#include + +#ifdef GPR_LINUX + +namespace grpc_core { + +// Creates a bundle slice containing the contents of all certificate files in +// a directory. +// Returns such slice. +// Exposed for testing purposes only. +grpc_slice CreateRootCertsBundle(const char* certs_directory); + +// Gets the absolute file path needed to load a certificate file. +// Populates path_buffer, which must be of size MAXPATHLEN. +// Exposed for testing purposes only. +void GetAbsoluteFilePath(const char* valid_file_dir, + const char* file_entry_name, char* path_buffer); + +} // namespace grpc_core + +#endif /* GPR_LINUX */ +#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H \ + */ diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc index 59cf3a0af18..04b4c87c713 100644 --- a/src/core/lib/security/security_connector/security_connector.cc +++ b/src/core/lib/security/security_connector/security_connector.cc @@ -21,7 +21,6 @@ #include "src/core/lib/security/security_connector/security_connector.h" #include -#include #include #include @@ -39,6 +38,7 @@ #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/credentials/fake/fake_credentials.h" #include "src/core/lib/security/credentials/ssl/ssl_credentials.h" +#include "src/core/lib/security/security_connector/load_system_roots.h" #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/security/transport/target_authority_table.h" @@ -57,6 +57,12 @@ static const char* installed_roots_path = INSTALL_PREFIX "/share/grpc/roots.pem"; #endif +/** Environment variable used as a flag to enable/disable loading system root + certificates from the OS trust store. */ +#ifndef GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR +#define GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_USE_SYSTEM_SSL_ROOTS" +#endif + #ifndef TSI_OPENSSL_ALPN_SUPPORT #define TSI_OPENSSL_ALPN_SUPPORT 1 #endif @@ -1186,6 +1192,10 @@ const char* DefaultSslRootStore::GetPemRootCerts() { grpc_slice DefaultSslRootStore::ComputePemRootCerts() { grpc_slice result = grpc_empty_slice(); + char* use_system_roots_env_value = + gpr_getenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR); + const bool use_system_roots = gpr_is_true(use_system_roots_env_value); + gpr_free(use_system_roots_env_value); // First try to load the roots from the environment. char* default_root_certs_path = gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR); @@ -1207,7 +1217,11 @@ grpc_slice DefaultSslRootStore::ComputePemRootCerts() { } gpr_free(pem_root_certs); } - // Fall back to installed certs if needed. + // Try loading roots from OS trust store if flag is enabled. + if (GRPC_SLICE_IS_EMPTY(result) && use_system_roots) { + result = LoadSystemRootCerts(); + } + // Fallback to roots manually shipped with gRPC. if (GRPC_SLICE_IS_EMPTY(result) && ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) { GRPC_LOG_IF_ERROR("load_file", diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index d6efb49750f..a8158311fb0 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -254,6 +254,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', 'src/core/lib/security/security_connector/alts_security_connector.cc', + 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', + 'src/core/lib/security/security_connector/load_system_roots_linux.cc', 'src/core/lib/security/security_connector/local_security_connector.cc', 'src/core/lib/security/security_connector/security_connector.cc', 'src/core/lib/security/transport/client_auth_filter.cc', diff --git a/test/core/security/BUILD b/test/core/security/BUILD index 12aa84d93b3..b7de955cdbe 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -128,6 +128,27 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "linux_system_roots_test", + srcs = ["linux_system_roots_test.cc"], + data = [ + "//test/core/security/etc:bundle.pem", + "//test/core/security/etc:test_roots/cert1.pem", + "//test/core/security/etc:test_roots/cert2.pem", + "//test/core/security/etc:test_roots/cert3.pem", + ], + language = "C++", + external_deps = [ + "gtest", + ], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], +) + grpc_cc_test( name = "ssl_credentials_test", srcs = ["ssl_credentials_test.cc"], @@ -219,9 +240,9 @@ grpc_cc_test( deps = [ "//:gpr", "//:grpc", - "//:grpc_base_c", + "//:grpc_base_c", "//:grpc_secure", - "//:tsi", + "//:tsi", "//:tsi_interface", "//test/core/util:gpr_test_util", ], diff --git a/test/core/security/etc/BUILD b/test/core/security/etc/BUILD new file mode 100644 index 00000000000..2c6ab64a3b8 --- /dev/null +++ b/test/core/security/etc/BUILD @@ -0,0 +1,22 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache v2 + +exports_files([ + "bundle.pem", + "test_roots/cert1.pem", + "test_roots/cert2.pem", + "test_roots/cert3.pem", +]) diff --git a/test/core/security/etc/README b/test/core/security/etc/README new file mode 100644 index 00000000000..6ba43825861 --- /dev/null +++ b/test/core/security/etc/README @@ -0,0 +1,2 @@ +These files are manual copies of a pem cert from the /etc/ssl/certs/ directory. +They serve only as dummy certificate test files. diff --git a/test/core/security/etc/bundle.pem b/test/core/security/etc/bundle.pem new file mode 100644 index 00000000000..07d7672f830 --- /dev/null +++ b/test/core/security/etc/bundle.pem @@ -0,0 +1,63 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/etc/test_roots/cert1.pem b/test/core/security/etc/test_roots/cert1.pem new file mode 100644 index 00000000000..988cc68aac2 --- /dev/null +++ b/test/core/security/etc/test_roots/cert1.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/etc/test_roots/cert2.pem b/test/core/security/etc/test_roots/cert2.pem new file mode 100644 index 00000000000..988cc68aac2 --- /dev/null +++ b/test/core/security/etc/test_roots/cert2.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/etc/test_roots/cert3.pem b/test/core/security/etc/test_roots/cert3.pem new file mode 100644 index 00000000000..988cc68aac2 --- /dev/null +++ b/test/core/security/etc/test_roots/cert3.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/linux_system_roots_test.cc b/test/core/security/linux_system_roots_test.cc new file mode 100644 index 00000000000..fce9c8dcc5d --- /dev/null +++ b/test/core/security/linux_system_roots_test.cc @@ -0,0 +1,104 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#ifdef GPR_LINUX +#include +#include +#include +#include +#include +#include + +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/gpr/tmpfile.h" +#include "src/core/lib/iomgr/load_file.h" +#include "src/core/lib/security/context/security_context.h" +#include "src/core/lib/security/security_connector/load_system_roots.h" +#include "src/core/lib/security/security_connector/load_system_roots_linux.h" +#include "src/core/lib/security/security_connector/security_connector.h" +#include "src/core/lib/slice/slice_string_helpers.h" +#include "src/core/tsi/ssl_transport_security.h" +#include "src/core/tsi/transport_security.h" +#include "test/core/util/test_config.h" + +#include "gtest/gtest.h" + +#ifndef GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR +#define GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_USE_SYSTEM_SSL_ROOTS" +#endif + +namespace grpc { +namespace { + +TEST(AbsoluteFilePathTest, ConcatenatesCorrectly) { + const char* directory = "nonexistent/test/directory"; + const char* filename = "doesnotexist.txt"; + char result_path[MAXPATHLEN]; + grpc_core::GetAbsoluteFilePath(directory, filename, result_path); + EXPECT_STREQ(result_path, "nonexistent/test/directory/doesnotexist.txt"); +} + +TEST(CreateRootCertsBundleTest, ReturnsEmpty) { + // Test that CreateRootCertsBundle returns an empty slice for null or + // nonexistent cert directories. + grpc_slice result_slice = grpc_core::CreateRootCertsBundle(nullptr); + EXPECT_TRUE(GRPC_SLICE_IS_EMPTY(result_slice)); + grpc_slice_unref(result_slice); + result_slice = grpc_core::CreateRootCertsBundle("does/not/exist"); + EXPECT_TRUE(GRPC_SLICE_IS_EMPTY(result_slice)); + grpc_slice_unref(result_slice); +} + +TEST(CreateRootCertsBundleTest, BundlesCorrectly) { + gpr_setenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR, "true"); + // Test that CreateRootCertsBundle returns a correct slice. + grpc_slice roots_bundle = grpc_empty_slice(); + GRPC_LOG_IF_ERROR( + "load_file", + grpc_load_file("test/core/security/etc/bundle.pem", 1, &roots_bundle)); + // result_slice should have the same content as roots_bundle. + grpc_slice result_slice = + grpc_core::CreateRootCertsBundle("test/core/security/etc/test_roots"); + char* result_str = grpc_slice_to_c_string(result_slice); + char* bundle_str = grpc_slice_to_c_string(roots_bundle); + EXPECT_STREQ(result_str, bundle_str); + // Clean up. + unsetenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR); + gpr_free(result_str); + gpr_free(bundle_str); + grpc_slice_unref(roots_bundle); + grpc_slice_unref(result_slice); +} + +} // namespace +} // namespace grpc + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} +#else +int main() { + printf("*** WARNING: this test is only supported on Linux systems ***\n"); + return 0; +} +#endif // GPR_LINUX diff --git a/test/core/security/security_connector_test.cc b/test/core/security/security_connector_test.cc index e4c3ace6b4a..82d77eef8bd 100644 --- a/test/core/security/security_connector_test.cc +++ b/test/core/security/security_connector_test.cc @@ -363,7 +363,7 @@ static void test_ipv6_address_san(void) { namespace grpc_core { namespace { -class TestDefafaultSllRootStore : public DefaultSslRootStore { +class TestDefaultSslRootStore : public DefaultSslRootStore { public: static grpc_slice ComputePemRootCertsForTesting() { return ComputePemRootCerts(); @@ -389,7 +389,7 @@ static void test_default_ssl_roots(void) { gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, ""); grpc_set_ssl_roots_override_callback(override_roots_success); grpc_slice roots = - grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); char* roots_contents = grpc_slice_to_c_string(roots); grpc_slice_unref(roots); GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0); @@ -398,7 +398,7 @@ static void test_default_ssl_roots(void) { /* Now let's set the env var: We should get the contents pointed value instead. */ gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, roots_env_var_file_path); - roots = grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); roots_contents = grpc_slice_to_c_string(roots); grpc_slice_unref(roots); GPR_ASSERT(strcmp(roots_contents, roots_for_env_var) == 0); @@ -407,7 +407,7 @@ static void test_default_ssl_roots(void) { /* Now reset the env var. We should fall back to the value overridden using the api. */ gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, ""); - roots = grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); roots_contents = grpc_slice_to_c_string(roots); grpc_slice_unref(roots); GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0); @@ -416,10 +416,10 @@ static void test_default_ssl_roots(void) { /* Now setup a permanent failure for the overridden roots and we should get an empty slice. */ grpc_set_ssl_roots_override_callback(override_roots_permanent_failure); - roots = grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); GPR_ASSERT(GRPC_SLICE_IS_EMPTY(roots)); const tsi_ssl_root_certs_store* root_store = - grpc_core::TestDefafaultSllRootStore::GetRootStore(); + grpc_core::TestDefaultSslRootStore::GetRootStore(); GPR_ASSERT(root_store == nullptr); /* Cleanup. */ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 18f56984fe2..ff76d5a1b41 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1355,6 +1355,10 @@ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.h \ src/core/lib/security/security_connector/alts_security_connector.cc \ src/core/lib/security/security_connector/alts_security_connector.h \ +src/core/lib/security/security_connector/load_system_roots.h \ +src/core/lib/security/security_connector/load_system_roots_fallback.cc \ +src/core/lib/security/security_connector/load_system_roots_linux.cc \ +src/core/lib/security/security_connector/load_system_roots_linux.h \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/local_security_connector.h \ src/core/lib/security/security_connector/security_connector.cc \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index a686dae8b4a..34e23f09c2d 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -3707,6 +3707,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "grpc_linux_system_roots_test", + "src": [ + "test/core/security/linux_system_roots_test.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "grpc_plugin_support" @@ -10358,6 +10375,8 @@ "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.h", "src/core/lib/security/transport/auth_filters.h", @@ -10406,6 +10425,10 @@ "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.cc", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_fallback.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.cc", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.cc", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index cf3b54e0440..de64097ec1c 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4333,6 +4333,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "grpc_linux_system_roots_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, From fc0b3073b11925bc51d4344458dd216d1b2e29c7 Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Thu, 9 Aug 2018 13:39:23 -0700 Subject: [PATCH 80/99] Add v1.14.1 to interop matrix --- tools/interop_matrix/client_matrix.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index dff33c88c02..bb9222d953f 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -97,7 +97,7 @@ LANG_RELEASE_MATRIX = { 'v1.13.0': None }, { - 'v1.14.0': None + 'v1.14.1': None }, ], 'go': [ @@ -226,7 +226,7 @@ LANG_RELEASE_MATRIX = { 'v1.13.0': None }, { - 'v1.14.0': None + 'v1.14.1': None }, ], 'node': [ @@ -314,7 +314,7 @@ LANG_RELEASE_MATRIX = { 'v1.13.0': None }, { - 'v1.14.0': None + 'v1.14.1': None }, ], 'php': [ @@ -358,7 +358,7 @@ LANG_RELEASE_MATRIX = { 'v1.13.0': None }, { - 'v1.14.0': None + 'v1.14.1': None }, ], 'csharp': [ @@ -407,7 +407,7 @@ LANG_RELEASE_MATRIX = { 'v1.13.0': None }, { - 'v1.14.0': None + 'v1.14.1': None }, ], } From 753f4dccd60bdb4f5886a866b90fe283bd335e40 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 10:39:11 +0200 Subject: [PATCH 81/99] Add PlatformApis.IsUnityIOS --- src/csharp/Grpc.Core/Internal/PlatformApis.cs | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs index c501aa89fb6..a8f147545b4 100644 --- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs +++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs @@ -42,6 +42,7 @@ namespace Grpc.Core.Internal static readonly bool isMono; static readonly bool isNetCore; static readonly bool isUnity; + static readonly bool isUnityIOS; static readonly bool isXamarin; static readonly bool isXamarinIOS; static readonly bool isXamarinAndroid; @@ -63,7 +64,25 @@ namespace Grpc.Core.Internal isNetCore = false; #endif isMono = Type.GetType("Mono.Runtime") != null; - isUnity = Type.GetType(UnityEngineApplicationClassName) != null; + + // Unity + var unityApplicationClass = Type.GetType(UnityEngineApplicationClassName); + if (unityApplicationClass != null) + { + isUnity = true; + // Consult value of Application.platform via reflection + // https://docs.unity3d.com/ScriptReference/Application-platform.html + var platformProperty = unityApplicationClass.GetTypeInfo().GetProperty("platform"); + var unityRuntimePlatform = platformProperty?.GetValue(null)?.ToString(); + isUnityIOS = (unityRuntimePlatform == "IPhonePlayer"); + } + else + { + isUnity = false; + isUnityIOS = false; + } + + // Xamarin isXamarinIOS = Type.GetType(XamarinIOSObjectClassName) != null; isXamarinAndroid = Type.GetType(XamarinAndroidObjectClassName) != null; isXamarin = isXamarinIOS || isXamarinAndroid; @@ -97,6 +116,14 @@ namespace Grpc.Core.Internal get { return isUnity; } } + /// + /// true if running on Unity iOS, false otherwise. + /// + public static bool IsUnityIOS + { + get { return isUnityIOS; } + } + /// /// true if running on a Xamarin platform (either Xamarin.Android or Xamarin.iOS), /// false otherwise. From 69984f66823e67259f422c8ef425329bd52ce6db Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 11:21:35 +0200 Subject: [PATCH 82/99] C#: avoid shutdown crash on iOS --- src/csharp/Grpc.Core/GrpcEnvironment.cs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs index a6a1d8af50e..db0fbb17ad5 100644 --- a/src/csharp/Grpc.Core/GrpcEnvironment.cs +++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs @@ -50,6 +50,7 @@ namespace Grpc.Core static int requestCallContextPoolThreadLocalCapacity = DefaultRequestCallContextPoolThreadLocalCapacity; static readonly HashSet registeredChannels = new HashSet(); static readonly HashSet registeredServers = new HashSet(); + volatile static bool alreadyInvokedNativeInit; static ILogger logger = new LogLevelFilterLogger(new ConsoleLogger(), LogLevel.Off, true); @@ -360,12 +361,26 @@ namespace Grpc.Core internal static void GrpcNativeInit() { + if (!IsNativeShutdownAllowed && alreadyInvokedNativeInit) + { + // Normally grpc_init and grpc_shutdown calls should come in pairs (C core does reference counting), + // but in case we avoid grpc_shutdown calls altogether, calling grpc_init has no effect + // besides incrementing an internal C core counter that could theoretically overflow. + // NOTE: synchronization not necessary here as we are only trying to avoid calling grpc_init + // so many times that it would causes an overflow, and thus "alreadyInvokedNativeInit" + // being eventually consistent is good enough. + return; + } NativeMethods.Get().grpcsharp_init(); + alreadyInvokedNativeInit = true; } internal static void GrpcNativeShutdown() { - NativeMethods.Get().grpcsharp_shutdown(); + if (IsNativeShutdownAllowed) + { + NativeMethods.Get().grpcsharp_shutdown(); + } } /// @@ -411,6 +426,14 @@ namespace Grpc.Core return GetThreadPoolSizeOrDefault(); } + // On some platforms (specifically iOS), thread local variables in native code + // require initialization/destruction. By skipping the grpc_shutdown() call, + // we avoid a potential crash where grpc_shutdown() has already destroyed + // the thread local variables, but some C core's *_destroy() methods still + // need to run (e.g. they may be run by finalizer thread which is out of our control) + // For more context, see https://github.com/grpc/grpc/issues/16294 + private static bool IsNativeShutdownAllowed => !PlatformApis.IsXamarinIOS && !PlatformApis.IsUnityIOS; + private static class ShutdownHooks { static object staticLock = new object(); From 152c413224a73be89fe83f4de3adf5530566b331 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 15:57:52 +0200 Subject: [PATCH 83/99] Improvements to ssl-performance.md --- doc/ssl-performance.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/doc/ssl-performance.md b/doc/ssl-performance.md index 2604e26a808..176c8d8f247 100644 --- a/doc/ssl-performance.md +++ b/doc/ssl-performance.md @@ -2,28 +2,35 @@ The SSL requirement of gRPC isn't necessarily making it easy to integrate. The HTTP/2 protocol requires ALPN support, which is a fairly new handshake protocol only supported by recent implementations. -As a result, we've tried hard to provide a smooth experience to our users when compiling and distributing gRPC, but this may come at performance costs due to this. More specifically, we will sometime build the SSL library by disabling assembly code, which can impact performances by an order of magnitude when processing encrypted streams. +As a result, we've tried hard to provide a smooth experience to our users when compiling and distributing gRPC, but this may come at performance costs due to this. More specifically, we will sometime build the SSL library by disabling assembly code +(by setting the `OPENSSL_NO_ASM` option), which can impact performance by an order of magnitude when processing encrypted streams. -Build system | Condition | Platform | Uses assembly code +## gRPC C++: Building from Source + +Build system | Condition | Platform | Uses assembly optimizations ---|---|---|-- Makefile | with OpenSSL 1.0.2 development files | all | :heavy_check_mark: Makefile | all other cases | all | :x: Bazel | | Linux | :heavy_check_mark: Bazel | | MacOS | :heavy_check_mark: Bazel | | Windows | :x: -CMake | | Windows | :x: -CMake | | all others | :heavy_check_mark: +CMake | boringssl from submodule (default) | all | :x: +CMake | pre-installed OpenSSL 1.0.2+ (`gRPC_SSL_PROVIDER=package`) | all | :heavy_check_mark: +## Other Languages: Binary/Source Packages In addition, we are shipping packages for language implementations. These packages are source packages, but also have pre-built binaries being distributed. Building packages from source may give a different result in some cases. -Language | From source | Platform | Uses assembly code +Language | From source | Platform | Uses assembly optimizations ---|---|---|--- +C# | n/a | all | :x: Node.JS | n/a | Linux | :heavy_check_mark: Node.JS | n/a | MacOS | :heavy_check_mark: Node.JS | n/a | Windows | :x: Electron | n/a | all | :heavy_check_mark: -Ruby | No | all | :x: +ObjC | Yes | iOS | :x: PHP | Yes | all | Same as the `Makefile` case from above PHP | No | all | :x: Python | n/a | all | :x: +Ruby | No | all | :x: + From 2583dc802a7e80ea87d77a06428196cdc81a6e6d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 16:11:52 +0200 Subject: [PATCH 84/99] regenerate doxygen --- tools/doxygen/Doxyfile.c++ | 1 + tools/doxygen/Doxyfile.c++.internal | 1 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + 4 files changed, 4 insertions(+) diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 2f06bda0168..688c271feac 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -791,6 +791,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index a46ebe61976..592b0b20e65 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -791,6 +791,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 4899eee3ea4..aa75bc68283 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -793,6 +793,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 18f56984fe2..6a6991ec508 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -793,6 +793,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ From c8c71657b04e055ce30354cfd74bca26de20ca13 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 17:14:47 +0200 Subject: [PATCH 85/99] fix mkdir race in build_packages task --- tools/run_tests/dockerize/build_and_run_docker.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index 4ef74085f94..3f01fbc7b7d 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -73,6 +73,10 @@ docker run \ # Copy output artifacts if [ "$OUTPUT_DIR" != "" ] then + # Create the artifact directory in advance to avoid a race in "docker cp" if tasks + # that were running in parallel finish at the same time. + # see https://github.com/grpc/grpc/issues/16155 + mkdir -p "$git_root/$OUTPUT_DIR" docker cp "$CONTAINER_NAME:/var/local/git/grpc/$OUTPUT_DIR" "$git_root" || FAILED="true" fi From cca1dbda74825bc8765c9ca720627860419d2185 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 18:20:44 +0200 Subject: [PATCH 86/99] fix macos interop_to_prod tests --- .../helper_scripts/prepare_build_macos_interop_rc | 11 ----------- tools/internal_ci/macos/grpc_interop_toprod.sh | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc index b0feeef363c..43bc9609c7e 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc @@ -17,17 +17,6 @@ # builds. This rc script must be used in the root directory of gRPC # and is expected to be used before prepare_build_macos_rc -export CONFIG=opt - -# Move gRPC repo to directory that Docker for Mac has drive access to -mkdir /Users/kbuilder/workspace -cp -R ./ /Users/kbuilder/workspace/grpc -cd /Users/kbuilder/workspace/grpc - -# Needed for identifying Docker image sha1 -brew update -brew install md5sha1sum - # Set up gRPC-Go and gRPC-Java to test git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java diff --git a/tools/internal_ci/macos/grpc_interop_toprod.sh b/tools/internal_ci/macos/grpc_interop_toprod.sh index 5ddabb9bf9c..e748a62e761 100755 --- a/tools/internal_ci/macos/grpc_interop_toprod.sh +++ b/tools/internal_ci/macos/grpc_interop_toprod.sh @@ -18,8 +18,8 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc source tools/internal_ci/helper_scripts/prepare_build_macos_rc +source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc # using run_interop_tests.py without --use_docker, so we need to build first tools/run_tests/run_tests.py -l c++ -c opt --build_only From 8e68ab855f6435914575a48887ddccdf789faf72 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 19:29:15 +0200 Subject: [PATCH 87/99] refresh certs in C# distribtest dockerfiles --- tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile | 3 +++ tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile | 4 ++++ tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile | 4 ++++ tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile | 4 ++++ tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile | 4 ++++ 5 files changed, 19 insertions(+) diff --git a/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile index 3e1faafdc04..e32b3cb5e25 100644 --- a/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile @@ -22,3 +22,6 @@ RUN yum install -y mono-devel RUN yum install -y nuget RUN yum install -y unzip + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile index 03fb7a53438..e95d781dfd1 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile index f2fa61a6916..aec936a5b8c 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile index 3edc31e1703..61ca1a08a46 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile @@ -38,3 +38,7 @@ RUN mkdir warmup \ && dotnet new \ && cd .. \ && rm -rf warmup + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile index 1a58f9784bf..93ee75cfcd4 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem From 9e5550a741c9132a68bfb7e93ff992fc0abb57ea Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 19:42:17 +0200 Subject: [PATCH 88/99] remove matt-kwong from OWNERS --- cmake/OWNERS | 2 +- tools/dockerfile/OWNERS | 2 +- tools/run_tests/performance/OWNERS | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/OWNERS b/cmake/OWNERS index 530a941c466..21981a7c555 100644 --- a/cmake/OWNERS +++ b/cmake/OWNERS @@ -1,4 +1,4 @@ set noparent @jtattermusch @nicolasnoble -@matt-kwong +@mehrdada diff --git a/tools/dockerfile/OWNERS b/tools/dockerfile/OWNERS index db4ab546a6d..5f0ad58d617 100644 --- a/tools/dockerfile/OWNERS +++ b/tools/dockerfile/OWNERS @@ -7,5 +7,5 @@ set noparent # for kokoro to be able to access the pre-built images. @jtattermusch -@matt-kwong +@mehrdada @nicolasnoble diff --git a/tools/run_tests/performance/OWNERS b/tools/run_tests/performance/OWNERS index fc1d6eec9c7..9cf8c131111 100644 --- a/tools/run_tests/performance/OWNERS +++ b/tools/run_tests/performance/OWNERS @@ -5,5 +5,5 @@ set noparent # to update the BigQuery schema @ncteisen -@matt-kwong +@apolcyn @jtattermusch From e08fa3379667a1fa3933486b7f98bebd27e5b40f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 19:44:06 +0200 Subject: [PATCH 89/99] regenerate CODEOWNERS --- .github/CODEOWNERS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 12f46d181e6..c28eb974f88 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # repository as the source of truth for module ownership. /**/OWNERS @markdroth @nicolasnoble @a11r /bazel/** @nicolasnoble @dgquintas @a11r @vjpai -/cmake/** @jtattermusch @nicolasnoble @matt-kwong +/cmake/** @jtattermusch @nicolasnoble @mehrdada /src/core/ext/filters/client_channel/** @markdroth @dgquintas @AspirinSJL -/tools/dockerfile/** @jtattermusch @matt-kwong @nicolasnoble -/tools/run_tests/performance/** @ncteisen @matt-kwong @jtattermusch +/tools/dockerfile/** @jtattermusch @mehrdada @nicolasnoble +/tools/run_tests/performance/** @ncteisen @apolcyn @jtattermusch From fbde7d37860fb95c775bc0d2e5d0fb1562f9bde1 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Fri, 10 Aug 2018 11:29:21 -0700 Subject: [PATCH 90/99] No need to build artifact for PHP MacOS --- tools/run_tests/artifacts/artifact_targets.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index a37098d9bf1..ef40c901b27 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -295,10 +295,6 @@ class PHPArtifact: self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( self.arch), 'tools/run_tests/artifacts/build_artifact_php.sh') - else: - return create_jobspec( - self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'], - use_workspace=True) class ProtocArtifact: @@ -400,6 +396,5 @@ def targets(): PythonArtifact('windows', 'x64', 'Python37'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), - PHPArtifact('linux', 'x64'), - PHPArtifact('macos', 'x64') + PHPArtifact('linux', 'x64') ]) From f621eee4cf102482703c188f0bf0ab97c0781175 Mon Sep 17 00:00:00 2001 From: Yihua Zhang Date: Fri, 10 Aug 2018 11:31:15 -0700 Subject: [PATCH 91/99] run cloud-to-prod interop tests with google default credentials --- test/cpp/interop/client_helper.cc | 20 +++---- tools/run_tests/run_interop_tests.py | 80 ++++++++++++++++++++++------ 2 files changed, 74 insertions(+), 26 deletions(-) diff --git a/test/cpp/interop/client_helper.cc b/test/cpp/interop/client_helper.cc index 29b5a1ed6c2..fb7b7bb7d03 100644 --- a/test/cpp/interop/client_helper.cc +++ b/test/cpp/interop/client_helper.cc @@ -88,20 +88,20 @@ std::shared_ptr CreateChannelForTestCase( std::shared_ptr creds; if (test_case == "compute_engine_creds") { - GPR_ASSERT(FLAGS_use_tls); - creds = GoogleComputeEngineCredentials(); - GPR_ASSERT(creds); + creds = FLAGS_custom_credentials_type == "google_default_credentials" + ? nullptr + : GoogleComputeEngineCredentials(); } else if (test_case == "jwt_token_creds") { - GPR_ASSERT(FLAGS_use_tls); grpc::string json_key = GetServiceAccountJsonKey(); std::chrono::seconds token_lifetime = std::chrono::hours(1); - creds = - ServiceAccountJWTAccessCredentials(json_key, token_lifetime.count()); - GPR_ASSERT(creds); + creds = FLAGS_custom_credentials_type == "google_default_credentials" + ? nullptr + : ServiceAccountJWTAccessCredentials(json_key, + token_lifetime.count()); } else if (test_case == "oauth2_auth_token") { - grpc::string raw_token = GetOauth2AccessToken(); - creds = AccessTokenCredentials(raw_token); - GPR_ASSERT(creds); + creds = FLAGS_custom_credentials_type == "google_default_credentials" + ? nullptr + : AccessTokenCredentials(GetOauth2AccessToken()); } if (FLAGS_custom_credentials_type.empty()) { transport_security security_type = diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index aa58107ced9..22055d58e8b 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -637,13 +637,13 @@ _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 'java', 'go', 'python', 'c++' ] -#TODO: Add c++ when c++ ALTS interop client is ready. _LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] -#TODO: Add c++ when c++ ALTS interop server is ready. _SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] -_TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure'] +_TRANSPORT_SECURITY_OPTIONS = [ + 'tls', 'alts', 'google_default_credentials', 'insecure' +] DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' @@ -724,6 +724,9 @@ def auth_options(language, test_case, service_account_key_file=None): key_file_arg = '--service_account_key_file=%s' % service_account_key_file default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' + # TODO: When using google_default_credentials outside of cloud-to-prod, the environment variable + # 'GOOGLE_APPLICATION_CREDENTIALS' needs to be set for the test case + # 'jwt_token_creds' to work. if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: if language in [ 'csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', @@ -763,15 +766,25 @@ def cloud_to_prod_jobspec(language, docker_image=None, auth=False, manual_cmd_log=None, - service_account_key_file=None): + service_account_key_file=None, + transport_security='tls'): """Creates jobspec for cloud-to-prod interop test""" container_name = None cmdargs = [ '--server_host=%s' % server_host, '--server_host_override=%s' % server_host, '--server_port=443', - '--use_tls=true', '--test_case=%s' % test_case ] + if transport_security == 'tls': + transport_security_options += ['--use_tls=true'] + elif transport_security == 'google_default_credentials' and language == 'c++': + transport_security_options += [ + '--custom_credentials_type=google_default_credentials' + ] + else: + print('Invalid transport security option.') + sys.exit(1) + cmdargs = cmdargs + transport_security_options environ = dict(language.cloud_to_prod_env(), **language.global_env()) if auth: auth_cmdargs, auth_env = auth_options(language, test_case, @@ -1285,14 +1298,16 @@ try: jobs = [] if args.cloud_to_prod: - if args.transport_security != 'tls': - print('TLS is always enabled for cloud_to_prod scenarios.') + if args.transport_security not in ['tls', 'google_default_credentials']: + print( + 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' + ) for server_host_nickname in args.prod_servers: for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: - test_job = cloud_to_prod_jobspec( + tls_test_job = cloud_to_prod_jobspec( language, test_case, server_host_nickname, @@ -1300,8 +1315,23 @@ try: docker_image=docker_images.get(str(language)), manual_cmd_log=client_manual_cmd_log, service_account_key_file=args. - service_account_key_file) - jobs.append(test_job) + service_account_key_file, + transport_security='tls') + jobs.append(tls_test_job) + if language == 'c++': + google_default_creds_test_job = cloud_to_prod_jobspec( + language, + test_case, + server_host_nickname, + prod_servers[server_host_nickname], + docker_image=docker_images.get( + str(language)), + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file, + transport_security= + 'google_default_credentials') + jobs.append(google_default_creds_test_job) if args.http2_interop: for test_case in _HTTP2_TEST_CASES: @@ -1312,12 +1342,15 @@ try: prod_servers[server_host_nickname], docker_image=docker_images.get(str(http2Interop)), manual_cmd_log=client_manual_cmd_log, - service_account_key_file=args.service_account_key_file) + service_account_key_file=args.service_account_key_file, + transport_security=args.transport_security) jobs.append(test_job) if args.cloud_to_prod_auth: - if args.transport_security != 'tls': - print('TLS is always enabled for cloud_to_prod scenarios.') + if args.transport_security not in ['tls', 'google_default_credentials']: + print( + 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' + ) for server_host_nickname in args.prod_servers: for language in languages: for test_case in _AUTH_TEST_CASES: @@ -1325,7 +1358,7 @@ try: not compute_engine_creds_required( language, test_case)): if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_prod_jobspec( + tls_test_job = cloud_to_prod_jobspec( language, test_case, server_host_nickname, @@ -1334,8 +1367,23 @@ try: auth=True, manual_cmd_log=client_manual_cmd_log, service_account_key_file=args. - service_account_key_file) - jobs.append(test_job) + service_account_key_file, + transport_security='tls') + jobs.append(tls_test_job) + if language == 'c++': + google_default_creds_test_job = cloud_to_prod_jobspec( + language, + test_case, + server_host_nickname, + prod_servers[server_host_nickname], + docker_image=docker_images.get( + str(language)), + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file, + transport_security= + 'google_default_credentials') + jobs.append(google_default_creds_test_job) for server in args.override_server: server_name = server[0] From 537503dc1a08171271f012d9b48770975d2abd1b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 22:28:55 +0200 Subject: [PATCH 92/99] C#: upgrade protobuf nugets to v3.6.1 --- src/csharp/Grpc.Core/Version.csproj.include | 2 +- templates/src/csharp/Grpc.Core/Version.csproj.include.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 6b0731eb401..45bd8ebd855 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -2,6 +2,6 @@ 1.15.0-dev - 3.6.0 + 3.6.1 diff --git a/templates/src/csharp/Grpc.Core/Version.csproj.include.template b/templates/src/csharp/Grpc.Core/Version.csproj.include.template index a950b1615d2..0ec0a08c499 100755 --- a/templates/src/csharp/Grpc.Core/Version.csproj.include.template +++ b/templates/src/csharp/Grpc.Core/Version.csproj.include.template @@ -4,6 +4,6 @@ ${settings.csharp_version} - 3.6.0 + 3.6.1 From 54e7b24b9a167cea766b03c8e559075ee7809927 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Fri, 10 Aug 2018 13:44:38 -0700 Subject: [PATCH 93/99] Remove if clause too --- tools/run_tests/artifacts/artifact_targets.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index ef40c901b27..34846202b86 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -290,11 +290,10 @@ class PHPArtifact: return [] def build_jobspec(self): - if self.platform == 'linux': - return create_docker_jobspec( - self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( - self.arch), - 'tools/run_tests/artifacts/build_artifact_php.sh') + return create_docker_jobspec( + self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( + self.arch), + 'tools/run_tests/artifacts/build_artifact_php.sh') class ProtocArtifact: From 9993869d042c59b79bac11c45f4c58ace5dbeccd Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Fri, 10 Aug 2018 14:37:09 -0700 Subject: [PATCH 94/99] lint fix --- tools/run_tests/artifacts/artifact_targets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 34846202b86..bdeb258e1fc 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -292,8 +292,7 @@ class PHPArtifact: def build_jobspec(self): return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( - self.arch), - 'tools/run_tests/artifacts/build_artifact_php.sh') + self.arch), 'tools/run_tests/artifacts/build_artifact_php.sh') class ProtocArtifact: From b7b1e77943e7ef59c987616516eaa21171e7f665 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Fri, 10 Aug 2018 14:38:33 -0700 Subject: [PATCH 95/99] Fix minor typo in documentation --- include/grpcpp/impl/codegen/async_generic_service.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/grpcpp/impl/codegen/async_generic_service.h b/include/grpcpp/impl/codegen/async_generic_service.h index 957bb776f1d..72e8178b764 100644 --- a/include/grpcpp/impl/codegen/async_generic_service.h +++ b/include/grpcpp/impl/codegen/async_generic_service.h @@ -52,7 +52,7 @@ class GenericServerContext final : public ServerContext { // ServerBuilder builder; // auto cq = builder.AddCompletionQueue(); // AsyncGenericService generic_service; -// builder.RegisterAsyncGeneicService(&generic_service); +// builder.RegisterAsyncGenericService(&generic_service); // auto server = builder.BuildAndStart(); // // // request a new call From 151e30567b407381af5134c9f5c3d782bf80c3e2 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 10 Aug 2018 14:43:12 -0700 Subject: [PATCH 96/99] Add newline to end of header --- src/core/lib/security/security_connector/load_system_roots.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/security/security_connector/load_system_roots.h b/src/core/lib/security/security_connector/load_system_roots.h index 8d4af5b2c6c..caeadb004f5 100644 --- a/src/core/lib/security/security_connector/load_system_roots.h +++ b/src/core/lib/security/security_connector/load_system_roots.h @@ -26,4 +26,5 @@ grpc_slice LoadSystemRootCerts(); } // namespace grpc_core -#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H */ \ No newline at end of file +#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H */ + From 1b59cc8606026b231394ef648885ef3650d165fe Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 10 Aug 2018 23:49:37 +0200 Subject: [PATCH 97/99] use AtomicCounter to avoid unnecessary grpc_init invocations --- src/csharp/Grpc.Core/GrpcEnvironment.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs index db0fbb17ad5..6ca694e0e46 100644 --- a/src/csharp/Grpc.Core/GrpcEnvironment.cs +++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs @@ -50,7 +50,7 @@ namespace Grpc.Core static int requestCallContextPoolThreadLocalCapacity = DefaultRequestCallContextPoolThreadLocalCapacity; static readonly HashSet registeredChannels = new HashSet(); static readonly HashSet registeredServers = new HashSet(); - volatile static bool alreadyInvokedNativeInit; + static readonly AtomicCounter nativeInitCounter = new AtomicCounter(); static ILogger logger = new LogLevelFilterLogger(new ConsoleLogger(), LogLevel.Off, true); @@ -361,18 +361,17 @@ namespace Grpc.Core internal static void GrpcNativeInit() { - if (!IsNativeShutdownAllowed && alreadyInvokedNativeInit) + if (!IsNativeShutdownAllowed && nativeInitCounter.Count > 0) { // Normally grpc_init and grpc_shutdown calls should come in pairs (C core does reference counting), // but in case we avoid grpc_shutdown calls altogether, calling grpc_init has no effect // besides incrementing an internal C core counter that could theoretically overflow. - // NOTE: synchronization not necessary here as we are only trying to avoid calling grpc_init - // so many times that it would causes an overflow, and thus "alreadyInvokedNativeInit" - // being eventually consistent is good enough. + // To avoid this theoretical possibility we guard repeated calls to grpc_init() + // with a 64-bit atomic counter (that can't realistically overflow). return; } NativeMethods.Get().grpcsharp_init(); - alreadyInvokedNativeInit = true; + nativeInitCounter.Increment(); } internal static void GrpcNativeShutdown() From d7a7c71849fd4c8b4ae41eb88ecaa3f8cb86bfa3 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 10 Aug 2018 15:04:32 -0700 Subject: [PATCH 98/99] clang_format --- src/core/lib/security/security_connector/load_system_roots.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/lib/security/security_connector/load_system_roots.h b/src/core/lib/security/security_connector/load_system_roots.h index caeadb004f5..5fdec154988 100644 --- a/src/core/lib/security/security_connector/load_system_roots.h +++ b/src/core/lib/security/security_connector/load_system_roots.h @@ -27,4 +27,3 @@ grpc_slice LoadSystemRootCerts(); } // namespace grpc_core #endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H */ - From e58d400ba2dcd8a0df41159bba3240ee96bb6d3f Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Fri, 10 Aug 2018 17:13:42 -0700 Subject: [PATCH 99/99] Fix type reference in documentation --- include/grpcpp/impl/codegen/async_generic_service.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/grpcpp/impl/codegen/async_generic_service.h b/include/grpcpp/impl/codegen/async_generic_service.h index 72e8178b764..2a0e1b40881 100644 --- a/include/grpcpp/impl/codegen/async_generic_service.h +++ b/include/grpcpp/impl/codegen/async_generic_service.h @@ -57,7 +57,7 @@ class GenericServerContext final : public ServerContext { // // // request a new call // GenericServerContext context; -// GenericAsyncReaderWriter stream; +// GenericServerAsyncReaderWriter stream; // generic_service.RequestCall(&context, &stream, cq.get(), cq.get(), tag); // // When tag is retrieved from cq->Next(), context.method() can be used to look