From 3a45e340289b351b0f33ef0000dcf55055e32791 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 30 Apr 2018 17:51:37 -0700 Subject: [PATCH 01/45] Update Node examples to use new @grpc/proto-loader package --- examples/node/dynamic_codegen/greeter_client.js | 11 ++++++++++- examples/node/dynamic_codegen/greeter_server.js | 11 ++++++++++- .../dynamic_codegen/route_guide/route_guide_client.js | 11 ++++++++++- .../dynamic_codegen/route_guide/route_guide_server.js | 11 ++++++++++- examples/node/package.json | 3 ++- 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/examples/node/dynamic_codegen/greeter_client.js b/examples/node/dynamic_codegen/greeter_client.js index 9fd1f88f4fb..c289f501ee1 100644 --- a/examples/node/dynamic_codegen/greeter_client.js +++ b/examples/node/dynamic_codegen/greeter_client.js @@ -19,7 +19,16 @@ var PROTO_PATH = __dirname + '/../../protos/helloworld.proto'; var grpc = require('grpc'); -var hello_proto = grpc.load(PROTO_PATH).helloworld; +var protoLoader = require('@grpc/proto-loader'); +var packageDefinition = protoLoader.loadSync( + PROTO_PATH, + {keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true + }); +var hello_proto = grpc.loadPackageDefinition(packageDefinition).helloworld; function main() { var client = new hello_proto.Greeter('localhost:50051', diff --git a/examples/node/dynamic_codegen/greeter_server.js b/examples/node/dynamic_codegen/greeter_server.js index 180f96c28b8..023257ecf08 100644 --- a/examples/node/dynamic_codegen/greeter_server.js +++ b/examples/node/dynamic_codegen/greeter_server.js @@ -19,7 +19,16 @@ var PROTO_PATH = __dirname + '/../../protos/helloworld.proto'; var grpc = require('grpc'); -var hello_proto = grpc.load(PROTO_PATH).helloworld; +var protoLoader = require('@grpc/proto-loader'); +var packageDefinition = protoLoader.loadSync( + PROTO_PATH, + {keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true + }); +var hello_proto = grpc.loadPackageDefinition(packageDefinition).helloworld; /** * Implements the SayHello RPC method. diff --git a/examples/node/dynamic_codegen/route_guide/route_guide_client.js b/examples/node/dynamic_codegen/route_guide/route_guide_client.js index 703cfd29028..34a44bd6a9e 100644 --- a/examples/node/dynamic_codegen/route_guide/route_guide_client.js +++ b/examples/node/dynamic_codegen/route_guide/route_guide_client.js @@ -24,7 +24,16 @@ var parseArgs = require('minimist'); var path = require('path'); var _ = require('lodash'); var grpc = require('grpc'); -var routeguide = grpc.load(PROTO_PATH).routeguide; +var protoLoader = require('@grpc/proto-loader'); +var packageDefinition = protoLoader.loadSync( + PROTO_PATH, + {keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true + }); +var routeguide = grpc.loadPackageDefinition(packageDefinition).routeguide; var client = new routeguide.RouteGuide('localhost:50051', grpc.credentials.createInsecure()); diff --git a/examples/node/dynamic_codegen/route_guide/route_guide_server.js b/examples/node/dynamic_codegen/route_guide/route_guide_server.js index 3819c092eb5..69fbc3c8840 100644 --- a/examples/node/dynamic_codegen/route_guide/route_guide_server.js +++ b/examples/node/dynamic_codegen/route_guide/route_guide_server.js @@ -23,7 +23,16 @@ var parseArgs = require('minimist'); var path = require('path'); var _ = require('lodash'); var grpc = require('grpc'); -var routeguide = grpc.load(PROTO_PATH).routeguide; +var protoLoader = require('@grpc/proto-loader'); +var packageDefinition = protoLoader.loadSync( + PROTO_PATH, + {keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true + }); +var routeguide = grpc.load(packageDefinition).routeguide; var COORD_FACTOR = 1e7; diff --git a/examples/node/package.json b/examples/node/package.json index 6317838295a..3af6a113fd5 100644 --- a/examples/node/package.json +++ b/examples/node/package.json @@ -2,9 +2,10 @@ "name": "grpc-examples", "version": "0.1.0", "dependencies": { + "@grpc/proto-loader": "^0.1.0", "async": "^1.5.2", "google-protobuf": "^3.0.0", - "grpc": "^1.0.0", + "grpc": "^1.11.0", "lodash": "^4.6.1", "minimist": "^1.2.0" } From 08ff4c1f69ad755e499fcf15417bc04282012593 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 1 May 2018 09:41:32 -0700 Subject: [PATCH 02/45] Fix one partially changed line in route guide examples --- examples/node/dynamic_codegen/route_guide/route_guide_server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/node/dynamic_codegen/route_guide/route_guide_server.js b/examples/node/dynamic_codegen/route_guide/route_guide_server.js index 69fbc3c8840..106491c1a10 100644 --- a/examples/node/dynamic_codegen/route_guide/route_guide_server.js +++ b/examples/node/dynamic_codegen/route_guide/route_guide_server.js @@ -32,7 +32,7 @@ var packageDefinition = protoLoader.loadSync( defaults: true, oneofs: true }); -var routeguide = grpc.load(packageDefinition).routeguide; +var routeguide = grpc.loadPackageDefinition(packageDefinition).routeguide; var COORD_FACTOR = 1e7; From 67bb4e30302cec45c9e05144a64ee6a38c0f9559 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 11 Jun 2018 08:54:30 -0700 Subject: [PATCH 03/45] Initial scaffolding --- include/grpc/grpc.h | 4 ++++ include/grpcpp/resource_quota.h | 10 ++++++++++ src/core/lib/iomgr/resource_quota.cc | 27 +++++++++++++++++++++++++++ src/core/lib/iomgr/resource_quota.h | 16 ++++++++++++++++ src/cpp/common/resource_quota_cc.cc | 4 ++++ 5 files changed, 61 insertions(+) diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index c129a66949f..bc3bc5fbbf6 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -450,6 +450,10 @@ GRPCAPI void grpc_resource_quota_unref(grpc_resource_quota* resource_quota); GRPCAPI void grpc_resource_quota_resize(grpc_resource_quota* resource_quota, size_t new_size); +/** Update the size of the maximum number of threads allowed */ +GRPCAPI void grpc_resource_quota_set_max_threads( + grpc_resource_quota* resource_quota, int new_max_threads); + /** Fetch a vtable for a grpc_channel_arg that points to a grpc_resource_quota */ GRPCAPI const grpc_arg_pointer_vtable* grpc_resource_quota_arg_vtable(void); diff --git a/include/grpcpp/resource_quota.h b/include/grpcpp/resource_quota.h index 554437a40d1..77cdd48dcc4 100644 --- a/include/grpcpp/resource_quota.h +++ b/include/grpcpp/resource_quota.h @@ -44,6 +44,16 @@ class ResourceQuota final : private GrpcLibraryCodegen { /// No time bound is given for this to occur however. ResourceQuota& Resize(size_t new_size); + /// Set the max number of threads that can be allocated from this + /// ResourceQuota object. + /// + /// If the new_max_threads value is smaller than the current value, no new + /// threads are allocated until the number of active threads fall below + /// new_max_threads. There is no time bound on when this may happen i.e none + /// of the current threads are forcefully destroyed and all threads run their + /// normal course. + ResourceQuota& SetMaxThreads(int new_max_threads); + grpc_resource_quota* c_resource_quota() const { return impl_; } private: diff --git a/src/core/lib/iomgr/resource_quota.cc b/src/core/lib/iomgr/resource_quota.cc index 539bc120cec..b50b2f2e468 100644 --- a/src/core/lib/iomgr/resource_quota.cc +++ b/src/core/lib/iomgr/resource_quota.cc @@ -96,6 +96,9 @@ struct grpc_resource_user { list, false otherwise */ bool added_to_free_pool; + /* The number of threads currently allocated to this resource user */ + gpr_atm num_threads; + /* Reclaimers: index 0 is the benign reclaimer, 1 is the destructive reclaimer */ grpc_closure* reclaimers[2]; @@ -135,12 +138,21 @@ struct grpc_resource_quota { gpr_atm last_size; + /* Max number of threads allowed */ + int max_threads; + + /* Number of threads currently allocated via this resource_quota object */ + gpr_atm num_threads; + /* Has rq_step been scheduled to occur? */ bool step_scheduled; + /* Are we currently reclaiming memory */ bool reclaiming; + /* Closure around rq_step */ grpc_closure rq_step_closure; + /* Closure around rq_reclamation_done */ grpc_closure rq_reclamation_done_closure; @@ -594,6 +606,8 @@ grpc_resource_quota* grpc_resource_quota_create(const char* name) { resource_quota->free_pool = INT64_MAX; resource_quota->size = INT64_MAX; gpr_atm_no_barrier_store(&resource_quota->last_size, GPR_ATM_MAX); + resource_quota->max_threads = INT_MAX; + gpr_atm_no_barrier_store(&resource_quota->num_threads, 0); resource_quota->step_scheduled = false; resource_quota->reclaiming = false; gpr_atm_no_barrier_store(&resource_quota->memory_usage_estimation, 0); @@ -646,6 +660,10 @@ double grpc_resource_quota_get_memory_pressure( (static_cast(MEMORY_USAGE_ESTIMATION_MAX)); } +/* Public API */ +void grpc_resource_quota_set_max_threads(grpc_resource_quota* resource_quota, + int new_max_threads) {} + /* Public API */ void grpc_resource_quota_resize(grpc_resource_quota* resource_quota, size_t size) { @@ -731,6 +749,7 @@ grpc_resource_user* grpc_resource_user_create( grpc_closure_list_init(&resource_user->on_allocated); resource_user->allocating = false; resource_user->added_to_free_pool = false; + gpr_atm_no_barrier_store(&resource_user->num_threads, 0); resource_user->reclaimers[0] = nullptr; resource_user->reclaimers[1] = nullptr; resource_user->new_reclaimers[0] = nullptr; @@ -785,6 +804,14 @@ void grpc_resource_user_shutdown(grpc_resource_user* resource_user) { } } +bool grpc_resource_user_alloc_threads(grpc_resource_user* resource_user, + int thd_count) { + return true; +} + +void grpc_resource_user_free_threads(grpc_resource_user* resource_user, + int thd_count) {} + void grpc_resource_user_alloc(grpc_resource_user* resource_user, size_t size, grpc_closure* optional_on_done) { gpr_mu_lock(&resource_user->mu); diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index 937daf87282..a111ebb4d89 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -93,6 +93,22 @@ void grpc_resource_user_ref(grpc_resource_user* resource_user); void grpc_resource_user_unref(grpc_resource_user* resource_user); void grpc_resource_user_shutdown(grpc_resource_user* resource_user); +/* Attempts to get quota (from the resource_user) to create 'thd_count' number + * of threads. Returns true if successful (i.e the caller is now free to create + * 'thd_count' number of threads or false if quota is not available */ +bool grpc_resource_user_alloc_threads(grpc_resource_user* resource_user, + int thd_count); +/* Releases 'thd_count' worth of quota back to the resource user. The quota + * should have been previously obtained successfully by calling + * grpc_resource_user_alloc_threads(). + * + * Note: There need not be an exact one-to-one correspondence between + * grpc_resource_user_alloc_threads() and grpc_resource_user_free_threads() + * calls. The only requirement is that the number of threads allocated should + * all be eventually released */ +void grpc_resource_user_free_threads(grpc_resource_user* resource_user, + int thd_count); + /* Allocate from the resource user (and its quota). If optional_on_done is NULL, then allocate immediately. This may push the quota over-limit, at which point reclamation will kick in. diff --git a/src/cpp/common/resource_quota_cc.cc b/src/cpp/common/resource_quota_cc.cc index daeb0ba171c..276e5f79548 100644 --- a/src/cpp/common/resource_quota_cc.cc +++ b/src/cpp/common/resource_quota_cc.cc @@ -33,4 +33,8 @@ ResourceQuota& ResourceQuota::Resize(size_t new_size) { return *this; } +ResourceQuota& ResourceQuota::SetMaxThreads(int new_max_threads) { + grpc_resource_quota_set_max_threads(impl_, new_max_threads); + return *this; +} } // namespace grpc From 913f9b930a7fb6a5377c1b5e15ec47f5645828e7 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 12 Jun 2018 16:17:54 -0700 Subject: [PATCH 04/45] Add Core resource quota implementation --- include/grpcpp/resource_quota.h | 6 ++-- src/core/lib/iomgr/resource_quota.cc | 48 +++++++++++++++++++++++++--- src/core/lib/iomgr/resource_quota.h | 2 +- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/include/grpcpp/resource_quota.h b/include/grpcpp/resource_quota.h index 77cdd48dcc4..50bd1cb849a 100644 --- a/include/grpcpp/resource_quota.h +++ b/include/grpcpp/resource_quota.h @@ -26,10 +26,10 @@ struct grpc_resource_quota; namespace grpc { -/// ResourceQuota represents a bound on memory usage by the gRPC library. -/// A ResourceQuota can be attached to a server (via \a ServerBuilder), +/// ResourceQuota represents a bound on memory and thread usage by the gRPC +/// library. A ResourceQuota can be attached to a server (via \a ServerBuilder), /// or a client channel (via \a ChannelArguments). -/// gRPC will attempt to keep memory used by all attached entities +/// gRPC will attempt to keep memory and threads used by all attached entities /// below the ResourceQuota bound. class ResourceQuota final : private GrpcLibraryCodegen { public: diff --git a/src/core/lib/iomgr/resource_quota.cc b/src/core/lib/iomgr/resource_quota.cc index b50b2f2e468..a30688bd873 100644 --- a/src/core/lib/iomgr/resource_quota.cc +++ b/src/core/lib/iomgr/resource_quota.cc @@ -138,11 +138,22 @@ struct grpc_resource_quota { gpr_atm last_size; + /* Mutex to protect max_threads and num_threads */ + /* Note: We could have used gpr_atm for max_threads and num_threads and avoid + * having this mutex; but in that case, each invocation of the function + * grpc_resource_user_alloc_threads() will have to do atleast two atomic loads + * (for max_threads and num_threads) followed by a CAS (on num_threads). + * Moreover, we expect grpc_resource_user_alloc_threads() to be often called + * concurrently thereby increasing the chances of failing the CAS operation. + * This additional complexity is not worth the tiny perf gain we may (or may + * not) have by using atomics */ + gpr_mu thd_mu; + /* Max number of threads allowed */ int max_threads; /* Number of threads currently allocated via this resource_quota object */ - gpr_atm num_threads; + int num_threads; /* Has rq_step been scheduled to occur? */ bool step_scheduled; @@ -606,8 +617,9 @@ grpc_resource_quota* grpc_resource_quota_create(const char* name) { resource_quota->free_pool = INT64_MAX; resource_quota->size = INT64_MAX; gpr_atm_no_barrier_store(&resource_quota->last_size, GPR_ATM_MAX); + gpr_mu_init(&resource_quota->thd_mu); resource_quota->max_threads = INT_MAX; - gpr_atm_no_barrier_store(&resource_quota->num_threads, 0); + resource_quota->num_threads = 0; resource_quota->step_scheduled = false; resource_quota->reclaiming = false; gpr_atm_no_barrier_store(&resource_quota->memory_usage_estimation, 0); @@ -662,7 +674,11 @@ double grpc_resource_quota_get_memory_pressure( /* Public API */ void grpc_resource_quota_set_max_threads(grpc_resource_quota* resource_quota, - int new_max_threads) {} + int new_max_threads) { + gpr_mu_lock(&resource_quota->thd_mu); + resource_quota->max_threads = new_max_threads; + gpr_mu_unlock(&resource_quota->thd_mu); +} /* Public API */ void grpc_resource_quota_resize(grpc_resource_quota* resource_quota, @@ -806,11 +822,33 @@ void grpc_resource_user_shutdown(grpc_resource_user* resource_user) { bool grpc_resource_user_alloc_threads(grpc_resource_user* resource_user, int thd_count) { - return true; + bool is_success = false; + gpr_mu_lock(&resource_user->resource_quota->thd_mu); + grpc_resource_quota* rq = resource_user->resource_quota; + if (rq->num_threads + thd_count <= rq->max_threads) { + rq->num_threads += thd_count; + gpr_atm_no_barrier_fetch_add(&resource_user->num_threads, thd_count); + is_success = true; + } + gpr_mu_unlock(&resource_user->resource_quota->thd_mu); + return is_success; } void grpc_resource_user_free_threads(grpc_resource_user* resource_user, - int thd_count) {} + int thd_count) { + gpr_mu_lock(&resource_user->resource_quota->thd_mu); + grpc_resource_quota* rq = resource_user->resource_quota; + rq->num_threads -= thd_count; + int old_cnt = static_cast( + gpr_atm_no_barrier_fetch_add(&resource_user->num_threads, -thd_count)); + if (old_cnt < thd_count || rq->num_threads < 0) { + gpr_log(GPR_ERROR, + "Releasing more threads (%d) that currently allocated (rq threads: " + "%d, ru threads: %d)", + thd_count, old_cnt, rq->num_threads + thd_count); + } + gpr_mu_unlock(&resource_user->resource_quota->thd_mu); +} void grpc_resource_user_alloc(grpc_resource_user* resource_user, size_t size, grpc_closure* optional_on_done) { diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index a111ebb4d89..7342ef84c84 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -95,7 +95,7 @@ void grpc_resource_user_shutdown(grpc_resource_user* resource_user); /* Attempts to get quota (from the resource_user) to create 'thd_count' number * of threads. Returns true if successful (i.e the caller is now free to create - * 'thd_count' number of threads or false if quota is not available */ + * 'thd_count' number of threads) or false if quota is not available */ bool grpc_resource_user_alloc_threads(grpc_resource_user* resource_user, int thd_count); /* Releases 'thd_count' worth of quota back to the resource user. The quota From 3ed81c8d652f8c904fceff99ec16b559232c467f Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Thu, 19 Jul 2018 18:30:27 +0200 Subject: [PATCH 05/45] Introduce --ssl_target flag to grpc_cli. This allows the client to specify the host name of the remote server for the purposes of TLS certificate validation, useful for test certificates and machines with ephemeral IP addresses with no associated DNS entries that have been assigned TLS certificates. --- test/cpp/util/cli_credentials.cc | 12 +++++++++++- test/cpp/util/cli_credentials.h | 1 + test/cpp/util/grpc_tool.cc | 17 +++++++++++++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index aa4eafb7569..1f047ca3daa 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -25,6 +25,10 @@ DEFINE_bool(use_auth, false, "Whether to create default google credentials."); DEFINE_string( access_token, "", "The access token that will be sent to the server to authenticate RPCs."); +DEFINE_string( + ssl_target, "", + "If not empty, treat the server host name as this for ssl/tls certificate " + "validation."); namespace grpc { namespace testing { @@ -58,7 +62,13 @@ const grpc::string CliCredentials::GetCredentialUsage() const { " --use_auth ; Set whether to create default google" " credentials\n" " --access_token ; Set the access token in metadata," - " overrides --use_auth\n"; + " overrides --use_auth\n" + " --ssl_target ; Set server host for tls validation\n"; +} + +const grpc::string CliCredentials::GetSslTargetNameOverride() const { + return FLAGS_enable_ssl ? FLAGS_ssl_target : ""; } + } // namespace testing } // namespace grpc diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index b1358e77d8b..8d662356de8 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -30,6 +30,7 @@ class CliCredentials { virtual ~CliCredentials() {} virtual std::shared_ptr GetCredentials() const; virtual const grpc::string GetCredentialUsage() const; + virtual const grpc::string GetSslTargetNameOverride() const; }; } // namespace testing diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index 840ca07d2bf..d21e8a30b8c 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -206,6 +206,15 @@ void ReadResponse(CliCall* call, const grpc::string& method_name, } } +std::shared_ptr CreateCliChannel( + grpc::string server_address, const CliCredentials& cred) { + grpc::ChannelArguments args; + if (!cred.GetSslTargetNameOverride().empty()) { + args.SetSslTargetNameOverride(cred.GetSslTargetNameOverride()); + } + return grpc::CreateCustomChannel(server_address, cred.GetCredentials(), args); +} + struct Command { const char* command; std::function channel = - grpc::CreateChannel(server_address, cred.GetCredentials()); + CreateCliChannel(server_address, cred); grpc::ProtoReflectionDescriptorDatabase desc_db(channel); grpc::protobuf::DescriptorPool desc_pool(&desc_db); @@ -422,7 +431,7 @@ bool GrpcTool::PrintType(int argc, const char** argv, grpc::string server_address(argv[0]); std::shared_ptr channel = - grpc::CreateChannel(server_address, cred.GetCredentials()); + CreateCliChannel(server_address, cred); grpc::ProtoReflectionDescriptorDatabase desc_db(channel); grpc::protobuf::DescriptorPool desc_pool(&desc_db); @@ -469,7 +478,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, bool print_mode = false; std::shared_ptr channel = - grpc::CreateChannel(server_address, cred.GetCredentials()); + CreateCliChannel(server_address, cred); if (!FLAGS_binary_input || !FLAGS_binary_output) { parser.reset( @@ -820,7 +829,7 @@ bool GrpcTool::ParseMessage(int argc, const char** argv, if (!FLAGS_binary_input || !FLAGS_binary_output) { std::shared_ptr channel = - grpc::CreateChannel(server_address, cred.GetCredentials()); + CreateCliChannel(server_address, cred); parser.reset( new grpc::testing::ProtoFileParser(FLAGS_remotedb ? channel : nullptr, FLAGS_proto_path, FLAGS_protofiles)); From 8c7ce557f8479fd0af421c2b3707905259293e04 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Thu, 19 Jul 2018 18:37:51 +0200 Subject: [PATCH 06/45] Add simple test for --ssl_target. This cribs from test_credentials_provider.cc, but without using `grpc::ChannelArguments`, as the grpc CLI is responsible for setting that according to the flag. --- test/cpp/util/BUILD | 1 + test/cpp/util/grpc_tool_test.cc | 51 +++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD index 9b42bb28b1d..c3bfeb76156 100644 --- a/test/cpp/util/BUILD +++ b/test/cpp/util/BUILD @@ -177,6 +177,7 @@ grpc_cc_test( "//:grpc++_reflection", "//src/proto/grpc/testing:echo_messages_proto", "//src/proto/grpc/testing:echo_proto", + "//test/core/end2end:ssl_test_data", "//test/core/util:grpc_test_util", ], ) diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 6574d1bb441..b741b50567a 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -35,6 +35,7 @@ #include "src/core/lib/gpr/env.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "src/proto/grpc/testing/echo.pb.h" +#include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" #include "test/cpp/util/cli_credentials.h" @@ -80,6 +81,9 @@ using grpc::testing::EchoResponse; " peer: \"peer\"\n" \ "}\n\n" +DECLARE_bool(enable_ssl); +DECLARE_string(ssl_target); + namespace grpc { namespace testing { @@ -97,10 +101,17 @@ const int kServerDefaultResponseStreamsToSend = 3; class TestCliCredentials final : public grpc::testing::CliCredentials { public: + TestCliCredentials(bool secure = false) : secure_(secure) {} std::shared_ptr GetCredentials() const override { - return InsecureChannelCredentials(); + if (!secure_) { + return InsecureChannelCredentials(); + } + SslCredentialsOptions ssl_opts = {test_root_cert, "", ""}; + return SslCredentials(grpc::SslCredentialsOptions(ssl_opts)); } const grpc::string GetCredentialUsage() const override { return ""; } + private: + const bool secure_; }; bool PrintStream(std::stringstream* ss, const grpc::string& output) { @@ -206,13 +217,24 @@ class GrpcToolTest : public ::testing::Test { // SetUpServer cannot be used with EXPECT_EXIT. grpc_pick_unused_port_or_die() // uses atexit() to free chosen ports, and it will spawn a new thread in // resolve_address_posix.c:192 at exit time. - const grpc::string SetUpServer() { + const grpc::string SetUpServer(bool secure = false) { std::ostringstream server_address; int port = grpc_pick_unused_port_or_die(); server_address << "localhost:" << port; // Setup server ServerBuilder builder; - builder.AddListeningPort(server_address.str(), InsecureServerCredentials()); + std::shared_ptr creds; + if (secure) { + SslServerCredentialsOptions::PemKeyCertPair pkcp = {test_server1_key, + test_server1_cert}; + SslServerCredentialsOptions ssl_opts; + ssl_opts.pem_root_certs = ""; + ssl_opts.pem_key_cert_pairs.push_back(pkcp); + creds = SslServerCredentials(ssl_opts); + } else { + creds = InsecureServerCredentials(); + } + builder.AddListeningPort(server_address.str(), creds); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); return server_address.str(); @@ -743,6 +765,29 @@ TEST_F(GrpcToolTest, CallCommandWithBadMetadata) { gpr_free(test_srcdir); } +TEST_F(GrpcToolTest, ListCommand_OverrideSslHostName) { + const grpc::string server_address = SetUpServer(true); + + // Test input "grpc_cli ls localhost: --enable_ssl + // --ssl_target=z.test.google.fr" + std::stringstream output_stream; + const char* argv[] = {"grpc_cli", "ls", server_address.c_str()}; + FLAGS_l = false; + FLAGS_enable_ssl = true; + FLAGS_ssl_target = "z.test.google.fr"; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, + TestCliCredentials(true), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), + "grpc.testing.EchoTestService\n" + "grpc.reflection.v1alpha.ServerReflection\n")); + + FLAGS_enable_ssl = false; + FLAGS_ssl_target = ""; + ShutdownServer(); +} + } // namespace testing } // namespace grpc From e1d94dfe606b32f53f3663dd69cf78db2cb0477b Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Thu, 19 Jul 2018 21:32:11 +0200 Subject: [PATCH 07/45] GoogleDefaultCredentials use SSL under the covers; include that case. --- test/cpp/util/cli_credentials.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index 1f047ca3daa..2494e22abb7 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -67,7 +67,7 @@ const grpc::string CliCredentials::GetCredentialUsage() const { } const grpc::string CliCredentials::GetSslTargetNameOverride() const { - return FLAGS_enable_ssl ? FLAGS_ssl_target : ""; + return (FLAGS_enable_ssl || FLAGS_use_auth) ? FLAGS_ssl_target : ""; } } // namespace testing From c7fae8812f4e44ae3c28aa5b62c2cd1e243d341a Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Thu, 19 Jul 2018 23:45:46 +0200 Subject: [PATCH 08/45] clang tidy --- test/cpp/util/grpc_tool.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index d21e8a30b8c..ccc60cca27a 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -207,7 +207,7 @@ void ReadResponse(CliCall* call, const grpc::string& method_name, } std::shared_ptr CreateCliChannel( - grpc::string server_address, const CliCredentials& cred) { + const grpc::string& server_address, const CliCredentials& cred) { grpc::ChannelArguments args; if (!cred.GetSslTargetNameOverride().empty()) { args.SetSslTargetNameOverride(cred.GetSslTargetNameOverride()); From adb8581456447c1468d161af90d9194c473d3fce Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Thu, 19 Jul 2018 23:49:02 +0200 Subject: [PATCH 09/45] clang format --- test/cpp/util/grpc_tool_test.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index b741b50567a..7e7f44551ef 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -110,6 +110,7 @@ class TestCliCredentials final : public grpc::testing::CliCredentials { return SslCredentials(grpc::SslCredentialsOptions(ssl_opts)); } const grpc::string GetCredentialUsage() const override { return ""; } + private: const bool secure_; }; @@ -775,10 +776,10 @@ TEST_F(GrpcToolTest, ListCommand_OverrideSslHostName) { FLAGS_l = false; FLAGS_enable_ssl = true; FLAGS_ssl_target = "z.test.google.fr"; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, - TestCliCredentials(true), - std::bind(PrintStream, &output_stream, - std::placeholders::_1))); + EXPECT_TRUE( + 0 == GrpcToolMainLib( + ArraySize(argv), argv, TestCliCredentials(true), + std::bind(PrintStream, &output_stream, std::placeholders::_1))); EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), "grpc.testing.EchoTestService\n" "grpc.reflection.v1alpha.ServerReflection\n")); From 6f1604b1eb873f71d72c93a483a9aca56a9f0418 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Fri, 20 Jul 2018 15:23:04 +0200 Subject: [PATCH 10/45] Fixup: Use a more exact logic for google default credentials. --- test/cpp/util/cli_credentials.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index 2494e22abb7..68eb0b17ce8 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -67,7 +67,9 @@ const grpc::string CliCredentials::GetCredentialUsage() const { } const grpc::string CliCredentials::GetSslTargetNameOverride() const { - return (FLAGS_enable_ssl || FLAGS_use_auth) ? FLAGS_ssl_target : ""; + bool use_tls = FLAGS_enable_ssl || + (FLAGS_access_token.empty() && FLAGS_use_auth); + return use_tls ? FLAGS_ssl_target : ""; } } // namespace testing From ec1c112cc17cd1290a901ca606ac916422d3342c Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 20 Jul 2018 10:28:40 -0700 Subject: [PATCH 11/45] Each ThreadManager is a resource user --- include/grpcpp/server.h | 3 +- src/cpp/server/server_builder.cc | 2 +- src/cpp/server/server_cc.cc | 23 ++++++++---- src/cpp/thread_manager/thread_manager.cc | 36 ++++++++++++++++--- src/cpp/thread_manager/thread_manager.h | 19 +++++++++- .../cpp/thread_manager/thread_manager_test.cc | 10 ++++-- 6 files changed, 76 insertions(+), 17 deletions(-) diff --git a/include/grpcpp/server.h b/include/grpcpp/server.h index 81c3907f86d..189cf8accf7 100644 --- a/include/grpcpp/server.h +++ b/include/grpcpp/server.h @@ -144,7 +144,8 @@ class Server : public ServerInterface, private GrpcLibraryCodegen { Server(int max_message_size, ChannelArguments* args, std::shared_ptr>> sync_server_cqs, - int min_pollers, int max_pollers, int sync_cq_timeout_msec); + grpc_resource_quota* server_rq, int min_pollers, int max_pollers, + int sync_cq_timeout_msec); /// Start the server. /// diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc index e0b9b7a62bd..0ab3cd0e323 100644 --- a/src/cpp/server/server_builder.cc +++ b/src/cpp/server/server_builder.cc @@ -261,7 +261,7 @@ std::unique_ptr ServerBuilder::BuildAndStart() { } std::unique_ptr server(new Server( - max_receive_message_size_, &args, sync_server_cqs, + max_receive_message_size_, &args, sync_server_cqs, resource_quota_, sync_server_settings_.min_pollers, sync_server_settings_.max_pollers, sync_server_settings_.cq_timeout_msec)); diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 0d77510e29d..6e6e0bfffe3 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -266,9 +266,9 @@ class Server::SyncRequestThreadManager : public ThreadManager { public: SyncRequestThreadManager(Server* server, CompletionQueue* server_cq, std::shared_ptr global_callbacks, - int min_pollers, int max_pollers, - int cq_timeout_msec) - : ThreadManager(min_pollers, max_pollers), + grpc_resource_quota* rq, int min_pollers, + int max_pollers, int cq_timeout_msec) + : ThreadManager("SyncServer", rq, min_pollers, max_pollers), server_(server), server_cq_(server_cq), cq_timeout_msec_(cq_timeout_msec), @@ -376,7 +376,8 @@ Server::Server( int max_receive_message_size, ChannelArguments* args, std::shared_ptr>> sync_server_cqs, - int min_pollers, int max_pollers, int sync_cq_timeout_msec) + grpc_resource_quota* server_rq, int min_pollers, int max_pollers, + int sync_cq_timeout_msec) : max_receive_message_size_(max_receive_message_size), sync_server_cqs_(std::move(sync_server_cqs)), started_(false), @@ -392,10 +393,20 @@ Server::Server( global_callbacks_->UpdateArguments(args); if (sync_server_cqs_ != nullptr) { + bool default_rq_created = false; + if (server_rq == nullptr) { + server_rq = grpc_resource_quota_create("SyncServer-Default"); + default_rq_created = true; + } + for (const auto& it : *sync_server_cqs_) { sync_req_mgrs_.emplace_back(new SyncRequestThreadManager( - this, it.get(), global_callbacks_, min_pollers, max_pollers, - sync_cq_timeout_msec)); + this, it.get(), global_callbacks_, server_rq, min_pollers, + max_pollers, sync_cq_timeout_msec)); + } + + if (default_rq_created) { + grpc_resource_quota_unref(server_rq); } } diff --git a/src/cpp/thread_manager/thread_manager.cc b/src/cpp/thread_manager/thread_manager.cc index 02ac56a3fdc..c0fa98798a0 100644 --- a/src/cpp/thread_manager/thread_manager.cc +++ b/src/cpp/thread_manager/thread_manager.cc @@ -48,12 +48,16 @@ ThreadManager::WorkerThread::~WorkerThread() { thd_.Join(); } -ThreadManager::ThreadManager(int min_pollers, int max_pollers) +ThreadManager::ThreadManager(const char* name, + grpc_resource_quota* resource_quota, + int min_pollers, int max_pollers) : shutdown_(false), num_pollers_(0), min_pollers_(min_pollers), max_pollers_(max_pollers == -1 ? INT_MAX : max_pollers), - num_threads_(0) {} + num_threads_(0) { + resource_user_ = grpc_resource_user_create(resource_quota, name); +} ThreadManager::~ThreadManager() { { @@ -61,6 +65,7 @@ ThreadManager::~ThreadManager() { GPR_ASSERT(num_threads_ == 0); } + grpc_resource_user_unref(resource_user_); CleanupCompletedThreads(); } @@ -113,9 +118,27 @@ void ThreadManager::Initialize() { } for (int i = 0; i < min_pollers_; i++) { - // Create a new thread (which ends up calling the MainWorkLoop() function - new WorkerThread(this); + if (!CreateNewThread(this)) { + gpr_log(GPR_ERROR, + "No quota available to create additional threads. Created %d (of " + "%d) threads", + i, min_pollers_); + break; + } + } +} + +bool ThreadManager::CreateNewThread(ThreadManager* thd_mgr) { + if (!grpc_resource_user_alloc_threads(thd_mgr->resource_user_, 1)) { + return false; } + // Create a new thread (which ends up calling the MainWorkLoop() function + new WorkerThread(thd_mgr); + return true; +} + +void ThreadManager::ReleaseThread(ThreadManager* thd_mgr) { + grpc_resource_user_free_threads(thd_mgr->resource_user_, 1); } void ThreadManager::MainWorkLoop() { @@ -146,7 +169,7 @@ void ThreadManager::MainWorkLoop() { num_threads_++; // Drop lock before spawning thread to avoid contention lock.unlock(); - new WorkerThread(this); + CreateNewThread(this); } else { // Drop lock for consistency with above branch lock.unlock(); @@ -196,7 +219,10 @@ void ThreadManager::MainWorkLoop() { } }; + // This thread is exiting. Do some cleanup work (i.e delete already completed + // worker threads and also release 1 thread back to the resource quota) CleanupCompletedThreads(); + ReleaseThread(this); // If we are here, either ThreadManager is shutting down or it already has // enough threads. diff --git a/src/cpp/thread_manager/thread_manager.h b/src/cpp/thread_manager/thread_manager.h index 5a40f2de477..23bd38ee4f1 100644 --- a/src/cpp/thread_manager/thread_manager.h +++ b/src/cpp/thread_manager/thread_manager.h @@ -27,12 +27,14 @@ #include #include "src/core/lib/gprpp/thd.h" +#include "src/core/lib/iomgr/resource_quota.h" namespace grpc { class ThreadManager { public: - explicit ThreadManager(int min_pollers, int max_pollers); + explicit ThreadManager(const char* name, grpc_resource_quota* resource_quota, + int min_pollers, int max_pollers); virtual ~ThreadManager(); // Initializes and Starts the Rpc Manager threads @@ -111,6 +113,13 @@ class ThreadManager { void MarkAsCompleted(WorkerThread* thd); void CleanupCompletedThreads(); + // Checks the resource quota and if available, creates a thread and returns + // true. If quota is not available, returns false (and thread is not created) + static bool CreateNewThread(ThreadManager* thd_mgr); + + // Give back a thread to the resource quota + static void ReleaseThread(ThreadManager* thd_mgr); + // Protects shutdown_, num_pollers_ and num_threads_ // TODO: sreek - Change num_pollers and num_threads_ to atomics std::mutex mu_; @@ -118,6 +127,14 @@ class ThreadManager { bool shutdown_; std::condition_variable shutdown_cv_; + // The resource user object to use when requesting quota to create threads + // + // Note: The user of this ThreadManager object must create grpc_resource_quota + // object (that contains the actual max thread quota) and a grpc_resource_user + // object through which quota is requested whenver new threads need to be + // created + grpc_resource_user* resource_user_; + // Number of threads doing polling int num_pollers_; diff --git a/test/cpp/thread_manager/thread_manager_test.cc b/test/cpp/thread_manager/thread_manager_test.cc index 7a95a9f17da..cf2cf770e6b 100644 --- a/test/cpp/thread_manager/thread_manager_test.cc +++ b/test/cpp/thread_manager/thread_manager_test.cc @@ -32,8 +32,8 @@ namespace grpc { class ThreadManagerTest final : public grpc::ThreadManager { public: - ThreadManagerTest() - : ThreadManager(kMinPollers, kMaxPollers), + ThreadManagerTest(const char* name, grpc_resource_quota* rq) + : ThreadManager(name, rq, kMinPollers, kMaxPollers), num_do_work_(0), num_poll_for_work_(0), num_work_found_(0) {} @@ -115,7 +115,11 @@ int main(int argc, char** argv) { std::srand(std::time(nullptr)); grpc::testing::InitTest(&argc, &argv, true); - grpc::ThreadManagerTest test_rpc_manager; + + grpc_resource_quota* rq = grpc_resource_quota_create("Test"); + grpc::ThreadManagerTest test_rpc_manager("TestThreadManager", rq); + grpc_resource_quota_unref(rq); + test_rpc_manager.PerformTest(); return 0; From 0fee91e002dfb6a9f255cf0e78666e81ea73ca5e Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Sat, 21 Jul 2018 08:21:18 +0200 Subject: [PATCH 12/45] More clang format --- test/cpp/util/cli_credentials.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index 68eb0b17ce8..d14dc18f168 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -67,8 +67,8 @@ const grpc::string CliCredentials::GetCredentialUsage() const { } const grpc::string CliCredentials::GetSslTargetNameOverride() const { - bool use_tls = FLAGS_enable_ssl || - (FLAGS_access_token.empty() && FLAGS_use_auth); + bool use_tls = + FLAGS_enable_ssl || (FLAGS_access_token.empty() && FLAGS_use_auth); return use_tls ? FLAGS_ssl_target : ""; } From b95772eeb926f78b8ac14e03b36ed3e73b2e1a2c Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 23 Jul 2018 23:38:13 -0700 Subject: [PATCH 13/45] Add Tests in Core and C++ and fix a few related bugs in thread_manager.cc --- src/core/lib/iomgr/resource_quota.cc | 7 + src/cpp/server/server_cc.cc | 10 +- src/cpp/thread_manager/thread_manager.cc | 70 +++++---- src/cpp/thread_manager/thread_manager.h | 42 +++-- test/core/iomgr/resource_quota_test.cc | 96 ++++++++++++ .../cpp/thread_manager/thread_manager_test.cc | 147 +++++++++++++----- 6 files changed, 288 insertions(+), 84 deletions(-) diff --git a/src/core/lib/iomgr/resource_quota.cc b/src/core/lib/iomgr/resource_quota.cc index a30688bd873..67d05aa202d 100644 --- a/src/core/lib/iomgr/resource_quota.cc +++ b/src/core/lib/iomgr/resource_quota.cc @@ -547,6 +547,11 @@ static void ru_shutdown(void* ru, grpc_error* error) { static void ru_destroy(void* ru, grpc_error* error) { grpc_resource_user* resource_user = static_cast(ru); GPR_ASSERT(gpr_atm_no_barrier_load(&resource_user->refs) == 0); + // Free all the remaining thread quota + grpc_resource_user_free_threads( + resource_user, + static_cast(gpr_atm_no_barrier_load(&resource_user->num_threads))); + for (int i = 0; i < GRPC_RULIST_COUNT; i++) { rulist_remove(resource_user, static_cast(i)); } @@ -642,6 +647,7 @@ grpc_resource_quota* grpc_resource_quota_create(const char* name) { void grpc_resource_quota_unref_internal(grpc_resource_quota* resource_quota) { if (gpr_unref(&resource_quota->refs)) { + GPR_ASSERT(resource_quota->num_threads == 0); // No outstanding thd quota GRPC_COMBINER_UNREF(resource_quota->combiner, "resource_quota"); gpr_free(resource_quota->name); gpr_free(resource_quota); @@ -846,6 +852,7 @@ void grpc_resource_user_free_threads(grpc_resource_user* resource_user, "Releasing more threads (%d) that currently allocated (rq threads: " "%d, ru threads: %d)", thd_count, old_cnt, rq->num_threads + thd_count); + abort(); } gpr_mu_unlock(&resource_user->resource_quota->thd_mu); } diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 6e6e0bfffe3..786ef44e3ef 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -47,6 +47,12 @@ namespace grpc { namespace { +// The default value for maximum number of threads that can be created in the +// sync server. This value of 1500 is empirically chosen. To increase the max +// number of threads in a sync server, pass a custom ResourceQuota object (with +// the desired number of max-threads set) to the server builder +#define DEFAULT_MAX_SYNC_SERVER_THREADS 1500 + class DefaultGlobalCallbacks final : public Server::GlobalCallbacks { public: ~DefaultGlobalCallbacks() override {} @@ -395,7 +401,9 @@ Server::Server( if (sync_server_cqs_ != nullptr) { bool default_rq_created = false; if (server_rq == nullptr) { - server_rq = grpc_resource_quota_create("SyncServer-Default"); + server_rq = grpc_resource_quota_create("SyncServer-default-rq"); + grpc_resource_quota_set_max_threads(server_rq, + DEFAULT_MAX_SYNC_SERVER_THREADS); default_rq_created = true; } diff --git a/src/cpp/thread_manager/thread_manager.cc b/src/cpp/thread_manager/thread_manager.cc index c0fa98798a0..5d367511e2e 100644 --- a/src/cpp/thread_manager/thread_manager.cc +++ b/src/cpp/thread_manager/thread_manager.cc @@ -22,8 +22,8 @@ #include #include - #include "src/core/lib/gprpp/thd.h" +#include "src/core/lib/iomgr/exec_ctx.h" namespace grpc { @@ -55,7 +55,8 @@ ThreadManager::ThreadManager(const char* name, num_pollers_(0), min_pollers_(min_pollers), max_pollers_(max_pollers == -1 ? INT_MAX : max_pollers), - num_threads_(0) { + num_threads_(0), + max_active_threads_sofar_(0) { resource_user_ = grpc_resource_user_create(resource_quota, name); } @@ -65,6 +66,7 @@ ThreadManager::~ThreadManager() { GPR_ASSERT(num_threads_ == 0); } + grpc_core::ExecCtx exec_ctx; // grpc_resource_user_unref needs an exec_ctx grpc_resource_user_unref(resource_user_); CleanupCompletedThreads(); } @@ -86,17 +88,27 @@ bool ThreadManager::IsShutdown() { return shutdown_; } +int ThreadManager::GetMaxActiveThreadsSoFar() { + std::lock_guard list_lock(list_mu_); + return max_active_threads_sofar_; +} + void ThreadManager::MarkAsCompleted(WorkerThread* thd) { { std::lock_guard list_lock(list_mu_); completed_threads_.push_back(thd); } - std::lock_guard lock(mu_); - num_threads_--; - if (num_threads_ == 0) { - shutdown_cv_.notify_one(); + { + std::lock_guard lock(mu_); + num_threads_--; + if (num_threads_ == 0) { + shutdown_cv_.notify_one(); + } } + + // Give a thread back to the resource quota + grpc_resource_user_free_threads(resource_user_, 1); } void ThreadManager::CleanupCompletedThreads() { @@ -111,34 +123,24 @@ void ThreadManager::CleanupCompletedThreads() { } void ThreadManager::Initialize() { + if (!grpc_resource_user_alloc_threads(resource_user_, min_pollers_)) { + gpr_log(GPR_ERROR, + "No thread quota available to even create the minimum required " + "polling threads (i.e %d). Unable to start the thread manager", + min_pollers_); + abort(); + } + { std::unique_lock lock(mu_); num_pollers_ = min_pollers_; num_threads_ = min_pollers_; + max_active_threads_sofar_ = min_pollers_; } for (int i = 0; i < min_pollers_; i++) { - if (!CreateNewThread(this)) { - gpr_log(GPR_ERROR, - "No quota available to create additional threads. Created %d (of " - "%d) threads", - i, min_pollers_); - break; - } - } -} - -bool ThreadManager::CreateNewThread(ThreadManager* thd_mgr) { - if (!grpc_resource_user_alloc_threads(thd_mgr->resource_user_, 1)) { - return false; + new WorkerThread(this); } - // Create a new thread (which ends up calling the MainWorkLoop() function - new WorkerThread(thd_mgr); - return true; -} - -void ThreadManager::ReleaseThread(ThreadManager* thd_mgr) { - grpc_resource_user_free_threads(thd_mgr->resource_user_, 1); } void ThreadManager::MainWorkLoop() { @@ -162,14 +164,17 @@ void ThreadManager::MainWorkLoop() { done = true; break; case WORK_FOUND: - // If we got work and there are now insufficient pollers, start a new - // one - if (!shutdown_ && num_pollers_ < min_pollers_) { + // If we got work and there are now insufficient pollers and there is + // quota available to create a new thread,start a new poller thread + if (!shutdown_ && num_pollers_ < min_pollers_ && + grpc_resource_user_alloc_threads(resource_user_, 1)) { num_pollers_++; num_threads_++; + max_active_threads_sofar_ = + std::max(max_active_threads_sofar_, num_threads_); // Drop lock before spawning thread to avoid contention lock.unlock(); - CreateNewThread(this); + new WorkerThread(this); } else { // Drop lock for consistency with above branch lock.unlock(); @@ -219,10 +224,9 @@ void ThreadManager::MainWorkLoop() { } }; - // This thread is exiting. Do some cleanup work (i.e delete already completed - // worker threads and also release 1 thread back to the resource quota) + // This thread is exiting. Do some cleanup work i.e delete already completed + // worker threads CleanupCompletedThreads(); - ReleaseThread(this); // If we are here, either ThreadManager is shutting down or it already has // enough threads. diff --git a/src/cpp/thread_manager/thread_manager.h b/src/cpp/thread_manager/thread_manager.h index 23bd38ee4f1..8332befed09 100644 --- a/src/cpp/thread_manager/thread_manager.h +++ b/src/cpp/thread_manager/thread_manager.h @@ -86,6 +86,11 @@ class ThreadManager { // all the threads have drained all the outstanding work virtual void Wait(); + // Max number of concurrent threads that were ever active in this thread + // manager so far. This is useful for debugging purposes (and in unit tests) + // to check if resource_quota is properly being enforced. + int GetMaxActiveThreadsSoFar(); + private: // Helper wrapper class around grpc_core::Thread. Takes a ThreadManager object // and starts a new grpc_core::Thread to calls the Run() function. @@ -93,6 +98,23 @@ class ThreadManager { // The Run() function calls ThreadManager::MainWorkLoop() function and once // that completes, it marks the WorkerThread completed by calling // ThreadManager::MarkAsCompleted() + // + // WHY IS THIS NEEDED?: + // When a thread terminates, some other tread *must* call Join() on that + // thread so that the resources are released. Having a WorkerThread wrapper + // will make this easier. Once Run() completes, each thread calls the + // following two functions: + // ThreadManager::CleanupCompletedThreads() + // ThreadManager::MarkAsCompleted() + // + // - MarkAsCompleted() puts the WorkerThread object in the ThreadManger's + // completed_threads_ list + // - CleanupCompletedThreads() calls "Join()" on the threads that are already + // in the completed_threads_ list (since a thread cannot call Join() on + // itself, it calls CleanupCompletedThreads() *before* calling + // MarkAsCompleted()) + // TODO: sreek - consider creating the threads 'detached' so that Join() need + // not be called class WorkerThread { public: WorkerThread(ThreadManager* thd_mgr); @@ -113,15 +135,8 @@ class ThreadManager { void MarkAsCompleted(WorkerThread* thd); void CleanupCompletedThreads(); - // Checks the resource quota and if available, creates a thread and returns - // true. If quota is not available, returns false (and thread is not created) - static bool CreateNewThread(ThreadManager* thd_mgr); - - // Give back a thread to the resource quota - static void ReleaseThread(ThreadManager* thd_mgr); - - // Protects shutdown_, num_pollers_ and num_threads_ - // TODO: sreek - Change num_pollers and num_threads_ to atomics + // Protects shutdown_, num_pollers_, num_threads_ and + // max_active_threads_sofar_ std::mutex mu_; bool shutdown_; @@ -142,10 +157,15 @@ class ThreadManager { int min_pollers_; int max_pollers_; - // The total number of threads (includes threads includes the threads that are - // currently polling i.e num_pollers_) + // The total number of threads currently active (includes threads includes the + // threads that are currently polling i.e num_pollers_) int num_threads_; + // See GetMaxActiveThreadsSoFar()'s description. + // To be more specific, this variable tracks the max value num_threads_ was + // ever set so far + int max_active_threads_sofar_; + std::mutex list_mu_; std::list completed_threads_; }; diff --git a/test/core/iomgr/resource_quota_test.cc b/test/core/iomgr/resource_quota_test.cc index 059ff7b5f8b..573e4010fa4 100644 --- a/test/core/iomgr/resource_quota_test.cc +++ b/test/core/iomgr/resource_quota_test.cc @@ -798,6 +798,97 @@ static void test_negative_rq_free_pool(void) { } } +// Simple test to check resource quota thread limits +static void test_thread_limit() { + grpc_core::ExecCtx exec_ctx; + + grpc_resource_quota* rq = grpc_resource_quota_create("test_thread_limit"); + grpc_resource_user* ru1 = grpc_resource_user_create(rq, "ru1"); + grpc_resource_user* ru2 = grpc_resource_user_create(rq, "ru2"); + + // Max threads = 100 + grpc_resource_quota_set_max_threads(rq, 100); + + // Request quota for 100 threads (50 for ru1, 50 for ru2) + GPR_ASSERT(grpc_resource_user_alloc_threads(ru1, 10)); + GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 10)); + GPR_ASSERT(grpc_resource_user_alloc_threads(ru1, 40)); + GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 40)); + + // Threads exhaused. Next request must fail + GPR_ASSERT(!grpc_resource_user_alloc_threads(ru2, 20)); + + // Free 20 threads from two different users + grpc_resource_user_free_threads(ru1, 10); + grpc_resource_user_free_threads(ru2, 10); + + // Next request to 20 threads must succeed + GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 20)); + + // No more thread quota again + GPR_ASSERT(!grpc_resource_user_alloc_threads(ru1, 20)); + + // Free 10 more + grpc_resource_user_free_threads(ru1, 10); + + GPR_ASSERT(grpc_resource_user_alloc_threads(ru1, 5)); + GPR_ASSERT(!grpc_resource_user_alloc_threads(ru2, 10)); // Only 5 available + GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 5)); + + // Teardown (ru1 and ru2 release all the quota back to rq) + grpc_resource_user_unref(ru1); + grpc_resource_user_unref(ru2); + grpc_resource_quota_unref(rq); +} + +// Change max quota in either directions dynamically +static void test_thread_maxquota_change() { + grpc_core::ExecCtx exec_ctx; + + grpc_resource_quota* rq = + grpc_resource_quota_create("test_thread_maxquota_change"); + grpc_resource_user* ru1 = grpc_resource_user_create(rq, "ru1"); + grpc_resource_user* ru2 = grpc_resource_user_create(rq, "ru2"); + + // Max threads = 100 + grpc_resource_quota_set_max_threads(rq, 100); + + // Request quota for 100 threads (50 for ru1, 50 for ru2) + GPR_ASSERT(grpc_resource_user_alloc_threads(ru1, 50)); + GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 50)); + + // Threads exhaused. Next request must fail + GPR_ASSERT(!grpc_resource_user_alloc_threads(ru2, 20)); + + // Increase maxquota and retry + // Max threads = 150; + grpc_resource_quota_set_max_threads(rq, 150); + GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 20)); // ru2 = 70, ru1 = 50 + + // Decrease maxquota (Note: Quota already given to ru1 and ru2 is unaffected) + // Max threads = 10; + grpc_resource_quota_set_max_threads(rq, 10); + + // New requests will fail until quota is available + GPR_ASSERT(!grpc_resource_user_alloc_threads(ru1, 10)); + + // Make quota available + grpc_resource_user_free_threads(ru1, 50); // ru1 now has 0 + GPR_ASSERT(!grpc_resource_user_alloc_threads(ru1, 10)); // Still not enough + + grpc_resource_user_free_threads(ru2, 70); // ru2 now has 0 + + // Now we can get quota up-to 10, the current max + GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 10)); + // No more thread quota again + GPR_ASSERT(!grpc_resource_user_alloc_threads(ru1, 10)); + + // Teardown (ru1 and ru2 release all the quota back to rq) + grpc_resource_user_unref(ru1); + grpc_resource_user_unref(ru2); + grpc_resource_quota_unref(rq); +} + int main(int argc, char** argv) { grpc_test_init(argc, argv); grpc_init(); @@ -827,6 +918,11 @@ int main(int argc, char** argv) { test_negative_rq_free_pool(); gpr_mu_destroy(&g_mu); gpr_cv_destroy(&g_cv); + + // Resource quota thread related + test_thread_limit(); + test_thread_maxquota_change(); + grpc_shutdown(); return 0; } diff --git a/test/cpp/thread_manager/thread_manager_test.cc b/test/cpp/thread_manager/thread_manager_test.cc index cf2cf770e6b..a7ed2dd380e 100644 --- a/test/cpp/thread_manager/thread_manager_test.cc +++ b/test/cpp/thread_manager/thread_manager_test.cc @@ -30,30 +30,44 @@ #include "test/cpp/util/test_config.h" namespace grpc { + +struct ThreadManagerTestSettings { + // The min number of pollers that SHOULD be active in ThreadManager + int min_pollers; + // The max number of pollers that could be active in ThreadManager + int max_pollers; + // The sleep duration in PollForWork() function to simulate "polling" + int poll_duration_ms; + // The sleep duration in DoWork() function to simulate "work" + int work_duration_ms; + // Max number of times PollForWork() is called before shutting down + int max_poll_calls; +}; + class ThreadManagerTest final : public grpc::ThreadManager { public: - ThreadManagerTest(const char* name, grpc_resource_quota* rq) - : ThreadManager(name, rq, kMinPollers, kMaxPollers), + ThreadManagerTest(const char* name, grpc_resource_quota* rq, + const ThreadManagerTestSettings& settings) + : ThreadManager(name, rq, settings.min_pollers, settings.max_pollers), + settings_(settings), num_do_work_(0), num_poll_for_work_(0), num_work_found_(0) {} grpc::ThreadManager::WorkStatus PollForWork(void** tag, bool* ok) override; void DoWork(void* tag, bool ok) override; - void PerformTest(); + + // Get number of times PollForWork() returned WORK_FOUND + int GetNumWorkFound(); + // Get number of times DoWork() was called + int GetNumDoWork(); private: void SleepForMs(int sleep_time_ms); - static const int kMinPollers = 2; - static const int kMaxPollers = 10; - - static const int kPollingTimeoutMsec = 10; - static const int kDoWorkDurationMsec = 1; - - // PollForWork will return SHUTDOWN after these many number of invocations - static const int kMaxNumPollForWork = 50; + ThreadManagerTestSettings settings_; + // Counters gpr_atm num_do_work_; // Number of calls to DoWork gpr_atm num_poll_for_work_; // Number of calls to PollForWork gpr_atm num_work_found_; // Number of times WORK_FOUND was returned @@ -69,58 +83,113 @@ void ThreadManagerTest::SleepForMs(int duration_ms) { grpc::ThreadManager::WorkStatus ThreadManagerTest::PollForWork(void** tag, bool* ok) { int call_num = gpr_atm_no_barrier_fetch_add(&num_poll_for_work_, 1); - - if (call_num >= kMaxNumPollForWork) { + if (call_num >= settings_.max_poll_calls) { Shutdown(); return SHUTDOWN; } - // Simulate "polling for work" by sleeping for sometime - SleepForMs(kPollingTimeoutMsec); - + SleepForMs(settings_.poll_duration_ms); // Simulate "polling" duration *tag = nullptr; *ok = true; - // Return timeout roughly 1 out of every 3 calls + // Return timeout roughly 1 out of every 3 calls just to make the test a bit + // more interesting if (call_num % 3 == 0) { return TIMEOUT; - } else { - gpr_atm_no_barrier_fetch_add(&num_work_found_, 1); - return WORK_FOUND; } + + gpr_atm_no_barrier_fetch_add(&num_work_found_, 1); + return WORK_FOUND; } void ThreadManagerTest::DoWork(void* tag, bool ok) { gpr_atm_no_barrier_fetch_add(&num_do_work_, 1); - SleepForMs(kDoWorkDurationMsec); // Simulate doing work by sleeping + SleepForMs(settings_.work_duration_ms); // Simulate work by sleeping } -void ThreadManagerTest::PerformTest() { - // Initialize() starts the ThreadManager - Initialize(); - - // Wait for all the threads to gracefully terminate - Wait(); +int ThreadManagerTest::GetNumWorkFound() { + return static_cast(gpr_atm_no_barrier_load(&num_work_found_)); +} - // The number of times DoWork() was called is equal to the number of times - // WORK_FOUND was returned - gpr_log(GPR_DEBUG, "DoWork() called %" PRIdPTR " times", - gpr_atm_no_barrier_load(&num_do_work_)); - GPR_ASSERT(gpr_atm_no_barrier_load(&num_do_work_) == - gpr_atm_no_barrier_load(&num_work_found_)); +int ThreadManagerTest::GetNumDoWork() { + return static_cast(gpr_atm_no_barrier_load(&num_do_work_)); } } // namespace grpc -int main(int argc, char** argv) { - std::srand(std::time(nullptr)); +// Test that the number of times DoWork() is called is equal to the number of +// times PollForWork() returned WORK_FOUND +static void TestPollAndWork() { + grpc_resource_quota* rq = grpc_resource_quota_create("Test-poll-and-work"); + grpc::ThreadManagerTestSettings settings = { + 2 /* min_pollers */, 10 /* max_pollers */, 10 /* poll_duration_ms */, + 1 /* work_duration_ms */, 50 /* max_poll_calls */}; - grpc::testing::InitTest(&argc, &argv, true); + grpc::ThreadManagerTest test_thd_mgr("TestThreadManager", rq, settings); + grpc_resource_quota_unref(rq); + + test_thd_mgr.Initialize(); // Start the thread manager + test_thd_mgr.Wait(); // Wait for all threads to finish + + // Verify that The number of times DoWork() was called is equal to the number + // of times WORK_FOUND was returned + gpr_log(GPR_DEBUG, "DoWork() called %d times", test_thd_mgr.GetNumDoWork()); + GPR_ASSERT(test_thd_mgr.GetNumDoWork() == test_thd_mgr.GetNumWorkFound()); +} - grpc_resource_quota* rq = grpc_resource_quota_create("Test"); - grpc::ThreadManagerTest test_rpc_manager("TestThreadManager", rq); +static void TestThreadQuota() { + const int kMaxNumThreads = 3; + grpc_resource_quota* rq = grpc_resource_quota_create("Test-thread-quota"); + grpc_resource_quota_set_max_threads(rq, kMaxNumThreads); + + // Set work_duration_ms to be much greater than poll_duration_ms. This way, + // the thread manager will be forced to create more 'polling' threads to + // honor the min_pollers guarantee + grpc::ThreadManagerTestSettings settings = { + 1 /* min_pollers */, 1 /* max_pollers */, 1 /* poll_duration_ms */, + 10 /* work_duration_ms */, 50 /* max_poll_calls */}; + + // Create two thread managers (but with same resource quota). This means + // that the max number of active threads across BOTH the thread managers + // cannot be greater than kMaxNumthreads + grpc::ThreadManagerTest test_thd_mgr_1("TestThreadManager-1", rq, settings); + grpc::ThreadManagerTest test_thd_mgr_2("TestThreadManager-2", rq, settings); + // It is ok to unref resource quota before starting thread managers. grpc_resource_quota_unref(rq); - test_rpc_manager.PerformTest(); + // Start both thread managers + test_thd_mgr_1.Initialize(); + test_thd_mgr_2.Initialize(); + + // Wait for both to finish + test_thd_mgr_1.Wait(); + test_thd_mgr_2.Wait(); + + // Now verify that the total number of active threads in either thread manager + // never exceeds kMaxNumThreads + // + // NOTE: Actually the total active threads across *both* thread managers at + // any point of time never exceeds kMaxNumThreads but unfortunately there is + // no easy way to verify it (i.e we can't just do (max1 + max2 <= k)) + // Its okay to not test this case here. The resource quota c-core tests + // provide enough coverage to resource quota object with multiple resource + // users + int max1 = test_thd_mgr_1.GetMaxActiveThreadsSoFar(); + int max2 = test_thd_mgr_2.GetMaxActiveThreadsSoFar(); + gpr_log( + GPR_DEBUG, + "MaxActiveThreads in TestThreadManager_1: %d, TestThreadManager_2: %d", + max1, max2); + GPR_ASSERT(max1 <= kMaxNumThreads && max2 <= kMaxNumThreads); +} + +int main(int argc, char** argv) { + std::srand(std::time(nullptr)); + grpc::testing::InitTest(&argc, &argv, true); + grpc_init(); + + TestPollAndWork(); + TestThreadQuota(); + grpc_shutdown(); return 0; } From 8f39834dd1d68c6fed18d2af2eb5f7acdb8549ab Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 25 Jul 2018 10:36:46 -0700 Subject: [PATCH 14/45] Change the default max threads to something more reasonable --- src/cpp/server/server_cc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 786ef44e3ef..472c5035fc1 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -48,10 +48,10 @@ namespace grpc { namespace { // The default value for maximum number of threads that can be created in the -// sync server. This value of 1500 is empirically chosen. To increase the max +// sync server. This value of 500 is empirically chosen. To increase the max // number of threads in a sync server, pass a custom ResourceQuota object (with // the desired number of max-threads set) to the server builder -#define DEFAULT_MAX_SYNC_SERVER_THREADS 1500 +#define DEFAULT_MAX_SYNC_SERVER_THREADS 500 class DefaultGlobalCallbacks final : public Server::GlobalCallbacks { public: From dd45987a5bf976079d9d89fa127740bd6f516a16 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 25 Jul 2018 10:39:16 -0700 Subject: [PATCH 15/45] fix comment --- src/core/lib/iomgr/resource_quota.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/resource_quota.cc b/src/core/lib/iomgr/resource_quota.cc index 67d05aa202d..47b7856e959 100644 --- a/src/core/lib/iomgr/resource_quota.cc +++ b/src/core/lib/iomgr/resource_quota.cc @@ -141,8 +141,8 @@ struct grpc_resource_quota { /* Mutex to protect max_threads and num_threads */ /* Note: We could have used gpr_atm for max_threads and num_threads and avoid * having this mutex; but in that case, each invocation of the function - * grpc_resource_user_alloc_threads() will have to do atleast two atomic loads - * (for max_threads and num_threads) followed by a CAS (on num_threads). + * grpc_resource_user_alloc_threads() would have had to do at least two atomic + * loads (for max_threads and num_threads) followed by a CAS (on num_threads). * Moreover, we expect grpc_resource_user_alloc_threads() to be often called * concurrently thereby increasing the chances of failing the CAS operation. * This additional complexity is not worth the tiny perf gain we may (or may From 9bc8ee42c20385214027ced5991b9973a0282655 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 25 Jul 2018 10:58:33 -0700 Subject: [PATCH 16/45] generate_projects.sh --- grpc.def | 1 + src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 ++ src/ruby/ext/grpc/rb_grpc_imports.generated.h | 3 +++ test/core/surface/public_headers_must_be_c89.c | 1 + 4 files changed, 7 insertions(+) diff --git a/grpc.def b/grpc.def index 5b98792662c..312e9166824 100644 --- a/grpc.def +++ b/grpc.def @@ -68,6 +68,7 @@ EXPORTS grpc_resource_quota_ref grpc_resource_quota_unref grpc_resource_quota_resize + grpc_resource_quota_set_max_threads grpc_resource_quota_arg_vtable grpc_channelz_get_top_channels grpc_channelz_get_channel diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 2443532bb8e..78090afd6c4 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -91,6 +91,7 @@ grpc_resource_quota_create_type grpc_resource_quota_create_import; grpc_resource_quota_ref_type grpc_resource_quota_ref_import; grpc_resource_quota_unref_type grpc_resource_quota_unref_import; grpc_resource_quota_resize_type grpc_resource_quota_resize_import; +grpc_resource_quota_set_max_threads_type grpc_resource_quota_set_max_threads_import; grpc_resource_quota_arg_vtable_type grpc_resource_quota_arg_vtable_import; grpc_channelz_get_top_channels_type grpc_channelz_get_top_channels_import; grpc_channelz_get_channel_type grpc_channelz_get_channel_import; @@ -341,6 +342,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_resource_quota_ref_import = (grpc_resource_quota_ref_type) GetProcAddress(library, "grpc_resource_quota_ref"); grpc_resource_quota_unref_import = (grpc_resource_quota_unref_type) GetProcAddress(library, "grpc_resource_quota_unref"); grpc_resource_quota_resize_import = (grpc_resource_quota_resize_type) GetProcAddress(library, "grpc_resource_quota_resize"); + grpc_resource_quota_set_max_threads_import = (grpc_resource_quota_set_max_threads_type) GetProcAddress(library, "grpc_resource_quota_set_max_threads"); grpc_resource_quota_arg_vtable_import = (grpc_resource_quota_arg_vtable_type) GetProcAddress(library, "grpc_resource_quota_arg_vtable"); grpc_channelz_get_top_channels_import = (grpc_channelz_get_top_channels_type) GetProcAddress(library, "grpc_channelz_get_top_channels"); grpc_channelz_get_channel_import = (grpc_channelz_get_channel_type) GetProcAddress(library, "grpc_channelz_get_channel"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index b08a1f94f7b..1807efa761e 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -248,6 +248,9 @@ extern grpc_resource_quota_unref_type grpc_resource_quota_unref_import; typedef void(*grpc_resource_quota_resize_type)(grpc_resource_quota* resource_quota, size_t new_size); extern grpc_resource_quota_resize_type grpc_resource_quota_resize_import; #define grpc_resource_quota_resize grpc_resource_quota_resize_import +typedef void(*grpc_resource_quota_set_max_threads_type)(grpc_resource_quota* resource_quota, int new_max_threads); +extern grpc_resource_quota_set_max_threads_type grpc_resource_quota_set_max_threads_import; +#define grpc_resource_quota_set_max_threads grpc_resource_quota_set_max_threads_import typedef const grpc_arg_pointer_vtable*(*grpc_resource_quota_arg_vtable_type)(void); extern grpc_resource_quota_arg_vtable_type grpc_resource_quota_arg_vtable_import; #define grpc_resource_quota_arg_vtable grpc_resource_quota_arg_vtable_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 9f4ad2b4d75..497f7194d57 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -130,6 +130,7 @@ int main(int argc, char **argv) { printf("%lx", (unsigned long) grpc_resource_quota_ref); printf("%lx", (unsigned long) grpc_resource_quota_unref); printf("%lx", (unsigned long) grpc_resource_quota_resize); + printf("%lx", (unsigned long) grpc_resource_quota_set_max_threads); printf("%lx", (unsigned long) grpc_resource_quota_arg_vtable); printf("%lx", (unsigned long) grpc_channelz_get_top_channels); printf("%lx", (unsigned long) grpc_channelz_get_channel); From 3ffde93f85af97d04ff06826948689091e29a00b Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 26 Jul 2018 10:51:08 -0700 Subject: [PATCH 17/45] BoringSSL commit and version boost --- src/objective-c/BoringSSL.podspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/BoringSSL.podspec b/src/objective-c/BoringSSL.podspec index 363983183e9..dc566803930 100644 --- a/src/objective-c/BoringSSL.podspec +++ b/src/objective-c/BoringSSL.podspec @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL' - version = '10.0.5' + version = '10.0.6' s.version = version s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google’s needs.' # Adapted from the homepage: @@ -68,7 +68,7 @@ Pod::Spec.new do |s| s.source = { :git => 'https://github.com/google/boringssl.git', - :commit => "0c1f336fba7c8cdbe8f32a8c75a8a9f8461feff1", + :commit => "b29b21a81b32ec273f118f589f46d56ad3332420", } s.ios.deployment_target = '5.0' From 9896c641860a70e41b006a982e44f40bb3f41053 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Fri, 27 Jul 2018 01:38:26 -0700 Subject: [PATCH 18/45] Revert "Merge pull request #16158 from apolcyn/revert_windows_breakage" This reverts commit d9e8f86675cf923714b3ead4b06087e14a88c08c, reversing changes made to 04847aeb1e62bc528d88fa3c47daa24a4cf629b3. --- BUILD | 3 + CMakeLists.txt | 44 ++++- Makefile | 42 +++++ build.yaml | 15 ++ config.m4 | 3 + config.w32 | 3 + gRPC-Core.podspec | 3 + grpc.gemspec | 3 + grpc.gyp | 6 + include/grpc/impl/codegen/port_platform.h | 4 - package.xml | 3 + .../resolver/dns/c_ares/dns_resolver_ares.cc | 5 +- .../dns/c_ares/grpc_ares_ev_driver.cc | 6 +- .../dns/c_ares/grpc_ares_ev_driver_windows.cc | 59 +++++++ .../resolver/dns/c_ares/grpc_ares_wrapper.cc | 9 +- .../resolver/dns/c_ares/grpc_ares_wrapper.h | 4 + .../dns/c_ares/grpc_ares_wrapper_posix.cc | 29 ++++ .../dns/c_ares/grpc_ares_wrapper_windows.cc | 29 ++++ src/core/lib/iomgr/socket_windows.cc | 29 ++++ src/core/lib/iomgr/socket_windows.h | 4 + src/python/grpcio/grpc_core_dependencies.py | 3 + test/core/iomgr/BUILD | 13 ++ .../grpc_ipv6_loopback_available_test.cc | 48 ++++++ test/cpp/naming/address_sorting_test.cc | 160 +++++++++++++----- test/cpp/naming/gen_build_yaml.py | 2 +- third_party/address_sorting/address_sorting.c | 9 +- .../address_sorting/address_sorting_windows.c | 46 ++++- .../include/address_sorting/address_sorting.h | 3 + tools/doxygen/Doxyfile.core.internal | 3 + .../generated/sources_and_headers.json | 22 ++- tools/run_tests/generated/tests.json | 38 ++++- 31 files changed, 577 insertions(+), 73 deletions(-) create mode 100644 src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc create mode 100644 src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc create mode 100644 src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc create mode 100644 test/core/iomgr/grpc_ipv6_loopback_available_test.cc diff --git a/BUILD b/BUILD index ee4b5dfaecc..81390dd1aa1 100644 --- a/BUILD +++ b/BUILD @@ -1433,7 +1433,10 @@ grpc_cc_library( "src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc", ], hdrs = [ diff --git a/CMakeLists.txt b/CMakeLists.txt index 84e9c08cb5b..e8e65d4b71d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -298,6 +298,7 @@ add_dependencies(buildtests_c grpc_completion_queue_test) add_dependencies(buildtests_c grpc_completion_queue_threading_test) add_dependencies(buildtests_c grpc_credentials_test) add_dependencies(buildtests_c grpc_fetch_oauth2) +add_dependencies(buildtests_c grpc_ipv6_loopback_available_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c grpc_json_token_test) endif() @@ -671,12 +672,8 @@ endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker) endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx address_sorting_test_unsecure) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx address_sorting_test) -endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx cancel_ares_query_test) endif() @@ -1236,8 +1233,11 @@ add_library(grpc src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc src/cpp/ext/filters/census/grpc_context.cc @@ -2538,8 +2538,11 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc @@ -7323,6 +7326,35 @@ target_link_libraries(grpc_fetch_oauth2 gpr ) +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + +add_executable(grpc_ipv6_loopback_available_test + test/core/iomgr/grpc_ipv6_loopback_available_test.cc +) + + +target_include_directories(grpc_ipv6_loopback_available_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} +) + +target_link_libraries(grpc_ipv6_loopback_available_test + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr +) + endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -16351,7 +16383,6 @@ target_link_libraries(resolver_component_tests_runner_invoker endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(address_sorting_test_unsecure test/cpp/naming/address_sorting_test.cc @@ -16391,10 +16422,8 @@ target_link_libraries(address_sorting_test_unsecure ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(address_sorting_test test/cpp/naming/address_sorting_test.cc @@ -16434,7 +16463,6 @@ target_link_libraries(address_sorting_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) diff --git a/Makefile b/Makefile index bad41975a05..5174ab6719c 100644 --- a/Makefile +++ b/Makefile @@ -1022,6 +1022,7 @@ grpc_completion_queue_threading_test: $(BINDIR)/$(CONFIG)/grpc_completion_queue_ grpc_create_jwt: $(BINDIR)/$(CONFIG)/grpc_create_jwt grpc_credentials_test: $(BINDIR)/$(CONFIG)/grpc_credentials_test grpc_fetch_oauth2: $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 +grpc_ipv6_loopback_available_test: $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test grpc_json_token_test: $(BINDIR)/$(CONFIG)/grpc_json_token_test grpc_jwt_verifier_test: $(BINDIR)/$(CONFIG)/grpc_jwt_verifier_test grpc_print_google_default_creds_token: $(BINDIR)/$(CONFIG)/grpc_print_google_default_creds_token @@ -1472,6 +1473,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/grpc_completion_queue_threading_test \ $(BINDIR)/$(CONFIG)/grpc_credentials_test \ $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 \ + $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test \ $(BINDIR)/$(CONFIG)/grpc_json_token_test \ $(BINDIR)/$(CONFIG)/grpc_jwt_verifier_test \ $(BINDIR)/$(CONFIG)/grpc_security_connector_test \ @@ -2028,6 +2030,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/grpc_completion_queue_threading_test || ( echo test grpc_completion_queue_threading_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_credentials_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_credentials_test || ( echo test grpc_credentials_test failed ; exit 1 ) + $(E) "[RUN] Testing grpc_ipv6_loopback_available_test" + $(Q) $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test || ( echo test grpc_ipv6_loopback_available_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_json_token_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_json_token_test || ( echo test grpc_json_token_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_jwt_verifier_test" @@ -3704,8 +3708,11 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ src/cpp/ext/filters/census/grpc_context.cc \ @@ -4972,8 +4979,11 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \ @@ -12365,6 +12375,38 @@ endif endif +GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_SRC = \ + test/core/iomgr/grpc_ipv6_loopback_available_test.cc \ + +GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test: $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/iomgr/grpc_ipv6_loopback_available_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_grpc_ipv6_loopback_available_test: $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS:.o=.dep) +endif +endif + + GRPC_JSON_TOKEN_TEST_SRC = \ test/core/security/json_token_test.cc \ diff --git a/build.yaml b/build.yaml index 30389ec1146..70af96046cb 100644 --- a/build.yaml +++ b/build.yaml @@ -740,8 +740,11 @@ filegroups: - src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc + - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc plugin: grpc_resolver_dns_ares uses: - grpc_base @@ -2730,6 +2733,18 @@ targets: - grpc - gpr_test_util - gpr +- name: grpc_ipv6_loopback_available_test + build: test + language: c + src: + - test/core/iomgr/grpc_ipv6_loopback_available_test.cc + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + exclude_iomgrs: + - uv - name: grpc_json_token_test build: test language: c diff --git a/config.m4 b/config.m4 index c277ccafc8c..aa40a698a64 100644 --- a/config.m4 +++ b/config.m4 @@ -380,8 +380,11 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ src/cpp/ext/filters/census/grpc_context.cc \ diff --git a/config.w32 b/config.w32 index 2857781dd57..5afa4466acd 100644 --- a/config.w32 +++ b/config.w32 @@ -355,8 +355,11 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\dns_resolver_ares.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_ev_driver.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_ev_driver_posix.cc " + + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_ev_driver_windows.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_fallback.cc " + + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_posix.cc " + + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_windows.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\native\\dns_resolver.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\sockaddr\\sockaddr_resolver.cc " + "src\\cpp\\ext\\filters\\census\\grpc_context.cc " + diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 23edaec656d..5c3649afbde 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -802,8 +802,11 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', 'src/cpp/ext/filters/census/grpc_context.cc', diff --git a/grpc.gemspec b/grpc.gemspec index b69d5a7c6fc..c250316b995 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -742,8 +742,11 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc ) s.files += %w( src/cpp/ext/filters/census/grpc_context.cc ) diff --git a/grpc.gyp b/grpc.gyp index e1485efa058..25082fe540a 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -572,8 +572,11 @@ 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', 'src/cpp/ext/filters/census/grpc_context.cc', @@ -1287,8 +1290,11 @@ 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc', diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 01ce5f03e96..2b61a8816d4 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -420,12 +420,8 @@ typedef unsigned __int64 uint64_t; #define GPR_MAX_ALIGNMENT 16 #ifndef GRPC_ARES -#ifdef GPR_WINDOWS -#define GRPC_ARES 0 -#else #define GRPC_ARES 1 #endif -#endif #ifndef GRPC_MUST_USE_RESULT #if defined(__GNUC__) && !defined(__MINGW32__) diff --git a/package.xml b/package.xml index 7f71536b1d0..acdc6ffdb38 100644 --- a/package.xml +++ b/package.xml @@ -747,8 +747,11 @@ + + + diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index f4f6444c5fa..7050e82121c 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -23,7 +23,6 @@ #include #include #include -#include #include #include @@ -142,8 +141,8 @@ AresDnsResolver::AresDnsResolver(const ResolverArgs& args) channel_args_ = grpc_channel_args_copy(args.args); const grpc_arg* arg = grpc_channel_args_find( channel_args_, GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION); - request_service_config_ = !grpc_channel_arg_get_integer( - arg, (grpc_integer_options){false, false, true}); + grpc_integer_options integer_options = {false, false, true}; + request_service_config_ = !grpc_channel_arg_get_integer(arg, integer_options); arg = grpc_channel_args_find(channel_args_, GRPC_ARG_DNS_MIN_TIME_BETWEEN_RESOLUTIONS_MS); min_time_between_resolutions_ = diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc index c886795608f..0068d0d5f4b 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc @@ -18,11 +18,10 @@ #include #include "src/core/lib/iomgr/port.h" -#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) +#if GRPC_ARES == 1 && !defined(GRPC_UV) #include #include -#include #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" @@ -32,7 +31,6 @@ #include #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" #include "src/core/lib/gpr/string.h" -#include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/sockaddr_utils.h" @@ -314,4 +312,4 @@ void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver) { } } -#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) */ +#endif /* GRPC_ARES == 1 && !defined(GRPC_UV) */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc new file mode 100644 index 00000000000..5d65ae3ab37 --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc @@ -0,0 +1,59 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include + +#include "src/core/lib/iomgr/port.h" +#if GRPC_ARES == 1 && defined(GPR_WINDOWS) + +#include +#include +#include "src/core/lib/gprpp/memory.h" + +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" + +namespace grpc_core { + +/* TODO: fill in the body of GrpcPolledFdWindows to enable c-ares on Windows. + This dummy implementation only allows grpc to compile on windows with + GRPC_ARES=1. */ +class GrpcPolledFdWindows : public GrpcPolledFd { + public: + GrpcPolledFdWindows() { abort(); } + ~GrpcPolledFdWindows() { abort(); } + void RegisterForOnReadableLocked(grpc_closure* read_closure) override { + abort(); + } + void RegisterForOnWriteableLocked(grpc_closure* write_closure) override { + abort(); + } + bool IsFdStillReadableLocked() override { abort(); } + void ShutdownLocked(grpc_error* error) override { abort(); } + ares_socket_t GetWrappedAresSocketLocked() override { abort(); } + const char* GetName() override { abort(); } +}; + +GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, + grpc_pollset_set* driver_pollset_set) { + return nullptr; +} + +void ConfigureAresChannelLocked(ares_channel* channel) { abort(); } + +} // namespace grpc_core + +#endif /* GRPC_ARES == 1 && defined(GPR_WINDOWS) */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc index 497ad998af1..b3d6437e9a6 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc @@ -22,7 +22,6 @@ #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" #include "src/core/lib/iomgr/sockaddr.h" -#include "src/core/lib/iomgr/socket_utils_posix.h" #include #include @@ -215,7 +214,7 @@ static void on_hostbyname_done_locked(void* arg, int status, int timeouts, memset(&addr, 0, addr_len); memcpy(&addr.sin6_addr, hostent->h_addr_list[i - prev_naddr], sizeof(struct in6_addr)); - addr.sin6_family = static_cast(hostent->h_addrtype); + addr.sin6_family = static_cast(hostent->h_addrtype); addr.sin6_port = hr->port; grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, @@ -236,7 +235,7 @@ static void on_hostbyname_done_locked(void* arg, int status, int timeouts, memset(&addr, 0, addr_len); memcpy(&addr.sin_addr, hostent->h_addr_list[i - prev_naddr], sizeof(struct in_addr)); - addr.sin_family = static_cast(hostent->h_addrtype); + addr.sin_family = static_cast(hostent->h_addrtype); addr.sin_port = hr->port; grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, @@ -281,7 +280,7 @@ static void on_srv_query_done_locked(void* arg, int status, int timeouts, grpc_ares_ev_driver_get_channel_locked(r->ev_driver); for (struct ares_srv_reply* srv_it = reply; srv_it != nullptr; srv_it = srv_it->next) { - if (grpc_ipv6_loopback_available()) { + if (grpc_ares_query_ipv6()) { grpc_ares_hostbyname_request* hr = create_hostbyname_request_locked( r, srv_it->host, htons(srv_it->port), true /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET6, @@ -452,7 +451,7 @@ static grpc_ares_request* grpc_dns_lookup_ares_locked_impl( } } r->pending_queries = 1; - if (grpc_ipv6_loopback_available()) { + if (grpc_ares_query_ipv6()) { hr = create_hostbyname_request_locked(r, host, strhtons(port), false /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_locked, diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h index ce26f5d524a..17eaa7ccf06 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h @@ -70,6 +70,10 @@ void grpc_ares_cleanup(void); * and destroys the grpc_ares_request */ void grpc_ares_complete_request_locked(grpc_ares_request* request); +/* Indicates whether or not AAAA queries should be attempted. */ +/* E.g., return false if ipv6 is known to not be available. */ +bool grpc_ares_query_ipv6(); + /* Exposed only for testing */ void grpc_cares_wrapper_test_only_address_sorting_sort( grpc_lb_addresses* lb_addrs); diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc new file mode 100644 index 00000000000..23c0fec74f3 --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc @@ -0,0 +1,29 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "src/core/lib/iomgr/port.h" +#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) + +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" +#include "src/core/lib/iomgr/socket_utils_posix.h" + +bool grpc_ares_query_ipv6() { return grpc_ipv6_loopback_available(); } + +#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc new file mode 100644 index 00000000000..ee827e284e2 --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc @@ -0,0 +1,29 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "src/core/lib/iomgr/port.h" +#if GRPC_ARES == 1 && defined(GPR_WINDOWS) + +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" +#include "src/core/lib/iomgr/socket_windows.h" + +bool grpc_ares_query_ipv6() { return grpc_ipv6_loopback_available(); } + +#endif /* GRPC_ARES == 1 && defined(GPR_WINDOWS) */ diff --git a/src/core/lib/iomgr/socket_windows.cc b/src/core/lib/iomgr/socket_windows.cc index 2e23409582b..4ad31cb35d1 100644 --- a/src/core/lib/iomgr/socket_windows.cc +++ b/src/core/lib/iomgr/socket_windows.cc @@ -36,6 +36,7 @@ #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_windows.h" +#include "src/core/lib/iomgr/sockaddr_windows.h" #include "src/core/lib/iomgr/socket_windows.h" grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name) { @@ -148,4 +149,32 @@ void grpc_socket_become_ready(grpc_winsocket* socket, if (should_destroy) destroy(socket); } +static gpr_once g_probe_ipv6_once = GPR_ONCE_INIT; +static bool g_ipv6_loopback_available = false; + +static void probe_ipv6_once(void) { + SOCKET s = socket(AF_INET6, SOCK_STREAM, 0); + g_ipv6_loopback_available = 0; + if (s == INVALID_SOCKET) { + gpr_log(GPR_INFO, "Disabling AF_INET6 sockets because socket() failed."); + } else { + grpc_sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_addr.s6_addr[15] = 1; /* [::1]:0 */ + if (bind(s, reinterpret_cast(&addr), sizeof(addr)) == 0) { + g_ipv6_loopback_available = 1; + } else { + gpr_log(GPR_INFO, + "Disabling AF_INET6 sockets because ::1 is not available."); + } + closesocket(s); + } +} + +int grpc_ipv6_loopback_available(void) { + gpr_once_init(&g_probe_ipv6_once, probe_ipv6_once); + return g_ipv6_loopback_available; +} + #endif /* GRPC_WINSOCK_SOCKET */ diff --git a/src/core/lib/iomgr/socket_windows.h b/src/core/lib/iomgr/socket_windows.h index 7bd01eded5a..b09b9da5628 100644 --- a/src/core/lib/iomgr/socket_windows.h +++ b/src/core/lib/iomgr/socket_windows.h @@ -108,6 +108,10 @@ void grpc_socket_notify_on_read(grpc_winsocket* winsocket, void grpc_socket_become_ready(grpc_winsocket* winsocket, grpc_winsocket_callback_info* ci); +/* Returns true if this system can create AF_INET6 sockets bound to ::1. + The value is probed once, and cached for the life of the process. */ +int grpc_ipv6_loopback_available(void); + #endif #endif /* GRPC_CORE_LIB_IOMGR_SOCKET_WINDOWS_H */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 49185cc6487..d6efb49750f 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -354,8 +354,11 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', 'src/cpp/ext/filters/census/grpc_context.cc', diff --git a/test/core/iomgr/BUILD b/test/core/iomgr/BUILD index fb0490a95f4..002671a5fae 100644 --- a/test/core/iomgr/BUILD +++ b/test/core/iomgr/BUILD @@ -124,6 +124,19 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "grpc_ipv6_loopback_available_test", + srcs = ["grpc_ipv6_loopback_available_test.cc"], + language = "C++", + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], +) + + grpc_cc_test( name = "load_file_test", srcs = ["load_file_test.cc"], diff --git a/test/core/iomgr/grpc_ipv6_loopback_available_test.cc b/test/core/iomgr/grpc_ipv6_loopback_available_test.cc new file mode 100644 index 00000000000..329aa9a8517 --- /dev/null +++ b/test/core/iomgr/grpc_ipv6_loopback_available_test.cc @@ -0,0 +1,48 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "src/core/lib/iomgr/port.h" + +// grpc_ipv6_loopback_available isn't currently available on UV. +#ifndef GRPC_UV + +#include +#include +#include "test/core/util/test_config.h" + +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/socket_windows.h" +#else +#include "src/core/lib/iomgr/socket_utils_posix.h" +#endif + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + grpc_init(); + // This test assumes that the ipv6 loopback is available + // in all environments in which grpc tests run in. + GPR_ASSERT(grpc_ipv6_loopback_available()); + grpc_shutdown(); + return 0; +} + +#else + +int main(int argc, char** argv) { return 0; } + +#endif /* GRPC_UV */ diff --git a/test/cpp/naming/address_sorting_test.cc b/test/cpp/naming/address_sorting_test.cc index a92e9e3b3e3..04c300876cc 100644 --- a/test/cpp/naming/address_sorting_test.cc +++ b/test/cpp/naming/address_sorting_test.cc @@ -24,10 +24,8 @@ #include #include -#include #include #include -#include #include #include @@ -51,6 +49,11 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#ifndef GPR_WINDOWS +#include +#include +#endif + namespace { struct TestAddress { @@ -190,10 +193,18 @@ void VerifyLbAddrOutputs(grpc_lb_addresses* lb_addrs, grpc_lb_addresses_destroy(lb_addrs); } -} // namespace +/* We need to run each test case inside of its own + * isolated grpc_init/grpc_shutdown pair, so that + * the "address sorting source addr factory" can be + * restored to its default for each test case. */ +class AddressSortingTest : public ::testing::Test { + protected: + void SetUp() override { grpc_init(); } + void TearDown() override { grpc_shutdown(); } +}; /* Tests for rule 1 */ -TEST(AddressSortingTest, TestDepriotizesUnreachableAddresses) { +TEST_F(AddressSortingTest, TestDepriotizesUnreachableAddresses) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -212,7 +223,7 @@ TEST(AddressSortingTest, TestDepriotizesUnreachableAddresses) { }); } -TEST(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv6) { +TEST_F(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv6) { bool ipv4_supported = true; bool ipv6_supported = false; OverrideAddressSortingSourceAddrFactory( @@ -231,7 +242,7 @@ TEST(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv6) { }); } -TEST(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv4) { +TEST_F(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv4) { bool ipv4_supported = false; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -253,7 +264,7 @@ TEST(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv4) { /* Tests for rule 2 */ -TEST(AddressSortingTest, TestDepriotizesNonMatchingScope) { +TEST_F(AddressSortingTest, TestDepriotizesNonMatchingScope) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -277,7 +288,7 @@ TEST(AddressSortingTest, TestDepriotizesNonMatchingScope) { /* Tests for rule 5 */ -TEST(AddressSortingTest, TestUsesLabelFromDefaultTable) { +TEST_F(AddressSortingTest, TestUsesLabelFromDefaultTable) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -300,7 +311,7 @@ TEST(AddressSortingTest, TestUsesLabelFromDefaultTable) { /* Flip the input on the test above to reorder the sort function's * comparator's inputs. */ -TEST(AddressSortingTest, TestUsesLabelFromDefaultTableInputFlipped) { +TEST_F(AddressSortingTest, TestUsesLabelFromDefaultTableInputFlipped) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -323,8 +334,8 @@ TEST(AddressSortingTest, TestUsesLabelFromDefaultTableInputFlipped) { /* Tests for rule 6 */ -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWithAnIpv4Address) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWithAnIpv4Address) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -348,8 +359,8 @@ TEST(AddressSortingTest, }); } -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWithV4CompatAndLocalhostAddress) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWithV4CompatAndLocalhostAddress) { bool ipv4_supported = true; bool ipv6_supported = true; // Handle unique observed behavior of inet_ntop(v4-compatible-address) on OS X. @@ -377,8 +388,8 @@ TEST(AddressSortingTest, }); } -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWithCatchAllAndLocalhostAddress) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWithCatchAllAndLocalhostAddress) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -403,8 +414,8 @@ TEST(AddressSortingTest, }); } -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWith2000PrefixedAddress) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWith2000PrefixedAddress) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -426,7 +437,7 @@ TEST(AddressSortingTest, }); } -TEST( +TEST_F( AddressSortingTest, TestUsesDestinationWithHigherPrecedenceWith2000PrefixedAddressEnsurePrefixMatchHasNoEffect) { bool ipv4_supported = true; @@ -448,8 +459,8 @@ TEST( }); } -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWithLinkAndSiteLocalAddresses) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWithLinkAndSiteLocalAddresses) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -469,19 +480,22 @@ TEST(AddressSortingTest, }); } -TEST( +TEST_F( AddressSortingTest, TestUsesDestinationWithHigherPrecedenceWithCatchAllAndAndV4MappedAddresses) { bool ipv4_supported = true; bool ipv6_supported = true; + // Use embedded ipv4 addresses with leading 1's instead of zero's to be + // compatible with inet_ntop implementations that can display such + // addresses with leading zero's as e.g.: "::ffff:0:2", as on windows. OverrideAddressSortingSourceAddrFactory( ipv4_supported, ipv6_supported, { - {"[::ffff:0.0.0.2]:443", {"[::ffff:0.0.0.3]:0", AF_INET6}}, + {"[::ffff:1.1.1.2]:443", {"[::ffff:1.1.1.3]:0", AF_INET6}}, {"[1234::2]:443", {"[1234::3]:0", AF_INET6}}, }); grpc_lb_addresses* lb_addrs = BuildLbAddrInputs({ - {"[::ffff:0.0.0.2]:443", AF_INET6}, + {"[::ffff:1.1.1.2]:443", AF_INET6}, {"[1234::2]:443", AF_INET6}, }); grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); @@ -489,13 +503,13 @@ TEST( // ::ffff:0:2 should match the v4-mapped // precedence entry and be deprioritized. "[1234::2]:443", - "[::ffff:0.0.0.2]:443", + "[::ffff:1.1.1.2]:443", }); } /* Tests for rule 8 */ -TEST(AddressSortingTest, TestPrefersSmallerScope) { +TEST_F(AddressSortingTest, TestPrefersSmallerScope) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -520,7 +534,7 @@ TEST(AddressSortingTest, TestPrefersSmallerScope) { /* Tests for rule 9 */ -TEST(AddressSortingTest, TestPrefersLongestMatchingSrcDstPrefix) { +TEST_F(AddressSortingTest, TestPrefersLongestMatchingSrcDstPrefix) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -543,8 +557,8 @@ TEST(AddressSortingTest, TestPrefersLongestMatchingSrcDstPrefix) { }); } -TEST(AddressSortingTest, - TestPrefersLongestMatchingSrcDstPrefixMatchesWholeAddress) { +TEST_F(AddressSortingTest, + TestPrefersLongestMatchingSrcDstPrefixMatchesWholeAddress) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -564,7 +578,7 @@ TEST(AddressSortingTest, }); } -TEST(AddressSortingTest, TestPrefersLongestPrefixStressInnerBytePrefix) { +TEST_F(AddressSortingTest, TestPrefersLongestPrefixStressInnerBytePrefix) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -584,7 +598,7 @@ TEST(AddressSortingTest, TestPrefersLongestPrefixStressInnerBytePrefix) { }); } -TEST(AddressSortingTest, TestPrefersLongestPrefixDiffersOnHighestBitOfByte) { +TEST_F(AddressSortingTest, TestPrefersLongestPrefixDiffersOnHighestBitOfByte) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -604,7 +618,7 @@ TEST(AddressSortingTest, TestPrefersLongestPrefixDiffersOnHighestBitOfByte) { }); } -TEST(AddressSortingTest, TestPrefersLongestPrefixDiffersByLastBit) { +TEST_F(AddressSortingTest, TestPrefersLongestPrefixDiffersByLastBit) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -628,7 +642,7 @@ TEST(AddressSortingTest, TestPrefersLongestPrefixDiffersByLastBit) { /* Tests for rule 10 */ -TEST(AddressSortingTest, TestStableSort) { +TEST_F(AddressSortingTest, TestStableSort) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -648,7 +662,7 @@ TEST(AddressSortingTest, TestStableSort) { }); } -TEST(AddressSortingTest, TestStableSortFiveElements) { +TEST_F(AddressSortingTest, TestStableSortFiveElements) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -677,7 +691,7 @@ TEST(AddressSortingTest, TestStableSortFiveElements) { }); } -TEST(AddressSortingTest, TestStableSortNoSrcAddrsExist) { +TEST_F(AddressSortingTest, TestStableSortNoSrcAddrsExist) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory(ipv4_supported, ipv6_supported, {}); @@ -698,7 +712,7 @@ TEST(AddressSortingTest, TestStableSortNoSrcAddrsExist) { }); } -TEST(AddressSortingTest, TestStableSortNoSrcAddrsExistWithIpv4) { +TEST_F(AddressSortingTest, TestStableSortNoSrcAddrsExistWithIpv4) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory(ipv4_supported, ipv6_supported, {}); @@ -713,7 +727,7 @@ TEST(AddressSortingTest, TestStableSortNoSrcAddrsExistWithIpv4) { }); } -TEST(AddressSortingTest, TestStableSortV4CompatAndSiteLocalAddresses) { +TEST_F(AddressSortingTest, TestStableSortV4CompatAndSiteLocalAddresses) { bool ipv4_supported = true; bool ipv6_supported = true; // Handle unique observed behavior of inet_ntop(v4-compatible-address) on OS X. @@ -744,6 +758,78 @@ TEST(AddressSortingTest, TestStableSortV4CompatAndSiteLocalAddresses) { }); } +/* TestPrefersIpv6Loopback tests the actual "address probing" code + * for the current platform, without any mocks. + * This test relies on the assumption that the ipv6 loopback address is + * available in the hosts/containers that grpc C/C++ tests run on + * (whether ipv4 loopback is available or not, an available ipv6 + * loopback should be preferred). */ +TEST_F(AddressSortingTest, TestPrefersIpv6Loopback) { + grpc_lb_addresses* lb_addrs = BuildLbAddrInputs({ + {"[::1]:443", AF_INET6}, + {"127.0.0.1:443", AF_INET}, + }); + grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + VerifyLbAddrOutputs(lb_addrs, { + "[::1]:443", + "127.0.0.1:443", + }); +} + +/* Flip the order of the inputs above and expect the same output order + * (try to rule out influence of arbitrary qsort ordering) */ +TEST_F(AddressSortingTest, TestPrefersIpv6LoopbackInputsFlipped) { + grpc_lb_addresses* lb_addrs = BuildLbAddrInputs({ + {"127.0.0.1:443", AF_INET}, + {"[::1]:443", AF_INET6}, + }); + grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + VerifyLbAddrOutputs(lb_addrs, { + "[::1]:443", + "127.0.0.1:443", + }); +} + +/* Try to rule out false positives in the above two tests in which + * the sorter might think that neither ipv6 or ipv4 loopback is + * available, but ipv6 loopback is still preferred only due + * to precedance table lookups. */ +TEST_F(AddressSortingTest, TestSorterKnowsIpv6LoopbackIsAvailable) { + sockaddr_in6 ipv6_loopback; + memset(&ipv6_loopback, 0, sizeof(ipv6_loopback)); + ipv6_loopback.sin6_family = AF_INET6; + ((char*)&ipv6_loopback.sin6_addr)[15] = 1; + ipv6_loopback.sin6_port = htons(443); + // Set up the source and destination parameters of + // address_sorting_get_source_addr + address_sorting_address sort_input_dest; + memcpy(&sort_input_dest.addr, &ipv6_loopback, sizeof(ipv6_loopback)); + sort_input_dest.len = sizeof(ipv6_loopback); + address_sorting_address source_for_sort_input_dest; + memset(&source_for_sort_input_dest, 0, sizeof(source_for_sort_input_dest)); + // address_sorting_get_source_addr returns true if a source address was found + // for the destination address, otherwise false. + EXPECT_TRUE(address_sorting_get_source_addr_for_testing( + &sort_input_dest, &source_for_sort_input_dest)); + // Now also check that the source address was filled in correctly. + EXPECT_GT(source_for_sort_input_dest.len, 0u); + sockaddr_in6* source_addr_output = + (sockaddr_in6*)source_for_sort_input_dest.addr; + EXPECT_EQ(source_addr_output->sin6_family, AF_INET6); + char* buf = static_cast(gpr_zalloc(100)); + EXPECT_NE(inet_ntop(AF_INET6, &source_addr_output->sin6_addr, buf, 100), + nullptr) + << "inet_ntop failed. Errno: " + std::to_string(errno); + std::string source_addr_str(buf); + gpr_free(buf); + // This test + // assumes that the source address for any loopback destination is also the + // loopback address. + EXPECT_EQ(source_addr_str, "::1"); +} + +} // namespace + int main(int argc, char** argv) { char* resolver = gpr_getenv("GRPC_DNS_RESOLVER"); if (resolver == nullptr || strlen(resolver) == 0) { @@ -754,9 +840,7 @@ int main(int argc, char** argv) { gpr_free(resolver); grpc_test_init(argc, argv); ::testing::InitGoogleTest(&argc, argv); - grpc_init(); auto result = RUN_ALL_TESTS(); - grpc_shutdown(); // Test sequential and nested inits and shutdowns. grpc_init(); grpc_init(); diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py index baa6512f624..5dad2ea7af6 100755 --- a/test/cpp/naming/gen_build_yaml.py +++ b/test/cpp/naming/gen_build_yaml.py @@ -110,7 +110,7 @@ def main(): 'gtest': True, 'run': True, 'src': ['test/cpp/naming/address_sorting_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util' + unsecure_build_config_suffix, 'grpc_test_util' + unsecure_build_config_suffix, diff --git a/third_party/address_sorting/address_sorting.c b/third_party/address_sorting/address_sorting.c index e4f3b537992..9aee0a5419f 100644 --- a/third_party/address_sorting/address_sorting.c +++ b/third_party/address_sorting/address_sorting.c @@ -55,12 +55,17 @@ static const int kIPv6AddrScopeGlobal = 3; static address_sorting_source_addr_factory* g_current_source_addr_factory = NULL; -static int address_sorting_get_source_addr(const address_sorting_address* dest, - address_sorting_address* source) { +static bool address_sorting_get_source_addr(const address_sorting_address* dest, + address_sorting_address* source) { return g_current_source_addr_factory->vtable->get_source_addr( g_current_source_addr_factory, dest, source); } +bool address_sorting_get_source_addr_for_testing( + const address_sorting_address* dest, address_sorting_address* source) { + return address_sorting_get_source_addr(dest, source); +} + static int ipv6_prefix_match_length(const struct sockaddr_in6* sa, const struct sockaddr_in6* sb) { unsigned char* a = (unsigned char*)&sa->sin6_addr; diff --git a/third_party/address_sorting/address_sorting_windows.c b/third_party/address_sorting/address_sorting_windows.c index b2f5708649e..662a88248e0 100644 --- a/third_party/address_sorting/address_sorting_windows.c +++ b/third_party/address_sorting/address_sorting_windows.c @@ -42,14 +42,54 @@ #if defined(ADDRESS_SORTING_WINDOWS) +#include +#include +#include #include +#include +#include -/* TODO : Add address sorting functionality to work on windows. */ +static bool windows_source_addr_factory_get_source_addr( + address_sorting_source_addr_factory* factory, + const address_sorting_address* dest_addr, + address_sorting_address* source_addr) { + bool source_addr_exists = false; + SOCKET s = socket(((struct sockaddr_in6*)dest_addr)->sin6_family, SOCK_DGRAM, + IPPROTO_UDP); + if (s != INVALID_SOCKET) { + if (connect(s, (struct sockaddr*)dest_addr, (int)dest_addr->len) == 0) { + address_sorting_address found_source_addr; + memset(&found_source_addr, 0, sizeof(found_source_addr)); + found_source_addr.len = sizeof(found_source_addr.addr); + if (getsockname(s, (struct sockaddr*)&found_source_addr.addr, + (socklen_t*)&found_source_addr.len) == 0) { + source_addr_exists = true; + *source_addr = found_source_addr; + } + } + closesocket(s); + } + return source_addr_exists; +} + +static void windows_source_addr_factory_destroy( + address_sorting_source_addr_factory* self) { + free(self); +} + +static const address_sorting_source_addr_factory_vtable + windows_source_addr_factory_vtable = { + windows_source_addr_factory_get_source_addr, + windows_source_addr_factory_destroy, +}; address_sorting_source_addr_factory* address_sorting_create_source_addr_factory_for_current_platform() { - abort(); - return NULL; + address_sorting_source_addr_factory* factory = + malloc(sizeof(address_sorting_source_addr_factory)); + memset(factory, 0, sizeof(address_sorting_source_addr_factory)); + factory->vtable = &windows_source_addr_factory_vtable; + return factory; } #endif // defined(ADDRESS_SORTING_WINDOWS) diff --git a/third_party/address_sorting/include/address_sorting/address_sorting.h b/third_party/address_sorting/include/address_sorting/address_sorting.h index f11cd424b53..c58fafe3f70 100644 --- a/third_party/address_sorting/include/address_sorting/address_sorting.h +++ b/third_party/address_sorting/include/address_sorting/address_sorting.h @@ -103,6 +103,9 @@ address_sorting_family address_sorting_abstract_get_family( void address_sorting_override_source_addr_factory_for_testing( address_sorting_source_addr_factory* factory); +bool address_sorting_get_source_addr_for_testing( + const address_sorting_address* dest, address_sorting_address* source); + #ifdef __cplusplus } #endif diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 576950934ec..18f56984fe2 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -924,9 +924,12 @@ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/native/README.md \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \ src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 072402b2cf6..a686dae8b4a 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -1032,6 +1032,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "grpc_ipv6_loopback_available_test", + "src": [ + "test/core/iomgr/grpc_ipv6_loopback_available_test.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -10254,9 +10271,12 @@ "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h", - "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc" + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc" ], "third_party": false, "type": "filegroup" diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index a5439a5db13..5815f82fefb 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1313,6 +1313,32 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "gtest": false, + "language": "c", + "name": "grpc_ipv6_loopback_available_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -5710,7 +5736,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5722,7 +5749,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, @@ -5732,7 +5760,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5744,7 +5773,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, From 3af1aaadabf49bc6274711a11f81627c0f351a9a Mon Sep 17 00:00:00 2001 From: Naresh Date: Fri, 27 Jul 2018 14:39:14 +0000 Subject: [PATCH 19/45] Basic setup to build gRPC Python with Bazel Building gRPC Python with Bazel has been one of the long requested additions to gRPC (#8079). Doing so had been made complex by the fact that Bazel itself is still in active development. There has been extensive work on building Cython code at tensorflow, which can be reused for gRPC's purposes as well. Major included changes required for building grpcio with Bazel are: - Include Cython as a third party Bazel package, to compile the Cython parts of gRPC Python. - Include rules for Python autoconfiguration so Python headers can be detected by cygrpc. --- bazel/cython_library.bzl | 74 +++++++ third_party/cython.BUILD | 29 +++ third_party/py/BUILD | 0 third_party/py/BUILD.tpl | 36 ++++ third_party/py/python_configure.bzl | 305 ++++++++++++++++++++++++++++ third_party/py/remote.BUILD.tpl | 10 + 6 files changed, 454 insertions(+) create mode 100644 bazel/cython_library.bzl create mode 100644 third_party/cython.BUILD create mode 100644 third_party/py/BUILD create mode 100644 third_party/py/BUILD.tpl create mode 100644 third_party/py/python_configure.bzl create mode 100644 third_party/py/remote.BUILD.tpl diff --git a/bazel/cython_library.bzl b/bazel/cython_library.bzl new file mode 100644 index 00000000000..48b41d74e8c --- /dev/null +++ b/bazel/cython_library.bzl @@ -0,0 +1,74 @@ +"""Custom rules for gRPC Python""" + + +# Adapted with modifications from +# tensorflow/tensorflow/core/platform/default/build_config.bzl +# Native Bazel rules don't exist yet to compile Cython code, but rules have +# been written at cython/cython and tensorflow/tensorflow. We branch from +# Tensorflow's version as it is more actively maintained and works for gRPC +# Python's needs. +def pyx_library(name, deps=[], py_deps=[], srcs=[], **kwargs): + """Compiles a group of .pyx / .pxd / .py files. + + First runs Cython to create .cpp files for each input .pyx or .py + .pxd + pair. Then builds a shared object for each, passing "deps" to each cc_binary + rule (includes Python headers by default). Finally, creates a py_library rule + with the shared objects and any pure Python "srcs", with py_deps as its + dependencies; the shared objects can be imported like normal Python files. + + Args: + name: Name for the rule. + deps: C/C++ dependencies of the Cython (e.g. Numpy headers). + py_deps: Pure Python dependencies of the final library. + srcs: .py, .pyx, or .pxd files to either compile or pass through. + **kwargs: Extra keyword arguments passed to the py_library. + """ + # First filter out files that should be run compiled vs. passed through. + py_srcs = [] + pyx_srcs = [] + pxd_srcs = [] + for src in srcs: + if src.endswith(".pyx") or (src.endswith(".py") and + src[:-3] + ".pxd" in srcs): + pyx_srcs.append(src) + elif src.endswith(".py"): + py_srcs.append(src) + else: + pxd_srcs.append(src) + if src.endswith("__init__.py"): + pxd_srcs.append(src) + + # Invoke cython to produce the shared object libraries. + for filename in pyx_srcs: + native.genrule( + name=filename + "_cython_translation", + srcs=[filename], + outs=[filename.split(".")[0] + ".cpp"], + # Optionally use PYTHON_BIN_PATH on Linux platforms so that python 3 + # works. Windows has issues with cython_binary so skip PYTHON_BIN_PATH. + cmd= + "PYTHONHASHSEED=0 $(location @cython//:cython_binary) --cplus $(SRCS) --output-file $(OUTS)", + tools=["@cython//:cython_binary"] + pxd_srcs, + ) + + shared_objects = [] + for src in pyx_srcs: + stem = src.split(".")[0] + shared_object_name = stem + ".so" + native.cc_binary( + name=shared_object_name, + srcs=[stem + ".cpp"], + deps=deps + ["@local_config_python//:python_headers"], + linkshared=1, + ) + shared_objects.append(shared_object_name) + + # Now create a py_library with these shared objects as data. + native.py_library( + name=name, + srcs=py_srcs, + deps=py_deps, + srcs_version="PY2AND3", + data=shared_objects, + **kwargs) + diff --git a/third_party/cython.BUILD b/third_party/cython.BUILD new file mode 100644 index 00000000000..ce9283e2a41 --- /dev/null +++ b/third_party/cython.BUILD @@ -0,0 +1,29 @@ +# Adapted with modifications from tensorflow/third_party/cython.BUILD + +py_library( + name="cython_lib", + srcs=glob( + ["Cython/**/*.py"], + exclude=[ + "**/Tests/*.py", + ], + ) + ["cython.py"], + data=glob([ + "Cython/**/*.pyx", + "Cython/Utility/*.*", + "Cython/Includes/**/*.pxd", + ]), + srcs_version="PY2AND3", + visibility=["//visibility:public"], +) + +# May not be named "cython", since that conflicts with Cython/ on OSX +py_binary( + name="cython_binary", + srcs=["cython.py"], + main="cython.py", + srcs_version="PY2AND3", + visibility=["//visibility:public"], + deps=["cython_lib"], +) + diff --git a/third_party/py/BUILD b/third_party/py/BUILD new file mode 100644 index 00000000000..e69de29bb2d diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl new file mode 100644 index 00000000000..2283c573bc3 --- /dev/null +++ b/third_party/py/BUILD.tpl @@ -0,0 +1,36 @@ +# Adapted with modifications from tensorflow/third_party/py/ + +package(default_visibility=["//visibility:public"]) + +# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib +# See https://docs.python.org/3/extending/windows.html +cc_import( + name="python_lib", + interface_library=select({ + ":windows": ":python_import_lib", + # A placeholder for Unix platforms which makes --no_build happy. + "//conditions:default": "not-existing.lib", + }), + system_provided=1, +) + +cc_library( + name="python_headers", + hdrs=[":python_include"], + deps=select({ + ":windows": [":python_lib"], + "//conditions:default": [], + }), + includes=["python_include"], +) + +config_setting( + name="windows", + values={"cpu": "x64_windows"}, + visibility=["//visibility:public"], +) + +%{PYTHON_INCLUDE_GENRULE} +%{PYTHON_IMPORT_LIB_GENRULE} + + diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl new file mode 100644 index 00000000000..2ba1e07049c --- /dev/null +++ b/third_party/py/python_configure.bzl @@ -0,0 +1,305 @@ +# Adapted with modifications from tensorflow/third_party/py/ +"""Repository rule for Python autoconfiguration. + +`python_configure` depends on the following environment variables: + + * `PYTHON_BIN_PATH`: location of python binary. + * `PYTHON_LIB_PATH`: Location of python libraries. +""" + +_BAZEL_SH = "BAZEL_SH" +_PYTHON_BIN_PATH = "PYTHON_BIN_PATH" +_PYTHON_LIB_PATH = "PYTHON_LIB_PATH" +_PYTHON_CONFIG_REPO = "PYTHON_CONFIG_REPO" + + +def _tpl(repository_ctx, tpl, substitutions={}, out=None): + if not out: + out = tpl + repository_ctx.template(out, Label("//third_party/py:%s.tpl" % tpl), + substitutions) + + +def _fail(msg): + """Output failure message when auto configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) + + +def _is_windows(repository_ctx): + """Returns true if the host operating system is windows.""" + os_name = repository_ctx.os.name.lower() + return os_name.find("windows") != -1 + + +def _execute(repository_ctx, + cmdline, + error_msg=None, + error_details=None, + empty_stdout_fine=False): + """Executes an arbitrary shell command. + + Args: + repository_ctx: the repository_ctx object + cmdline: list of strings, the command to execute + error_msg: string, a summary of the error if the command fails + error_details: string, details about the error or steps to fix it + empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise + it's an error + Return: + the result of repository_ctx.execute(cmdline) + """ + result = repository_ctx.execute(cmdline) + if result.stderr or not (empty_stdout_fine or result.stdout): + _fail("\n".join([ + error_msg.strip() if error_msg else "Repository command failed", + result.stderr.strip(), error_details if error_details else "" + ])) + else: + return result + + +def _read_dir(repository_ctx, src_dir): + """Returns a string with all files in a directory. + + Finds all files inside a directory, traversing subfolders and following + symlinks. The returned string contains the full path of all files + separated by line breaks. + """ + if _is_windows(repository_ctx): + src_dir = src_dir.replace("/", "\\") + find_result = _execute( + repository_ctx, + ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], + empty_stdout_fine=True) + # src_files will be used in genrule.outs where the paths must + # use forward slashes. + return find_result.stdout.replace("\\", "/") + else: + find_result = _execute( + repository_ctx, ["find", src_dir, "-follow", "-type", "f"], + empty_stdout_fine=True) + return find_result.stdout + + +def _genrule(src_dir, genrule_name, command, outs): + """Returns a string with a genrule. + + Genrule executes the given command and produces the given outputs. + """ + return ('genrule(\n' + ' name = "' + genrule_name + '",\n' + + ' outs = [\n' + outs + '\n ],\n' + ' cmd = """\n' + + command + '\n """,\n' + ')\n') + + +def _normalize_path(path): + """Returns a path with '/' and remove the trailing slash.""" + path = path.replace("\\", "/") + if path[-1] == "/": + path = path[:-1] + return path + + +def _symlink_genrule_for_dir(repository_ctx, + src_dir, + dest_dir, + genrule_name, + src_files=[], + dest_files=[]): + """Returns a genrule to symlink(or copy if on Windows) a set of files. + + If src_dir is passed, files will be read from the given directory; otherwise + we assume files are in src_files and dest_files + """ + if src_dir != None: + src_dir = _normalize_path(src_dir) + dest_dir = _normalize_path(dest_dir) + files = '\n'.join( + sorted(_read_dir(repository_ctx, src_dir).splitlines())) + # Create a list with the src_dir stripped to use for outputs. + dest_files = files.replace(src_dir, '').splitlines() + src_files = files.splitlines() + command = [] + outs = [] + for i in range(len(dest_files)): + if dest_files[i] != "": + # If we have only one file to link we do not want to use the dest_dir, as + # $(@D) will include the full path to the file. + dest = '$(@D)/' + dest_dir + dest_files[i] if len( + dest_files) != 1 else '$(@D)/' + dest_files[i] + # On Windows, symlink is not supported, so we just copy all the files. + cmd = 'cp -f' if _is_windows(repository_ctx) else 'ln -s' + command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) + outs.append(' "' + dest_dir + dest_files[i] + '",') + return _genrule(src_dir, genrule_name, " && ".join(command), + "\n".join(outs)) + + +def _get_python_bin(repository_ctx): + """Gets the python bin path.""" + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) + if python_bin != None: + return python_bin + python_bin_path = repository_ctx.which("python") + if python_bin_path != None: + return str(python_bin_path) + _fail("Cannot find python in PATH, please make sure " + + "python is installed and add its directory in PATH, or --define " + + "%s='/something/else'.\nPATH=%s" % + (_PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) + + +def _get_bash_bin(repository_ctx): + """Gets the bash bin path.""" + bash_bin = repository_ctx.os.environ.get(_BAZEL_SH) + if bash_bin != None: + return bash_bin + else: + bash_bin_path = repository_ctx.which("bash") + if bash_bin_path != None: + return str(bash_bin_path) + else: + _fail( + "Cannot find bash in PATH, please make sure " + + "bash is installed and add its directory in PATH, or --define " + + "%s='/path/to/bash'.\nPATH=%s" % + (_BAZEL_SH, repository_ctx.os.environ.get("PATH", ""))) + + +def _get_python_lib(repository_ctx, python_bin): + """Gets the python lib path.""" + python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) + if python_lib != None: + return python_lib + print_lib = ( + "<=1:\n" + + " print(paths[0])\n" + "END") + cmd = '%s - %s' % (python_bin, print_lib) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + return result.stdout.strip('\n') + + +def _check_python_lib(repository_ctx, python_lib): + """Checks the python lib path.""" + cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if result.return_code == 1: + _fail("Invalid python library path: %s" % python_lib) + + +def _check_python_bin(repository_ctx, python_bin): + """Checks the python bin path.""" + cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if result.return_code == 1: + _fail("--define %s='%s' is not executable. Is it the python binary?" % + (_PYTHON_BIN_PATH, python_bin)) + + +def _get_python_include(repository_ctx, python_bin): + """Gets the python include path.""" + result = _execute( + repository_ctx, [ + python_bin, "-c", 'from __future__ import print_function;' + + 'from distutils import sysconfig;' + + 'print(sysconfig.get_python_inc())' + ], + error_msg="Problem getting python include path.", + error_details=( + "Is the Python binary path set up right? " + "(See ./configure or " + + _PYTHON_BIN_PATH + ".) " + "Is distutils installed?")) + return result.stdout.splitlines()[0] + + +def _get_python_import_lib_name(repository_ctx, python_bin): + """Get Python import library name (pythonXY.lib) on Windows.""" + result = _execute( + repository_ctx, [ + python_bin, "-c", + 'import sys;' + 'print("python" + str(sys.version_info[0]) + ' + + ' str(sys.version_info[1]) + ".lib")' + ], + error_msg="Problem getting python import library.", + error_details=("Is the Python binary path set up right? " + + "(See ./configure or " + _PYTHON_BIN_PATH + ".) ")) + return result.stdout.splitlines()[0] + + +def _create_local_python_repository(repository_ctx): + """Creates the repository containing files set up to build with Python.""" + python_bin = _get_python_bin(repository_ctx) + _check_python_bin(repository_ctx, python_bin) + python_lib = _get_python_lib(repository_ctx, python_bin) + _check_python_lib(repository_ctx, python_lib) + python_include = _get_python_include(repository_ctx, python_bin) + python_include_rule = _symlink_genrule_for_dir( + repository_ctx, python_include, 'python_include', 'python_include') + python_import_lib_genrule = "" + # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib + # See https://docs.python.org/3/extending/windows.html + if _is_windows(repository_ctx): + python_include = _normalize_path(python_include) + python_import_lib_name = _get_python_import_lib_name( + repository_ctx, python_bin) + python_import_lib_src = python_include.rsplit( + '/', 1)[0] + "/libs/" + python_import_lib_name + python_import_lib_genrule = _symlink_genrule_for_dir( + repository_ctx, None, '', 'python_import_lib', + [python_import_lib_src], [python_import_lib_name]) + _tpl( + repository_ctx, "BUILD", { + "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, + "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, + }) + + +def _create_remote_python_repository(repository_ctx, remote_config_repo): + """Creates pointers to a remotely configured repo set up to build with Python. + """ + _tpl(repository_ctx, "remote.BUILD", { + "%{REMOTE_PYTHON_REPO}": remote_config_repo, + }, "BUILD") + + +def _python_autoconf_impl(repository_ctx): + """Implementation of the python_autoconf repository rule.""" + if _PYTHON_CONFIG_REPO in repository_ctx.os.environ: + _create_remote_python_repository( + repository_ctx, repository_ctx.os.environ[_PYTHON_CONFIG_REPO]) + else: + _create_local_python_repository(repository_ctx) + + +python_configure = repository_rule( + implementation=_python_autoconf_impl, + environ=[ + _BAZEL_SH, + _PYTHON_BIN_PATH, + _PYTHON_LIB_PATH, + _PYTHON_CONFIG_REPO, + ], +) +"""Detects and configures the local Python. + +Add the following to your WORKSPACE FILE: + +```python +python_configure(name = "local_config_python") +``` + +Args: + name: A unique name for this workspace rule. +""" + diff --git a/third_party/py/remote.BUILD.tpl b/third_party/py/remote.BUILD.tpl new file mode 100644 index 00000000000..1bfe1f0bf65 --- /dev/null +++ b/third_party/py/remote.BUILD.tpl @@ -0,0 +1,10 @@ +# Adapted with modifications from tensorflow/third_party/py/ + +package(default_visibility=["//visibility:public"]) + +alias( + name="python_headers", + actual="%{REMOTE_PYTHON_REPO}:python_headers", +) + + From 11f199e34dc416a2bd8b56391b242a867bedade4 Mon Sep 17 00:00:00 2001 From: Naresh Date: Fri, 27 Jul 2018 14:44:09 +0000 Subject: [PATCH 20/45] Workspace changes to build gRPC Python with Bazel - Include new rules from bazelbuild/rules_python, which allow for importing pip dependencies with Bazel. - Make Cython BUILD file visible to WORKSPACE. - Use a custom requirements file for Bazel python. This is added to allow for requirements previously added only during build steps. --- WORKSPACE | 41 ++++++++++++++++++++++++++++++++++++++++- requirements.bazel.txt | 10 ++++++++++ third_party/BUILD | 1 + 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 requirements.bazel.txt diff --git a/WORKSPACE b/WORKSPACE index 8b68c0d2b19..9a0c41977f0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,5 +1,44 @@ -workspace(name = "com_github_grpc_grpc") +workspace(name="com_github_grpc_grpc") load("//bazel:grpc_deps.bzl", "grpc_deps", "grpc_test_only_deps") grpc_deps() grpc_test_only_deps() + +new_http_archive( + name="cython", + sha256="d68138a2381afbdd0876c3cb2a22389043fa01c4badede1228ee073032b07a27", + urls=[ + "https://github.com/cython/cython/archive/c2b80d87658a8525ce091cbe146cb7eaa29fed5c.tar.gz", + ], + strip_prefix="cython-c2b80d87658a8525ce091cbe146cb7eaa29fed5c", + build_file="//third_party:cython.BUILD", +) + +load("//third_party/py:python_configure.bzl", "python_configure") +python_configure(name="local_config_python") + +git_repository( + name="io_bazel_rules_python", + remote="https://github.com/bazelbuild/rules_python.git", + commit="8b5d0683a7d878b28fffe464779c8a53659fc645", +) + +load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip_import") + +pip_repositories() +pip_import( + name="grpc_python_dependencies", + requirements="//:requirements.bazel.txt", +) + +load("@grpc_python_dependencies//:requirements.bzl", "pip_install") +pip_install() + +git_repository( + name="org_pubref_rules_protobuf", + remote="https://github.com/pubref/rules_protobuf", + tag="v0.8.2", +) + +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_repositories") +py_proto_repositories() diff --git a/requirements.bazel.txt b/requirements.bazel.txt new file mode 100644 index 00000000000..16f31f9e94b --- /dev/null +++ b/requirements.bazel.txt @@ -0,0 +1,10 @@ +# GRPC Python setup requirements +coverage>=4.0 +cython==0.28.3 +enum34>=1.0.4 +protobuf>=3.5.0.post1 +six>=1.10 +wheel>=0.29 +futures>=2.2.0 +google-auth>=1.0.0 +oauth2client==4.1.0 diff --git a/third_party/BUILD b/third_party/BUILD index f06c5e9c67d..5ec919dc48d 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -8,4 +8,5 @@ exports_files([ "incremental.BUILD", "zope_interface.BUILD", "constantly.BUILD", + "cython.BUILD", ]) From c2a22a1ab8e8221c95f8874668eb6260c1e171b4 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 27 Jul 2018 16:19:03 -0700 Subject: [PATCH 21/45] Address core review comments --- src/core/lib/iomgr/resource_quota.cc | 80 ++++++++++--------- src/core/lib/iomgr/resource_quota.h | 8 +- src/cpp/thread_manager/thread_manager.cc | 6 +- src/cpp/thread_manager/thread_manager.h | 7 +- test/core/iomgr/resource_quota_test.cc | 45 ++++++----- .../cpp/thread_manager/thread_manager_test.cc | 30 ++++--- 6 files changed, 94 insertions(+), 82 deletions(-) diff --git a/src/core/lib/iomgr/resource_quota.cc b/src/core/lib/iomgr/resource_quota.cc index 47b7856e959..b6fc7579f7e 100644 --- a/src/core/lib/iomgr/resource_quota.cc +++ b/src/core/lib/iomgr/resource_quota.cc @@ -97,7 +97,7 @@ struct grpc_resource_user { bool added_to_free_pool; /* The number of threads currently allocated to this resource user */ - gpr_atm num_threads; + gpr_atm num_threads_allocated; /* Reclaimers: index 0 is the benign reclaimer, 1 is the destructive reclaimer */ @@ -138,22 +138,23 @@ struct grpc_resource_quota { gpr_atm last_size; - /* Mutex to protect max_threads and num_threads */ - /* Note: We could have used gpr_atm for max_threads and num_threads and avoid - * having this mutex; but in that case, each invocation of the function - * grpc_resource_user_alloc_threads() would have had to do at least two atomic - * loads (for max_threads and num_threads) followed by a CAS (on num_threads). - * Moreover, we expect grpc_resource_user_alloc_threads() to be often called - * concurrently thereby increasing the chances of failing the CAS operation. - * This additional complexity is not worth the tiny perf gain we may (or may - * not) have by using atomics */ - gpr_mu thd_mu; + /* Mutex to protect max_threads and num_threads_allocated */ + /* Note: We could have used gpr_atm for max_threads and num_threads_allocated + * and avoid having this mutex; but in that case, each invocation of the + * function grpc_resource_user_allocate_threads() would have had to do at + * least two atomic loads (for max_threads and num_threads_allocated) followed + * by a CAS (on num_threads_allocated). + * Moreover, we expect grpc_resource_user_allocate_threads() to be often + * called concurrently thereby increasing the chances of failing the CAS + * operation. This additional complexity is not worth the tiny perf gain we + * may (or may not) have by using atomics */ + gpr_mu thread_count_mu; /* Max number of threads allowed */ int max_threads; /* Number of threads currently allocated via this resource_quota object */ - int num_threads; + int num_threads_allocated; /* Has rq_step been scheduled to occur? */ bool step_scheduled; @@ -548,9 +549,9 @@ static void ru_destroy(void* ru, grpc_error* error) { grpc_resource_user* resource_user = static_cast(ru); GPR_ASSERT(gpr_atm_no_barrier_load(&resource_user->refs) == 0); // Free all the remaining thread quota - grpc_resource_user_free_threads( - resource_user, - static_cast(gpr_atm_no_barrier_load(&resource_user->num_threads))); + grpc_resource_user_free_threads(resource_user, + static_cast(gpr_atm_no_barrier_load( + &resource_user->num_threads_allocated))); for (int i = 0; i < GRPC_RULIST_COUNT; i++) { rulist_remove(resource_user, static_cast(i)); @@ -622,9 +623,9 @@ grpc_resource_quota* grpc_resource_quota_create(const char* name) { resource_quota->free_pool = INT64_MAX; resource_quota->size = INT64_MAX; gpr_atm_no_barrier_store(&resource_quota->last_size, GPR_ATM_MAX); - gpr_mu_init(&resource_quota->thd_mu); + gpr_mu_init(&resource_quota->thread_count_mu); resource_quota->max_threads = INT_MAX; - resource_quota->num_threads = 0; + resource_quota->num_threads_allocated = 0; resource_quota->step_scheduled = false; resource_quota->reclaiming = false; gpr_atm_no_barrier_store(&resource_quota->memory_usage_estimation, 0); @@ -647,7 +648,8 @@ grpc_resource_quota* grpc_resource_quota_create(const char* name) { void grpc_resource_quota_unref_internal(grpc_resource_quota* resource_quota) { if (gpr_unref(&resource_quota->refs)) { - GPR_ASSERT(resource_quota->num_threads == 0); // No outstanding thd quota + // No outstanding thread quota + GPR_ASSERT(resource_quota->num_threads_allocated == 0); GRPC_COMBINER_UNREF(resource_quota->combiner, "resource_quota"); gpr_free(resource_quota->name); gpr_free(resource_quota); @@ -681,9 +683,10 @@ double grpc_resource_quota_get_memory_pressure( /* Public API */ void grpc_resource_quota_set_max_threads(grpc_resource_quota* resource_quota, int new_max_threads) { - gpr_mu_lock(&resource_quota->thd_mu); + GPR_ASSERT(new_max_threads >= 0); + gpr_mu_lock(&resource_quota->thread_count_mu); resource_quota->max_threads = new_max_threads; - gpr_mu_unlock(&resource_quota->thd_mu); + gpr_mu_unlock(&resource_quota->thread_count_mu); } /* Public API */ @@ -771,7 +774,7 @@ grpc_resource_user* grpc_resource_user_create( grpc_closure_list_init(&resource_user->on_allocated); resource_user->allocating = false; resource_user->added_to_free_pool = false; - gpr_atm_no_barrier_store(&resource_user->num_threads, 0); + gpr_atm_no_barrier_store(&resource_user->num_threads_allocated, 0); resource_user->reclaimers[0] = nullptr; resource_user->reclaimers[1] = nullptr; resource_user->new_reclaimers[0] = nullptr; @@ -826,35 +829,38 @@ void grpc_resource_user_shutdown(grpc_resource_user* resource_user) { } } -bool grpc_resource_user_alloc_threads(grpc_resource_user* resource_user, - int thd_count) { +bool grpc_resource_user_allocate_threads(grpc_resource_user* resource_user, + int thread_count) { + GPR_ASSERT(thread_count >= 0); bool is_success = false; - gpr_mu_lock(&resource_user->resource_quota->thd_mu); + gpr_mu_lock(&resource_user->resource_quota->thread_count_mu); grpc_resource_quota* rq = resource_user->resource_quota; - if (rq->num_threads + thd_count <= rq->max_threads) { - rq->num_threads += thd_count; - gpr_atm_no_barrier_fetch_add(&resource_user->num_threads, thd_count); + if (rq->num_threads_allocated + thread_count <= rq->max_threads) { + rq->num_threads_allocated += thread_count; + gpr_atm_no_barrier_fetch_add(&resource_user->num_threads_allocated, + thread_count); is_success = true; } - gpr_mu_unlock(&resource_user->resource_quota->thd_mu); + gpr_mu_unlock(&resource_user->resource_quota->thread_count_mu); return is_success; } void grpc_resource_user_free_threads(grpc_resource_user* resource_user, - int thd_count) { - gpr_mu_lock(&resource_user->resource_quota->thd_mu); + int thread_count) { + GPR_ASSERT(thread_count >= 0); + gpr_mu_lock(&resource_user->resource_quota->thread_count_mu); grpc_resource_quota* rq = resource_user->resource_quota; - rq->num_threads -= thd_count; - int old_cnt = static_cast( - gpr_atm_no_barrier_fetch_add(&resource_user->num_threads, -thd_count)); - if (old_cnt < thd_count || rq->num_threads < 0) { + rq->num_threads_allocated -= thread_count; + int old_count = static_cast(gpr_atm_no_barrier_fetch_add( + &resource_user->num_threads_allocated, -thread_count)); + if (old_count < thread_count || rq->num_threads_allocated < 0) { gpr_log(GPR_ERROR, - "Releasing more threads (%d) that currently allocated (rq threads: " + "Releasing more threads (%d) than currently allocated (rq threads: " "%d, ru threads: %d)", - thd_count, old_cnt, rq->num_threads + thd_count); + thread_count, rq->num_threads_allocated + thread_count, old_count); abort(); } - gpr_mu_unlock(&resource_user->resource_quota->thd_mu); + gpr_mu_unlock(&resource_user->resource_quota->thread_count_mu); } void grpc_resource_user_alloc(grpc_resource_user* resource_user, size_t size, diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index 7342ef84c84..1d5e95e04a4 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -96,14 +96,14 @@ void grpc_resource_user_shutdown(grpc_resource_user* resource_user); /* Attempts to get quota (from the resource_user) to create 'thd_count' number * of threads. Returns true if successful (i.e the caller is now free to create * 'thd_count' number of threads) or false if quota is not available */ -bool grpc_resource_user_alloc_threads(grpc_resource_user* resource_user, - int thd_count); +bool grpc_resource_user_allocate_threads(grpc_resource_user* resource_user, + int thd_count); /* Releases 'thd_count' worth of quota back to the resource user. The quota * should have been previously obtained successfully by calling - * grpc_resource_user_alloc_threads(). + * grpc_resource_user_allocate_threads(). * * Note: There need not be an exact one-to-one correspondence between - * grpc_resource_user_alloc_threads() and grpc_resource_user_free_threads() + * grpc_resource_user_allocate_threads() and grpc_resource_user_free_threads() * calls. The only requirement is that the number of threads allocated should * all be eventually released */ void grpc_resource_user_free_threads(grpc_resource_user* resource_user, diff --git a/src/cpp/thread_manager/thread_manager.cc b/src/cpp/thread_manager/thread_manager.cc index 5d367511e2e..57067d46960 100644 --- a/src/cpp/thread_manager/thread_manager.cc +++ b/src/cpp/thread_manager/thread_manager.cc @@ -123,7 +123,7 @@ void ThreadManager::CleanupCompletedThreads() { } void ThreadManager::Initialize() { - if (!grpc_resource_user_alloc_threads(resource_user_, min_pollers_)) { + if (!grpc_resource_user_allocate_threads(resource_user_, min_pollers_)) { gpr_log(GPR_ERROR, "No thread quota available to even create the minimum required " "polling threads (i.e %d). Unable to start the thread manager", @@ -165,9 +165,9 @@ void ThreadManager::MainWorkLoop() { break; case WORK_FOUND: // If we got work and there are now insufficient pollers and there is - // quota available to create a new thread,start a new poller thread + // quota available to create a new thread, start a new poller thread if (!shutdown_ && num_pollers_ < min_pollers_ && - grpc_resource_user_alloc_threads(resource_user_, 1)) { + grpc_resource_user_allocate_threads(resource_user_, 1)) { num_pollers_++; num_threads_++; max_active_threads_sofar_ = diff --git a/src/cpp/thread_manager/thread_manager.h b/src/cpp/thread_manager/thread_manager.h index 8332befed09..01043edb31e 100644 --- a/src/cpp/thread_manager/thread_manager.h +++ b/src/cpp/thread_manager/thread_manager.h @@ -100,7 +100,7 @@ class ThreadManager { // ThreadManager::MarkAsCompleted() // // WHY IS THIS NEEDED?: - // When a thread terminates, some other tread *must* call Join() on that + // When a thread terminates, some other thread *must* call Join() on that // thread so that the resources are released. Having a WorkerThread wrapper // will make this easier. Once Run() completes, each thread calls the // following two functions: @@ -113,8 +113,9 @@ class ThreadManager { // in the completed_threads_ list (since a thread cannot call Join() on // itself, it calls CleanupCompletedThreads() *before* calling // MarkAsCompleted()) - // TODO: sreek - consider creating the threads 'detached' so that Join() need - // not be called + // + // TODO(sreek): Consider creating the threads 'detached' so that Join() need + // not be called (and the need for this WorkerThread class is eliminated) class WorkerThread { public: WorkerThread(ThreadManager* thd_mgr); diff --git a/test/core/iomgr/resource_quota_test.cc b/test/core/iomgr/resource_quota_test.cc index 573e4010fa4..f3b35fed327 100644 --- a/test/core/iomgr/resource_quota_test.cc +++ b/test/core/iomgr/resource_quota_test.cc @@ -810,30 +810,31 @@ static void test_thread_limit() { grpc_resource_quota_set_max_threads(rq, 100); // Request quota for 100 threads (50 for ru1, 50 for ru2) - GPR_ASSERT(grpc_resource_user_alloc_threads(ru1, 10)); - GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 10)); - GPR_ASSERT(grpc_resource_user_alloc_threads(ru1, 40)); - GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 40)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru1, 10)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 10)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru1, 40)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 40)); - // Threads exhaused. Next request must fail - GPR_ASSERT(!grpc_resource_user_alloc_threads(ru2, 20)); + // Threads exhausted. Next request must fail + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru2, 20)); // Free 20 threads from two different users grpc_resource_user_free_threads(ru1, 10); grpc_resource_user_free_threads(ru2, 10); // Next request to 20 threads must succeed - GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 20)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 20)); // No more thread quota again - GPR_ASSERT(!grpc_resource_user_alloc_threads(ru1, 20)); + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru1, 20)); // Free 10 more grpc_resource_user_free_threads(ru1, 10); - GPR_ASSERT(grpc_resource_user_alloc_threads(ru1, 5)); - GPR_ASSERT(!grpc_resource_user_alloc_threads(ru2, 10)); // Only 5 available - GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 5)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru1, 5)); + GPR_ASSERT( + !grpc_resource_user_allocate_threads(ru2, 10)); // Only 5 available + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 5)); // Teardown (ru1 and ru2 release all the quota back to rq) grpc_resource_user_unref(ru1); @@ -841,7 +842,7 @@ static void test_thread_limit() { grpc_resource_quota_unref(rq); } -// Change max quota in either directions dynamically +// Change max quota in either direction dynamically static void test_thread_maxquota_change() { grpc_core::ExecCtx exec_ctx; @@ -854,34 +855,34 @@ static void test_thread_maxquota_change() { grpc_resource_quota_set_max_threads(rq, 100); // Request quota for 100 threads (50 for ru1, 50 for ru2) - GPR_ASSERT(grpc_resource_user_alloc_threads(ru1, 50)); - GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 50)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru1, 50)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 50)); - // Threads exhaused. Next request must fail - GPR_ASSERT(!grpc_resource_user_alloc_threads(ru2, 20)); + // Threads exhausted. Next request must fail + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru2, 20)); // Increase maxquota and retry // Max threads = 150; grpc_resource_quota_set_max_threads(rq, 150); - GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 20)); // ru2 = 70, ru1 = 50 + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 20)); // ru2=70, ru1=50 // Decrease maxquota (Note: Quota already given to ru1 and ru2 is unaffected) // Max threads = 10; grpc_resource_quota_set_max_threads(rq, 10); // New requests will fail until quota is available - GPR_ASSERT(!grpc_resource_user_alloc_threads(ru1, 10)); + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru1, 10)); // Make quota available - grpc_resource_user_free_threads(ru1, 50); // ru1 now has 0 - GPR_ASSERT(!grpc_resource_user_alloc_threads(ru1, 10)); // Still not enough + grpc_resource_user_free_threads(ru1, 50); // ru1 now has 0 + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru1, 10)); // not enough grpc_resource_user_free_threads(ru2, 70); // ru2 now has 0 // Now we can get quota up-to 10, the current max - GPR_ASSERT(grpc_resource_user_alloc_threads(ru2, 10)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 10)); // No more thread quota again - GPR_ASSERT(!grpc_resource_user_alloc_threads(ru1, 10)); + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru1, 10)); // Teardown (ru1 and ru2 release all the quota back to rq) grpc_resource_user_unref(ru1); diff --git a/test/cpp/thread_manager/thread_manager_test.cc b/test/cpp/thread_manager/thread_manager_test.cc index a7ed2dd380e..838f5f72adb 100644 --- a/test/cpp/thread_manager/thread_manager_test.cc +++ b/test/cpp/thread_manager/thread_manager_test.cc @@ -124,16 +124,18 @@ static void TestPollAndWork() { 2 /* min_pollers */, 10 /* max_pollers */, 10 /* poll_duration_ms */, 1 /* work_duration_ms */, 50 /* max_poll_calls */}; - grpc::ThreadManagerTest test_thd_mgr("TestThreadManager", rq, settings); + grpc::ThreadManagerTest test_thread_mgr("TestThreadManager", rq, settings); grpc_resource_quota_unref(rq); - test_thd_mgr.Initialize(); // Start the thread manager - test_thd_mgr.Wait(); // Wait for all threads to finish + test_thread_mgr.Initialize(); // Start the thread manager + test_thread_mgr.Wait(); // Wait for all threads to finish // Verify that The number of times DoWork() was called is equal to the number // of times WORK_FOUND was returned - gpr_log(GPR_DEBUG, "DoWork() called %d times", test_thd_mgr.GetNumDoWork()); - GPR_ASSERT(test_thd_mgr.GetNumDoWork() == test_thd_mgr.GetNumWorkFound()); + gpr_log(GPR_DEBUG, "DoWork() called %d times", + test_thread_mgr.GetNumDoWork()); + GPR_ASSERT(test_thread_mgr.GetNumDoWork() == + test_thread_mgr.GetNumWorkFound()); } static void TestThreadQuota() { @@ -151,18 +153,20 @@ static void TestThreadQuota() { // Create two thread managers (but with same resource quota). This means // that the max number of active threads across BOTH the thread managers // cannot be greater than kMaxNumthreads - grpc::ThreadManagerTest test_thd_mgr_1("TestThreadManager-1", rq, settings); - grpc::ThreadManagerTest test_thd_mgr_2("TestThreadManager-2", rq, settings); + grpc::ThreadManagerTest test_thread_mgr_1("TestThreadManager-1", rq, + settings); + grpc::ThreadManagerTest test_thread_mgr_2("TestThreadManager-2", rq, + settings); // It is ok to unref resource quota before starting thread managers. grpc_resource_quota_unref(rq); // Start both thread managers - test_thd_mgr_1.Initialize(); - test_thd_mgr_2.Initialize(); + test_thread_mgr_1.Initialize(); + test_thread_mgr_2.Initialize(); // Wait for both to finish - test_thd_mgr_1.Wait(); - test_thd_mgr_2.Wait(); + test_thread_mgr_1.Wait(); + test_thread_mgr_2.Wait(); // Now verify that the total number of active threads in either thread manager // never exceeds kMaxNumThreads @@ -173,8 +177,8 @@ static void TestThreadQuota() { // Its okay to not test this case here. The resource quota c-core tests // provide enough coverage to resource quota object with multiple resource // users - int max1 = test_thd_mgr_1.GetMaxActiveThreadsSoFar(); - int max2 = test_thd_mgr_2.GetMaxActiveThreadsSoFar(); + int max1 = test_thread_mgr_1.GetMaxActiveThreadsSoFar(); + int max2 = test_thread_mgr_2.GetMaxActiveThreadsSoFar(); gpr_log( GPR_DEBUG, "MaxActiveThreads in TestThreadManager_1: %d, TestThreadManager_2: %d", From 6eac5e41b1bc238bfb4acc456b2f6c906b09e87e Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 27 Jul 2018 17:23:02 -0700 Subject: [PATCH 22/45] std::max is not available on some windows platforms --- src/cpp/thread_manager/thread_manager.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cpp/thread_manager/thread_manager.cc b/src/cpp/thread_manager/thread_manager.cc index 57067d46960..fa9eec5f9ba 100644 --- a/src/cpp/thread_manager/thread_manager.cc +++ b/src/cpp/thread_manager/thread_manager.cc @@ -170,8 +170,9 @@ void ThreadManager::MainWorkLoop() { grpc_resource_user_allocate_threads(resource_user_, 1)) { num_pollers_++; num_threads_++; - max_active_threads_sofar_ = - std::max(max_active_threads_sofar_, num_threads_); + if (num_threads_ > max_active_threads_sofar_) { + max_active_threads_sofar_ = num_threads_; + } // Drop lock before spawning thread to avoid contention lock.unlock(); new WorkerThread(this); From 6bd510bf6bda90e6a3fbd07a881b1afa0afab6a5 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 30 Jul 2018 11:46:04 -0700 Subject: [PATCH 23/45] Make more tests in /tmpfs --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index cbc42f52956..3212e80854e 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -90,6 +90,9 @@ date git submodule update --init -# Store intermediate build files of ios binary size test into /tmpfs +# Store intermediate build files of ObjC tests into /tmpfs mkdir /tmpfs/Build-ios-binary-size ln -s /tmpfs/Build-ios-binary-size src/objective-c/examples/Sample/Build +mkdir /tmpfs/DerivedData +rm -rf ~/Library/Developer/Xcode/DerivedData +ln -s /tmpfs/DerivedData ~/Library/Developer/Xcode/DerivedData From d4c5c84f96cc49c615cef6dbb755fb3c8675ca28 Mon Sep 17 00:00:00 2001 From: kpayson64 Date: Mon, 30 Jul 2018 11:39:34 -0700 Subject: [PATCH 24/45] Immediately run write closures for failed stream --- src/core/ext/transport/chttp2/transport/chttp2_transport.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 9ad271753ce..bd6fec6fbe2 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1208,7 +1208,7 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t, grpc_error_add_child(closure->error_data.error, error); } if (closure->next_data.scratch < CLOSURE_BARRIER_FIRST_REF_BIT) { - if ((t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) || + if (s->seen_error || (t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) || !(closure->next_data.scratch & CLOSURE_BARRIER_MAY_COVER_WRITE)) { GRPC_CLOSURE_RUN(closure, closure->error_data.error); } else { From f22dc13c5bd7f215c22cd5656f82d8a76277829a Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 30 Jul 2018 12:29:50 -0700 Subject: [PATCH 25/45] Eliminate PingOneLocked() from LB policy API. --- .../filters/client_channel/client_channel.cc | 33 +++-- .../ext/filters/client_channel/lb_policy.h | 20 +-- .../client_channel/lb_policy/grpclb/grpclb.cc | 124 +++++------------- .../lb_policy/pick_first/pick_first.cc | 26 ++-- .../lb_policy/round_robin/round_robin.cc | 31 ++--- 5 files changed, 85 insertions(+), 149 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 024c9d737e8..45bca2738c1 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -571,15 +571,27 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) { if (op->send_ping.on_initiate != nullptr || op->send_ping.on_ack != nullptr) { if (chand->lb_policy == nullptr) { - GRPC_CLOSURE_SCHED( - op->send_ping.on_initiate, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Ping with no load balancing")); - GRPC_CLOSURE_SCHED( - op->send_ping.on_ack, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Ping with no load balancing")); + grpc_error* error = + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Ping with no load balancing"); + GRPC_CLOSURE_SCHED(op->send_ping.on_initiate, GRPC_ERROR_REF(error)); + GRPC_CLOSURE_SCHED(op->send_ping.on_ack, error); } else { - chand->lb_policy->PingOneLocked(op->send_ping.on_initiate, - op->send_ping.on_ack); + grpc_error* error = GRPC_ERROR_NONE; + grpc_core::LoadBalancingPolicy::PickState pick_state; + memset(&pick_state, 0, sizeof(pick_state)); + // Pick must return synchronously, because pick_state.on_complete is null. + GPR_ASSERT(chand->lb_policy->PickLocked(&pick_state, &error)); + if (pick_state.connected_subchannel != nullptr) { + pick_state.connected_subchannel->Ping(op->send_ping.on_initiate, + op->send_ping.on_ack); + } else { + if (error == GRPC_ERROR_NONE) { + error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "LB policy dropped call on ping"); + } + GRPC_CLOSURE_SCHED(op->send_ping.on_initiate, GRPC_ERROR_REF(error)); + GRPC_CLOSURE_SCHED(op->send_ping.on_ack, error); + } op->bind_pollset = nullptr; } op->send_ping.on_initiate = nullptr; @@ -2684,14 +2696,15 @@ class LbPicker { grpc_combiner_scheduler(chand->combiner)); calld->pick.on_complete = &calld->pick_closure; GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback"); - const bool pick_done = chand->lb_policy->PickLocked(&calld->pick); + grpc_error* error = GRPC_ERROR_NONE; + const bool pick_done = chand->lb_policy->PickLocked(&calld->pick, &error); if (GPR_LIKELY(pick_done)) { // Pick completed synchronously. if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: pick completed synchronously", chand, calld); } - pick_done_locked(elem, GRPC_ERROR_NONE); + pick_done_locked(elem, error); GRPC_CALL_STACK_UNREF(calld->owning_call, "pick_callback"); } else { // Pick will be returned asynchronously. diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 3150df8847b..31c08246ae4 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -71,6 +71,7 @@ class LoadBalancingPolicy /// Storage for LB token in \a initial_metadata, or nullptr if not used. grpc_linked_mdelem lb_token_mdelem_storage; /// Closure to run when pick is complete, if not completed synchronously. + /// If null, pick will fail if a result is not available synchronously. grpc_closure* on_complete; /// Will be set to the selected subchannel, or nullptr on failure or when /// the LB policy decides to drop the call. @@ -99,10 +100,15 @@ class LoadBalancingPolicy /// Finds an appropriate subchannel for a call, based on data in \a pick. /// \a pick must remain alive until the pick is complete. /// - /// If the pick succeeds and a result is known immediately, returns true. - /// Otherwise, \a pick->on_complete will be invoked once the pick is - /// complete with its error argument set to indicate success or failure. - virtual bool PickLocked(PickState* pick) GRPC_ABSTRACT; + /// If a result is known immediately, returns true, setting \a *error + /// upon failure. Otherwise, \a pick->on_complete will be invoked once + /// the pick is complete with its error argument set to indicate success + /// or failure. + /// + /// If \a pick->on_complete is null and no result is known immediately, + /// a synchronous failure will be returned (i.e., \a *error will be + /// set and true will be returned). + virtual bool PickLocked(PickState* pick, grpc_error** error) GRPC_ABSTRACT; /// Cancels \a pick. /// The \a on_complete callback of the pending pick will be invoked with @@ -133,12 +139,6 @@ class LoadBalancingPolicy virtual void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) GRPC_ABSTRACT; - /// Performs a connected subchannel ping via \a ConnectedSubchannel::Ping() - /// against one of the connected subchannels managed by the policy. - /// Note: This is intended only for use in tests. - virtual void PingOneLocked(grpc_closure* on_initiate, - grpc_closure* on_ack) GRPC_ABSTRACT; - /// Tries to enter a READY connectivity state. /// TODO(roth): As part of restructuring how we handle IDLE state, /// consider whether this method is still needed. diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 959c7441a37..2d1f7774748 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -123,7 +123,7 @@ class GrpcLb : public LoadBalancingPolicy { GrpcLb(const grpc_lb_addresses* addresses, const Args& args); void UpdateLocked(const grpc_channel_args& args) override; - bool PickLocked(PickState* pick) override; + bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, uint32_t initial_metadata_flags_eq, @@ -133,7 +133,6 @@ class GrpcLb : public LoadBalancingPolicy { grpc_connectivity_state CheckConnectivityLocked( grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; - void PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) override; void ExitIdleLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* child_channels) override; @@ -167,13 +166,6 @@ class GrpcLb : public LoadBalancingPolicy { PendingPick* next = nullptr; }; - /// A linked list of pending pings waiting for the RR policy to be created. - struct PendingPing { - grpc_closure* on_initiate; - grpc_closure* on_ack; - PendingPing* next = nullptr; - }; - /// Contains a call to the LB server and all the data related to the call. class BalancerCallState : public InternallyRefCountedWithTracing { @@ -272,14 +264,12 @@ class GrpcLb : public LoadBalancingPolicy { void AddPendingPick(PendingPick* pp); static void OnPendingPickComplete(void* arg, grpc_error* error); - // Pending ping methods. - void AddPendingPing(grpc_closure* on_initiate, grpc_closure* on_ack); - // Methods for dealing with the RR policy. void CreateOrUpdateRoundRobinPolicyLocked(); grpc_channel_args* CreateRoundRobinPolicyArgsLocked(); void CreateRoundRobinPolicyLocked(const Args& args); - bool PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp); + bool PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp, + grpc_error** error); void UpdateConnectivityStateFromRoundRobinPolicyLocked( grpc_error* rr_state_error); static void OnRoundRobinConnectivityChangedLocked(void* arg, @@ -342,9 +332,8 @@ class GrpcLb : public LoadBalancingPolicy { grpc_timer lb_fallback_timer_; grpc_closure lb_on_fallback_; - // Pending picks and pings that are waiting on the RR policy's connectivity. + // Pending picks that are waiting on the RR policy's connectivity. PendingPick* pending_picks_ = nullptr; - PendingPing* pending_pings_ = nullptr; // The RR policy to use for the backends. OrphanablePtr rr_policy_; @@ -1080,7 +1069,6 @@ GrpcLb::GrpcLb(const grpc_lb_addresses* addresses, GrpcLb::~GrpcLb() { GPR_ASSERT(pending_picks_ == nullptr); - GPR_ASSERT(pending_pings_ == nullptr); gpr_mu_destroy(&lb_channel_mu_); gpr_free((void*)server_name_); grpc_channel_args_destroy(args_); @@ -1126,14 +1114,6 @@ void GrpcLb::ShutdownLocked() { // Note: pp is deleted in this callback. GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error)); } - // Clear pending pings. - PendingPing* pping; - while ((pping = pending_pings_) != nullptr) { - pending_pings_ = pping->next; - GRPC_CLOSURE_SCHED(pping->on_initiate, GRPC_ERROR_REF(error)); - GRPC_CLOSURE_SCHED(pping->on_ack, GRPC_ERROR_REF(error)); - Delete(pping); - } GRPC_ERROR_UNREF(error); } @@ -1147,9 +1127,10 @@ void GrpcLb::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) { pending_picks_ = pp->next; pp->pick->on_complete = pp->original_on_complete; pp->pick->user_data = nullptr; - if (new_policy->PickLocked(pp->pick)) { + grpc_error* error = GRPC_ERROR_NONE; + if (new_policy->PickLocked(pp->pick, &error)) { // Synchronous return; schedule closure. - GRPC_CLOSURE_SCHED(pp->pick->on_complete, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(pp->pick->on_complete, error); } Delete(pp); } @@ -1233,58 +1214,37 @@ void GrpcLb::ExitIdleLocked() { } } -bool GrpcLb::PickLocked(PickState* pick) { +bool GrpcLb::PickLocked(PickState* pick, grpc_error** error) { PendingPick* pp = PendingPickCreate(pick); bool pick_done = false; if (rr_policy_ != nullptr) { - const grpc_connectivity_state rr_connectivity_state = - rr_policy_->CheckConnectivityLocked(nullptr); - // The RR policy may have transitioned to SHUTDOWN but the callback - // registered to capture this event (on_rr_connectivity_changed_) may not - // have been invoked yet. We need to make sure we aren't trying to pick - // from an RR policy instance that's in shutdown. - if (rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) { + if (grpc_lb_glb_trace.enabled()) { + gpr_log(GPR_INFO, "[grpclb %p] about to PICK from RR %p", this, + rr_policy_.get()); + } + pick_done = + PickFromRoundRobinPolicyLocked(false /* force_async */, pp, error); + } else { // rr_policy_ == NULL + if (pick->on_complete == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "No pick result available but synchronous result required."); + pick_done = true; + } else { if (grpc_lb_glb_trace.enabled()) { gpr_log(GPR_INFO, - "[grpclb %p] NOT picking from from RR %p: RR conn state=%s", - this, rr_policy_.get(), - grpc_connectivity_state_name(rr_connectivity_state)); + "[grpclb %p] No RR policy. Adding to grpclb's pending picks", + this); } AddPendingPick(pp); - pick_done = false; - } else { // RR not in shutdown - if (grpc_lb_glb_trace.enabled()) { - gpr_log(GPR_INFO, "[grpclb %p] about to PICK from RR %p", this, - rr_policy_.get()); + if (!started_picking_) { + StartPickingLocked(); } - pick_done = PickFromRoundRobinPolicyLocked(false /* force_async */, pp); - } - } else { // rr_policy_ == NULL - if (grpc_lb_glb_trace.enabled()) { - gpr_log(GPR_INFO, - "[grpclb %p] No RR policy. Adding to grpclb's pending picks", - this); - } - AddPendingPick(pp); - if (!started_picking_) { - StartPickingLocked(); + pick_done = false; } - pick_done = false; } return pick_done; } -void GrpcLb::PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) { - if (rr_policy_ != nullptr) { - rr_policy_->PingOneLocked(on_initiate, on_ack); - } else { - AddPendingPing(on_initiate, on_ack); - if (!started_picking_) { - StartPickingLocked(); - } - } -} - void GrpcLb::FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* child_channels) { // delegate to the RoundRobin to fill the children subchannels. @@ -1598,18 +1558,6 @@ void GrpcLb::AddPendingPick(PendingPick* pp) { pending_picks_ = pp; } -// -// PendingPing -// - -void GrpcLb::AddPendingPing(grpc_closure* on_initiate, grpc_closure* on_ack) { - PendingPing* pping = New(); - pping->on_initiate = on_initiate; - pping->on_ack = on_ack; - pping->next = pending_pings_; - pending_pings_ = pping; -} - // // code for interacting with the RR policy // @@ -1619,7 +1567,8 @@ void GrpcLb::AddPendingPing(grpc_closure* on_initiate, grpc_closure* on_ack) { // cleanups this callback would otherwise be responsible for. // If \a force_async is true, then we will manually schedule the // completion callback even if the pick is available immediately. -bool GrpcLb::PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp) { +bool GrpcLb::PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp, + grpc_error** error) { // Check for drops if we are not using fallback backend addresses. if (serverlist_ != nullptr) { // Look at the index into the serverlist to see if we should drop this call. @@ -1653,11 +1602,12 @@ bool GrpcLb::PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp) { GPR_ASSERT(pp->pick->user_data == nullptr); pp->pick->user_data = (void**)&pp->lb_token; // Pick via the RR policy. - bool pick_done = rr_policy_->PickLocked(pp->pick); + bool pick_done = rr_policy_->PickLocked(pp->pick, error); if (pick_done) { PendingPickSetMetadataAndContext(pp); if (force_async) { - GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(pp->original_on_complete, *error); + *error = GRPC_ERROR_NONE; pick_done = false; } Delete(pp); @@ -1709,18 +1659,8 @@ void GrpcLb::CreateRoundRobinPolicyLocked(const Args& args) { "[grpclb %p] Pending pick about to (async) PICK from RR %p", this, rr_policy_.get()); } - PickFromRoundRobinPolicyLocked(true /* force_async */, pp); - } - // Send pending pings to RR policy. - PendingPing* pping; - while ((pping = pending_pings_)) { - pending_pings_ = pping->next; - if (grpc_lb_glb_trace.enabled()) { - gpr_log(GPR_INFO, "[grpclb %p] Pending ping about to PING from RR %p", - this, rr_policy_.get()); - } - rr_policy_->PingOneLocked(pping->on_initiate, pping->on_ack); - Delete(pping); + grpc_error* error = GRPC_ERROR_NONE; + PickFromRoundRobinPolicyLocked(true /* force_async */, pp, &error); } } diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 023281db974..a74edd37bb2 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -46,7 +46,7 @@ class PickFirst : public LoadBalancingPolicy { explicit PickFirst(const Args& args); void UpdateLocked(const grpc_channel_args& args) override; - bool PickLocked(PickState* pick) override; + bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, uint32_t initial_metadata_flags_eq, @@ -56,7 +56,6 @@ class PickFirst : public LoadBalancingPolicy { grpc_connectivity_state CheckConnectivityLocked( grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; - void PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) override; void ExitIdleLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -173,9 +172,10 @@ void PickFirst::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) { PickState* pick; while ((pick = pending_picks_) != nullptr) { pending_picks_ = pick->next; - if (new_policy->PickLocked(pick)) { + grpc_error* error = GRPC_ERROR_NONE; + if (new_policy->PickLocked(pick, &error)) { // Synchronous return, schedule closure. - GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(pick->on_complete, error); } } } @@ -259,13 +259,18 @@ void PickFirst::ExitIdleLocked() { } } -bool PickFirst::PickLocked(PickState* pick) { +bool PickFirst::PickLocked(PickState* pick, grpc_error** error) { // If we have a selected subchannel already, return synchronously. if (selected_ != nullptr) { pick->connected_subchannel = selected_->connected_subchannel()->Ref(); return true; } // No subchannel selected yet, so handle asynchronously. + if (pick->on_complete == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "No pick result available but synchronous result required."); + return true; + } if (!started_picking_) { StartPickingLocked(); } @@ -293,17 +298,6 @@ void PickFirst::NotifyOnStateChangeLocked(grpc_connectivity_state* current, notify); } -void PickFirst::PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) { - if (selected_ != nullptr) { - selected_->connected_subchannel()->Ping(on_initiate, on_ack); - } else { - GRPC_CLOSURE_SCHED(on_initiate, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Not connected")); - GRPC_CLOSURE_SCHED(on_ack, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Not connected")); - } -} - void PickFirst::FillChildRefsForChannelz( ChildRefsList* child_subchannels_to_fill, ChildRefsList* ignored) { mu_guard guard(&child_refs_mu_); diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index fc56a4961f5..9c3a15c67b0 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -57,7 +57,7 @@ class RoundRobin : public LoadBalancingPolicy { explicit RoundRobin(const Args& args); void UpdateLocked(const grpc_channel_args& args) override; - bool PickLocked(PickState* pick) override; + bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, uint32_t initial_metadata_flags_eq, @@ -67,7 +67,6 @@ class RoundRobin : public LoadBalancingPolicy { grpc_connectivity_state CheckConnectivityLocked( grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; - void PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) override; void ExitIdleLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -253,9 +252,10 @@ void RoundRobin::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) { PickState* pick; while ((pick = pending_picks_) != nullptr) { pending_picks_ = pick->next; - if (new_policy->PickLocked(pick)) { + grpc_error* error = GRPC_ERROR_NONE; + if (new_policy->PickLocked(pick, &error)) { // Synchronous return, schedule closure. - GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(pick->on_complete, error); } } } @@ -368,7 +368,7 @@ void RoundRobin::DrainPendingPicksLocked() { } } -bool RoundRobin::PickLocked(PickState* pick) { +bool RoundRobin::PickLocked(PickState* pick, grpc_error** error) { if (grpc_lb_round_robin_trace.enabled()) { gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", this, shutdown_); } @@ -376,6 +376,11 @@ bool RoundRobin::PickLocked(PickState* pick) { if (subchannel_list_ != nullptr) { if (DoPickLocked(pick)) return true; } + if (pick->on_complete == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "No pick result available but synchronous result required."); + return true; + } /* no pick currently available. Save for later in list of pending picks */ pick->next = pending_picks_; pending_picks_ = pick; @@ -647,22 +652,6 @@ void RoundRobin::NotifyOnStateChangeLocked(grpc_connectivity_state* current, notify); } -void RoundRobin::PingOneLocked(grpc_closure* on_initiate, - grpc_closure* on_ack) { - const size_t next_ready_index = - subchannel_list_->GetNextReadySubchannelIndexLocked(); - if (next_ready_index < subchannel_list_->num_subchannels()) { - RoundRobinSubchannelData* selected = - subchannel_list_->subchannel(next_ready_index); - selected->connected_subchannel()->Ping(on_initiate, on_ack); - } else { - GRPC_CLOSURE_SCHED(on_initiate, GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Round Robin not connected")); - GRPC_CLOSURE_SCHED(on_ack, GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Round Robin not connected")); - } -} - void RoundRobin::UpdateLocked(const grpc_channel_args& args) { const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES); AutoChildRefsUpdater guard(this); From 2a3d32819944a1fc29b0fbf79f0f2935a9374abe Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Mon, 30 Jul 2018 18:29:24 -0700 Subject: [PATCH 26/45] Bump version to v1.14.0-pre2 --- BUILD | 4 ++-- build.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/BUILD b/BUILD index 0c6748767da..2d2722780bf 100644 --- a/BUILD +++ b/BUILD @@ -66,9 +66,9 @@ config_setting( # This should be updated along with build.yaml g_stands_for = "gladiolus" -core_version = "6.0.0-pre1" +core_version = "6.0.0-pre2" -version = "1.14.0-pre1" +version = "1.14.0-pre2" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build.yaml b/build.yaml index b0113c81926..d3bc84ffb94 100644 --- a/build.yaml +++ b/build.yaml @@ -12,9 +12,9 @@ settings: '#08': Use "-preN" suffixes to identify pre-release versions '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here - core_version: 6.0.0-pre1 + core_version: 6.0.0-pre2 g_stands_for: gladiolus - version: 1.14.0-pre1 + version: 1.14.0-pre2 filegroups: - name: alts_proto headers: From deeb781374e7c6165b39fc92cd9bf8971fabf93c Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Mon, 30 Jul 2018 18:34:35 -0700 Subject: [PATCH 27/45] Regenrate projects --- CMakeLists.txt | 2 +- Makefile | 6 +++--- gRPC-C++.podspec | 4 ++-- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 4 ++-- src/core/lib/surface/version.cc | 2 +- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 2 +- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 6 +++--- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/objective-c/tests/version.h | 4 ++-- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.core | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- 31 files changed, 38 insertions(+), 38 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 02e01da2838..b4c0226e178 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.14.0-pre1") +set(PACKAGE_VERSION "1.14.0-pre2") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index bd583bdf561..8b6cd4554ed 100644 --- a/Makefile +++ b/Makefile @@ -436,9 +436,9 @@ E = @echo Q = @ endif -CORE_VERSION = 6.0.0-pre1 -CPP_VERSION = 1.14.0-pre1 -CSHARP_VERSION = 1.14.0-pre1 +CORE_VERSION = 6.0.0-pre2 +CPP_VERSION = 1.14.0-pre2 +CSHARP_VERSION = 1.14.0-pre2 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index d331b6eac78..e38d5b48f00 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.14.0-pre1' + # version = '1.14.0-pre2' version = '0.0.3' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.14.0-pre1' + grpc_version = '1.14.0-pre2' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 5ce491dfff7..1e725abf223 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.14.0-pre1' + version = '1.14.0-pre2' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 889a1efa325..412ae7adeee 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.14.0-pre1' + version = '1.14.0-pre2' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index f642382580e..1c05a318dc0 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.14.0-pre1' + version = '1.14.0-pre2' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index d431bea27ab..d2657b35664 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.14.0-pre1' + version = '1.14.0-pre2' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index 0b07fb0639d..d5e8aaf6916 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2018-01-19 - 1.14.0RC1 - 1.14.0RC1 + 1.14.0RC2 + 1.14.0RC2 beta diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index 211cabdcbdb..ad1a4e0370e 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -23,6 +23,6 @@ #include -const char* grpc_version_string(void) { return "6.0.0-pre1"; } +const char* grpc_version_string(void) { return "6.0.0-pre2"; } const char* grpc_g_stands_for(void) { return "gladiolus"; } diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 91f1e549392..21ceb1cc183 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.14.0-pre1"; } +grpc::string Version() { return "1.14.0-pre2"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index ed31435a8ee..a1fba5fffe8 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.14.0-pre1 + 1.14.0-pre2 3.5.1 diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index b091f701a73..d888937b10e 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -38,6 +38,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.14.0-pre1"; + public const string CurrentVersion = "1.14.0-pre2"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index d8e6d0deede..b1dbd9e69bb 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.14.0-pre1 +set VERSION=1.14.0-pre2 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 92c0140fc0f..59ee72a05b9 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -45,8 +45,8 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.14.0-pre1" -OutputDirectory ../../artifacts -nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.0-pre1" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.14.0-pre1" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.14.0-pre2" -OutputDirectory ../../artifacts +nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.0-pre2" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.14.0-pre2" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index ef38adad238..6ebec95ba2e 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.14.0-pre1' + v = '1.14.0-pre2' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index d84ebbb2374..6f8ee8dab94 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-pre1" +#define GRPC_OBJC_VERSION_STRING @"1.14.0-pre2" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index f5c18e36277..74a952fe838 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-pre1" -#define GRPC_C_VERSION_STRING @"6.0.0-pre1" +#define GRPC_OBJC_VERSION_STRING @"1.14.0-pre2" +#define GRPC_C_VERSION_STRING @"6.0.0-pre2" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 0ac887dcd4a..9b916b466ed 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.14.0RC1" +#define PHP_GRPC_VERSION "1.14.0RC2" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index c9bea9c7111..e474a6b50e4 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.14.0rc1""" +__version__ = """1.14.0rc2""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index c66641985b4..61545652fa0 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.14.0rc1' +VERSION = '1.14.0rc2' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index d577106402f..61d27e12b39 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.14.0rc1' +VERSION = '1.14.0rc2' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 659fb69eb14..69caf17a54b 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.14.0rc1' +VERSION = '1.14.0rc2' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 8d80c625097..21bb29fff2e 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.14.0rc1' +VERSION = '1.14.0rc2' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index b292aebd9fb..7bdce8175cf 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.14.0rc1' +VERSION = '1.14.0rc2' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index d2a4bdc53f6..9698d31dab4 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.14.0.pre1' + VERSION = '1.14.0.pre2' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 0e4c3c5f5e7..88169f3a2b3 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.14.0.pre1' + VERSION = '1.14.0.pre2' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 11891edb5a3..cf5a7298665 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.14.0rc1' +VERSION = '1.14.0rc2' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 6558f99419b..dc8a38658eb 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-pre1 +PROJECT_NUMBER = 1.14.0-pre2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index e9b6c2dc3ce..addfc2da7c0 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-pre1 +PROJECT_NUMBER = 1.14.0-pre2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 501f07d4a3d..1759195c01f 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-pre1 +PROJECT_NUMBER = 6.0.0-pre2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 0e92946497d..643ee66402f 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-pre1 +PROJECT_NUMBER = 6.0.0-pre2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From fbfebd8e4e92067145127e23bbdbe3d596234634 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 30 Jul 2018 22:24:36 -0700 Subject: [PATCH 28/45] Add Cython functionality to directly wrap grpc_arg --- .../grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi | 5 ++++- .../grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi index 853bf6f8e04..f5baf8db3f1 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi @@ -1,4 +1,4 @@ -# Copyright 2018 gRPC authors. +# Copyright 2018 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,6 +22,9 @@ cdef void _destroy_pointer(void* pointer) cdef int _compare_pointer(void* first_pointer, void* second_pointer) +cdef tuple _wrap_grpc_arg(grpc_arg arg) + + cdef class _ArgumentProcessor: cdef grpc_arg c_argument diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi index aecd3d7b116..c94c03eee87 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi @@ -34,6 +34,18 @@ cdef int _compare_pointer(void* first_pointer, void* second_pointer): return 0 +cdef class _GrpcArgWrapper: + + cdef grpc_arg arg + + +cdef tuple _wrap_grpc_arg(grpc_arg arg): + + wrapped = _GrpcArgWrapper() + wrapped.arg = arg + return ("grpc.python._cygrpc._GrpcArgWrapper", wrapped) + + cdef class _ArgumentProcessor: cdef void c(self, argument, grpc_arg_pointer_vtable *vtable, references): @@ -51,6 +63,8 @@ cdef class _ArgumentProcessor: if encoded_value is not value: references.append(encoded_value) self.c_argument.value.string = encoded_value + elif isinstance(value, _GrpcArgWrapper): + self.c_argument = (<_GrpcArgWrapper>value).arg elif hasattr(value, '__int__'): # Pointer objects must override __int__() to return # the underlying C address (Python ints are word size). The From 3c497bf4c9259781f5e85442e4ea56924a16fd2d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 25 Jul 2018 15:49:50 +0200 Subject: [PATCH 29/45] add HelloworldXamarin scaffolding --- examples/csharp/HelloworldXamarin/.gitignore | 41 ++++ .../Droid/Assets/AboutAssets.txt | 19 ++ .../Droid/HelloworldXamarin.Droid.csproj | 67 ++++++ .../HelloworldXamarin/Droid/MainActivity.cs | 27 +++ .../Droid/Properties/AndroidManifest.xml | 6 + .../Droid/Properties/AssemblyInfo.cs | 27 +++ .../Droid/Resources/AboutResources.txt | 44 ++++ .../Droid/Resources/Resource.designer.cs | 63 ++++++ .../Droid/Resources/layout/Main.axml | 4 + .../Droid/Resources/mipmap-hdpi/Icon.png | Bin 0 -> 2201 bytes .../Droid/Resources/mipmap-mdpi/Icon.png | Bin 0 -> 1410 bytes .../Droid/Resources/mipmap-xhdpi/Icon.png | Bin 0 -> 3237 bytes .../Droid/Resources/mipmap-xxhdpi/Icon.png | Bin 0 -> 5414 bytes .../Droid/Resources/mipmap-xxxhdpi/Icon.png | Bin 0 -> 7825 bytes .../Droid/Resources/values/Strings.xml | 5 + .../HelloworldXamarin/HelloworldXamarin.sln | 45 ++++ .../HelloworldXamarin.projitems | 14 ++ .../HelloworldXamarin.shproj | 11 + .../HelloworldXamarin/MyClass.cs | 10 + .../HelloworldXamarin/iOS/AppDelegate.cs | 59 +++++ .../AppIcon.appiconset/Contents.json | 202 ++++++++++++++++++ .../iOS/Assets.xcassets/Contents.json | 6 + .../HelloworldXamarin/iOS/Entitlements.plist | 6 + .../iOS/HelloworldXamarin.iOS.csproj | 110 ++++++++++ .../csharp/HelloworldXamarin/iOS/Info.plist | 48 +++++ .../iOS/LaunchScreen.storyboard | 27 +++ examples/csharp/HelloworldXamarin/iOS/Main.cs | 20 ++ .../HelloworldXamarin/iOS/Main.storyboard | 40 ++++ .../HelloworldXamarin/iOS/ViewController.cs | 34 +++ .../iOS/ViewController.designer.cs | 25 +++ 30 files changed, 960 insertions(+) create mode 100644 examples/csharp/HelloworldXamarin/.gitignore create mode 100644 examples/csharp/HelloworldXamarin/Droid/Assets/AboutAssets.txt create mode 100644 examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj create mode 100644 examples/csharp/HelloworldXamarin/Droid/MainActivity.cs create mode 100644 examples/csharp/HelloworldXamarin/Droid/Properties/AndroidManifest.xml create mode 100644 examples/csharp/HelloworldXamarin/Droid/Properties/AssemblyInfo.cs create mode 100644 examples/csharp/HelloworldXamarin/Droid/Resources/AboutResources.txt create mode 100644 examples/csharp/HelloworldXamarin/Droid/Resources/Resource.designer.cs create mode 100644 examples/csharp/HelloworldXamarin/Droid/Resources/layout/Main.axml create mode 100644 examples/csharp/HelloworldXamarin/Droid/Resources/mipmap-hdpi/Icon.png create mode 100644 examples/csharp/HelloworldXamarin/Droid/Resources/mipmap-mdpi/Icon.png create mode 100644 examples/csharp/HelloworldXamarin/Droid/Resources/mipmap-xhdpi/Icon.png create mode 100644 examples/csharp/HelloworldXamarin/Droid/Resources/mipmap-xxhdpi/Icon.png create mode 100644 examples/csharp/HelloworldXamarin/Droid/Resources/mipmap-xxxhdpi/Icon.png create mode 100644 examples/csharp/HelloworldXamarin/Droid/Resources/values/Strings.xml create mode 100644 examples/csharp/HelloworldXamarin/HelloworldXamarin.sln create mode 100644 examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldXamarin.projitems create mode 100644 examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldXamarin.shproj create mode 100644 examples/csharp/HelloworldXamarin/HelloworldXamarin/MyClass.cs create mode 100644 examples/csharp/HelloworldXamarin/iOS/AppDelegate.cs create mode 100644 examples/csharp/HelloworldXamarin/iOS/Assets.xcassets/AppIcon.appiconset/Contents.json create mode 100644 examples/csharp/HelloworldXamarin/iOS/Assets.xcassets/Contents.json create mode 100644 examples/csharp/HelloworldXamarin/iOS/Entitlements.plist create mode 100644 examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj create mode 100644 examples/csharp/HelloworldXamarin/iOS/Info.plist create mode 100644 examples/csharp/HelloworldXamarin/iOS/LaunchScreen.storyboard create mode 100644 examples/csharp/HelloworldXamarin/iOS/Main.cs create mode 100644 examples/csharp/HelloworldXamarin/iOS/Main.storyboard create mode 100644 examples/csharp/HelloworldXamarin/iOS/ViewController.cs create mode 100644 examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs diff --git a/examples/csharp/HelloworldXamarin/.gitignore b/examples/csharp/HelloworldXamarin/.gitignore new file mode 100644 index 00000000000..bf793edcc66 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/.gitignore @@ -0,0 +1,41 @@ +# Autosave files +*~ + +# build +[Oo]bj/ +[Bb]in/ +packages/ +TestResults/ + +# globs +Makefile.in +*.DS_Store +*.sln.cache +*.suo +*.cache +*.pidb +*.userprefs +*.usertasks +config.log +config.make +config.status +aclocal.m4 +install-sh +autom4te.cache/ +*.user +*.tar.gz +tarballs/ +test-results/ +Thumbs.db +.vs/ + +# Mac bundle stuff +*.dmg +*.app + +# resharper +*_Resharper.* +*.Resharper + +# dotCover +*.dotCover diff --git a/examples/csharp/HelloworldXamarin/Droid/Assets/AboutAssets.txt b/examples/csharp/HelloworldXamarin/Droid/Assets/AboutAssets.txt new file mode 100644 index 00000000000..a9b0638eb1b --- /dev/null +++ b/examples/csharp/HelloworldXamarin/Droid/Assets/AboutAssets.txt @@ -0,0 +1,19 @@ +Any raw assets you want to be deployed with your application can be placed in +this directory (and child directories) and given a Build Action of "AndroidAsset". + +These files will be deployed with your package and will be accessible using Android's +AssetManager, like this: + +public class ReadAsset : Activity +{ + protected override void OnCreate (Bundle bundle) + { + base.OnCreate (bundle); + + InputStream input = Assets.Open ("my_asset.txt"); + } +} + +Additionally, some Android functions will automatically load asset files: + +Typeface tf = Typeface.CreateFromAsset (Context.Assets, "fonts/samplefont.ttf"); diff --git a/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj b/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj new file mode 100644 index 00000000000..20f1cedb872 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj @@ -0,0 +1,67 @@ + + + + Debug + AnyCPU + {B9B0D41C-1C07-4590-A919-5865E741B2EA} + {EFBA0AD7-5A72-4C68-AF49-83D382785DCF};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + Library + HelloworldXamarin.Droid + HelloworldXamarin.Droid + v8.1 + True + Resources\Resource.designer.cs + Resource + Properties\AndroidManifest.xml + Resources + Assets + true + + + true + full + false + bin\Debug + DEBUG; + prompt + 4 + None + + + true + pdbonly + true + bin\Release + prompt + 4 + true + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs b/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs new file mode 100644 index 00000000000..20982422f28 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs @@ -0,0 +1,27 @@ +using Android.App; +using Android.Widget; +using Android.OS; + +namespace HelloworldXamarin.Droid +{ + [Activity(Label = "HelloworldXamarin", MainLauncher = true, Icon = "@mipmap/icon")] + public class MainActivity : Activity + { + int count = 1; + + protected override void OnCreate(Bundle savedInstanceState) + { + base.OnCreate(savedInstanceState); + + // Set our view from the "main" layout resource + SetContentView(Resource.Layout.Main); + + // Get our button from the layout resource, + // and attach an event to it + Button button = FindViewById + + + + + + + + + + + + + + + + diff --git a/examples/csharp/HelloworldXamarin/iOS/ViewController.cs b/examples/csharp/HelloworldXamarin/iOS/ViewController.cs new file mode 100644 index 00000000000..dfbaf084b6d --- /dev/null +++ b/examples/csharp/HelloworldXamarin/iOS/ViewController.cs @@ -0,0 +1,34 @@ +using System; + +using UIKit; + +namespace HelloworldXamarin.iOS +{ + public partial class ViewController : UIViewController + { + int count = 1; + + public ViewController(IntPtr handle) : base(handle) + { + } + + public override void ViewDidLoad() + { + base.ViewDidLoad(); + + // Perform any additional setup after loading the view, typically from a nib. + Button.AccessibilityIdentifier = "myButton"; + Button.TouchUpInside += delegate + { + var title = string.Format("{0} clicks!", count++); + Button.SetTitle(title, UIControlState.Normal); + }; + } + + public override void DidReceiveMemoryWarning() + { + base.DidReceiveMemoryWarning(); + // Release any cached data, images, etc that aren't in use. + } + } +} diff --git a/examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs b/examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs new file mode 100644 index 00000000000..2677a068dca --- /dev/null +++ b/examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs @@ -0,0 +1,25 @@ +// +// This file has been generated automatically by MonoDevelop to store outlets and +// actions made in the Xcode designer. If it is removed, they will be lost. +// Manual changes to this file may not be handled correctly. +// +using Foundation; + +namespace HelloworldXamarin.iOS +{ + [Register("ViewController")] + partial class ViewController + { + [Outlet] + UIKit.UIButton Button { get; set; } + + void ReleaseDesignerOutlets() + { + if (Button != null) + { + Button.Dispose(); + Button = null; + } + } + } +} From fd91656429ba68ce7af88a866f2ee1545677c33c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 25 Jul 2018 15:53:44 +0200 Subject: [PATCH 30/45] HelloworldXamaring: add Grpc.Core dependency --- .../Droid/HelloworldXamarin.Droid.csproj | 13 +++++ .../HelloworldXamarin/Droid/packages.config | 53 +++++++++++++++++++ .../iOS/HelloworldXamarin.iOS.csproj | 13 +++++ .../HelloworldXamarin/iOS/packages.config | 53 +++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 examples/csharp/HelloworldXamarin/Droid/packages.config create mode 100644 examples/csharp/HelloworldXamarin/iOS/packages.config diff --git a/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj b/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj index 20f1cedb872..157fa3f0ef4 100644 --- a/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj +++ b/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj @@ -42,6 +42,17 @@ + + + + ..\packages\System.Runtime.Loader.4.0.0\lib\netstandard1.5\System.Runtime.Loader.dll + + + ..\packages\System.Interactive.Async.3.1.1\lib\netstandard1.3\System.Interactive.Async.dll + + + ..\packages\Grpc.Core.1.15.0-dev\lib\netstandard1.5\Grpc.Core.dll + @@ -52,6 +63,7 @@ + @@ -64,4 +76,5 @@ + \ No newline at end of file diff --git a/examples/csharp/HelloworldXamarin/Droid/packages.config b/examples/csharp/HelloworldXamarin/Droid/packages.config new file mode 100644 index 00000000000..4fee491e7e9 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/Droid/packages.config @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj b/examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj index 27cb1c41ab4..f968398278f 100644 --- a/examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj +++ b/examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj @@ -81,6 +81,17 @@ + + + + ..\packages\System.Runtime.Loader.4.0.0\lib\netstandard1.5\System.Runtime.Loader.dll + + + ..\packages\System.Interactive.Async.3.1.1\lib\netstandard1.3\System.Interactive.Async.dll + + + ..\packages\Grpc.Core.1.15.0-dev\lib\netstandard1.5\Grpc.Core.dll + @@ -96,6 +107,7 @@ + @@ -107,4 +119,5 @@ + \ No newline at end of file diff --git a/examples/csharp/HelloworldXamarin/iOS/packages.config b/examples/csharp/HelloworldXamarin/iOS/packages.config new file mode 100644 index 00000000000..ad960e401e5 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/iOS/packages.config @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From ecaba8a8d30d227d3ba23c23d43eb4ec12e75954 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 25 Jul 2018 15:56:46 +0200 Subject: [PATCH 31/45] HelloworldXamaring: add Google.Protobuf dependency --- .../HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj | 3 +++ examples/csharp/HelloworldXamarin/Droid/packages.config | 1 + .../csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj | 3 +++ examples/csharp/HelloworldXamarin/iOS/packages.config | 1 + 4 files changed, 8 insertions(+) diff --git a/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj b/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj index 157fa3f0ef4..b5ca8490a48 100644 --- a/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj +++ b/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj @@ -53,6 +53,9 @@ ..\packages\Grpc.Core.1.15.0-dev\lib\netstandard1.5\Grpc.Core.dll + + ..\packages\Google.Protobuf.3.6.0\lib\netstandard1.0\Google.Protobuf.dll + diff --git a/examples/csharp/HelloworldXamarin/Droid/packages.config b/examples/csharp/HelloworldXamarin/Droid/packages.config index 4fee491e7e9..3574b6782b3 100644 --- a/examples/csharp/HelloworldXamarin/Droid/packages.config +++ b/examples/csharp/HelloworldXamarin/Droid/packages.config @@ -1,5 +1,6 @@  + diff --git a/examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj b/examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj index f968398278f..b5c0d1d1192 100644 --- a/examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj +++ b/examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj @@ -92,6 +92,9 @@ ..\packages\Grpc.Core.1.15.0-dev\lib\netstandard1.5\Grpc.Core.dll + + ..\packages\Google.Protobuf.3.6.0\lib\netstandard1.0\Google.Protobuf.dll + diff --git a/examples/csharp/HelloworldXamarin/iOS/packages.config b/examples/csharp/HelloworldXamarin/iOS/packages.config index ad960e401e5..ce4bceb62a8 100644 --- a/examples/csharp/HelloworldXamarin/iOS/packages.config +++ b/examples/csharp/HelloworldXamarin/iOS/packages.config @@ -1,5 +1,6 @@  + From b335d58cb726d930a77e7756054e7b6c754d54c3 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 25 Jul 2018 16:25:47 +0200 Subject: [PATCH 32/45] add generated protos --- .../HelloworldXamarin/Helloworld.cs | 286 ++++++++++++++++++ .../HelloworldXamarin/HelloworldGrpc.cs | 150 +++++++++ .../HelloworldXamarin.projitems | 3 +- .../HelloworldXamarin/MyClass.cs | 10 - 4 files changed, 438 insertions(+), 11 deletions(-) create mode 100644 examples/csharp/HelloworldXamarin/HelloworldXamarin/Helloworld.cs create mode 100644 examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldGrpc.cs delete mode 100644 examples/csharp/HelloworldXamarin/HelloworldXamarin/MyClass.cs diff --git a/examples/csharp/HelloworldXamarin/HelloworldXamarin/Helloworld.cs b/examples/csharp/HelloworldXamarin/HelloworldXamarin/Helloworld.cs new file mode 100644 index 00000000000..ecfc8e131cb --- /dev/null +++ b/examples/csharp/HelloworldXamarin/HelloworldXamarin/Helloworld.cs @@ -0,0 +1,286 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: helloworld.proto +#pragma warning disable 1591, 0612, 3021 +#region Designer generated code + +using pb = global::Google.Protobuf; +using pbc = global::Google.Protobuf.Collections; +using pbr = global::Google.Protobuf.Reflection; +using scg = global::System.Collections.Generic; +namespace Helloworld { + + /// Holder for reflection information generated from helloworld.proto + public static partial class HelloworldReflection { + + #region Descriptor + /// File descriptor for helloworld.proto + public static pbr::FileDescriptor Descriptor { + get { return descriptor; } + } + private static pbr::FileDescriptor descriptor; + + static HelloworldReflection() { + byte[] descriptorData = global::System.Convert.FromBase64String( + string.Concat( + "ChBoZWxsb3dvcmxkLnByb3RvEgpoZWxsb3dvcmxkIhwKDEhlbGxvUmVxdWVz", + "dBIMCgRuYW1lGAEgASgJIh0KCkhlbGxvUmVwbHkSDwoHbWVzc2FnZRgBIAEo", + "CTJJCgdHcmVldGVyEj4KCFNheUhlbGxvEhguaGVsbG93b3JsZC5IZWxsb1Jl", + "cXVlc3QaFi5oZWxsb3dvcmxkLkhlbGxvUmVwbHkiAEI2Chtpby5ncnBjLmV4", + "YW1wbGVzLmhlbGxvd29ybGRCD0hlbGxvV29ybGRQcm90b1ABogIDSExXYgZw", + "cm90bzM=")); + descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, + new pbr::FileDescriptor[] { }, + new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] { + new pbr::GeneratedClrTypeInfo(typeof(global::Helloworld.HelloRequest), global::Helloworld.HelloRequest.Parser, new[]{ "Name" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Helloworld.HelloReply), global::Helloworld.HelloReply.Parser, new[]{ "Message" }, null, null, null) + })); + } + #endregion + + } + #region Messages + /// + /// The request message containing the user's name. + /// + public sealed partial class HelloRequest : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new HelloRequest()); + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pbr::MessageDescriptor Descriptor { + get { return global::Helloworld.HelloworldReflection.Descriptor.MessageTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public HelloRequest() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public HelloRequest(HelloRequest other) : this() { + name_ = other.name_; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public HelloRequest Clone() { + return new HelloRequest(this); + } + + /// Field number for the "name" field. + public const int NameFieldNumber = 1; + private string name_ = ""; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string Name { + get { return name_; } + set { + name_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override bool Equals(object other) { + return Equals(other as HelloRequest); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool Equals(HelloRequest other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Name != other.Name) return false; + return true; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override int GetHashCode() { + int hash = 1; + if (Name.Length != 0) hash ^= Name.GetHashCode(); + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(pb::CodedOutputStream output) { + if (Name.Length != 0) { + output.WriteRawTag(10); + output.WriteString(Name); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int CalculateSize() { + int size = 0; + if (Name.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Name); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(HelloRequest other) { + if (other == null) { + return; + } + if (other.Name.Length != 0) { + Name = other.Name; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + input.SkipLastField(); + break; + case 10: { + Name = input.ReadString(); + break; + } + } + } + } + + } + + /// + /// The response message containing the greetings + /// + public sealed partial class HelloReply : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new HelloReply()); + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pbr::MessageDescriptor Descriptor { + get { return global::Helloworld.HelloworldReflection.Descriptor.MessageTypes[1]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public HelloReply() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public HelloReply(HelloReply other) : this() { + message_ = other.message_; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public HelloReply Clone() { + return new HelloReply(this); + } + + /// Field number for the "message" field. + public const int MessageFieldNumber = 1; + private string message_ = ""; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public string Message { + get { return message_; } + set { + message_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override bool Equals(object other) { + return Equals(other as HelloReply); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool Equals(HelloReply other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Message != other.Message) return false; + return true; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override int GetHashCode() { + int hash = 1; + if (Message.Length != 0) hash ^= Message.GetHashCode(); + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(pb::CodedOutputStream output) { + if (Message.Length != 0) { + output.WriteRawTag(10); + output.WriteString(Message); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int CalculateSize() { + int size = 0; + if (Message.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Message); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(HelloReply other) { + if (other == null) { + return; + } + if (other.Message.Length != 0) { + Message = other.Message; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + input.SkipLastField(); + break; + case 10: { + Message = input.ReadString(); + break; + } + } + } + } + + } + + #endregion + +} + +#endregion Designer generated code diff --git a/examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldGrpc.cs b/examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldGrpc.cs new file mode 100644 index 00000000000..c808884e579 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldGrpc.cs @@ -0,0 +1,150 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: helloworld.proto +// Original file comments: +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#pragma warning disable 1591 +#region Designer generated code + +using System; +using System.Threading; +using System.Threading.Tasks; +using grpc = global::Grpc.Core; + +namespace Helloworld { + /// + /// The greeting service definition. + /// + public static partial class Greeter + { + static readonly string __ServiceName = "helloworld.Greeter"; + + static readonly grpc::Marshaller __Marshaller_HelloRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Helloworld.HelloRequest.Parser.ParseFrom); + static readonly grpc::Marshaller __Marshaller_HelloReply = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Helloworld.HelloReply.Parser.ParseFrom); + + static readonly grpc::Method __Method_SayHello = new grpc::Method( + grpc::MethodType.Unary, + __ServiceName, + "SayHello", + __Marshaller_HelloRequest, + __Marshaller_HelloReply); + + /// Service descriptor + public static global::Google.Protobuf.Reflection.ServiceDescriptor Descriptor + { + get { return global::Helloworld.HelloworldReflection.Descriptor.Services[0]; } + } + + /// Base class for server-side implementations of Greeter + public abstract partial class GreeterBase + { + /// + /// Sends a greeting + /// + /// The request received from the client. + /// The context of the server-side call handler being invoked. + /// The response to send back to the client (wrapped by a task). + public virtual global::System.Threading.Tasks.Task SayHello(global::Helloworld.HelloRequest request, grpc::ServerCallContext context) + { + throw new grpc::RpcException(new grpc::Status(grpc::StatusCode.Unimplemented, "")); + } + + } + + /// Client for Greeter + public partial class GreeterClient : grpc::ClientBase + { + /// Creates a new client for Greeter + /// The channel to use to make remote calls. + public GreeterClient(grpc::Channel channel) : base(channel) + { + } + /// Creates a new client for Greeter that uses a custom CallInvoker. + /// The callInvoker to use to make remote calls. + public GreeterClient(grpc::CallInvoker callInvoker) : base(callInvoker) + { + } + /// Protected parameterless constructor to allow creation of test doubles. + protected GreeterClient() : base() + { + } + /// Protected constructor to allow creation of configured clients. + /// The client configuration. + protected GreeterClient(ClientBaseConfiguration configuration) : base(configuration) + { + } + + /// + /// Sends a greeting + /// + /// The request to send to the server. + /// The initial metadata to send with the call. This parameter is optional. + /// An optional deadline for the call. The call will be cancelled if deadline is hit. + /// An optional token for canceling the call. + /// The response received from the server. + public virtual global::Helloworld.HelloReply SayHello(global::Helloworld.HelloRequest request, grpc::Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + { + return SayHello(request, new grpc::CallOptions(headers, deadline, cancellationToken)); + } + /// + /// Sends a greeting + /// + /// The request to send to the server. + /// The options for the call. + /// The response received from the server. + public virtual global::Helloworld.HelloReply SayHello(global::Helloworld.HelloRequest request, grpc::CallOptions options) + { + return CallInvoker.BlockingUnaryCall(__Method_SayHello, null, options, request); + } + /// + /// Sends a greeting + /// + /// The request to send to the server. + /// The initial metadata to send with the call. This parameter is optional. + /// An optional deadline for the call. The call will be cancelled if deadline is hit. + /// An optional token for canceling the call. + /// The call object. + public virtual grpc::AsyncUnaryCall SayHelloAsync(global::Helloworld.HelloRequest request, grpc::Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + { + return SayHelloAsync(request, new grpc::CallOptions(headers, deadline, cancellationToken)); + } + /// + /// Sends a greeting + /// + /// The request to send to the server. + /// The options for the call. + /// The call object. + public virtual grpc::AsyncUnaryCall SayHelloAsync(global::Helloworld.HelloRequest request, grpc::CallOptions options) + { + return CallInvoker.AsyncUnaryCall(__Method_SayHello, null, options, request); + } + /// Creates a new instance of client from given ClientBaseConfiguration. + protected override GreeterClient NewInstance(ClientBaseConfiguration configuration) + { + return new GreeterClient(configuration); + } + } + + /// Creates service definition that can be registered with a server + /// An object implementing the server-side handling logic. + public static grpc::ServerServiceDefinition BindService(GreeterBase serviceImpl) + { + return grpc::ServerServiceDefinition.CreateBuilder() + .AddMethod(__Method_SayHello, serviceImpl.SayHello).Build(); + } + + } +} +#endregion diff --git a/examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldXamarin.projitems b/examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldXamarin.projitems index 87466cb4a72..25f5478c538 100644 --- a/examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldXamarin.projitems +++ b/examples/csharp/HelloworldXamarin/HelloworldXamarin/HelloworldXamarin.projitems @@ -9,6 +9,7 @@ HelloworldXamarin - + + \ No newline at end of file diff --git a/examples/csharp/HelloworldXamarin/HelloworldXamarin/MyClass.cs b/examples/csharp/HelloworldXamarin/HelloworldXamarin/MyClass.cs deleted file mode 100644 index e3fa67641ab..00000000000 --- a/examples/csharp/HelloworldXamarin/HelloworldXamarin/MyClass.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; -namespace HelloworldXamarin -{ - public class MyClass - { - public MyClass() - { - } - } -} From 705843c041f24ce5b2737fc15dfd45c34f26cf82 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 25 Jul 2018 17:30:19 +0200 Subject: [PATCH 33/45] experimental helloworld client for android --- .../HelloworldXamarin/Droid/MainActivity.cs | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs b/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs index 20982422f28..2c158e9d7cc 100644 --- a/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs +++ b/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs @@ -1,13 +1,15 @@ using Android.App; using Android.Widget; using Android.OS; +using Grpc.Core; +using Helloworld; namespace HelloworldXamarin.Droid { [Activity(Label = "HelloworldXamarin", MainLauncher = true, Icon = "@mipmap/icon")] public class MainActivity : Activity { - int count = 1; + //int count = 1; protected override void OnCreate(Bundle savedInstanceState) { @@ -20,8 +22,26 @@ namespace HelloworldXamarin.Droid // and attach an event to it Button button = FindViewById