From e5e0131c8d4e7d8b99bedbfe7902aadf5eace3bf Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 17 Feb 2016 13:06:01 -0800 Subject: [PATCH 01/97] Add more information to precompiled name --- src/python/grpcio/precompiled.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio/precompiled.py b/src/python/grpcio/precompiled.py index 05c651b506f..6d9a4b85c26 100644 --- a/src/python/grpcio/precompiled.py +++ b/src/python/grpcio/precompiled.py @@ -31,6 +31,7 @@ import os import platform import shutil import sys +import sysconfig import setuptools @@ -51,9 +52,15 @@ USE_PRECOMPILED_BINARIES = bool(int(os.environ.get( def _tagged_ext_name(base): uname = platform.uname() - tags = '-'.join((grpc_version.VERSION, uname[0], uname[4])) - flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4' - return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor) + tags = ( + grpc_version.VERSION, + 'py{}'.format(sysconfig.get_python_version()), + uname[0], + uname[4], + ) + ucs = 'ucs{}'.format(sysconfig.get_config_var('Py_UNICODE_SIZE')) + return '{base}-{tags}-{ucs}'.format( + base=base, tags='-'.join(tags), ucs=ucs) class BuildTaggedExt(setuptools.Command): From c0a97a9cfe257138d293774868b4157151d64339 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 22 Feb 2016 17:51:02 -0800 Subject: [PATCH 02/97] Update Python doc with pip version notes --- src/python/grpcio/README.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index 698c760ebe2..bb6d99c4b15 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -23,11 +23,15 @@ Else system wide (on Ubuntu)... $ sudo pip install grpcio +n.b. On Windows and on Mac OS X one *must* have a recent release of :code:`pip` +to retrieve the proper wheel from PyPI. Be sure to upgrade to the latest +version! + From Source ~~~~~~~~~~~ Building from source requires that you have the Python headers (usually a -package named `python-dev`). +package named :code:`python-dev`). :: From fe8ad714d452b27a66058e111b9832ab08c1036a Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 22 Feb 2016 17:51:02 -0800 Subject: [PATCH 03/97] Update Python doc w.r.t. Windows availability --- src/python/grpcio/README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index bb6d99c4b15..d491311886b 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -6,7 +6,7 @@ Package for gRPC Python. Installation ------------ -gRPC Python is available for Linux and Mac OS X running Python 2.7. +gRPC Python is available for Linux, Mac OS X, and Windows running Python 2.7. From PyPI ~~~~~~~~~ @@ -40,5 +40,5 @@ package named :code:`python-dev`). $ cd $REPO_ROOT $ pip install . -Note that `$REPO_ROOT` can be assigned to whatever directory name floats your -fancy. +Note that :code:`$REPO_ROOT` can be assigned to whatever directory name floats +your fancy. From 3ee1f9b7784b680b03d28d92efdffae9b7ad476d Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 22 Feb 2016 18:37:18 -0800 Subject: [PATCH 04/97] Update Python docgen --- setup.py | 1 + src/python/grpcio/commands.py | 3 +-- tools/distrib/python/docgen.py | 12 +++++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index cfb578e2158..cf5ab4920fb 100644 --- a/setup.py +++ b/setup.py @@ -154,6 +154,7 @@ INSTALL_REQUIRES = ( SETUP_REQUIRES = ( 'sphinx>=1.3', + 'sphinx_rtd_theme>=0.1.8' ) + INSTALL_REQUIRES COMMAND_CLASS = { diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index aa29c728f25..99ba41b614f 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -119,8 +119,7 @@ class SphinxDocumentation(setuptools.Command): import sphinx import sphinx.apidoc metadata = self.distribution.metadata - src_dir = os.path.join( - PYTHON_STEM, self.distribution.package_dir[''], 'grpc') + src_dir = os.path.join(PYTHON_STEM, 'grpc') sys.path.append(src_dir) sphinx.apidoc.main([ '', '--force', '--full', '-H', metadata.name, '-A', metadata.author, diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index 4ac8f9c64a8..72c65ad14a8 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -51,29 +51,35 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..', '..')) CONFIG = args.config -SETUP_PATH = os.path.join(PROJECT_ROOT, 'src/python/grpcio/setup.py') -DOC_PATH = os.path.join(PROJECT_ROOT, 'src/python/grpcio/doc/build') +SETUP_PATH = os.path.join(PROJECT_ROOT, 'setup.py') +REQUIREMENTS_PATH = os.path.join(PROJECT_ROOT, 'requirements.txt') +DOC_PATH = os.path.join(PROJECT_ROOT, 'doc/build') INCLUDE_PATH = os.path.join(PROJECT_ROOT, 'include') LIBRARY_PATH = os.path.join(PROJECT_ROOT, 'libs/{}'.format(CONFIG)) VIRTUALENV_DIR = os.path.join(SCRIPT_DIR, 'distrib_virtualenv') VIRTUALENV_PYTHON_PATH = os.path.join(VIRTUALENV_DIR, 'bin', 'python') +VIRTUALENV_PIP_PATH = os.path.join(VIRTUALENV_DIR, 'bin', 'pip') environment = os.environ.copy() environment.update({ 'CONFIG': CONFIG, 'CFLAGS': '-I{}'.format(INCLUDE_PATH), 'LDFLAGS': '-L{}'.format(LIBRARY_PATH), - 'LD_LIBRARY_PATH': LIBRARY_PATH + 'LD_LIBRARY_PATH': LIBRARY_PATH, + 'GRPC_PYTHON_BUILD_WITH_CYTHON': '1', }) subprocess_arguments_list = [ {'args': ['make'], 'cwd': PROJECT_ROOT}, {'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment}, + {'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH], + 'env': environment}, {'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'build'], 'env': environment}, {'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'doc'], 'env': environment}, ] for subprocess_arguments in subprocess_arguments_list: + print('Running command: {}'.format(subprocess_arguments['args'])) subprocess.check_call(**subprocess_arguments) if args.submit: From 071aff284f0ba700e1296688ed2b52dddfce051e Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Feb 2016 18:31:27 -0800 Subject: [PATCH 05/97] Accept instance name as argument --- tools/gce/create_linux_worker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh index 2a9e77ab173..399e3ec4e43 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_worker.sh @@ -37,7 +37,7 @@ cd $(dirname $0) CLOUD_PROJECT=grpc-testing ZONE=us-central1-a -INSTANCE_NAME=grpc-jenkins-worker1 +INSTANCE_NAME="${1:-grpc-jenkins-worker1}" gcloud compute instances create $INSTANCE_NAME \ --project="$CLOUD_PROJECT" \ From 02eda41d215d2e49f5626e0ecd2c2d16e9c9a1f2 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 25 Feb 2016 19:43:57 -0800 Subject: [PATCH 06/97] Do stream operations in a more idiomatic way: Write, WritesDone, Read until fail, Finish --- test/cpp/qps/driver.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 1c7fdf87960..18a9ae4d256 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -351,9 +351,11 @@ std::unique_ptr RunScenario( gpr_log(GPR_INFO, "Finishing"); for (auto server = &servers[0]; server != &servers[num_servers]; server++) { GPR_ASSERT(server->stream->Write(server_mark)); + GPR_ASSERT(server->stream->WritesDone()); } for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Write(client_mark)); + GPR_ASSERT(client->stream->WritesDone()); } for (auto server = &servers[0]; server != &servers[num_servers]; server++) { GPR_ASSERT(server->stream->Read(&server_status)); @@ -361,6 +363,7 @@ std::unique_ptr RunScenario( result->server_resources.emplace_back( stats.time_elapsed(), stats.time_user(), stats.time_system(), server_status.cores()); + GPR_ASSERT(!server->stream->Read(&server_status)); } for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Read(&client_status)); @@ -368,14 +371,13 @@ std::unique_ptr RunScenario( result->latencies.MergeProto(stats.latencies()); result->client_resources.emplace_back( stats.time_elapsed(), stats.time_user(), stats.time_system(), -1); + GPR_ASSERT(!client->stream->Read(&client_status)); } for (auto client = &clients[0]; client != &clients[num_clients]; client++) { - GPR_ASSERT(client->stream->WritesDone()); GPR_ASSERT(client->stream->Finish().ok()); } for (auto server = &servers[0]; server != &servers[num_servers]; server++) { - GPR_ASSERT(server->stream->WritesDone()); GPR_ASSERT(server->stream->Finish().ok()); } delete[] clients; From e7042b5ddf4f72dbba79e53621f28ba7d3cd9df4 Mon Sep 17 00:00:00 2001 From: vjpai Date: Thu, 25 Feb 2016 20:28:05 -0800 Subject: [PATCH 07/97] Simplify Mark processing code --- test/cpp/qps/client.h | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 2dc83f0f292..6a822e35c78 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -123,15 +123,13 @@ class Client { if (reset) { Histogram* to_merge = new Histogram[threads_.size()]; for (size_t i = 0; i < threads_.size(); i++) { - threads_[i]->BeginSwap(&to_merge[i]); - } - std::unique_ptr timer(new UsageTimer); - timer_.swap(timer); - for (size_t i = 0; i < threads_.size(); i++) { - threads_[i]->EndSwap(); + threads_[i]->Swap(&to_merge[i]); latencies.Merge(to_merge[i]); } delete[] to_merge; + + std::unique_ptr timer(new UsageTimer); + timer_.swap(timer); timer_result = timer->Mark(); } else { // merge snapshots of each thread histogram @@ -227,7 +225,6 @@ class Client { public: Thread(Client* client, size_t idx) : done_(false), - new_stats_(nullptr), client_(client), idx_(idx), impl_(&Thread::ThreadFunc, this) {} @@ -240,16 +237,9 @@ class Client { impl_.join(); } - void BeginSwap(Histogram* n) { + void Swap(Histogram* n) { std::lock_guard g(mu_); - new_stats_ = n; - } - - void EndSwap() { - std::unique_lock g(mu_); - while (new_stats_ != nullptr) { - cv_.wait(g); - }; + n->Swap(&histogram_); } void MergeStatsInto(Histogram* hist) { @@ -263,10 +253,11 @@ class Client { void ThreadFunc() { for (;;) { + // lock since the thread should only be doing one thing at a time + std::lock_guard g(mu_); // run the loop body const bool thread_still_ok = client_->ThreadFunc(&histogram_, idx_); - // lock, see if we're done - std::lock_guard g(mu_); + // see if we're done if (!thread_still_ok) { gpr_log(GPR_ERROR, "Finishing client thread due to RPC error"); done_ = true; @@ -274,19 +265,11 @@ class Client { if (done_) { return; } - // check if we're resetting stats, swap out the histogram if so - if (new_stats_) { - new_stats_->Swap(&histogram_); - new_stats_ = nullptr; - cv_.notify_one(); - } } } std::mutex mu_; - std::condition_variable cv_; bool done_; - Histogram* new_stats_; Histogram histogram_; Client* client_; const size_t idx_; From 762ed4368afa513ef468eac4cf16e4c91ff9b61a Mon Sep 17 00:00:00 2001 From: Chris Bacon Date: Fri, 26 Feb 2016 12:44:10 +0000 Subject: [PATCH 08/97] Support for coreclr Use .ToLowerInvariant, which is supported by coreclr --- src/csharp/Grpc.Core/Metadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Metadata.cs b/src/csharp/Grpc.Core/Metadata.cs index aa22f840d6c..52cef96f405 100644 --- a/src/csharp/Grpc.Core/Metadata.cs +++ b/src/csharp/Grpc.Core/Metadata.cs @@ -323,7 +323,7 @@ namespace Grpc.Core private static string NormalizeKey(string key) { - var normalized = GrpcPreconditions.CheckNotNull(key, "key").ToLower(CultureInfo.InvariantCulture); + var normalized = GrpcPreconditions.CheckNotNull(key, "key").ToLowerInvariant(); GrpcPreconditions.CheckArgument(ValidKeyRegex.IsMatch(normalized), "Metadata entry key not valid. Keys can only contain lowercase alphanumeric characters, underscores and hyphens."); return normalized; From 60a7e3336df1554891bae57b3f8d6e7304f6e039 Mon Sep 17 00:00:00 2001 From: vjpai Date: Fri, 26 Feb 2016 10:30:17 -0800 Subject: [PATCH 09/97] Finish clients fully before finishing servers --- test/cpp/qps/driver.cc | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 18a9ae4d256..bc8780f74d5 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -348,23 +348,11 @@ std::unique_ptr RunScenario( std::unique_ptr result(new ScenarioResult); result->client_config = result_client_config; result->server_config = result_server_config; - gpr_log(GPR_INFO, "Finishing"); - for (auto server = &servers[0]; server != &servers[num_servers]; server++) { - GPR_ASSERT(server->stream->Write(server_mark)); - GPR_ASSERT(server->stream->WritesDone()); - } + gpr_log(GPR_INFO, "Finishing clients"); for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Write(client_mark)); GPR_ASSERT(client->stream->WritesDone()); } - for (auto server = &servers[0]; server != &servers[num_servers]; server++) { - GPR_ASSERT(server->stream->Read(&server_status)); - const auto& stats = server_status.stats(); - result->server_resources.emplace_back( - stats.time_elapsed(), stats.time_user(), stats.time_system(), - server_status.cores()); - GPR_ASSERT(!server->stream->Read(&server_status)); - } for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Read(&client_status)); const auto& stats = client_status.stats(); @@ -373,14 +361,28 @@ std::unique_ptr RunScenario( stats.time_elapsed(), stats.time_user(), stats.time_system(), -1); GPR_ASSERT(!client->stream->Read(&client_status)); } - for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Finish().ok()); } + delete[] clients; + + gpr_log(GPR_INFO, "Finishing servers"); + for (auto server = &servers[0]; server != &servers[num_servers]; server++) { + GPR_ASSERT(server->stream->Write(server_mark)); + GPR_ASSERT(server->stream->WritesDone()); + } + for (auto server = &servers[0]; server != &servers[num_servers]; server++) { + GPR_ASSERT(server->stream->Read(&server_status)); + const auto& stats = server_status.stats(); + result->server_resources.emplace_back( + stats.time_elapsed(), stats.time_user(), stats.time_system(), + server_status.cores()); + GPR_ASSERT(!server->stream->Read(&server_status)); + } for (auto server = &servers[0]; server != &servers[num_servers]; server++) { GPR_ASSERT(server->stream->Finish().ok()); } - delete[] clients; + delete[] servers; return result; } From ef3528e2a19f96a17434fcc6a672e1279ea4c262 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 26 Feb 2016 13:25:49 -0800 Subject: [PATCH 10/97] Make client properly report when message deserialization fails --- src/node/src/client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/src/client.js b/src/node/src/client.js index c65dd736503..d6aa999502c 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -336,7 +336,7 @@ function makeUnaryRequestFunction(method, serialize, deserialize) { } } if (status.code !== grpc.status.OK) { - error = new Error(response.status.details); + error = new Error(status.details); error.code = status.code; error.metadata = status.metadata; callback(error); From c47c9497fa6518205a598c2bf74517df09541fed Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 26 Feb 2016 14:39:59 -0800 Subject: [PATCH 11/97] clang-format --- test/cpp/qps/client.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 6a822e35c78..92e77eed9b2 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -253,7 +253,7 @@ class Client { void ThreadFunc() { for (;;) { - // lock since the thread should only be doing one thing at a time + // lock since the thread should only be doing one thing at a time std::lock_guard g(mu_); // run the loop body const bool thread_still_ok = client_->ThreadFunc(&histogram_, idx_); From 6156f0962efdd065c53f550293db7bb1aaa3355b Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Sun, 28 Feb 2016 19:14:07 -0800 Subject: [PATCH 12/97] Simplify PHP installation for release 0.13 --- build.yaml | 27 + config.m4 | 599 ++++++++++++++++++++ package.xml | 964 +++++++++++++++++++++++++++++++++ src/php/README.md | 32 -- templates/config.m4.template | 59 ++ templates/package.xml.template | 161 ++++++ tools/distrib/python/docgen.py | 2 +- 7 files changed, 1811 insertions(+), 33 deletions(-) create mode 100644 config.m4 create mode 100644 package.xml create mode 100644 templates/config.m4.template create mode 100644 templates/package.xml.template diff --git a/build.yaml b/build.yaml index 23853a8f83d..68912fc07ea 100644 --- a/build.yaml +++ b/build.yaml @@ -2788,6 +2788,33 @@ node_modules: - src/node/ext/server.cc - src/node/ext/server_credentials.cc - src/node/ext/timeval.cc +php_config_m4: + deps: + - grpc + - gpr + - boringssl + headers: + - src/php/ext/grpc/byte_buffer.h + - src/php/ext/grpc/call.h + - src/php/ext/grpc/call_credentials.h + - src/php/ext/grpc/channel.h + - src/php/ext/grpc/channel_credentials.h + - src/php/ext/grpc/completion_queue.h + - src/php/ext/grpc/php_grpc.h + - src/php/ext/grpc/server.h + - src/php/ext/grpc/server_credentials.h + - src/php/ext/grpc/timeval.h + src: + - src/php/ext/grpc/byte_buffer.c + - src/php/ext/grpc/call.c + - src/php/ext/grpc/call_credentials.c + - src/php/ext/grpc/channel.c + - src/php/ext/grpc/channel_credentials.c + - src/php/ext/grpc/completion_queue.c + - src/php/ext/grpc/php_grpc.c + - src/php/ext/grpc/server.c + - src/php/ext/grpc/server_credentials.c + - src/php/ext/grpc/timeval.c python_dependencies: deps: - grpc diff --git a/config.m4 b/config.m4 new file mode 100644 index 00000000000..ab614d19ae5 --- /dev/null +++ b/config.m4 @@ -0,0 +1,599 @@ +PHP_ARG_ENABLE(grpc, whether to enable grpc support, +[ --enable-grpc Enable grpc support]) + +if test "$PHP_GRPC" != "no"; then + dnl Write more examples of tests here... + + dnl # --with-grpc -> add include path + PHP_ADD_INCLUDE(../../grpc/include) + PHP_ADD_INCLUDE(../../grpc/src/php/ext/grpc) + PHP_ADD_INCLUDE(../../grpc/third_party/boringssl/include) + + LIBS="-lpthread $LIBS" + + GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" + PHP_ADD_LIBRARY(pthread) + + PHP_ADD_LIBRARY(dl,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(dl) + + case $host in + *darwin*) ;; + *) + PHP_ADD_LIBRARY(rt,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(rt) + ;; + esac + + PHP_NEW_EXTENSION(grpc, + src/php/ext/grpc/byte_buffer.c \ + src/php/ext/grpc/call.c \ + src/php/ext/grpc/call_credentials.c \ + src/php/ext/grpc/channel.c \ + src/php/ext/grpc/channel_credentials.c \ + src/php/ext/grpc/completion_queue.c \ + src/php/ext/grpc/php_grpc.c \ + src/php/ext/grpc/server.c \ + src/php/ext/grpc/server_credentials.c \ + src/php/ext/grpc/timeval.c \ + src/core/profiling/basic_timers.c \ + src/core/profiling/stap_timers.c \ + src/core/support/alloc.c \ + src/core/support/avl.c \ + src/core/support/cmdline.c \ + src/core/support/cpu_iphone.c \ + src/core/support/cpu_linux.c \ + src/core/support/cpu_posix.c \ + src/core/support/cpu_windows.c \ + src/core/support/env_linux.c \ + src/core/support/env_posix.c \ + src/core/support/env_win32.c \ + src/core/support/file.c \ + src/core/support/file_posix.c \ + src/core/support/file_win32.c \ + src/core/support/histogram.c \ + src/core/support/host_port.c \ + src/core/support/log.c \ + src/core/support/log_android.c \ + src/core/support/log_linux.c \ + src/core/support/log_posix.c \ + src/core/support/log_win32.c \ + src/core/support/murmur_hash.c \ + src/core/support/slice.c \ + src/core/support/slice_buffer.c \ + src/core/support/stack_lockfree.c \ + src/core/support/string.c \ + src/core/support/string_posix.c \ + src/core/support/string_win32.c \ + src/core/support/subprocess_posix.c \ + src/core/support/subprocess_windows.c \ + src/core/support/sync.c \ + src/core/support/sync_posix.c \ + src/core/support/sync_win32.c \ + src/core/support/thd.c \ + src/core/support/thd_posix.c \ + src/core/support/thd_win32.c \ + src/core/support/time.c \ + src/core/support/time_posix.c \ + src/core/support/time_precise.c \ + src/core/support/time_win32.c \ + src/core/support/tls_pthread.c \ + src/core/support/wrap_memcpy.c \ + src/core/census/grpc_context.c \ + src/core/census/grpc_filter.c \ + src/core/channel/channel_args.c \ + src/core/channel/channel_stack.c \ + src/core/channel/client_channel.c \ + src/core/channel/client_uchannel.c \ + src/core/channel/compress_filter.c \ + src/core/channel/connected_channel.c \ + src/core/channel/http_client_filter.c \ + src/core/channel/http_server_filter.c \ + src/core/channel/subchannel_call_holder.c \ + src/core/client_config/client_config.c \ + src/core/client_config/connector.c \ + src/core/client_config/default_initial_connect_string.c \ + src/core/client_config/initial_connect_string.c \ + src/core/client_config/lb_policies/pick_first.c \ + src/core/client_config/lb_policies/round_robin.c \ + src/core/client_config/lb_policy.c \ + src/core/client_config/lb_policy_factory.c \ + src/core/client_config/lb_policy_registry.c \ + src/core/client_config/resolver.c \ + src/core/client_config/resolver_factory.c \ + src/core/client_config/resolver_registry.c \ + src/core/client_config/resolvers/dns_resolver.c \ + src/core/client_config/resolvers/sockaddr_resolver.c \ + src/core/client_config/subchannel.c \ + src/core/client_config/subchannel_factory.c \ + src/core/client_config/subchannel_index.c \ + src/core/client_config/uri_parser.c \ + src/core/compression/algorithm.c \ + src/core/compression/message_compress.c \ + src/core/debug/trace.c \ + src/core/httpcli/format_request.c \ + src/core/httpcli/httpcli.c \ + src/core/httpcli/parser.c \ + src/core/iomgr/closure.c \ + src/core/iomgr/endpoint.c \ + src/core/iomgr/endpoint_pair_posix.c \ + src/core/iomgr/endpoint_pair_windows.c \ + src/core/iomgr/exec_ctx.c \ + src/core/iomgr/executor.c \ + src/core/iomgr/fd_posix.c \ + src/core/iomgr/iocp_windows.c \ + src/core/iomgr/iomgr.c \ + src/core/iomgr/iomgr_posix.c \ + src/core/iomgr/iomgr_windows.c \ + src/core/iomgr/pollset_multipoller_with_epoll.c \ + src/core/iomgr/pollset_multipoller_with_poll_posix.c \ + src/core/iomgr/pollset_posix.c \ + src/core/iomgr/pollset_set_posix.c \ + src/core/iomgr/pollset_set_windows.c \ + src/core/iomgr/pollset_windows.c \ + src/core/iomgr/resolve_address_posix.c \ + src/core/iomgr/resolve_address_windows.c \ + src/core/iomgr/sockaddr_utils.c \ + src/core/iomgr/socket_utils_common_posix.c \ + src/core/iomgr/socket_utils_linux.c \ + src/core/iomgr/socket_utils_posix.c \ + src/core/iomgr/socket_windows.c \ + src/core/iomgr/tcp_client_posix.c \ + src/core/iomgr/tcp_client_windows.c \ + src/core/iomgr/tcp_posix.c \ + src/core/iomgr/tcp_server_posix.c \ + src/core/iomgr/tcp_server_windows.c \ + src/core/iomgr/tcp_windows.c \ + src/core/iomgr/time_averaged_stats.c \ + src/core/iomgr/timer.c \ + src/core/iomgr/timer_heap.c \ + src/core/iomgr/udp_server.c \ + src/core/iomgr/wakeup_fd_eventfd.c \ + src/core/iomgr/wakeup_fd_nospecial.c \ + src/core/iomgr/wakeup_fd_pipe.c \ + src/core/iomgr/wakeup_fd_posix.c \ + src/core/iomgr/workqueue_posix.c \ + src/core/iomgr/workqueue_windows.c \ + src/core/json/json.c \ + src/core/json/json_reader.c \ + src/core/json/json_string.c \ + src/core/json/json_writer.c \ + src/core/surface/alarm.c \ + src/core/surface/api_trace.c \ + src/core/surface/byte_buffer.c \ + src/core/surface/byte_buffer_reader.c \ + src/core/surface/call.c \ + src/core/surface/call_details.c \ + src/core/surface/call_log_batch.c \ + src/core/surface/channel.c \ + src/core/surface/channel_connectivity.c \ + src/core/surface/channel_create.c \ + src/core/surface/channel_ping.c \ + src/core/surface/completion_queue.c \ + src/core/surface/event_string.c \ + src/core/surface/init.c \ + src/core/surface/lame_client.c \ + src/core/surface/metadata_array.c \ + src/core/surface/server.c \ + src/core/surface/server_chttp2.c \ + src/core/surface/server_create.c \ + src/core/surface/validate_metadata.c \ + src/core/surface/version.c \ + src/core/transport/byte_stream.c \ + src/core/transport/chttp2/alpn.c \ + src/core/transport/chttp2/bin_encoder.c \ + src/core/transport/chttp2/frame_data.c \ + src/core/transport/chttp2/frame_goaway.c \ + src/core/transport/chttp2/frame_ping.c \ + src/core/transport/chttp2/frame_rst_stream.c \ + src/core/transport/chttp2/frame_settings.c \ + src/core/transport/chttp2/frame_window_update.c \ + src/core/transport/chttp2/hpack_encoder.c \ + src/core/transport/chttp2/hpack_parser.c \ + src/core/transport/chttp2/hpack_table.c \ + src/core/transport/chttp2/huffsyms.c \ + src/core/transport/chttp2/incoming_metadata.c \ + src/core/transport/chttp2/parsing.c \ + src/core/transport/chttp2/status_conversion.c \ + src/core/transport/chttp2/stream_lists.c \ + src/core/transport/chttp2/stream_map.c \ + src/core/transport/chttp2/timeout_encoding.c \ + src/core/transport/chttp2/varint.c \ + src/core/transport/chttp2/writing.c \ + src/core/transport/chttp2_transport.c \ + src/core/transport/connectivity_state.c \ + src/core/transport/metadata.c \ + src/core/transport/metadata_batch.c \ + src/core/transport/static_metadata.c \ + src/core/transport/transport.c \ + src/core/transport/transport_op_string.c \ + src/core/httpcli/httpcli_security_connector.c \ + src/core/security/base64.c \ + src/core/security/client_auth_filter.c \ + src/core/security/credentials.c \ + src/core/security/credentials_metadata.c \ + src/core/security/credentials_posix.c \ + src/core/security/credentials_win32.c \ + src/core/security/google_default_credentials.c \ + src/core/security/handshake.c \ + src/core/security/json_token.c \ + src/core/security/jwt_verifier.c \ + src/core/security/secure_endpoint.c \ + src/core/security/security_connector.c \ + src/core/security/security_context.c \ + src/core/security/server_auth_filter.c \ + src/core/security/server_secure_chttp2.c \ + src/core/surface/init_secure.c \ + src/core/surface/secure_channel_create.c \ + src/core/tsi/fake_transport_security.c \ + src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security.c \ + src/core/census/context.c \ + src/core/census/initialize.c \ + src/core/census/mlog.c \ + src/core/census/operation.c \ + src/core/census/placeholders.c \ + src/core/census/tracing.c \ + src/boringssl/err_data.c \ + third_party/boringssl/crypto/aes/aes.c \ + third_party/boringssl/crypto/aes/mode_wrappers.c \ + third_party/boringssl/crypto/asn1/a_bitstr.c \ + third_party/boringssl/crypto/asn1/a_bool.c \ + third_party/boringssl/crypto/asn1/a_bytes.c \ + third_party/boringssl/crypto/asn1/a_d2i_fp.c \ + third_party/boringssl/crypto/asn1/a_dup.c \ + third_party/boringssl/crypto/asn1/a_enum.c \ + third_party/boringssl/crypto/asn1/a_gentm.c \ + third_party/boringssl/crypto/asn1/a_i2d_fp.c \ + third_party/boringssl/crypto/asn1/a_int.c \ + third_party/boringssl/crypto/asn1/a_mbstr.c \ + third_party/boringssl/crypto/asn1/a_object.c \ + third_party/boringssl/crypto/asn1/a_octet.c \ + third_party/boringssl/crypto/asn1/a_print.c \ + third_party/boringssl/crypto/asn1/a_strnid.c \ + third_party/boringssl/crypto/asn1/a_time.c \ + third_party/boringssl/crypto/asn1/a_type.c \ + third_party/boringssl/crypto/asn1/a_utctm.c \ + third_party/boringssl/crypto/asn1/a_utf8.c \ + third_party/boringssl/crypto/asn1/asn1_lib.c \ + third_party/boringssl/crypto/asn1/asn1_par.c \ + third_party/boringssl/crypto/asn1/asn_pack.c \ + third_party/boringssl/crypto/asn1/bio_asn1.c \ + third_party/boringssl/crypto/asn1/bio_ndef.c \ + third_party/boringssl/crypto/asn1/f_enum.c \ + third_party/boringssl/crypto/asn1/f_int.c \ + third_party/boringssl/crypto/asn1/f_string.c \ + third_party/boringssl/crypto/asn1/t_bitst.c \ + third_party/boringssl/crypto/asn1/t_pkey.c \ + third_party/boringssl/crypto/asn1/tasn_dec.c \ + third_party/boringssl/crypto/asn1/tasn_enc.c \ + third_party/boringssl/crypto/asn1/tasn_fre.c \ + third_party/boringssl/crypto/asn1/tasn_new.c \ + third_party/boringssl/crypto/asn1/tasn_prn.c \ + third_party/boringssl/crypto/asn1/tasn_typ.c \ + third_party/boringssl/crypto/asn1/tasn_utl.c \ + third_party/boringssl/crypto/asn1/x_bignum.c \ + third_party/boringssl/crypto/asn1/x_long.c \ + third_party/boringssl/crypto/base64/base64.c \ + third_party/boringssl/crypto/bio/bio.c \ + third_party/boringssl/crypto/bio/bio_mem.c \ + third_party/boringssl/crypto/bio/buffer.c \ + third_party/boringssl/crypto/bio/connect.c \ + third_party/boringssl/crypto/bio/fd.c \ + third_party/boringssl/crypto/bio/file.c \ + third_party/boringssl/crypto/bio/hexdump.c \ + third_party/boringssl/crypto/bio/pair.c \ + third_party/boringssl/crypto/bio/printf.c \ + third_party/boringssl/crypto/bio/socket.c \ + third_party/boringssl/crypto/bio/socket_helper.c \ + third_party/boringssl/crypto/bn/add.c \ + third_party/boringssl/crypto/bn/asm/x86_64-gcc.c \ + third_party/boringssl/crypto/bn/bn.c \ + third_party/boringssl/crypto/bn/bn_asn1.c \ + third_party/boringssl/crypto/bn/cmp.c \ + third_party/boringssl/crypto/bn/convert.c \ + third_party/boringssl/crypto/bn/ctx.c \ + third_party/boringssl/crypto/bn/div.c \ + third_party/boringssl/crypto/bn/exponentiation.c \ + third_party/boringssl/crypto/bn/gcd.c \ + third_party/boringssl/crypto/bn/generic.c \ + third_party/boringssl/crypto/bn/kronecker.c \ + third_party/boringssl/crypto/bn/montgomery.c \ + third_party/boringssl/crypto/bn/mul.c \ + third_party/boringssl/crypto/bn/prime.c \ + third_party/boringssl/crypto/bn/random.c \ + third_party/boringssl/crypto/bn/rsaz_exp.c \ + third_party/boringssl/crypto/bn/shift.c \ + third_party/boringssl/crypto/bn/sqrt.c \ + third_party/boringssl/crypto/buf/buf.c \ + third_party/boringssl/crypto/bytestring/ber.c \ + third_party/boringssl/crypto/bytestring/cbb.c \ + third_party/boringssl/crypto/bytestring/cbs.c \ + third_party/boringssl/crypto/chacha/chacha_generic.c \ + third_party/boringssl/crypto/chacha/chacha_vec.c \ + third_party/boringssl/crypto/cipher/aead.c \ + third_party/boringssl/crypto/cipher/cipher.c \ + third_party/boringssl/crypto/cipher/derive_key.c \ + third_party/boringssl/crypto/cipher/e_aes.c \ + third_party/boringssl/crypto/cipher/e_chacha20poly1305.c \ + third_party/boringssl/crypto/cipher/e_des.c \ + third_party/boringssl/crypto/cipher/e_null.c \ + third_party/boringssl/crypto/cipher/e_rc2.c \ + third_party/boringssl/crypto/cipher/e_rc4.c \ + third_party/boringssl/crypto/cipher/e_ssl3.c \ + third_party/boringssl/crypto/cipher/e_tls.c \ + third_party/boringssl/crypto/cipher/tls_cbc.c \ + third_party/boringssl/crypto/cmac/cmac.c \ + third_party/boringssl/crypto/conf/conf.c \ + third_party/boringssl/crypto/cpu-arm.c \ + third_party/boringssl/crypto/cpu-intel.c \ + third_party/boringssl/crypto/crypto.c \ + third_party/boringssl/crypto/curve25519/curve25519.c \ + third_party/boringssl/crypto/des/des.c \ + third_party/boringssl/crypto/dh/check.c \ + third_party/boringssl/crypto/dh/dh.c \ + third_party/boringssl/crypto/dh/dh_asn1.c \ + third_party/boringssl/crypto/dh/params.c \ + third_party/boringssl/crypto/digest/digest.c \ + third_party/boringssl/crypto/digest/digests.c \ + third_party/boringssl/crypto/directory_posix.c \ + third_party/boringssl/crypto/directory_win.c \ + third_party/boringssl/crypto/dsa/dsa.c \ + third_party/boringssl/crypto/dsa/dsa_asn1.c \ + third_party/boringssl/crypto/ec/ec.c \ + third_party/boringssl/crypto/ec/ec_asn1.c \ + third_party/boringssl/crypto/ec/ec_key.c \ + third_party/boringssl/crypto/ec/ec_montgomery.c \ + third_party/boringssl/crypto/ec/oct.c \ + third_party/boringssl/crypto/ec/p224-64.c \ + third_party/boringssl/crypto/ec/p256-64.c \ + third_party/boringssl/crypto/ec/p256-x86_64.c \ + third_party/boringssl/crypto/ec/simple.c \ + third_party/boringssl/crypto/ec/util-64.c \ + third_party/boringssl/crypto/ec/wnaf.c \ + third_party/boringssl/crypto/ecdh/ecdh.c \ + third_party/boringssl/crypto/ecdsa/ecdsa.c \ + third_party/boringssl/crypto/ecdsa/ecdsa_asn1.c \ + third_party/boringssl/crypto/engine/engine.c \ + third_party/boringssl/crypto/err/err.c \ + third_party/boringssl/crypto/evp/algorithm.c \ + third_party/boringssl/crypto/evp/digestsign.c \ + third_party/boringssl/crypto/evp/evp.c \ + third_party/boringssl/crypto/evp/evp_asn1.c \ + third_party/boringssl/crypto/evp/evp_ctx.c \ + third_party/boringssl/crypto/evp/p_dsa_asn1.c \ + third_party/boringssl/crypto/evp/p_ec.c \ + third_party/boringssl/crypto/evp/p_ec_asn1.c \ + third_party/boringssl/crypto/evp/p_rsa.c \ + third_party/boringssl/crypto/evp/p_rsa_asn1.c \ + third_party/boringssl/crypto/evp/pbkdf.c \ + third_party/boringssl/crypto/evp/sign.c \ + third_party/boringssl/crypto/ex_data.c \ + third_party/boringssl/crypto/hkdf/hkdf.c \ + third_party/boringssl/crypto/hmac/hmac.c \ + third_party/boringssl/crypto/lhash/lhash.c \ + third_party/boringssl/crypto/md4/md4.c \ + third_party/boringssl/crypto/md5/md5.c \ + third_party/boringssl/crypto/mem.c \ + third_party/boringssl/crypto/modes/cbc.c \ + third_party/boringssl/crypto/modes/cfb.c \ + third_party/boringssl/crypto/modes/ctr.c \ + third_party/boringssl/crypto/modes/gcm.c \ + third_party/boringssl/crypto/modes/ofb.c \ + third_party/boringssl/crypto/obj/obj.c \ + third_party/boringssl/crypto/obj/obj_xref.c \ + third_party/boringssl/crypto/pem/pem_all.c \ + third_party/boringssl/crypto/pem/pem_info.c \ + third_party/boringssl/crypto/pem/pem_lib.c \ + third_party/boringssl/crypto/pem/pem_oth.c \ + third_party/boringssl/crypto/pem/pem_pk8.c \ + third_party/boringssl/crypto/pem/pem_pkey.c \ + third_party/boringssl/crypto/pem/pem_x509.c \ + third_party/boringssl/crypto/pem/pem_xaux.c \ + third_party/boringssl/crypto/pkcs8/p5_pbe.c \ + third_party/boringssl/crypto/pkcs8/p5_pbev2.c \ + third_party/boringssl/crypto/pkcs8/p8_pkey.c \ + third_party/boringssl/crypto/pkcs8/pkcs8.c \ + third_party/boringssl/crypto/poly1305/poly1305.c \ + third_party/boringssl/crypto/poly1305/poly1305_arm.c \ + third_party/boringssl/crypto/poly1305/poly1305_vec.c \ + third_party/boringssl/crypto/rand/rand.c \ + third_party/boringssl/crypto/rand/urandom.c \ + third_party/boringssl/crypto/rand/windows.c \ + third_party/boringssl/crypto/rc4/rc4.c \ + third_party/boringssl/crypto/refcount_c11.c \ + third_party/boringssl/crypto/refcount_lock.c \ + third_party/boringssl/crypto/rsa/blinding.c \ + third_party/boringssl/crypto/rsa/padding.c \ + third_party/boringssl/crypto/rsa/rsa.c \ + third_party/boringssl/crypto/rsa/rsa_asn1.c \ + third_party/boringssl/crypto/rsa/rsa_impl.c \ + third_party/boringssl/crypto/sha/sha1.c \ + third_party/boringssl/crypto/sha/sha256.c \ + third_party/boringssl/crypto/sha/sha512.c \ + third_party/boringssl/crypto/stack/stack.c \ + third_party/boringssl/crypto/thread.c \ + third_party/boringssl/crypto/thread_none.c \ + third_party/boringssl/crypto/thread_pthread.c \ + third_party/boringssl/crypto/thread_win.c \ + third_party/boringssl/crypto/time_support.c \ + third_party/boringssl/crypto/x509/a_digest.c \ + third_party/boringssl/crypto/x509/a_sign.c \ + third_party/boringssl/crypto/x509/a_strex.c \ + third_party/boringssl/crypto/x509/a_verify.c \ + third_party/boringssl/crypto/x509/asn1_gen.c \ + third_party/boringssl/crypto/x509/by_dir.c \ + third_party/boringssl/crypto/x509/by_file.c \ + third_party/boringssl/crypto/x509/i2d_pr.c \ + third_party/boringssl/crypto/x509/pkcs7.c \ + third_party/boringssl/crypto/x509/t_crl.c \ + third_party/boringssl/crypto/x509/t_req.c \ + third_party/boringssl/crypto/x509/t_x509.c \ + third_party/boringssl/crypto/x509/t_x509a.c \ + third_party/boringssl/crypto/x509/x509.c \ + third_party/boringssl/crypto/x509/x509_att.c \ + third_party/boringssl/crypto/x509/x509_cmp.c \ + third_party/boringssl/crypto/x509/x509_d2.c \ + third_party/boringssl/crypto/x509/x509_def.c \ + third_party/boringssl/crypto/x509/x509_ext.c \ + third_party/boringssl/crypto/x509/x509_lu.c \ + third_party/boringssl/crypto/x509/x509_obj.c \ + third_party/boringssl/crypto/x509/x509_r2x.c \ + third_party/boringssl/crypto/x509/x509_req.c \ + third_party/boringssl/crypto/x509/x509_set.c \ + third_party/boringssl/crypto/x509/x509_trs.c \ + third_party/boringssl/crypto/x509/x509_txt.c \ + third_party/boringssl/crypto/x509/x509_v3.c \ + third_party/boringssl/crypto/x509/x509_vfy.c \ + third_party/boringssl/crypto/x509/x509_vpm.c \ + third_party/boringssl/crypto/x509/x509cset.c \ + third_party/boringssl/crypto/x509/x509name.c \ + third_party/boringssl/crypto/x509/x509rset.c \ + third_party/boringssl/crypto/x509/x509spki.c \ + third_party/boringssl/crypto/x509/x509type.c \ + third_party/boringssl/crypto/x509/x_algor.c \ + third_party/boringssl/crypto/x509/x_all.c \ + third_party/boringssl/crypto/x509/x_attrib.c \ + third_party/boringssl/crypto/x509/x_crl.c \ + third_party/boringssl/crypto/x509/x_exten.c \ + third_party/boringssl/crypto/x509/x_info.c \ + third_party/boringssl/crypto/x509/x_name.c \ + third_party/boringssl/crypto/x509/x_pkey.c \ + third_party/boringssl/crypto/x509/x_pubkey.c \ + third_party/boringssl/crypto/x509/x_req.c \ + third_party/boringssl/crypto/x509/x_sig.c \ + third_party/boringssl/crypto/x509/x_spki.c \ + third_party/boringssl/crypto/x509/x_val.c \ + third_party/boringssl/crypto/x509/x_x509.c \ + third_party/boringssl/crypto/x509/x_x509a.c \ + third_party/boringssl/crypto/x509v3/pcy_cache.c \ + third_party/boringssl/crypto/x509v3/pcy_data.c \ + third_party/boringssl/crypto/x509v3/pcy_lib.c \ + third_party/boringssl/crypto/x509v3/pcy_map.c \ + third_party/boringssl/crypto/x509v3/pcy_node.c \ + third_party/boringssl/crypto/x509v3/pcy_tree.c \ + third_party/boringssl/crypto/x509v3/v3_akey.c \ + third_party/boringssl/crypto/x509v3/v3_akeya.c \ + third_party/boringssl/crypto/x509v3/v3_alt.c \ + third_party/boringssl/crypto/x509v3/v3_bcons.c \ + third_party/boringssl/crypto/x509v3/v3_bitst.c \ + third_party/boringssl/crypto/x509v3/v3_conf.c \ + third_party/boringssl/crypto/x509v3/v3_cpols.c \ + third_party/boringssl/crypto/x509v3/v3_crld.c \ + third_party/boringssl/crypto/x509v3/v3_enum.c \ + third_party/boringssl/crypto/x509v3/v3_extku.c \ + third_party/boringssl/crypto/x509v3/v3_genn.c \ + third_party/boringssl/crypto/x509v3/v3_ia5.c \ + third_party/boringssl/crypto/x509v3/v3_info.c \ + third_party/boringssl/crypto/x509v3/v3_int.c \ + third_party/boringssl/crypto/x509v3/v3_lib.c \ + third_party/boringssl/crypto/x509v3/v3_ncons.c \ + third_party/boringssl/crypto/x509v3/v3_pci.c \ + third_party/boringssl/crypto/x509v3/v3_pcia.c \ + third_party/boringssl/crypto/x509v3/v3_pcons.c \ + third_party/boringssl/crypto/x509v3/v3_pku.c \ + third_party/boringssl/crypto/x509v3/v3_pmaps.c \ + third_party/boringssl/crypto/x509v3/v3_prn.c \ + third_party/boringssl/crypto/x509v3/v3_purp.c \ + third_party/boringssl/crypto/x509v3/v3_skey.c \ + third_party/boringssl/crypto/x509v3/v3_sxnet.c \ + third_party/boringssl/crypto/x509v3/v3_utl.c \ + third_party/boringssl/ssl/custom_extensions.c \ + third_party/boringssl/ssl/d1_both.c \ + third_party/boringssl/ssl/d1_clnt.c \ + third_party/boringssl/ssl/d1_lib.c \ + third_party/boringssl/ssl/d1_meth.c \ + third_party/boringssl/ssl/d1_pkt.c \ + third_party/boringssl/ssl/d1_srtp.c \ + third_party/boringssl/ssl/d1_srvr.c \ + third_party/boringssl/ssl/dtls_record.c \ + third_party/boringssl/ssl/pqueue/pqueue.c \ + third_party/boringssl/ssl/s3_both.c \ + third_party/boringssl/ssl/s3_clnt.c \ + third_party/boringssl/ssl/s3_enc.c \ + third_party/boringssl/ssl/s3_lib.c \ + third_party/boringssl/ssl/s3_meth.c \ + third_party/boringssl/ssl/s3_pkt.c \ + third_party/boringssl/ssl/s3_srvr.c \ + third_party/boringssl/ssl/ssl_aead_ctx.c \ + third_party/boringssl/ssl/ssl_asn1.c \ + third_party/boringssl/ssl/ssl_buffer.c \ + third_party/boringssl/ssl/ssl_cert.c \ + third_party/boringssl/ssl/ssl_cipher.c \ + third_party/boringssl/ssl/ssl_file.c \ + third_party/boringssl/ssl/ssl_lib.c \ + third_party/boringssl/ssl/ssl_rsa.c \ + third_party/boringssl/ssl/ssl_session.c \ + third_party/boringssl/ssl/ssl_stat.c \ + third_party/boringssl/ssl/t1_enc.c \ + third_party/boringssl/ssl/t1_lib.c \ + third_party/boringssl/ssl/tls_record.c \ + , $ext_shared, , -Wall -Werror \ + -Wno-parentheses-equality -Wno-unused-value -std=c11 \ + -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN \ + -D_HAS_EXCEPTIONS=0 -DNOMINMAX) + + PHP_ADD_BUILD_DIR($ext_builddir/src/php/ext/grpc) + + PHP_ADD_BUILD_DIR($ext_builddir/src/boringssl) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/census) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/channel) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_config) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_config/lb_policies) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_config/resolvers) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/compression) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/debug) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/httpcli) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/iomgr) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/json) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/profiling) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/security) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/support) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/surface) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/transport) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/transport/chttp2) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/aes) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/asn1) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/base64) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bio) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bn) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bn/asm) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/buf) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bytestring) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/chacha) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/cipher) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/cmac) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/conf) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/curve25519) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/des) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/dh) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/digest) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/dsa) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/ec) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/ecdh) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/ecdsa) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/engine) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/err) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/evp) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/hkdf) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/hmac) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/lhash) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/md4) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/md5) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/modes) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/obj) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/pem) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/pkcs8) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/poly1305) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/rand) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/rc4) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/rsa) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/sha) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/stack) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/x509) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/x509v3) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/ssl) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/ssl/pqueue) +fi diff --git a/package.xml b/package.xml new file mode 100644 index 00000000000..68d6dfb9402 --- /dev/null +++ b/package.xml @@ -0,0 +1,964 @@ + + + grpc + pecl.php.net + A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. + Remote Procedure Calls (RPCs) provide a useful abstraction for building distributed applications and services. The libraries in this repository provide a concrete implementation of the gRPC protocol, layered over HTTP/2. These libraries enable communication between clients and servers using any combination of the supported languages. + + Stanley Cheung + stanleycheung + grpc-packages@google.com + yes + + 2016-02-24 + + + 0.8.0 + 0.8.0 + + + beta + beta + + BSD + +- Simplify gRPC PHP installation #4517 +- Wrap gRPC core library version 0.13 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5.5.0 + + + 1.4.0 + + + + grpc + + + + + 0.5.0 + 0.5.0 + + + alpha + alpha + + 2015-06-16 + BSD + +First alpha release + + + + + 0.5.1 + 0.5.1 + + + alpha + alpha + + 2015-07-09 + BSD + +Update to wrap gRPC C Core version 0.10.0 + + + + + 0.6.0 + 0.6.0 + + + beta + beta + + 2015-09-24 + BSD + +- support per message compression disable +- expose per-call host override option +- expose connectivity API +- expose channel target and call peer +- add user-agent +- update to wrap gRPC C core library beta version 0.11.0 + + + + + 0.6.1 + 0.6.0 + + + beta + beta + + 2015-10-21 + BSD + +- fixed undefined constant fatal error when run with apache/nginx #2275 + + + + + 0.7.0 + 0.7.0 + + + beta + beta + + 2016-01-13 + BSD + +- Breaking change to Credentials class (removed) #3765 +- Replaced by ChannelCredentials and CallCredentials class #3765 +- New plugin based metadata auth API #4394 +- Explicit ChannelCredentials::createInsecure() call + + + + + 0.8.0 + 0.8.0 + + + beta + beta + + 2016-02-24 + BSD + +- Simplify gRPC PHP installation #4517 +- Wrap gRPC core library version 0.13 + + + + diff --git a/src/php/README.md b/src/php/README.md index b1823b92261..08d0b413c95 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -33,44 +33,12 @@ $ sudo mv phpunit.phar /usr/local/bin/phpunit ## Quick Install -**Linux (Debian):** - -Add [Debian jessie-backports][] to your `sources.list` file. Example: - -```sh -echo "deb http://http.debian.net/debian jessie-backports main" | \ -sudo tee -a /etc/apt/sources.list -``` - -Install the gRPC Debian package - -```sh -sudo apt-get update -sudo apt-get install libgrpc-dev -``` - Install the gRPC PHP extension ```sh sudo pecl install grpc-beta ``` -**Mac OS X:** - -Install [homebrew][]. Example: - -```sh -ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -``` - -Install the gRPC core library and the PHP extension in one step - -```sh -$ curl -fsSL https://goo.gl/getgrpc | bash -s php -``` - -This will download and run the [gRPC install script][] and compile the gRPC PHP extension. - ## Build from Source diff --git a/templates/config.m4.template b/templates/config.m4.template new file mode 100644 index 00000000000..5847d456f51 --- /dev/null +++ b/templates/config.m4.template @@ -0,0 +1,59 @@ +%YAML 1.2 +--- | + PHP_ARG_ENABLE(grpc, whether to enable grpc support, + [ --enable-grpc Enable grpc support]) + + if test "$PHP_GRPC" != "no"; then + dnl Write more examples of tests here... + + dnl # --with-grpc -> add include path + PHP_ADD_INCLUDE(../../grpc/include) + PHP_ADD_INCLUDE(../../grpc/src/php/ext/grpc) + PHP_ADD_INCLUDE(../../grpc/third_party/boringssl/include) + + LIBS="-lpthread $LIBS" + + GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" + PHP_ADD_LIBRARY(pthread) + + PHP_ADD_LIBRARY(dl,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(dl) + + case $host in + *darwin*) ;; + *) + PHP_ADD_LIBRARY(rt,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(rt) + ;; + esac + + PHP_NEW_EXTENSION(grpc, + % for source in php_config_m4.src: + ${source} ${"\\"} + % endfor + % for lib in libs: + % if lib.name in php_config_m4.get('deps', []): + % for source in lib.src: + ${source} ${"\\"} + % endfor + % endif + % endfor + , $ext_shared, , -Wall -Werror ${"\\"} + -Wno-parentheses-equality -Wno-unused-value -std=c11 ${"\\"} + -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN ${"\\"} + -D_HAS_EXCEPTIONS=0 -DNOMINMAX) + + PHP_ADD_BUILD_DIR($ext_builddir/src/php/ext/grpc) + <% + dirs = {} + for lib in libs: + if lib.name in php_config_m4.get('deps', []): + for source in lib.src: + dirs[source[:source.rfind('/')]] = 1 + dirs = dirs.keys() + dirs.sort() + %> + % for dir in dirs: + PHP_ADD_BUILD_DIR($ext_builddir/${dir}) + % endfor + fi diff --git a/templates/package.xml.template b/templates/package.xml.template new file mode 100644 index 00000000000..058f5064715 --- /dev/null +++ b/templates/package.xml.template @@ -0,0 +1,161 @@ +%YAML 1.2 +--- | + + + grpc + pecl.php.net + A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. + Remote Procedure Calls (RPCs) provide a useful abstraction for building distributed applications and services. The libraries in this repository provide a concrete implementation of the gRPC protocol, layered over HTTP/2. These libraries enable communication between clients and servers using any combination of the supported languages. + + Stanley Cheung + stanleycheung + grpc-packages@google.com + yes + + 2016-02-24 + + + 0.8.0 + 0.8.0 + + + beta + beta + + BSD + + - Simplify gRPC PHP installation #4517 + - Wrap gRPC core library version 0.13 + + + + + + + + % for source in php_config_m4.src + php_config_m4.headers: + + % endfor + % for lib in libs: + % if lib.name in php_config_m4.get('deps', []): + % for source in lib.get('public_headers', []) + lib.headers + lib.src: + + % endfor + % endif + % endfor + + + + + + 5.5.0 + + + 1.4.0 + + + + grpc + + + + + 0.5.0 + 0.5.0 + + + alpha + alpha + + 2015-06-16 + BSD + + First alpha release + + + + + 0.5.1 + 0.5.1 + + + alpha + alpha + + 2015-07-09 + BSD + + Update to wrap gRPC C Core version 0.10.0 + + + + + 0.6.0 + 0.6.0 + + + beta + beta + + 2015-09-24 + BSD + + - support per message compression disable + - expose per-call host override option + - expose connectivity API + - expose channel target and call peer + - add user-agent + - update to wrap gRPC C core library beta version 0.11.0 + + + + + 0.6.1 + 0.6.0 + + + beta + beta + + 2015-10-21 + BSD + + - fixed undefined constant fatal error when run with apache/nginx #2275 + + + + + 0.7.0 + 0.7.0 + + + beta + beta + + 2016-01-13 + BSD + + - Breaking change to Credentials class (removed) #3765 + - Replaced by ChannelCredentials and CallCredentials class #3765 + - New plugin based metadata auth API #4394 + - Explicit ChannelCredentials::createInsecure() call + + + + + 0.8.0 + 0.8.0 + + + beta + beta + + 2016-02-24 + BSD + + - Simplify gRPC PHP installation #4517 + - Wrap gRPC core library version 0.13 + + + + diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index 72c65ad14a8..161d83e3b7f 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From 093ff5db8b4d9fa9965b505fc8e33316c3daf2bb Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 3 Mar 2016 12:18:27 -0800 Subject: [PATCH 13/97] Fix copyright --- test/cpp/interop/reconnect_interop_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1f6b352db17..79a60cc8602 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From ebf1fcff7cf3ba0ad515fccab5dcf0ab75c5bd31 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 13:53:49 -0800 Subject: [PATCH 14/97] Update base INSTALL to markdown and remove outdated content --- CONTRIBUTING.md | 2 +- INSTALL | 225 ------------------------------ INSTALL.md | 45 ++++++ README.md | 2 +- examples/cpp/README.md | 2 +- examples/cpp/cpptutorial.md | 2 +- examples/cpp/helloworld/README.md | 2 +- 7 files changed, 50 insertions(+), 230 deletions(-) delete mode 100644 INSTALL create mode 100644 INSTALL.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9423c46547a..03d2e276a82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ In order to protect both you and ourselves, you will need to sign the ### Technical requirements You will need several tools to work with this repository. In addition to all of -the packages described in the [INSTALL](INSTALL) file, you will also need +the packages described in the [INSTALL](INSTALL.md) file, you will also need python, and the mako template renderer. To install the latter, using pip, one should simply be able to do `pip install mako`. diff --git a/INSTALL b/INSTALL deleted file mode 100644 index a0df57dcd35..00000000000 --- a/INSTALL +++ /dev/null @@ -1,225 +0,0 @@ -These instructions only cover building grpc C and C++ libraries under -typical unix systems. If you need more information, please try grpc's -wiki pages: - - https://github.com/google/grpc/wiki - - -************************* -* If you are in a hurry * -************************* - -On Linux (Debian): - - Note: you will need to add the Debian 'jessie-backports' distribution to your sources - file first. - - Add the following line to your `/etc/apt/sources.list` file: - - deb http://http.debian.net/debian jessie-backports main - - Install the gRPC library: - - $ [sudo] apt-get install libgrpc-dev - -OR - - $ git clone https://github.com/grpc/grpc.git - $ cd grpc - $ git submodule update --init - $ make - $ [sudo] make install - -You don't need anything else than GNU Make, gcc and autotools. Under a Debian -or Ubuntu system, this should boil down to the following packages: - - $ [sudo] apt-get install build-essential autoconf libtool - -Building the python wrapper requires the following: - - $ [sudo] apt-get install python-all-dev python-virtualenv - -If you want to install in a different directory than the default /usr/lib, you can -override it on the command line: - - $ [sudo] make install prefix=/opt - - -******************************* -* More detailled instructions * -******************************* - -Setting up dependencies -======================= - -Dependencies to compile the libraries -------------------------------------- - -grpc libraries have few external dependencies. If you need to compile and -install them, they are present in the third_party directory if you have -cloned the github repository recursively. If you didn't clone recursively, -you can still get them later by running the following command: - - $ git submodule update --init - -Note that the Makefile makes it much easier for you to compile from sources -if you were to clone recursively our git repository: it will automatically -compile zlib and OpenSSL, which are core requirements for grpc. Note this -creates grpc libraries that will have zlib and OpenSSL built-in inside of them, -which significantly increases the libraries' size. - -In order to decrease that size, you can manually install zlib and OpenSSL on -your system, so that the Makefile can use them instead. - -Under a Debian or Ubuntu system, one can acquire the development package -for zlib this way: - - # apt-get install zlib1g-dev - -To the best of our knowledge, no distribution has an OpenSSL package that -supports ALPN yet, so you would still have to depend on installing from source -for that particular dependency if you want to reduce the libraries' size. - -The recommended version of OpenSSL that provides ALPN support is available -at this URL: - - https://www.openssl.org/source/openssl-1.0.2.tar.gz - - -Dependencies to compile and run the tests ------------------------------------------ - -Compiling and running grpc plain-C tests dont't require any more dependency. - - -Compiling and running grpc C++ tests depend on protobuf 3.0.0, gtest and -gflags. Although gflags is provided in third_party, you will need to manually -install that dependency on your system to run these tests. - -Under a Debian or Ubuntu system, you can install the gtests and gflags packages -using apt-get: - - # apt-get install libgflags-dev libgtest-dev - -However, protobuf 3.0.0 isn't in a debian package yet, but the Makefile will -automatically try and compile the one present in third_party if you cloned the -repository recursively, and that it detects your system is lacking it. - -Compiling and installing protobuf 3.0.0 requires a few more dependencies in -itself, notably the autoconf suite. If you have apt-get, you can install -these dependencies this way: - - # apt-get install autoconf libtool - -If you want to run the tests using one of the sanitized configurations, you -will need clang and its instrumented libc++: - - # apt-get install clang libc++-dev - -Mac-specific notes: -------------------- - -For a Mac system, git is not available by default. You will first need to -install Xcode from the Mac AppStore and then run the following command from a -terminal: - - $ sudo xcode-select --install - -You should also install "port" following the instructions at -https://www.macports.org . This will reside in /opt/local/bin/port for -most Mac installations. Do the "git submodule" command listed above. - -Then execute the following for all the needed build dependencies - - $ sudo /opt/local/bin/port install autoconf automake libtool gflags cmake - $ mkdir ~/gtest-svn - $ svn checkout http://googletest.googlecode.com/svn/trunk/ gtest-svn - $ mkdir mybuild - $ cd mybuild - $ cmake ../gtest-svn - $ make - $ make gtest.a gtest_main.a - $ sudo cp libgtest.a libgtest_main.a /opt/local/lib - $ sudo mkdir /opt/local/include/gtest - $ sudo cp -pr ../gtest-svn/include/gtest /opt/local/include/gtest - -We will also need to make openssl and install it appropriately - - $ cd - $ cd third_party/openssl - $ ./config - $ sudo make install - $ cd ../../ - -If you are going to make changes and need to regenerate the projects file, -you will need to install certain modules for python. - - $ sudo easy_install simplejson mako - -Mingw-specific notes: ---------------------- - -While gRPC compiles properly under mingw, some more preparation work is needed. -The recommendation is to use msys2. The installation instructions are available -at that address: http://msys2.github.io/ - -Once this is installed, make sure you are using the following: MinGW-w64 Win64. -You'll be required to install a few more packages: - - $ pacman -S make mingw-w64-x86_64-gcc mingw-w64-x86_64-zlib autoconf automake libtool - -Please also install OpenSSL from that website: - - http://slproweb.com/products/Win32OpenSSL.html - -The package Win64 OpenSSL v1.0.2a should do. At that point you should be able -to compile gRPC with the following: - - $ export LDFLAGS="-L/mingw64/lib -L/c/OpenSSL-Win64" - $ export CPPFLAGS="-I/mingw64/include -I/c/OpenSSL-Win64/include" - $ make - -A word on OpenSSL ------------------ - -Secure HTTP2 requires the TLS extension ALPN (see rfc 7301 and -http://http2.github.io/http2-spec/ section 3.3). Our HTTP2 implementation -relies on OpenSSL's implementation. OpenSSL 1.0.2 is the first released version -of OpenSSL that has ALPN support, and this explains our dependency on it. - -Note that the Makefile supports compiling only the unsecure elements of grpc, -and if you do not have OpenSSL and do not want it, you can still proceed -with installing only the elements you require. However, we strongly recommend -the use of encryption for all network traffic, and discourage the use of grpc -without TLS. - - -Compiling -========= - -If you have all the dependencies mentioned above, you should simply be able -to go ahead and run "make" to compile grpc's C and C++ libraries: - - $ make - - -Testing -======= - -To build and run the tests, you can run the command: - - $ make test - -If you want to be able to run them in parallel, and get better output, you can -also use the python tool we have written: - - $ ./tools/run_tests/run_tests.py - - -Installing -========== - -Once everything is compiled, you should be able to install grpc C and C++ -libraries and headers: - - # make install diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 00000000000..aa809f23c8f --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,45 @@ +#If you are in a hurry + +For language-specific installation instructions, please refer to these +documents + + * [C++](examples/cpp) + * [C#](src/csharp): NuGet package `Grpc` + * [Go](https://github.com/grpc/grpc-go): `go get google.golang.org/grpc` + * [Java](https://github.com/grpc/grpc-java) + * [Node](src/node): `npm install grpc` + * [Objective-C](src/objective-c) + * [PHP](src/php): `pecl install grpc-beta` + * [Python](src/python/grpcio): `pip install grpcio` + * [Ruby](src/ruby): `gem install grpc` + + +#Pre-requisites + +##Linux + +```sh + $ [sudo] apt-get install build-essential autoconf libtool +``` + +##Mac OSX + +For a Mac system, git is not available by default. You will first need to +install Xcode from the Mac AppStore and then run the following command from a +terminal: + +```sh + $ [sudo] xcode-select --install +``` + +#Build from Source + +This is for compiling just the gRPC C Core library. + +```sh + $ git clone https://github.com/grpc/grpc.git + $ cd grpc + $ git submodule update --init + $ make + $ [sudo] make install +``` diff --git a/README.md b/README.md index 033e09b91b7..abb4905392c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You can find more detailed documentation and examples in the [doc](doc) and [exa #Installation -See [grpc/INSTALL](INSTALL) for installation instructions for various platforms. +See [INSTALL](INSTALL.md) for installation instructions for various platforms. #Repository Structure & Status diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 85c495099b7..979ba55dbb5 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -2,7 +2,7 @@ ## Installation -To install gRPC on your system, follow the instructions [here](../../INSTALL). +To install gRPC on your system, follow the instructions [here](../../INSTALL.md). ## Hello C++ gRPC! diff --git a/examples/cpp/cpptutorial.md b/examples/cpp/cpptutorial.md index cd1cddb1115..ef9ca99c0fe 100644 --- a/examples/cpp/cpptutorial.md +++ b/examples/cpp/cpptutorial.md @@ -91,7 +91,7 @@ message Point { Next we need to generate the gRPC client and server interfaces from our .proto service definition. We do this using the protocol buffer compiler `protoc` with a special gRPC C++ plugin. -For simplicity, we've provided a [makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL) first): +For simplicity, we've provided a [makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL.md) first): ```shell $ make route_guide.grpc.pb.cc route_guide.pb.cc diff --git a/examples/cpp/helloworld/README.md b/examples/cpp/helloworld/README.md index 90f3d6b1475..8e11f7cdf3e 100644 --- a/examples/cpp/helloworld/README.md +++ b/examples/cpp/helloworld/README.md @@ -2,7 +2,7 @@ ### Install gRPC Make sure you have installed gRPC on your system. Follow the instructions here: -[https://github.com/grpc/grpc/blob/master/INSTALL](../../../INSTALL). +[https://github.com/grpc/grpc/blob/master/INSTALL](../../../INSTALL.md). ### Get the tutorial source code From 1279bd7934e3c5522b0b6ee6dffcb5b944bf752d Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 14:49:03 -0800 Subject: [PATCH 15/97] bit of text change --- INSTALL.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index aa809f23c8f..d9411db021d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,7 +1,7 @@ #If you are in a hurry -For language-specific installation instructions, please refer to these -documents +For language-specific installation instructions for gRPC runtime, please +refer to these documents * [C++](examples/cpp) * [C#](src/csharp): NuGet package `Grpc` @@ -34,7 +34,8 @@ terminal: #Build from Source -This is for compiling just the gRPC C Core library. +For developers who are interested to contribute, here is how to compile the +gRPC C Core library. ```sh $ git clone https://github.com/grpc/grpc.git From f2f7d57c01037993220132817a2897777288f3ca Mon Sep 17 00:00:00 2001 From: Dan Born Date: Thu, 3 Mar 2016 17:26:12 -0800 Subject: [PATCH 16/97] Allow use of alternative credential types. --- src/proto/grpc/testing/echo_messages.proto | 1 + test/cpp/end2end/end2end_test.cc | 33 ++++++++++++++-------- test/cpp/end2end/test_service_impl.cc | 10 ++++--- test/cpp/util/test_credentials_provider.h | 5 +++- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/proto/grpc/testing/echo_messages.proto b/src/proto/grpc/testing/echo_messages.proto index d05a35548d1..5ce0a1fd642 100644 --- a/src/proto/grpc/testing/echo_messages.proto +++ b/src/proto/grpc/testing/echo_messages.proto @@ -42,6 +42,7 @@ message RequestParams { bool echo_peer = 7; string expected_client_identity = 8; // will force check_auth_context. bool skip_cancelled_check = 9; + string expected_transport_security_type = 10; } message EchoRequest { diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index dc2c4f6426f..8131a14ff76 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -59,6 +59,7 @@ using grpc::testing::EchoRequest; using grpc::testing::EchoResponse; +using grpc::testing::kTlsCredentialsType; using std::chrono::system_clock; namespace grpc { @@ -1194,6 +1195,8 @@ TEST_P(SecureEnd2endTest, BlockingAuthMetadataPluginAndProcessorSuccess) { request.mutable_param()->set_echo_metadata(true); request.mutable_param()->set_expected_client_identity( TestAuthMetadataProcessor::kGoodGuy); + request.mutable_param()->set_expected_transport_security_type( + GetParam().credentials_type); Status s = stub_->Echo(&context, request, &response); EXPECT_EQ(request.message(), response.message()); @@ -1301,6 +1304,8 @@ TEST_P(SecureEnd2endTest, NonBlockingAuthMetadataPluginAndProcessorSuccess) { request.mutable_param()->set_echo_metadata(true); request.mutable_param()->set_expected_client_identity( TestAuthMetadataProcessor::kGoodGuy); + request.mutable_param()->set_expected_transport_security_type( + GetParam().credentials_type); Status s = stub_->Echo(&context, request, &response); EXPECT_EQ(request.message(), response.message()); @@ -1349,25 +1354,29 @@ TEST_P(SecureEnd2endTest, ClientAuthContext) { EchoRequest request; EchoResponse response; request.set_message("Hello"); - request.mutable_param()->set_check_auth_context(true); - + request.mutable_param()->set_check_auth_context(GetParam().credentials_type == + kTlsCredentialsType); + request.mutable_param()->set_expected_transport_security_type( + GetParam().credentials_type); ClientContext context; Status s = stub_->Echo(&context, request, &response); EXPECT_EQ(response.message(), request.message()); EXPECT_TRUE(s.ok()); std::shared_ptr auth_ctx = context.auth_context(); - std::vector ssl = + std::vector tst = auth_ctx->FindPropertyValues("transport_security_type"); - EXPECT_EQ(1u, ssl.size()); - EXPECT_EQ("ssl", ToString(ssl[0])); - EXPECT_EQ("x509_subject_alternative_name", - auth_ctx->GetPeerIdentityPropertyName()); - EXPECT_EQ(3u, auth_ctx->GetPeerIdentity().size()); - EXPECT_EQ("*.test.google.fr", ToString(auth_ctx->GetPeerIdentity()[0])); - EXPECT_EQ("waterzooi.test.google.be", - ToString(auth_ctx->GetPeerIdentity()[1])); - EXPECT_EQ("*.test.youtube.com", ToString(auth_ctx->GetPeerIdentity()[2])); + EXPECT_EQ(1u, tst.size()); + EXPECT_EQ(GetParam().credentials_type, ToString(tst[0])); + if (GetParam().credentials_type == kTlsCredentialsType) { + EXPECT_EQ("x509_subject_alternative_name", + auth_ctx->GetPeerIdentityPropertyName()); + EXPECT_EQ(3u, auth_ctx->GetPeerIdentity().size()); + EXPECT_EQ("*.test.google.fr", ToString(auth_ctx->GetPeerIdentity()[0])); + EXPECT_EQ("waterzooi.test.google.be", + ToString(auth_ctx->GetPeerIdentity()[1])); + EXPECT_EQ("*.test.youtube.com", ToString(auth_ctx->GetPeerIdentity()[2])); + } } std::vector CreateTestScenarios(bool use_proxy, diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc index 7c3e514effa..698d4678c87 100644 --- a/test/cpp/end2end/test_service_impl.cc +++ b/test/cpp/end2end/test_service_impl.cc @@ -63,13 +63,14 @@ void MaybeEchoDeadline(ServerContext* context, const EchoRequest* request, } void CheckServerAuthContext(const ServerContext* context, + const grpc::string& expected_transport_security_type, const grpc::string& expected_client_identity) { std::shared_ptr auth_ctx = context->auth_context(); - std::vector ssl = + std::vector tst = auth_ctx->FindPropertyValues("transport_security_type"); - EXPECT_EQ(1u, ssl.size()); - EXPECT_EQ("ssl", ToString(ssl[0])); - if (expected_client_identity.length() == 0) { + EXPECT_EQ(1u, tst.size()); + EXPECT_EQ(expected_transport_security_type, ToString(tst[0])); + if (expected_client_identity.empty()) { EXPECT_TRUE(auth_ctx->GetPeerIdentityPropertyName().empty()); EXPECT_TRUE(auth_ctx->GetPeerIdentity().empty()); EXPECT_FALSE(auth_ctx->IsPeerAuthenticated()); @@ -139,6 +140,7 @@ Status TestServiceImpl::Echo(ServerContext* context, const EchoRequest* request, (request->param().expected_client_identity().length() > 0 || request->param().check_auth_context())) { CheckServerAuthContext(context, + request->param().expected_transport_security_type(), request->param().expected_client_identity()); } if (request->has_param() && request->param().response_message_length() > 0) { diff --git a/test/cpp/util/test_credentials_provider.h b/test/cpp/util/test_credentials_provider.h index 50fadb53a24..1fb311e556e 100644 --- a/test/cpp/util/test_credentials_provider.h +++ b/test/cpp/util/test_credentials_provider.h @@ -44,7 +44,10 @@ namespace grpc { namespace testing { const char kInsecureCredentialsType[] = "INSECURE_CREDENTIALS"; -const char kTlsCredentialsType[] = "TLS_CREDENTIALS"; + +// For real credentials, like tls/ssl, this name should match the AuthContext +// property "transport_security_type". +const char kTlsCredentialsType[] = "ssl"; // Provide test credentials of a particular type. class CredentialTypeProvider { From 51221f1e7e65231539614f9b020c331f234663a5 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Thu, 3 Mar 2016 17:44:50 -0800 Subject: [PATCH 17/97] clang reformat. --- test/cpp/end2end/test_service_impl.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc index 698d4678c87..fe29c4afe9e 100644 --- a/test/cpp/end2end/test_service_impl.cc +++ b/test/cpp/end2end/test_service_impl.cc @@ -62,9 +62,10 @@ void MaybeEchoDeadline(ServerContext* context, const EchoRequest* request, } } -void CheckServerAuthContext(const ServerContext* context, - const grpc::string& expected_transport_security_type, - const grpc::string& expected_client_identity) { +void CheckServerAuthContext( + const ServerContext* context, + const grpc::string& expected_transport_security_type, + const grpc::string& expected_client_identity) { std::shared_ptr auth_ctx = context->auth_context(); std::vector tst = auth_ctx->FindPropertyValues("transport_security_type"); From 93744fccb784c2cab8a9f008348f1e87cc1293e5 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 20:41:38 -0800 Subject: [PATCH 18/97] update PHP composer.json files --- composer.json | 11 +++++++++-- src/php/composer.json | 6 ++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index 61f81e02bf4..9ba45fd3a36 100644 --- a/composer.json +++ b/composer.json @@ -2,13 +2,20 @@ "name": "grpc/grpc", "type": "library", "description": "gRPC library for PHP", - "version": "0.6.0", + "version": "0.13.0", "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/stanley-cheung/Protobuf-PHP" + } + ], "require": { "php": ">=5.5.0", - "google/auth": "dev-master" + "datto/protobuf-php": "dev-master", + "google/auth": "v0.7" }, "autoload": { "psr-4": { diff --git a/src/php/composer.json b/src/php/composer.json index 1d41f847ac0..59f846fdcc6 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -1,7 +1,9 @@ { "name": "grpc/grpc", + "type": "library", "description": "gRPC library for PHP", - "version": "0.6.0", + "version": "0.13.0", + "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", "repositories": [ @@ -13,7 +15,7 @@ "require": { "php": ">=5.5.0", "datto/protobuf-php": "dev-master", - "google/auth": "dev-master" + "google/auth": "v0.7" }, "autoload": { "psr-4": { From 4cbf32ff66b92162da3553e254ea7a8ca0d82057 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Fri, 4 Mar 2016 12:21:05 -0800 Subject: [PATCH 19/97] mention how to get protoc compiler in base INSTALL.md --- INSTALL.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index d9411db021d..ee4bc2b73d1 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -32,6 +32,16 @@ terminal: $ [sudo] xcode-select --install ``` +##Protoc + +By default gRPC uses [protocol buffers](https://github.com/google/protobuf), +you will need the `protoc` compiler to generate stub server and client code. + +If you compile from source, see below, the Makefile will automatically try +and compile the one present in third_party if you cloned the repository +recursively, and that it detects your system is lacking it. + + #Build from Source For developers who are interested to contribute, here is how to compile the From 11ce4ffc75fef5a790ae55285457d787cbb50cea Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Fri, 4 Mar 2016 13:47:32 -0800 Subject: [PATCH 20/97] update min node version --- src/node/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/README.md b/src/node/README.md index b46b9862432..3501b54a665 100644 --- a/src/node/README.md +++ b/src/node/README.md @@ -5,7 +5,7 @@ Beta ## PREREQUISITES -- `node`: This requires `node` to be installed. If you instead have the `nodejs` executable on Debian, you should install the [`nodejs-legacy`](https://packages.debian.org/sid/nodejs-legacy) package. +- `node`: This requires `node` to be installed, version `0.12` or above. If you instead have the `nodejs` executable on Debian, you should install the [`nodejs-legacy`](https://packages.debian.org/sid/nodejs-legacy) package. ## INSTALLATION From b0d1567e8ea9c4ed528e63df9a8649c953b4e349 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 7 Mar 2016 10:51:02 -0800 Subject: [PATCH 21/97] Revert "Revert "Properly integrate async API with server-side cancellations."" --- .../grpc++/impl/codegen/completion_queue.h | 1 + include/grpc++/impl/codegen/server_context.h | 3 + src/cpp/server/server_context.cc | 26 +- test/cpp/end2end/async_end2end_test.cc | 232 +++++++++++++----- 4 files changed, 198 insertions(+), 64 deletions(-) diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h index 102831e1c9b..928ab2db317 100644 --- a/include/grpc++/impl/codegen/completion_queue.h +++ b/include/grpc++/impl/codegen/completion_queue.h @@ -184,6 +184,7 @@ class CompletionQueue : private GrpcLibrary { bool Pluck(CompletionQueueTag* tag); /// Performs a single polling pluck on \a tag. + /// \warning Must not be mixed with calls to \a Next. void TryPluck(CompletionQueueTag* tag); grpc_completion_queue* cq_; // owned diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h index ad08b8210d6..91ebe574b14 100644 --- a/include/grpc++/impl/codegen/server_context.h +++ b/include/grpc++/impl/codegen/server_context.h @@ -103,6 +103,9 @@ class ServerContext { void AddInitialMetadata(const grpc::string& key, const grpc::string& value); void AddTrailingMetadata(const grpc::string& key, const grpc::string& value); + // IsCancelled is always safe to call when using sync API + // When using async API, it is only safe to call IsCancelled after + // the AsyncNotifyWhenDone tag has been delivered bool IsCancelled() const; // Cancel the Call from the server. This is a best-effort API and depending on diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index e205a1969b3..eb49b210379 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -62,7 +62,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void FillOps(grpc_op* ops, size_t* nops) GRPC_OVERRIDE; bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE; - bool CheckCancelled(CompletionQueue* cq); + bool CheckCancelled(CompletionQueue* cq) { + cq->TryPluck(this); + return CheckCancelledNoPluck(); + } + bool CheckCancelledAsync() { return CheckCancelledNoPluck(); } void set_tag(void* tag) { has_tag_ = true; @@ -72,6 +76,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void Unref(); private: + bool CheckCancelledNoPluck() { + grpc::lock_guard g(mu_); + return finalized_ ? (cancelled_ != 0) : false; + } + bool has_tag_; void* tag_; grpc::mutex mu_; @@ -88,12 +97,6 @@ void ServerContext::CompletionOp::Unref() { } } -bool ServerContext::CompletionOp::CheckCancelled(CompletionQueue* cq) { - cq->TryPluck(this); - grpc::lock_guard g(mu_); - return finalized_ ? cancelled_ != 0 : false; -} - void ServerContext::CompletionOp::FillOps(grpc_op* ops, size_t* nops) { ops->op = GRPC_OP_RECV_CLOSE_ON_SERVER; ops->data.recv_close_on_server.cancelled = &cancelled_; @@ -182,7 +185,14 @@ void ServerContext::TryCancel() const { } bool ServerContext::IsCancelled() const { - return completion_op_ && completion_op_->CheckCancelled(cq_); + if (has_notify_when_done_tag_) { + // when using async API, but the result is only valid + // if the tag has already been delivered at the completion queue + return completion_op_ && completion_op_->CheckCancelledAsync(); + } else { + // when using sync API + return completion_op_ && completion_op_->CheckCancelled(cq_); + } } void ServerContext::set_compression_level(grpc_compression_level level) { diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 9ca3bf98f85..dc8c2bb6e5b 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -68,6 +68,7 @@ namespace testing { namespace { void* tag(int i) { return (void*)(intptr_t)i; } +int detag(void* p) { return static_cast(reinterpret_cast(p)); } #ifdef GPR_POSIX_SOCKET static int maybe_assert_non_blocking_poll(struct pollfd* pfds, nfds_t nfds, @@ -106,37 +107,50 @@ class PollingOverrider { class Verifier { public: explicit Verifier(bool spin) : spin_(spin) {} + // Expect sets the expected ok value for a specific tag Verifier& Expect(int i, bool expect_ok) { expectations_[tag(i)] = expect_ok; return *this; } + // Next waits for 1 async tag to complete, checks its + // expectations, and returns the tag + int Next(CompletionQueue* cq, bool ignore_ok) { + bool ok; + void* got_tag; + if (spin_) { + for (;;) { + auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); + if (r == CompletionQueue::TIMEOUT) continue; + if (r == CompletionQueue::GOT_EVENT) break; + gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); + abort(); + } + } else { + EXPECT_TRUE(cq->Next(&got_tag, &ok)); + } + auto it = expectations_.find(got_tag); + EXPECT_TRUE(it != expectations_.end()); + if (!ignore_ok) { + EXPECT_EQ(it->second, ok); + } + expectations_.erase(it); + return detag(got_tag); + } + + // Verify keeps calling Next until all currently set + // expected tags are complete void Verify(CompletionQueue* cq) { Verify(cq, false); } + // This version of Verify allows optionally ignoring the + // outcome of the expectation void Verify(CompletionQueue* cq, bool ignore_ok) { GPR_ASSERT(!expectations_.empty()); while (!expectations_.empty()) { - bool ok; - void* got_tag; - if (spin_) { - for (;;) { - auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); - if (r == CompletionQueue::TIMEOUT) continue; - if (r == CompletionQueue::GOT_EVENT) break; - gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); - abort(); - } - } else { - EXPECT_TRUE(cq->Next(&got_tag, &ok)); - } - auto it = expectations_.find(got_tag); - EXPECT_TRUE(it != expectations_.end()); - if (!ignore_ok) { - EXPECT_EQ(it->second, ok); - } - expectations_.erase(it); + Next(cq, ignore_ok); } } + // This version of Verify stops after a certain deadline void Verify(CompletionQueue* cq, std::chrono::system_clock::time_point deadline) { if (expectations_.empty()) { @@ -793,7 +807,8 @@ TEST_P(AsyncEnd2endTest, UnimplementedRpc) { } // This class is for testing scenarios where RPCs are cancelled on the server -// by calling ServerContext::TryCancel() +// by calling ServerContext::TryCancel(). Server uses AsyncNotifyWhenDone +// API to check for cancellation class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { protected: typedef enum { @@ -803,13 +818,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { CANCEL_AFTER_PROCESSING } ServerTryCancelRequestPhase; - void ServerTryCancel(ServerContext* context) { - EXPECT_FALSE(context->IsCancelled()); - context->TryCancel(); - gpr_log(GPR_INFO, "Server called TryCancel()"); - EXPECT_TRUE(context->IsCancelled()); - } - // Helper for testing client-streaming RPCs which are cancelled on the server. // Depending on the value of server_try_cancel parameter, this will test one // of the following three scenarios: @@ -843,6 +851,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of 'RequestStream' calls // and receive the 'RequestStream' call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestRequestStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -858,9 +867,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_server_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // Since cancellation is done before server reads any results, we know // for sure that all cq results will return false from this point forward @@ -868,22 +880,39 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while reading the // requests from the client. Since the cancellation can happen at anytime, // some of the cq results (i.e those until cancellation) might be true but // its non deterministic. So better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } // Server reads 3 messages (tags 6, 7 and 8) + // But if want_done_tag is true, we might also see tag 11 for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { srv_stream.Read(&recv_request, tag(tag_idx)); - Verifier(GetParam()) - .Expect(tag_idx, expected_server_cq_result) - .Verify(cq_.get(), ignore_cq_result); + // Note that we'll add something to the verifier and verify that + // something was seen, but it might be tag 11 and not what we + // just added + int got_tag = verif.Expect(tag_idx, expected_server_cq_result) + .Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); + } } if (server_try_cancel_thd != NULL) { @@ -892,7 +921,15 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); + } + + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; } // The RPC has been cancelled at this point for sure (i.e irrespective of @@ -945,6 +982,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { Verifier(GetParam()).Expect(1, true).Verify(cq_.get()); // On the server, request to be notified of 'ResponseStream' calls and // receive the call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -952,9 +990,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -962,24 +1003,41 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while writing responses // to the client. Since the cancellation can happen at anytime, some of // the cq results (i.e those until cancellation) might be true but it is // non deterministic. So better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } // Server sends three messages (tags 3, 4 and 5) + // But if want_done tag is true, we might also see tag 11 for (int tag_idx = 3; tag_idx <= 5; tag_idx++) { send_response.set_message("Pong " + std::to_string(tag_idx)); srv_stream.Write(send_response, tag(tag_idx)); - Verifier(GetParam()) - .Expect(tag_idx, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + // Note that we'll add something to the verifier and verify that + // something was seen, but it might be tag 11 and not what we + // just added + int got_tag = verif.Expect(tag_idx, expected_cq_result) + .Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); + } } if (server_try_cancel_thd != NULL) { @@ -988,13 +1046,21 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); // Client reads may fail bacause it is notified that the stream is // cancelled. ignore_cq_result = true; } + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + } + // Client attemts to read the three messages from the server for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { cli_stream->Read(&recv_response, tag(tag_idx)); @@ -1052,6 +1118,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of the 'BidiStream' call and // receive the call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestBidiStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -1063,9 +1130,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -1073,42 +1143,84 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Since server is going to cancel the RPC in a parallel thread, some of // the cq results (i.e those until the cancellation) might be true. Since // that number is non-deterministic, it is better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } + int got_tag; srv_stream.Read(&recv_request, tag(4)); - Verifier(GetParam()) - .Expect(4, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(4, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 4) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 4); + } send_response.set_message("Pong"); srv_stream.Write(send_response, tag(5)); - Verifier(GetParam()) - .Expect(5, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(5, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 5) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 5); + } cli_stream->Read(&recv_response, tag(6)); - Verifier(GetParam()) - .Expect(6, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(6, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 6) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 6); + } // This is expected to succeed in all cases cli_stream->WritesDone(tag(7)); - Verifier(GetParam()).Expect(7, true).Verify(cq_.get()); + verif.Expect(7, true); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 7) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 7); + } // This is expected to fail in all cases i.e for all values of // server_try_cancel. This is because at this point, either there are no // more msgs from the client (because client called WritesDone) or the RPC // is cancelled on the server srv_stream.Read(&recv_request, tag(8)); - Verifier(GetParam()).Expect(8, false).Verify(cq_.get()); + verif.Expect(8, false); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 8) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 8); + } if (server_try_cancel_thd != NULL) { server_try_cancel_thd->join(); @@ -1116,7 +1228,15 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); + } + + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; } // The RPC has been cancelled at this point for sure (i.e irrespective of From de14e50a914c9db99d90812cbaec0110b3e1a52d Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 7 Mar 2016 12:11:47 -0800 Subject: [PATCH 22/97] text update --- INSTALL.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index ee4bc2b73d1..3c694ba5822 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -37,9 +37,10 @@ terminal: By default gRPC uses [protocol buffers](https://github.com/google/protobuf), you will need the `protoc` compiler to generate stub server and client code. -If you compile from source, see below, the Makefile will automatically try -and compile the one present in third_party if you cloned the repository -recursively, and that it detects your system is lacking it. +If you compile gRPC from source, as described below, the Makefile will +automatically try and compile the `protoc` in third_party if you cloned the +repository recursively and it detects that you don't already have it +installed. #Build from Source From 67b05a2513d739c5ab8fc63829ef5fa0340dee59 Mon Sep 17 00:00:00 2001 From: Harsh Vardhan Date: Tue, 8 Mar 2016 02:18:57 +0530 Subject: [PATCH 23/97] updates contributing.md --- CONTRIBUTING.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9423c46547a..518ba2c4809 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,9 +36,25 @@ In order to run most of the available tests, one would need to run: `./tools/run_tests/run_tests.py` -If you want to run all the possible tests for any of the languages {c, c++, node, php, python}, do this: - -`./tools/run_tests/run_tests.py -l -c all` +If you want to run tests for any of the languages {c, c++, node, php, python}, do this: + +`./tools/run_tests/run_tests.py -l ` + +For list of available commands, refer to this: + +``` +run_tests.py [-h] + [-c {asan,asan-noleaks,basicprof,dbg,gcov,helgrind,memcheck,msan,mutrace,opt,stapprof,tsan,ubsan}] + [-n RUNS_PER_TEST] [-r REGEX] [-j JOBS] [-s SLOWDOWN] [-f] + [-t] [--newline_on_success] + [-l {all,c,c++,csharp,node,objc,php,python,ruby,sanity} [{all,c,c++,csharp,node,objc,php,python,ruby,sanity} ...]] + [-S] [--use_docker] [--allow_flakes] + [--arch {default,x86,x64}] + [--compiler {default,gcc4.4,gcc4.9,gcc5.3,vs2010,vs2013,vs2015}] + [--build_only] [--measure_cpu_costs] + [--update_submodules [UPDATE_SUBMODULES [UPDATE_SUBMODULES ...]]] + [-a ANTAGONISTS] [-x XML_REPORT] +``` ## Adding or removing source code From a41a6775d4487cce3eea5084928da39ceff923fd Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 7 Mar 2016 17:10:07 -0800 Subject: [PATCH 24/97] php: update readme --- examples/php/README.md | 10 ++-- src/php/README.md | 115 +++++++++++++++++++++++++++++------------ 2 files changed, 87 insertions(+), 38 deletions(-) diff --git a/examples/php/README.md b/examples/php/README.md index 8fb060863a3..ea9ccb67908 100644 --- a/examples/php/README.md +++ b/examples/php/README.md @@ -4,16 +4,15 @@ gRPC in 3 minutes (PHP) PREREQUISITES ------------- -This requires PHP 5.5 or greater. +This requires `php` >=5.5, `phpize`, `pecl`, `phpunit` INSTALL ------- - - On Mac OS X, install [homebrew][]. Run the following command to install gRPC. + - Install the gRPC PHP extension ```sh - $ curl -fsSL https://goo.gl/getgrpc | bash -s php + $ [sudo] pecl install grpc-beta ``` - This will download and run the [gRPC install script][] and compile the gRPC PHP extension. - Clone this repository @@ -37,6 +36,7 @@ TRY IT! Please follow the instruction in [Node][] to run the server ``` $ cd examples/node + $ npm install $ nodejs greeter_server.js ``` @@ -58,7 +58,5 @@ TUTORIAL You can find a more detailed tutorial in [gRPC Basics: PHP][] -[homebrew]:http://brew.sh -[gRPC install script]:https://raw.githubusercontent.com/grpc/homebrew-grpc/master/scripts/install [Node]:https://github.com/grpc/grpc/tree/master/examples/node [gRPC Basics: PHP]:http://www.grpc.io/docs/tutorials/basic/php.html diff --git a/src/php/README.md b/src/php/README.md index b368482f068..c97c3796cda 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -9,14 +9,21 @@ Beta ## Environment -Prerequisite: PHP 5.5 or later, `phpunit`, `pecl` +Prerequisite: `php` >=5.5, `phpize`, `pecl`, `phpunit` -**Linux:** +**Linux (Debian):** ```sh $ sudo apt-get install php5 php5-dev php-pear ``` +**Linux (CentOS):** + +```sh +$ yum install php55w +$ yum --enablerepo=remi,remi-php55 install php-devel php-pear +``` + **Mac OS X:** ```sh @@ -24,11 +31,11 @@ $ curl -O http://pear.php.net/go-pear.phar $ sudo php -d detect_unicode=0 go-pear.phar ``` -**PHPUnit: (Both Linux and Mac OS X)** +**PHPUnit:** ```sh -$ curl https://phar.phpunit.de/phpunit.phar -o phpunit.phar -$ chmod +x phpunit.phar -$ sudo mv phpunit.phar /usr/local/bin/phpunit +$ wget https://phar.phpunit.de/phpunit-old.phar +$ chmod +x phpunit-old.phar +$ sudo mv phpunit-old.phar /usr/bin/phpunit ``` ## Quick Install @@ -39,15 +46,22 @@ Install the gRPC PHP extension sudo pecl install grpc-beta ``` +This will compile and install the gRPC PHP extension into the standard PHP extension directory. You should be able to run the [unit tests](#unit-tests), with the PHP extension installed. + +To run tests with generated stub code from `.proto` files, you will need the `composer`, `protoc` and `protoc-gen-php` binaries additionally. See sections [below](#generated-code-tests). + ## Build from Source + +### gRPC C core library + Clone this repository ```sh $ git clone https://github.com/grpc/grpc.git ``` -Build and install the gRPC C core libraries +Build and install the gRPC C core library ```sh $ cd grpc @@ -56,20 +70,15 @@ $ make $ sudo make install ``` -Note: you may encounter a warning about the Protobuf compiler `protoc` 3.0.0+ not being installed. The following might help, and will be useful later on when we need to compile the `protoc-gen-php` tool. +### gRPC PHP extension -```sh -$ cd grpc/third_party/protobuf -$ sudo make install # 'make' should have been run by core grpc -``` - -Install the gRPC PHP extension +Install the gRPC PHP extension from PECL ```sh $ sudo pecl install grpc-beta ``` -OR +Or, compile from source ```sh $ cd grpc/src/php/ext/grpc @@ -79,58 +88,96 @@ $ make $ sudo make install ``` +### Update php.ini + Add this line to your `php.ini` file, e.g. `/etc/php5/cli/php.ini` ```sh extension=grpc.so ``` -Install Composer +## Unit Tests + +You will need the source code to run tests + +```sh +$ git clone https://github.com/grpc/grpc.git +$ cd grpc +$ git pull --recurse-submodules && git submodule update --init --recursive +``` + +Run unit tests ```sh $ cd grpc/src/php +$ ./bin/run_tests.sh +``` + +## Generated Code Tests + +### Composer + +You need to install `composer`, to pull in some runtime dependencies based on the `composer.json` file. + +```sh $ curl -sS https://getcomposer.org/installer | php $ sudo mv composer.phar /usr/local/bin/composer + +$ cd grpc/src/php $ composer install ``` -## Unit Tests +### Protobuf compiler -Run unit tests +You need the install the protobuf compiler, `protoc`, 3.0.0+. + +If you had compiled the gRPC C core library from source above, the `protoc` binary should have been installed as well. In the case it wasn't, you can run the following commands to install it. ```sh -$ cd grpc/src/php -$ ./bin/run_tests.sh +$ cd grpc/third_party/protobuf +$ sudo make install # 'make' should have been run by core grpc ``` -## Generated Code Tests +Or you can download a `protoc` binaries from [here](https://github.com/google/protobuf/releases). + -Install `protoc-gen-php` +### PHP protobuf compiler + +You need to install `protoc-gen-php`, so that you can generate stub classes `.php` files from service definition `.proto` files. ```sh -$ cd grpc/src/php/vendor/datto/protobuf-php +$ cd grpc/src/php/vendor/datto/protobuf-php # if you had run `composer install` in the previous step + +OR + +$ git clone https://github.com/stanley-cheung/Protobuf-PHP # clone from github repo + $ gem install rake ronn $ rake pear:package version=1.0 $ sudo pear install Protobuf-1.0.tgz ``` -Generate client stub code +### Client Stub + +Generate client stub classes from `.proto` files ```sh $ cd grpc/src/php $ ./bin/generate_proto_php.sh ``` -Run a local server serving the math services +### Run test server - - Please see [Node][] on how to run an example server +Run a local server serving the math services. Please see [Node][] on how to run an example server ```sh -$ cd grpc/src/node +$ cd grpc $ npm install -$ nodejs examples/math_server.js +$ nodejs src/node/test/math/math_server.js ``` +### Run test client + Run the generated code tests ```sh @@ -161,13 +208,15 @@ $ sudo service apache2 restart Make sure the Node math server is still running, as above. ```sh -$ cd grpc/src/node -$ nodejs examples/math_server.js +$ cd grpc +$ npm install +$ nodejs src/node/test/math/math_server.js ``` Make sure you have run `composer install` to generate the `vendor/autoload.php` file ```sh +$ cd grpc/src/php $ composer install ``` @@ -229,13 +278,15 @@ $ sudo service php5-fpm restart Make sure the Node math server is still running, as above. ```sh -$ cd grpc/src/node -$ nodejs examples/math_server.js +$ cd grpc +$ npm install +$ nodejs src/node/test/math/math_server.js ``` Make sure you have run `composer install` to generate the `vendor/autoload.php` file ```sh +$ cd grpc/src/php $ composer install ``` From 0e65fb36f24bceb05b51a9970068d6ba6b1f409d Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 7 Mar 2016 17:49:37 -0800 Subject: [PATCH 25/97] Node: propagate read errors back down to core --- src/node/src/client.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/node/src/client.js b/src/node/src/client.js index 9acf51bd98b..81299b337ca 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -149,6 +149,9 @@ function _readsDone(status) { if (!status) { status = {code: grpc.status.OK, details: 'OK'}; } + if (status.code !== grpc.status.OK) { + this.call.cancelWithStatus(status.code, status.details); + } this.finished = true; this.read_status = status; this._emitStatusIfDone(); From 56188658d771ecdb4981e8a175313fecc78a61df Mon Sep 17 00:00:00 2001 From: Harsh Vardhan Date: Tue, 8 Mar 2016 12:55:02 +0530 Subject: [PATCH 26/97] updates contributing.md --- CONTRIBUTING.md | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 518ba2c4809..e147074943c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,21 +40,9 @@ If you want to run tests for any of the languages {c, c++, node, php, python}, d `./tools/run_tests/run_tests.py -l ` -For list of available commands, refer to this: - -``` -run_tests.py [-h] - [-c {asan,asan-noleaks,basicprof,dbg,gcov,helgrind,memcheck,msan,mutrace,opt,stapprof,tsan,ubsan}] - [-n RUNS_PER_TEST] [-r REGEX] [-j JOBS] [-s SLOWDOWN] [-f] - [-t] [--newline_on_success] - [-l {all,c,c++,csharp,node,objc,php,python,ruby,sanity} [{all,c,c++,csharp,node,objc,php,python,ruby,sanity} ...]] - [-S] [--use_docker] [--allow_flakes] - [--arch {default,x86,x64}] - [--compiler {default,gcc4.4,gcc4.9,gcc5.3,vs2010,vs2013,vs2015}] - [--build_only] [--measure_cpu_costs] - [--update_submodules [UPDATE_SUBMODULES [UPDATE_SUBMODULES ...]]] - [-a ANTAGONISTS] [-x XML_REPORT] -``` +To know about the list of available commands, do this: + +`./tools/run_tests/run_tests.py -h` ## Adding or removing source code From a2c8b20ac785d88e4df9bfe1a832e48173458c3e Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 8 Mar 2016 08:21:56 -0800 Subject: [PATCH 27/97] review changes --- src/php/README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/php/README.md b/src/php/README.md index c97c3796cda..cf8f2c11b0f 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -48,7 +48,7 @@ sudo pecl install grpc-beta This will compile and install the gRPC PHP extension into the standard PHP extension directory. You should be able to run the [unit tests](#unit-tests), with the PHP extension installed. -To run tests with generated stub code from `.proto` files, you will need the `composer`, `protoc` and `protoc-gen-php` binaries additionally. See sections [below](#generated-code-tests). +To run tests with generated stub code from `.proto` files, you will also need the `composer`, `protoc` and `protoc-gen-php` binaries. You can find out how to get these [below](#generated-code-tests). ## Build from Source @@ -115,9 +115,11 @@ $ ./bin/run_tests.sh ## Generated Code Tests +This section specifies the prerequisites for running the generated code tests, as well as how to run the tests themselves. + ### Composer -You need to install `composer`, to pull in some runtime dependencies based on the `composer.json` file. +If you don't have it already, install `composer` to pull in some runtime dependencies based on the `composer.json` file. ```sh $ curl -sS https://getcomposer.org/installer | php @@ -129,21 +131,21 @@ $ composer install ### Protobuf compiler -You need the install the protobuf compiler, `protoc`, 3.0.0+. +Again if you don't have it already, you need to install the protobuf compiler `protoc`, version 3.0.0+. -If you had compiled the gRPC C core library from source above, the `protoc` binary should have been installed as well. In the case it wasn't, you can run the following commands to install it. +If you compiled the gRPC C core library from source above, the `protoc` binary should have been installed as well. If it hasn't been installed, you can run the following commands to install it. ```sh $ cd grpc/third_party/protobuf $ sudo make install # 'make' should have been run by core grpc ``` -Or you can download a `protoc` binaries from [here](https://github.com/google/protobuf/releases). +Alternatively, you can download `protoc` binaries from [the protocol buffers Github repository](https://github.com/google/protobuf/releases). ### PHP protobuf compiler -You need to install `protoc-gen-php`, so that you can generate stub classes `.php` files from service definition `.proto` files. +You need to install `protoc-gen-php` to generate stub class `.php` files from service definition `.proto` files. ```sh $ cd grpc/src/php/vendor/datto/protobuf-php # if you had run `composer install` in the previous step @@ -168,7 +170,7 @@ $ ./bin/generate_proto_php.sh ### Run test server -Run a local server serving the math services. Please see [Node][] on how to run an example server +Run a local server serving the math services. Please see [Node][] for how to run an example server. ```sh $ cd grpc From 3be6801b72e91b0e3a1c6b497db02e144e32f7af Mon Sep 17 00:00:00 2001 From: Lisa Carey Date: Tue, 8 Mar 2016 17:47:14 +0000 Subject: [PATCH 28/97] Clarifying how to install for gRPC C++ --- INSTALL.md | 2 +- examples/cpp/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 3c694ba5822..454a8b7b2fd 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -3,7 +3,7 @@ For language-specific installation instructions for gRPC runtime, please refer to these documents - * [C++](examples/cpp) + * [C++](examples/cpp): Currently to install gRPC for C++, you need to build from source as described below. * [C#](src/csharp): NuGet package `Grpc` * [Go](https://github.com/grpc/grpc-go): `go get google.golang.org/grpc` * [Java](https://github.com/grpc/grpc-java) diff --git a/examples/cpp/README.md b/examples/cpp/README.md index e4b0eb698bf..d93cbacf7b2 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -2,7 +2,7 @@ ## Installation -To install gRPC on your system, follow the instructions [here](../../INSTALL.md). +To install gRPC on your system, follow the instructions to build from source [here](../../INSTALL.md). This also installs the protocol buffer compiler `protoc` (if you don't have it already), and the C++ gRPC plugin for `protoc`. ## Hello C++ gRPC! From 9d1476364a002372f2ab375c3aceeb29b7a227cd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 8 Mar 2016 17:00:53 -0800 Subject: [PATCH 29/97] Fix timers --- src/core/iomgr/timer.c | 2 +- src/core/iomgr/timer_heap.c | 14 +++--- test/core/iomgr/timer_heap_test.c | 84 ++----------------------------- 3 files changed, 12 insertions(+), 88 deletions(-) diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 8379fffad02..1bf8387d74b 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -41,7 +41,7 @@ #define INVALID_HEAP_INDEX 0xffffffffu -#define LOG2_NUM_SHARDS 5 +#define LOG2_NUM_SHARDS 2 #define NUM_SHARDS (1 << LOG2_NUM_SHARDS) #define ADD_DEADLINE_SCALE 0.33 #define MIN_QUEUE_WINDOW_DURATION 0.01 diff --git a/src/core/iomgr/timer_heap.c b/src/core/iomgr/timer_heap.c index 9d8be5c1fcb..167bb7c4d11 100644 --- a/src/core/iomgr/timer_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -46,7 +46,7 @@ static void adjust_upwards(grpc_timer **first, uint32_t i, grpc_timer *t) { while (i > 0) { uint32_t parent = (uint32_t)(((int)i - 1) / 2); - if (gpr_time_cmp(first[parent]->deadline, t->deadline) >= 0) break; + if (gpr_time_cmp(first[parent]->deadline, t->deadline) <= 0) break; first[i] = first[parent]; first[i]->heap_index = i; i = parent; @@ -62,16 +62,14 @@ static void adjust_downwards(grpc_timer **first, uint32_t i, uint32_t length, grpc_timer *t) { for (;;) { uint32_t left_child = 1u + 2u * i; - uint32_t right_child; - uint32_t next_i; if (left_child >= length) break; - right_child = left_child + 1; - next_i = right_child < length && + uint32_t right_child = left_child + 1; + uint32_t next_i = right_child < length && gpr_time_cmp(first[left_child]->deadline, - first[right_child]->deadline) < 0 + first[right_child]->deadline) > 0 ? right_child : left_child; - if (gpr_time_cmp(t->deadline, first[next_i]->deadline) >= 0) break; + if (gpr_time_cmp(t->deadline, first[next_i]->deadline) <= 0) break; first[i] = first[next_i]; first[i]->heap_index = i; i = next_i; @@ -95,7 +93,7 @@ static void maybe_shrink(grpc_timer_heap *heap) { static void note_changed_priority(grpc_timer_heap *heap, grpc_timer *timer) { uint32_t i = timer->heap_index; uint32_t parent = (uint32_t)(((int)i - 1) / 2); - if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) < 0) { + if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) > 0) { adjust_upwards(heap->timers, i, timer); } else { adjust_downwards(heap->timers, i, heap->timer_count, timer); diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 077a9fd6bd8..9b51b912003 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -57,79 +57,6 @@ static grpc_timer *create_test_elements(size_t num_elements) { return elems; } -static int cmp_elem(const void *a, const void *b) { - int i = *(const int *)a; - int j = *(const int *)b; - return i - j; -} - -static size_t *all_top(grpc_timer_heap *pq, size_t *n) { - size_t *vec = NULL; - size_t *need_to_check_children; - size_t num_need_to_check_children = 0; - - *n = 0; - if (pq->timer_count == 0) return vec; - need_to_check_children = - gpr_malloc(pq->timer_count * sizeof(*need_to_check_children)); - need_to_check_children[num_need_to_check_children++] = 0; - vec = gpr_malloc(pq->timer_count * sizeof(*vec)); - while (num_need_to_check_children > 0) { - size_t ind = need_to_check_children[0]; - size_t leftchild, rightchild; - num_need_to_check_children--; - memmove(need_to_check_children, need_to_check_children + 1, - num_need_to_check_children * sizeof(*need_to_check_children)); - vec[(*n)++] = ind; - leftchild = 1u + 2u * ind; - if (leftchild < pq->timer_count) { - if (gpr_time_cmp(pq->timers[leftchild]->deadline, - pq->timers[ind]->deadline) >= 0) { - need_to_check_children[num_need_to_check_children++] = leftchild; - } - rightchild = leftchild + 1; - if (rightchild < pq->timer_count && - gpr_time_cmp(pq->timers[rightchild]->deadline, - pq->timers[ind]->deadline) >= 0) { - need_to_check_children[num_need_to_check_children++] = rightchild; - } - } - } - - gpr_free(need_to_check_children); - - return vec; -} - -static void check_pq_top(grpc_timer *elements, grpc_timer_heap *pq, - uint8_t *inpq, size_t num_elements) { - gpr_timespec max_deadline = gpr_inf_past(GPR_CLOCK_REALTIME); - size_t *max_deadline_indices = - gpr_malloc(num_elements * sizeof(*max_deadline_indices)); - size_t *top_elements; - size_t num_max_deadline_indices = 0; - size_t num_top_elements; - size_t i; - for (i = 0; i < num_elements; ++i) { - if (inpq[i] && gpr_time_cmp(elements[i].deadline, max_deadline) >= 0) { - if (gpr_time_cmp(elements[i].deadline, max_deadline) > 0) { - num_max_deadline_indices = 0; - max_deadline = elements[i].deadline; - } - max_deadline_indices[num_max_deadline_indices++] = elements[i].heap_index; - } - } - qsort(max_deadline_indices, num_max_deadline_indices, - sizeof(*max_deadline_indices), cmp_elem); - top_elements = all_top(pq, &num_top_elements); - GPR_ASSERT(num_top_elements == num_max_deadline_indices); - for (i = 0; i < num_top_elements; i++) { - GPR_ASSERT(max_deadline_indices[i] == top_elements[i]); - } - gpr_free(max_deadline_indices); - gpr_free(top_elements); -} - static int contains(grpc_timer_heap *pq, grpc_timer *el) { size_t i; for (i = 0; i < pq->timer_count; i++) { @@ -145,11 +72,11 @@ static void check_valid(grpc_timer_heap *pq) { size_t right_child = left_child + 1u; if (left_child < pq->timer_count) { GPR_ASSERT(gpr_time_cmp(pq->timers[i]->deadline, - pq->timers[left_child]->deadline) >= 0); + pq->timers[left_child]->deadline) <= 0); } if (right_child < pq->timer_count) { GPR_ASSERT(gpr_time_cmp(pq->timers[i]->deadline, - pq->timers[right_child]->deadline) >= 0); + pq->timers[right_child]->deadline) <= 0); } } } @@ -162,6 +89,8 @@ static void test1(void) { grpc_timer *test_elements = create_test_elements(num_test_elements); uint8_t *inpq = gpr_malloc(num_test_elements); + gpr_log(GPR_DEBUG, "******************************************* test1"); + grpc_timer_heap_init(&pq); memset(inpq, 0, num_test_elements); GPR_ASSERT(grpc_timer_heap_is_empty(&pq)); @@ -172,7 +101,6 @@ static void test1(void) { check_valid(&pq); GPR_ASSERT(contains(&pq, &test_elements[i])); inpq[i] = 1; - check_pq_top(test_elements, &pq, inpq, num_test_elements); } for (i = 0; i < num_test_elements; ++i) { /* Test that check still succeeds even for element that wasn't just @@ -182,7 +110,7 @@ static void test1(void) { GPR_ASSERT(pq.timer_count == num_test_elements); - check_pq_top(test_elements, &pq, inpq, num_test_elements); + check_valid(&pq); for (i = 0; i < num_test_operations; ++i) { size_t elem_num = (size_t)rand() % num_test_elements; @@ -193,14 +121,12 @@ static void test1(void) { grpc_timer_heap_add(&pq, el); GPR_ASSERT(contains(&pq, el)); inpq[elem_num] = 1; - check_pq_top(test_elements, &pq, inpq, num_test_elements); check_valid(&pq); } else { GPR_ASSERT(contains(&pq, el)); grpc_timer_heap_remove(&pq, el); GPR_ASSERT(!contains(&pq, el)); inpq[elem_num] = 0; - check_pq_top(test_elements, &pq, inpq, num_test_elements); check_valid(&pq); } } From 08e0c191beaa9fc818ae2363c7684307fde084bf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 8 Mar 2016 17:04:50 -0800 Subject: [PATCH 30/97] Revert mistaken change --- src/core/iomgr/timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 1bf8387d74b..8379fffad02 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -41,7 +41,7 @@ #define INVALID_HEAP_INDEX 0xffffffffu -#define LOG2_NUM_SHARDS 2 +#define LOG2_NUM_SHARDS 5 #define NUM_SHARDS (1 << LOG2_NUM_SHARDS) #define ADD_DEADLINE_SCALE 0.33 #define MIN_QUEUE_WINDOW_DURATION 0.01 From 341b5abef5a039a731c9da260c4e9b76b1849415 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 8 Mar 2016 17:16:29 -0800 Subject: [PATCH 31/97] Doc Fixit: src/cpp/README need more information --- src/cpp/README.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/cpp/README.md b/src/cpp/README.md index baeba083155..a13669b99e1 100644 --- a/src/cpp/README.md +++ b/src/cpp/README.md @@ -6,3 +6,62 @@ This directory contains source code for C++ implementation of gRPC. #Status Beta + +#Pre-requisites + +##Linux + +```sh + $ [sudo] apt-get install build-essential autoconf libtool +``` + +##Mac OSX + +For a Mac system, git is not available by default. You will first need to +install Xcode from the Mac AppStore and then run the following command from a +terminal: + +```sh + $ [sudo] xcode-select --install +``` + +##Protoc + +By default gRPC uses [protocol buffers](https://github.com/google/protobuf), +you will need the `protoc` compiler to generate stub server and client code. + +If you compile gRPC from source, as described below, the Makefile will +automatically try and compile the `protoc` in third party if you cloned the +repository recursively and it detects that you don't already have it +installed. + +If it hasn't been installed, you can run the following commands to install it. + +```sh +$ cd grpc/third_party/protobuf +$ sudo make install # 'make' should have been run by core grpc +``` + +Alternatively, you can download `protoc` binaries from +[the protocol buffers Github repository](https://github.com/google/protobuf/releases). + +#INSTALLATION + +Currently to install gRPC for C++, you need to build from source as described below. + +#Build from Source + +```sh + $ git clone https://github.com/grpc/grpc.git + $ cd grpc + $ git submodule update --init + $ make + $ [sudo] make install +``` + +#DOCUMENTATION + +- The gRPC C++ refenrence documentation is available online at + [grpc.io](http://www.grpc.io/docs/tutorials/basic/c.html) +- [Helloworld example](../../examples/cpp/helloworld) + From 72dbc4aa2d84cdf03dee863af22b4c5870615806 Mon Sep 17 00:00:00 2001 From: Harsh Vardhan Date: Wed, 9 Mar 2016 10:50:58 +0530 Subject: [PATCH 32/97] updates contributing.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e147074943c..011ce43bcdc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ In order to run most of the available tests, one would need to run: `./tools/run_tests/run_tests.py` -If you want to run tests for any of the languages {c, c++, node, php, python}, do this: +If you want to run tests for any of the languages {all, c, c++, csharp, node, objc, php, python, ruby}, do this: `./tools/run_tests/run_tests.py -l ` From 1b393da4332ad97aac62ee8ecb422d6d0fe108f3 Mon Sep 17 00:00:00 2001 From: Lisa Carey Date: Wed, 9 Mar 2016 15:16:56 +0000 Subject: [PATCH 33/97] Updated main examples README to link to main docs and quickstarts --- examples/README.md | 447 ++------------------------------------------- 1 file changed, 12 insertions(+), 435 deletions(-) diff --git a/examples/README.md b/examples/README.md index cd85417ca20..287a80266c9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,450 +1,27 @@ +# Examples -# Getting started +This directory contains code examples for all the C-based gRPC implementations: C++, Node.js, Python, Ruby, Objective-C, PHP, and C#. You can find examples and instructions specific to your +favourite language in the relevant subdirectory. + +Examples for Go and Java gRPC live in their own repositories: -Welcome to the developer documentation for gRPC, a language-neutral, -platform-neutral remote procedure call (RPC) system developed at Google. +* [Java](https://github.com/grpc/grpc-java/tree/master/examples) +* [Android Java](https://github.com/grpc/grpc-java/tree/master/examples/android) +* [Go](https://github.com/grpc/grpc-go/tree/master/examples) -This document introduces you to gRPC with a quick overview and a simple -Hello World example. You'll find more tutorials and reference docs in this repository - more documentation is coming soon! +For more comprehensive documentation, including an [overview](http://www.grpc.io/docs/) and tutorials that use this example code, visit [grpc.io](http://www.grpc.io/docs/). - ## Quick start -You can find quick start guides for each language, including installation instructions, examples, and tutorials here: + +Each example directory has quick start instructions for the appropriate language, including installation instructions and how to run our simplest Hello World example: + * [C++](cpp) -* [Java](https://github.com/grpc/grpc-java/tree/master/examples) -* [Go](https://github.com/grpc/grpc-go/tree/master/examples) * [Ruby](ruby) * [Node.js](node) -* [Android Java](https://github.com/grpc/grpc-java/tree/master/examples/android) * [Python](python/helloworld) * [C#](csharp) * [Objective-C](objective-c/helloworld) * [PHP](php) -## What's in this repository? - -The `examples` directory contains documentation, resources, and examples -for all gRPC users. You can find examples and instructions specific to your -favourite language in the relevant subdirectory. - -You can find out about the gRPC source code repositories in -[`grpc`](https://github.com/grpc/grpc). Each repository provides instructions -for building the appropriate libraries for your language. - - -## What is gRPC? - -In gRPC a *client* application can directly call -methods on a *server* application on a different machine as if it was a -local object, making it easier for you to create distributed applications and -services. As in many RPC systems, gRPC is based around the idea of defining -a *service*, specifying the methods that can be called remotely with their -parameters and return types. On the server side, the server implements this -interface and runs a gRPC server to handle client calls. On the client side, -the client has a *stub* that provides exactly the same methods as the server. - - - -gRPC clients and servers can run and talk to each other in a variety of -environments - from servers inside Google to your own desktop - and can -be written in any of gRPC's [supported languages](#quickstart). So, for -example, you can easily create a gRPC server in Java with clients in Go, -Python, or Ruby. In addition, the latest Google APIs will have gRPC versions -of their interfaces, letting you easily build Google functionality into -your applications. - - -### Working with protocol buffers - -By default gRPC uses *protocol buffers*, Google’s -mature open source mechanism for serializing structured data (although it -can be used with other data formats such as JSON). As you'll -see in our example below, you define gRPC services using *proto files*, -with method parameters and return types specified as protocol buffer message -types. You -can find out lots more about protocol buffers in the [Protocol Buffers -documentation](https://developers.google.com/protocol-buffers/docs/overview). - -#### Protocol buffer versions - -While protocol buffers have been available for open source users for some -time, our examples use a new flavour of protocol buffers called proto3, -which has a slightly simplified syntax, some useful new features, and supports -lots more languages. This is currently available as an alpha release in -Java, C++, Java_nano (Android Java), Python, and Ruby from [the protocol buffers Github -repo](https://github.com/google/protobuf/releases), as well as a Go language -generator from [the golang/protobuf Github repo](https://github.com/golang/protobuf), with more languages in development. You can find out more in the [proto3 language guide](https://developers.google.com/protocol-buffers/docs/proto3), and see -the major differences from the current default version in the [release notes](https://github.com/google/protobuf/releases). More proto3 documentation is coming soon. - -In general, while you *can* use proto2 (the current default protocol buffers version), we recommend that you use proto3 with gRPC as it lets you use the full range of gRPC-supported languages, as well as avoiding compatibility -issues with proto2 clients talking to proto3 servers and vice versa. - - -## Hello gRPC! - -Now that you know a bit more about gRPC, the easiest way to see how it -works is to look at a simple example. Our Hello World walks you through the -construction of a simple gRPC client-server application, showing you how to: - -- Create a protocol buffers schema that defines a simple RPC service with -a single -Hello World method. -- Create a Java server that implements this interface. -- Create a Java client that accesses the Java server. -- Create a Go client that accesses -the same Java server. - -The complete code for the example is available in the `examples` -directory. We use the Git versioning system for source code management: -however, you don't need to know anything about Git to follow along other -than how to install and run a few git commands. - -This is an introductory example rather than a comprehensive tutorial, so -don't worry if you're not a Go or -Java developer - the concepts are similar for all languages, and you can -find more implementations of our Hello World example in other languages (and full tutorials where available) in -the [language-specific folders](#quickstart) in this repository. Complete tutorials and -reference documentation for all gRPC languages are coming soon. - - -### Setup - -This section explains how to set up your local machine to work with -the example code. If you just want to read the example, you can go straight -to the [next step](#servicedef). - -#### Install Git - -You can download and install Git from http://git-scm.com/download. Once -installed you should have access to the git command line tool. The main -commands that you will need to use are: - -- git clone ... : clone a remote repository onto your local machine -- git checkout ... : check out a particular branch or a tagged version of -the code to hack on - -#### Install gRPC - -To build and install gRPC plugins and related tools: -- For Java, see the [Java quick start](https://github.com/grpc/grpc-java). -- For Go, see the [Go quick start](https://github.com/grpc/grpc-go). - -#### Get the source code - -The example code for our Java example lives in the `grpc-java` -GitHub repository. Clone this repository to your local machine by running the -following command: - - -``` -git clone https://github.com/grpc/grpc-java.git -``` - -Change your current directory to grpc-java/examples - -``` -cd grpc-java/examples -``` - - - - -### Defining a service - -The first step in creating our example is to define a *service*: an RPC -service specifies the methods that can be called remotely with their parameters -and return types. As you saw in the -[overview](#protocolbuffers) above, gRPC does this using [protocol -buffers](https://developers.google.com/protocol-buffers/docs/overview). We -use the protocol buffers interface definition language (IDL) to define our -service methods, and define the parameters and return -types as protocol buffer message types. Both the client and the -server use interface code generated from the service definition. - -Here's our example service definition, defined using protocol buffers IDL in -[helloworld.proto](https://github.com/grpc/grpc-java/tree/master/examples/src/main/proto). The `Greeter` -service has one method, `SayHello`, that lets the server receive a single -`HelloRequest` -message from the remote client containing the user's name, then send back -a greeting in a single `HelloReply`. This is the simplest type of RPC you -can specify in gRPC - you can find out about other types in the tutorial for your chosen language. - -```proto -syntax = "proto3"; - -option java_package = "io.grpc.examples"; - -package helloworld; - -// The greeter service definition. -service Greeter { - // Sends a greeting - rpc SayHello (HelloRequest) returns (HelloReply) {} -} - -// The request message containing the user's name. -message HelloRequest { - string name = 1; -} - -// The response message containing the greetings -message HelloReply { - string message = 1; -} - -``` - - -### Generating gRPC code - -Once we've defined our service, we use the protocol buffer compiler -`protoc` to generate the special client and server code we need to create -our application - right now we're going to generate Java code, though you -can generate gRPC code in any gRPC-supported language (as you'll see later -in this example). The generated code contains both stub code for clients to -use and an abstract interface for servers to implement, both with the method -defined in our `Greeter` service. - -(If you didn't install the gRPC plugins and protoc on your system and are just reading along with -the example, you can skip this step and move -onto the next one where we examine the generated code.) - -For simplicity, we've provided a [Gradle build file](https://github.com/grpc/grpc-java/blob/master/examples/build.gradle) with our Java examples that runs `protoc` for you with the appropriate plugin, input, and output: - -```shell -../gradlew build -``` - -This generates the following classes from our .proto, which contain all the generated code -we need to create our example: - -- `Helloworld.java`, which -has all the protocol buffer code to populate, serialize, and retrieve our -`HelloRequest` and `HelloReply` message types -- `GreeterGrpc.java`, which contains (along with some other useful code): - - an interface for `Greeter` servers to implement - - ```java - public static interface Greeter { - public void sayHello(io.grpc.examples.Helloworld.HelloRequest request, - io.grpc.stub.StreamObserver responseObserver); - } - ``` - - - _stub_ classes that clients can use to talk to a `Greeter` server. As you can see, they also implement the `Greeter` interface. - - ```java - public static class GreeterStub extends - io.grpc.stub.AbstractStub - implements Greeter { - ... - } - ``` - - -### Writing a server - -Now let's write some code! First we'll create a server application to implement -our service. Note that we're not going to go into a lot of detail about how -to create a server in this section. More detailed information will be in the -tutorial for your chosen language: check if there's one available yet in the relevant [quick start](#quickstart). - -Our server application has two classes: - -- a main server class that hosts the service implementation and allows access over the -network: [HelloWorldServer.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java). - - -- a simple service implementation class [GreeterImpl.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java#L51). - - -#### Service implementation - -[GreeterImpl.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java#L51) -actually implements our `Greeter` service's required behaviour. - -As you can see, the class `GreeterImpl` implements the interface -`GreeterGrpc.Greeter` that we [generated](#generating) from our proto -[IDL](https://github.com/grpc/grpc-java/tree/master/examples/src/main/proto) by implementing the method `sayHello`: - -```java - @Override - public void sayHello(HelloRequest req, StreamObserver responseObserver) { - HelloReply reply = HelloReply.newBuilder().setMessage("Hello " + req.getName()).build(); - responseObserver.onValue(reply); - responseObserver.onCompleted(); - } -``` -- `sayHello` takes two parameters: - - `HelloRequest`: the request - - `StreamObserver`: a response observer, which is - a special interface for the server to call with its response - -To return our response to the client and complete the call: - -1. We construct and populate a `HelloReply` response object with our exciting -message, as specified in our interface definition. -2. We return the `HelloReply` to the client and then specify that we've finished dealing with the RPC. - - -#### Server implementation - -[HelloWorldServer.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java) -shows the other main feature required to provide a gRPC service; making the service -implementation available from the network. - -```java - /* The port on which the server should run */ - private int port = 50051; - private ServerImpl server; - - private void start() throws Exception { - server = NettyServerBuilder.forPort(port) - .addService(GreeterGrpc.bindService(new GreeterImpl())) - .build().start(); - logger.info("Server started, listening on " + port); - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - // Use stderr here since the logger may have been reset by its JVM shutdown hook. - System.err.println("*** shutting down gRPC server since JVM is shutting down"); - HelloWorldServer.this.stop(); - System.err.println("*** server shut down"); - } - }); - } - -``` - -Here we create an appropriate gRPC server, binding the `Greeter` service -implementation that we created to a port. Then we start the server running: the server is now ready to receive -requests from `Greeter` service clients on our specified port. We'll cover -how all this works in a bit more detail in our language-specific documentation. - - -### Writing a client - -Client-side gRPC is pretty simple. In this step, we'll use the generated code -to write a simple client that can access the `Greeter` server we created -in the [previous section](#server). You can see the complete client code in -[HelloWorldClient.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldClient.java). - -Again, we're not going to go into much detail about how to implement a client; -we'll leave that for the tutorial. - -#### Connecting to the service - -First let's look at how we connect to the `Greeter` server. First we need -to create a gRPC channel, specifying the hostname and port of the server we -want to connect to. Then we use the channel to construct the stub instance. - - -```java - private final ChannelImpl channel; - private final GreeterGrpc.GreeterBlockingStub blockingStub; - - public HelloWorldClient(String host, int port) { - channel = - NettyChannelBuilder.forAddress(host, port).negotiationType(NegotiationType.PLAINTEXT) - .build(); - blockingStub = GreeterGrpc.newBlockingStub(channel); - } - -``` - -In this case, we create a blocking stub. This means that the RPC call waits -for the server to respond, and will either return a response or raise an -exception. gRPC Java has other kinds of stubs that make non-blocking calls -to the server, where the response is returned asynchronously. - -#### Calling an RPC - -Now we can contact the service and obtain a greeting: - -1. We construct and fill in a `HelloRequest` to send to the service. -2. We call the stub's `hello()` RPC with our request and get a `HelloReply` -back, from which we can get our greeting. - - -```java - HelloRequest req = HelloRequest.newBuilder().setName(name).build(); - HelloReply reply = blockingStub.sayHello(req); - -``` - - -### Try it out! - -Our [Gradle build file](https://github.com/grpc/grpc-java/blob/master/examples/build.gradle) simplifies building and running the examples. - -You can build and run the server from the `grpc-java` root folder with: - -```sh -$ ./gradlew :grpc-examples:helloWorldServer -``` - -and in another terminal window confirm that it receives a message. - -```sh -$ ./gradlew :grpc-examples:helloWorldClient -``` - -### Adding another client - -Finally, let's look at one of gRPC's most useful features - interoperability -between code in different languages. So far, we've just looked at Java code -generated from and implementing our `Greeter` service definition. However, -as you'll see if you look at the language-specific subdirectories -in this repository, we've also generated and implemented `Greeter` -in some of gRPC's other supported languages. Each service -and client uses interface code generated from the same proto -that we used for the Java example. - -So, for example, if we visit the [`go` example -directory](https://github.com/grpc/grpc-go/tree/master/examples) and look at the -[`greeter_client`](https://github.com/grpc/grpc-go/blob/master/examples/greeter_client/main.go), -we can see that like the Java client, it connects to a `Greeter` service -at `localhost:50051` and uses a stub to call the `SayHello` method with a -`HelloRequest`: - -```go -const ( - address = "localhost:50051" - defaultName = "world" -) - -func main() { - // Set up a connection to the server. - conn, err := grpc.Dial(address) - if err != nil { - log.Fatalf("did not connect: %v", err) - } - defer conn.Close() - c := pb.NewGreeterClient(conn) - - // Contact the server and print out its response. - name := defaultName - if len(os.Args) > 1 { - name = os.Args[1] - } - r, err := c.SayHello(context.Background(), &pb.HelloRequest{Name: - name}) - if err != nil { - log.Fatalf("could not greet: %v", err) - } - log.Printf("Greeting: %s", r.Message) -} -``` - - -If we run the Java server from earlier in another terminal window, we can -run the Go client and connect to it just like the Java client, even though -it's written in a different language. -``` -$ greeter_client -``` -## Read more! -- You can find links to language-specific tutorials, examples, and other docs in each language's [quick start](#quickstart). -- [gRPC Authentication Support](http://www.grpc.io/docs/guides/auth.html) introduces authentication support in gRPC with supported mechanisms and examples. From 0e6e34e4fae218bbff1d586fe42f2f4151615282 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:21:43 -0800 Subject: [PATCH 34/97] Add an additional test --- test/core/iomgr/timer_heap_test.c | 108 +++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 9b51b912003..3c78a6c1171 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -38,6 +38,8 @@ #include #include +#include + #include "test/core/util/test_config.h" static gpr_timespec random_deadline(void) { @@ -81,6 +83,10 @@ static void check_valid(grpc_timer_heap *pq) { } } +/******************************************************************************* + * test1 + */ + static void test1(void) { grpc_timer_heap pq; const size_t num_test_elements = 200; @@ -89,7 +95,7 @@ static void test1(void) { grpc_timer *test_elements = create_test_elements(num_test_elements); uint8_t *inpq = gpr_malloc(num_test_elements); - gpr_log(GPR_DEBUG, "******************************************* test1"); + gpr_log(GPR_INFO, "test1"); grpc_timer_heap_init(&pq); memset(inpq, 0, num_test_elements); @@ -136,7 +142,106 @@ static void test1(void) { gpr_free(inpq); } +/******************************************************************************* + * test2 + */ + +typedef struct { + grpc_timer elem; + bool inserted; +} elem_struct; + +static elem_struct *search_elems(elem_struct *elems, size_t count, bool inserted) { + size_t *search_order = gpr_malloc(count * sizeof(*search_order)); + for (size_t i = 0; i < count; i++) { + search_order[i] = i; + } + for (size_t i = 0; i < count * 2; i++) { + size_t a = (size_t)rand() % count; + size_t b = (size_t)rand() % count; + GPR_SWAP(size_t, search_order[a], search_order[b]); + } + elem_struct *out = NULL; + for (size_t i = 0; out == NULL && i < count; i++) { + if (elems[search_order[i]].inserted == inserted) { + out = &elems[search_order[i]]; + } + } + gpr_free(search_order); + return out; +} + +static void test2(void) { +gpr_log(GPR_INFO, "test2"); + + grpc_timer_heap pq; + + elem_struct elems[1000]; + size_t num_inserted = 0; + + grpc_timer_heap_init(&pq); + memset(elems, 0, sizeof(elems)); + + for (size_t round = 0; round < 10000; round++) { + int r = (size_t)rand() % 1000; + if (r <= 550) { + /* 55% of the time we try to add something */ + elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), false); + if (el != NULL) { + el->elem.deadline = random_deadline(); + grpc_timer_heap_add(&pq, &el->elem); + el->inserted = true; + num_inserted++; + check_valid(&pq); + } + } else if (r <= 650) { + /* 10% of the time we try to remove something */ + elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), true); + if (el != NULL) { + grpc_timer_heap_remove(&pq, &el->elem); + el->inserted = false; + num_inserted--; + check_valid(&pq); + } + } else { + /* the remaining times we pop */ + if (num_inserted > 0) { + grpc_timer *top = grpc_timer_heap_top(&pq); + grpc_timer_heap_pop(&pq); + for (size_t i = 0; i < GPR_ARRAY_SIZE(elems); i++) { + if (top == &elems[i].elem) { + GPR_ASSERT(elems[i].inserted); + elems[i].inserted = false; + } + } + num_inserted--; + check_valid(&pq); + } + } + + if (num_inserted) { + gpr_timespec *min_deadline = NULL; + for (size_t i = 0; i < GPR_ARRAY_SIZE(elems); i++) { + if (elems[i].inserted) { + if (min_deadline == NULL) { + min_deadline = &elems[i].elem.deadline; + } else { + if (gpr_time_cmp(elems[i].elem.deadline, *min_deadline) < 0) { + min_deadline = &elems[i].elem.deadline; + } + } + } + } + GPR_ASSERT(0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline)); + } + } + + grpc_timer_heap_destroy(&pq); +} + static void shrink_test(void) { + gpr_log(GPR_INFO, "shrink_test"); + grpc_timer_heap pq; size_t i; size_t expected_size; @@ -200,6 +305,7 @@ int main(int argc, char **argv) { for (i = 0; i < 5; i++) { test1(); + test2(); shrink_test(); } From 3a951bd975cb14ba7fe2b59e7c6b598932998a73 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:23:57 -0800 Subject: [PATCH 35/97] clang-format --- src/core/iomgr/timer_heap.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/iomgr/timer_heap.c b/src/core/iomgr/timer_heap.c index 167bb7c4d11..9d4aaf3eee7 100644 --- a/src/core/iomgr/timer_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -65,10 +65,10 @@ static void adjust_downwards(grpc_timer **first, uint32_t i, uint32_t length, if (left_child >= length) break; uint32_t right_child = left_child + 1; uint32_t next_i = right_child < length && - gpr_time_cmp(first[left_child]->deadline, - first[right_child]->deadline) > 0 - ? right_child - : left_child; + gpr_time_cmp(first[left_child]->deadline, + first[right_child]->deadline) > 0 + ? right_child + : left_child; if (gpr_time_cmp(t->deadline, first[next_i]->deadline) <= 0) break; first[i] = first[next_i]; first[i]->heap_index = i; From a39c1995623ed5ef48c49fc5212f2140b3390462 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:26:52 -0800 Subject: [PATCH 36/97] clang-format --- test/core/iomgr/timer_heap_test.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 3c78a6c1171..4510d51bd70 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -151,7 +151,8 @@ typedef struct { bool inserted; } elem_struct; -static elem_struct *search_elems(elem_struct *elems, size_t count, bool inserted) { +static elem_struct *search_elems(elem_struct *elems, size_t count, + bool inserted) { size_t *search_order = gpr_malloc(count * sizeof(*search_order)); for (size_t i = 0; i < count; i++) { search_order[i] = i; @@ -172,7 +173,7 @@ static elem_struct *search_elems(elem_struct *elems, size_t count, bool inserted } static void test2(void) { -gpr_log(GPR_INFO, "test2"); + gpr_log(GPR_INFO, "test2"); grpc_timer_heap pq; @@ -232,7 +233,8 @@ gpr_log(GPR_INFO, "test2"); } } } - GPR_ASSERT(0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline)); + GPR_ASSERT( + 0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline)); } } From 5f3ba8c834b829254f85fedee6441fc82f14bcfc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:44:51 -0800 Subject: [PATCH 37/97] Copyright, casting fixes --- src/core/iomgr/timer_heap.c | 2 +- test/core/iomgr/timer_heap_test.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/timer_heap.c b/src/core/iomgr/timer_heap.c index 9d4aaf3eee7..b5df566c453 100644 --- a/src/core/iomgr/timer_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 4510d51bd70..cd34696f7dd 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -184,7 +184,7 @@ static void test2(void) { memset(elems, 0, sizeof(elems)); for (size_t round = 0; round < 10000; round++) { - int r = (size_t)rand() % 1000; + int r = rand() % 1000; if (r <= 550) { /* 55% of the time we try to add something */ elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), false); From 76a5c0e433eb0a4a941e0bf9172e55c6f8058ed4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 09:05:30 -0800 Subject: [PATCH 38/97] Fix memory leak: keep all resolver calls under the same lock --- src/core/channel/client_channel.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index d4ba9508185..f021a8ae329 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -165,7 +165,6 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, channel_data *chand = arg; grpc_lb_policy *lb_policy = NULL; grpc_lb_policy *old_lb_policy; - grpc_resolver *old_resolver; grpc_connectivity_state state = GRPC_CHANNEL_TRANSIENT_FAILURE; int exit_idle = 0; @@ -201,28 +200,25 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, } if (iomgr_success && chand->resolver) { - grpc_resolver *resolver = chand->resolver; - GRPC_RESOLVER_REF(resolver, "channel-next"); grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, state, "new_lb+resolver"); if (lb_policy != NULL) { watch_lb_policy(exec_ctx, chand, lb_policy, state); } - gpr_mu_unlock(&chand->mu_config); GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver"); - grpc_resolver_next(exec_ctx, resolver, &chand->incoming_configuration, + grpc_resolver_next(exec_ctx, chand->resolver, + &chand->incoming_configuration, &chand->on_config_changed); - GRPC_RESOLVER_UNREF(exec_ctx, resolver, "channel-next"); + gpr_mu_unlock(&chand->mu_config); } else { - old_resolver = chand->resolver; - chand->resolver = NULL; + if (chand->resolver != NULL) { + grpc_resolver_shutdown(exec_ctx, chand->resolver); + GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel"); + chand->resolver = NULL; + } grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, GRPC_CHANNEL_FATAL_FAILURE, "resolver_gone"); gpr_mu_unlock(&chand->mu_config); - if (old_resolver != NULL) { - grpc_resolver_shutdown(exec_ctx, old_resolver); - GRPC_RESOLVER_UNREF(exec_ctx, old_resolver, "channel"); - } } if (exit_idle) { @@ -247,7 +243,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_transport_op *op) { channel_data *chand = elem->channel_data; - grpc_resolver *destroy_resolver = NULL; grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL); @@ -279,7 +274,8 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, if (op->disconnect && chand->resolver != NULL) { grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, GRPC_CHANNEL_FATAL_FAILURE, "disconnect"); - destroy_resolver = chand->resolver; + grpc_resolver_shutdown(exec_ctx, chand->resolver); + GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel"); chand->resolver = NULL; if (chand->lb_policy != NULL) { grpc_pollset_set_del_pollset_set(exec_ctx, @@ -290,11 +286,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, } } gpr_mu_unlock(&chand->mu_config); - - if (destroy_resolver) { - grpc_resolver_shutdown(exec_ctx, destroy_resolver); - GRPC_RESOLVER_UNREF(exec_ctx, destroy_resolver, "channel"); - } } typedef struct { From d426d9dfdbd1712b1b241a17452ed15441b502a1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 09:30:18 -0800 Subject: [PATCH 39/97] Fix memory leak if call is already cancelled Obvious in hindsight, if the cas failed, we still created the subchannel call object and then left it dangling. Fixes a leak observed on master in the past 48 hours. --- src/core/channel/subchannel_call_holder.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c index 8f46885a043..9c087dc2a1d 100644 --- a/src/core/channel/subchannel_call_holder.c +++ b/src/core/channel/subchannel_call_holder.c @@ -174,17 +174,15 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) { holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; if (holder->connected_subchannel == NULL) { fail_locked(exec_ctx, holder); + } else if (1 == gpr_atm_acq_load(&holder->subchannel_call)) { + /* already cancelled before subchannel became ready */ + fail_locked(exec_ctx, holder); } else { - if (!gpr_atm_rel_cas( - &holder->subchannel_call, 0, - (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( - exec_ctx, holder->connected_subchannel, holder->pollset))) { - GPR_ASSERT(gpr_atm_acq_load(&holder->subchannel_call) == 1); - /* if this cas fails, the call was cancelled before the pick completed */ - fail_locked(exec_ctx, holder); - } else { - retry_waiting_locked(exec_ctx, holder); - } + gpr_atm_rel_store( + &holder->subchannel_call, + (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( + exec_ctx, holder->connected_subchannel, holder->pollset)); + retry_waiting_locked(exec_ctx, holder); } gpr_mu_unlock(&holder->mu); GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel"); From d48d84da195642c89d1108567398d576f2d3382d Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 9 Mar 2016 11:10:20 -0800 Subject: [PATCH 40/97] Ruby: fix some synchronization code in server implementation --- src/ruby/.rubocop.yml | 3 + src/ruby/lib/grpc/generic/rpc_server.rb | 96 ++++++++++++++---------- src/ruby/spec/generic/rpc_server_spec.rb | 23 +----- 3 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/ruby/.rubocop.yml b/src/ruby/.rubocop.yml index ff5cf8db831..d13ce426553 100644 --- a/src/ruby/.rubocop.yml +++ b/src/ruby/.rubocop.yml @@ -15,3 +15,6 @@ Metrics/CyclomaticComplexity: Metrics/PerceivedComplexity: Max: 8 + +Metrics/ClassLength: + Max: 250 diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index ef2997c9918..eefa9577c47 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -107,7 +107,9 @@ module GRPC # Starts running the jobs in the thread pool. def start - fail 'already stopped' if @stopped + @stop_mutex.synchronize do + fail 'already stopped' if @stopped + end until @workers.size == @size.to_i next_thread = Thread.new do catch(:exit) do # allows { throw :exit } to kill a thread @@ -264,10 +266,13 @@ module GRPC @pool = Pool.new(@pool_size) @run_cond = ConditionVariable.new @run_mutex = Mutex.new - @running = false + # running_state can take 4 values: :not_started, :running, :stopping, and + # :stopped. State transitions can only proceed in that order. + @running_state = :not_started @server = RpcServer.setup_srv(server_override, @cq, **kw) - @stopped = false - @stop_mutex = Mutex.new + # Mutex to synchronize registration of services and registered service + # count. @run_mutex should not be acquired while holding @handle_mutex + @handle_mutex = Mutex.new end # stops a running server @@ -275,27 +280,28 @@ module GRPC # the call has no impact if the server is already stopped, otherwise # server's current call loop is it's last. def stop - return unless @running - @stop_mutex.synchronize do - @stopped = true + @run_mutex.synchronize do + fail 'Cannot stop before starting' if @running_state == :not_started + return if @running_state != :running + @running_state = :stopping end deadline = from_relative_time(@poll_period) - return if @server.close(@cq, deadline) - deadline = from_relative_time(@poll_period) @server.close(@cq, deadline) @pool.stop end - # determines if the server has been stopped - def stopped? - @stop_mutex.synchronize do - return @stopped + def running_state + @run_mutex.synchronize do + return @running_state end end - # determines if the server is currently running def running? - @running + running_state == :running + end + + def stopped? + running_state == :stopped end # Is called from other threads to wait for #run to start up the server. @@ -304,13 +310,11 @@ module GRPC # # @param timeout [Numeric] number of seconds to wait # @result [true, false] true if the server is running, false otherwise - def wait_till_running(timeout = 0.1) - end_time, sleep_period = Time.now + timeout, (1.0 * timeout) / 100 - while Time.now < end_time - @run_mutex.synchronize { @run_cond.wait(@run_mutex) } unless running? - sleep(sleep_period) + def wait_till_running(timeout = nil) + @run_mutex.synchronize do + @run_cond.wait(@run_mutex, timeout) if @running_state == :not_started + return @running_state == :running end - running? end # Runs the server in its own thread, then waits for signal INT or TERM on @@ -360,11 +364,16 @@ module GRPC # @param service [Object|Class] a service class or object as described # above def handle(service) - fail 'cannot add services if the server is running' if running? - fail 'cannot add services if the server is stopped' if stopped? - cls = service.is_a?(Class) ? service : service.class - assert_valid_service_class(cls) - add_rpc_descs_for(service) + @run_mutex.synchronize do + unless @running_state == :not_started + fail 'cannot add services if the server has been started' + end + cls = service.is_a?(Class) ? service : service.class + assert_valid_service_class(cls) + @handle_mutex.synchronize do + add_rpc_descs_for(service) + end + end end # runs the server @@ -375,16 +384,13 @@ module GRPC # - #running? returns true after this is called, until #stop cause the # the server to stop. def run - if rpc_descs.size.zero? - GRPC.logger.warn('did not run as no services were present') - return - end @run_mutex.synchronize do - @running = true - @run_cond.signal + fail 'cannot run without registering services' if rpc_descs.size.zero? + @pool.start + @server.start + @running_state = :running + @run_cond.broadcast end - @pool.start - @server.start loop_handle_server_calls end @@ -413,9 +419,9 @@ module GRPC # handles calls to the server def loop_handle_server_calls - fail 'not running' unless @running + fail 'not started' if running_state == :not_started loop_tag = Object.new - until stopped? + while running_state == :running begin an_rpc = @server.request_call(@cq, loop_tag, INFINITE_FUTURE) break if (!an_rpc.nil?) && an_rpc.call.nil? @@ -430,11 +436,14 @@ module GRPC rescue Core::CallError, RuntimeError => e # these might happen for various reasonse. The correct behaviour of # the server is to log them and continue, if it's not shutting down. - GRPC.logger.warn("server call failed: #{e}") unless stopped? + if running_state == :running + GRPC.logger.warn("server call failed: #{e}") + end next end end - @running = false + # @running_state should be :stopping here + @run_mutex.synchronize { @running_state = :stopped } GRPC.logger.info("stopped: #{self}") end @@ -467,11 +476,15 @@ module GRPC protected def rpc_descs - @rpc_descs ||= {} + @handle_mutex.synchronize do + return @rpc_descs ||= {} + end end def rpc_handlers - @rpc_handlers ||= {} + @handle_mutex.synchronize do + @rpc_handlers ||= {} + end end def assert_valid_service_class(cls) @@ -484,9 +497,10 @@ module GRPC cls.assert_rpc_descs_have_methods end + # This should be called while holding @handle_mutex def add_rpc_descs_for(service) cls = service.is_a?(Class) ? service : service.class - specs, handlers = rpc_descs, rpc_handlers + specs, handlers = (@rpc_descs ||= {}), (@rpc_handlers ||= {}) cls.rpc_descs.each_pair do |name, spec| route = "/#{cls.service_name}/#{name}".to_sym fail "already registered: rpc #{route} from #{spec}" if specs.key? route diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index be6331d68bb..e16ba60387e 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -220,19 +220,10 @@ describe GRPC::RpcServer do @srv = RpcServer.new(**opts) end - after(:each) do - @srv.stop - end - it 'starts out false' do expect(@srv.stopped?).to be(false) end - it 'stays false after a #stop is called before #run' do - @srv.stop - expect(@srv.stopped?).to be(false) - end - it 'stays false after the server starts running', server: true do @srv.handle(EchoService) t = Thread.new { @srv.run } @@ -247,8 +238,8 @@ describe GRPC::RpcServer do t = Thread.new { @srv.run } @srv.wait_till_running @srv.stop - expect(@srv.stopped?).to be(true) t.join + expect(@srv.stopped?).to be(true) end end @@ -266,9 +257,7 @@ describe GRPC::RpcServer do server_override: @server } r = RpcServer.new(**opts) - r.run - expect(r.running?).to be(false) - r.stop + expect { r.run }.to raise_error(RuntimeError) end it 'is true after run is called with a registered service' do @@ -293,10 +282,6 @@ describe GRPC::RpcServer do @srv = RpcServer.new(**@opts) end - after(:each) do - @srv.stop - end - it 'raises if #run has already been called' do @srv.handle(EchoService) t = Thread.new { @srv.run } @@ -528,10 +513,6 @@ describe GRPC::RpcServer do @srv = RpcServer.new(**server_opts) end - after(:each) do - @srv.stop - end - it 'should be added to BadStatus when requests fail', server: true do service = FailingService.new @srv.handle(service) From 3c77ff452fce5265c3454d26613bc6df529475ed Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 9 Mar 2016 11:44:56 -0800 Subject: [PATCH 41/97] Fix copyright --- src/ruby/spec/generic/rpc_server_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index e16ba60387e..dfaec6d6edc 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From ec49e156fe62c4c451cd65686efaa07999178166 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 11:51:23 -0800 Subject: [PATCH 42/97] Fix typo --- src/python/grpcio/tests/_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py index 38a5432e791..b0dbd92a495 100644 --- a/src/python/grpcio/tests/_runner.py +++ b/src/python/grpcio/tests/_runner.py @@ -144,7 +144,7 @@ class Runner(object): def run(self, suite): """See setuptools' test_runner setup argument for information.""" # only run test cases with id starting with given prefix - testcase_filter = os.getenv('GPRC_PYTHON_TESTRUNNER_FILTER') + testcase_filter = os.getenv('GRPC_PYTHON_TESTRUNNER_FILTER') filtered_cases = [] for case in _loader.iterate_suite_cases(suite): if not testcase_filter or case.id().startswith(testcase_filter): From 798b0dc4426f4485ad108a04c2dd44285ba2e700 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 9 Mar 2016 12:00:09 -0800 Subject: [PATCH 43/97] Doc Fixit: src/cpp/README need more information, revised 03092016 - Revised 'protoc' section and 'Documentation' section as LisaFC suggested. - Added 'Examples' section as LisaFC suggested. Resolves: #5661 --- src/cpp/README.md | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/cpp/README.md b/src/cpp/README.md index a13669b99e1..83d37aa2ed8 100644 --- a/src/cpp/README.md +++ b/src/cpp/README.md @@ -30,10 +30,8 @@ terminal: By default gRPC uses [protocol buffers](https://github.com/google/protobuf), you will need the `protoc` compiler to generate stub server and client code. -If you compile gRPC from source, as described below, the Makefile will -automatically try and compile the `protoc` in third party if you cloned the -repository recursively and it detects that you don't already have it -installed. +If you compile gRPC from source, as described below, this also installs the +`protoc` compiler. If it hasn't been installed, you can run the following commands to install it. @@ -45,9 +43,10 @@ $ sudo make install # 'make' should have been run by core grpc Alternatively, you can download `protoc` binaries from [the protocol buffers Github repository](https://github.com/google/protobuf/releases). -#INSTALLATION +#Installation -Currently to install gRPC for C++, you need to build from source as described below. +Currently to install gRPC for C++, you need to build from source as described +below. #Build from Source @@ -59,9 +58,25 @@ Currently to install gRPC for C++, you need to build from source as described be $ [sudo] make install ``` -#DOCUMENTATION +#Documentation -- The gRPC C++ refenrence documentation is available online at - [grpc.io](http://www.grpc.io/docs/tutorials/basic/c.html) -- [Helloworld example](../../examples/cpp/helloworld) +You can find out how to build and run our simplest gRPC C++ example in our +[C++ quick start](https://github.com/grpc/grpc/tree/{{ site.data.config.grpc_release_branch }}/examples/cpp). +For more detailed documentation on using gRPC in C++ , see our main +documentation site at [grpc.io](http://grpc.io), specifically: + +* [Overview](http://www.grpc.io/docs/): An introduction to gRPC with a simple + Hello World example in all our supported languages, including C++. +* [gRPC Basics - C++](http://www.grpc.io/docs/tutorials/basic/c.html): + A tutorial that steps you through creating a simple gRPC C++ example + application. +* [Asynchronous Basics - C++](http://www.grpc.io/docs/tutorials/async/helloasync-cpp.html): + A tutorial that shows you how to use gRPC C++'s asynchronous/non-blocking + APIs. + + +# Examples + +Code examples for gRPC C++ live in this repository's +[examples/cpp](https://github.com/grpc/grpc/tree/{{ site.data.config.grpc_release_branch }}/examples/cpp) directory. From 1f646dc8873db8ce9bc1aac189cc2cc471e25bb1 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 14:46:45 -0800 Subject: [PATCH 44/97] Release GIL in queue __dealloc__ --- .../grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index c139147114b..e11138b1cd8 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -140,7 +140,8 @@ cdef class CompletionQueue: grpc_completion_queue_shutdown(self.c_completion_queue) # Pump the queue while not self.is_shutdown: - event = grpc_completion_queue_next( - self.c_completion_queue, c_deadline, NULL) + with nogil: + event = grpc_completion_queue_next( + self.c_completion_queue, c_deadline, NULL) self._interpret_event(event) grpc_completion_queue_destroy(self.c_completion_queue) From f7ff8be43a721511b5b4cc0e277fe32e894f794d Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 15:27:28 -0800 Subject: [PATCH 45/97] Fix other typo to match previous typo fix --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index cc004f38d7f..2df82adf0a7 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -360,7 +360,7 @@ class PythonLanguage(object): ['tools/run_tests/run_python.sh'], None, environ=dict(environment.items() + - [('GPRC_PYTHON_TESTRUNNER_FILTER', suite_name)]), + [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]), shortname='py.test.%s' % suite_name, timeout_seconds=5*60) for suite_name in tests_json] From 68a94c8f072f14764130215fa0da45d5965471b2 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 11:45:07 -0800 Subject: [PATCH 46/97] Don't set linetrace unless testing coverage --- setup.py | 7 ++++++- tools/run_tests/build_python.sh | 6 +++++- tools/run_tests/run_python.sh | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e1abe7fd8ff..b21265c5756 100644 --- a/setup.py +++ b/setup.py @@ -108,8 +108,13 @@ if "linux" in sys.platform or "darwin" in sys.platform: def cython_extensions(package_names, module_names, extra_sources, include_dirs, libraries, define_macros, build_with_cython=False): + # Set compiler directives linetrace argument only if we care about tracing; + # this is due to Cython having different behavior between linetrace being + # False and linetrace being unset. See issue #5689. + cython_compiler_directives = {} if ENABLE_CYTHON_TRACING: define_macros = define_macros + [('CYTHON_TRACE_NOGIL', 1)] + cython_compiler_directives['linetrace'] = True file_extension = 'pyx' if build_with_cython else 'c' module_files = [os.path.join(PYTHON_STEM, name.replace('.', '/') + '.' + file_extension) @@ -129,7 +134,7 @@ def cython_extensions(package_names, module_names, extra_sources, include_dirs, return Cython.Build.cythonize( extensions, include_path=include_dirs, - compiler_directives={'linetrace': bool(ENABLE_CYTHON_TRACING)}) + compiler_directives=cython_compiler_directives) else: return extensions diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index f120fc7ed69..79a148faf1b 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -40,7 +40,11 @@ export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH export CFLAGS="-I$ROOT/include -std=gnu99" export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 -export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 + +if [ "$CONFIG" = "gcov" ] +then + export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 +fi tox --notest diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index ace49e15142..d4b7250cbb7 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -40,10 +40,10 @@ export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH export CFLAGS="-I$ROOT/include -std=c89" export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 -export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 if [ "$CONFIG" = "gcov" ] then + export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 tox else $ROOT/.tox/py27/bin/python $ROOT/setup.py test_lite From 1d68520c22fc708cdd6043962a0f59964a8aaf60 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 9 Mar 2016 17:30:37 -0800 Subject: [PATCH 47/97] Remove extraneous mutex, restrict state transitions --- src/ruby/lib/grpc/generic/rpc_server.rb | 37 ++++++++++++++----------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index eefa9577c47..b30d19dd2bb 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -270,9 +270,6 @@ module GRPC # :stopped. State transitions can only proceed in that order. @running_state = :not_started @server = RpcServer.setup_srv(server_override, @cq, **kw) - # Mutex to synchronize registration of services and registered service - # count. @run_mutex should not be acquired while holding @handle_mutex - @handle_mutex = Mutex.new end # stops a running server @@ -283,7 +280,7 @@ module GRPC @run_mutex.synchronize do fail 'Cannot stop before starting' if @running_state == :not_started return if @running_state != :running - @running_state = :stopping + transition_running_state(:stopping) end deadline = from_relative_time(@poll_period) @server.close(@cq, deadline) @@ -296,6 +293,20 @@ module GRPC end end + # Can only be called while holding @run_mutex + def transition_running_state(target_state) + state_transitions = { + not_started: :running, + running: :stopping, + stopping: :stopped + } + if state_transitions[@running_state] == target_state + @running_state = target_state + else + fail "Bad server state transition: #{@running_state}->#{target_state}" + end + end + def running? running_state == :running end @@ -370,9 +381,7 @@ module GRPC end cls = service.is_a?(Class) ? service : service.class assert_valid_service_class(cls) - @handle_mutex.synchronize do - add_rpc_descs_for(service) - end + add_rpc_descs_for(service) end end @@ -388,7 +397,7 @@ module GRPC fail 'cannot run without registering services' if rpc_descs.size.zero? @pool.start @server.start - @running_state = :running + transition_running_state(:running) @run_cond.broadcast end loop_handle_server_calls @@ -443,7 +452,7 @@ module GRPC end end # @running_state should be :stopping here - @run_mutex.synchronize { @running_state = :stopped } + @run_mutex.synchronize { transition_running_state(:stopped) } GRPC.logger.info("stopped: #{self}") end @@ -476,15 +485,11 @@ module GRPC protected def rpc_descs - @handle_mutex.synchronize do - return @rpc_descs ||= {} - end + @rpc_descs ||= {} end def rpc_handlers - @handle_mutex.synchronize do - @rpc_handlers ||= {} - end + @rpc_handlers ||= {} end def assert_valid_service_class(cls) @@ -497,7 +502,7 @@ module GRPC cls.assert_rpc_descs_have_methods end - # This should be called while holding @handle_mutex + # This should be called while holding @run_mutex def add_rpc_descs_for(service) cls = service.is_a?(Class) ? service : service.class specs, handlers = (@rpc_descs ||= {}), (@rpc_handlers ||= {}) From d4726c1ae63315726faa88e924c5749f13816ead Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 23 Feb 2016 16:57:36 -0800 Subject: [PATCH 48/97] add support for clang3.4 and 3.6 --- .../cxx_ubuntu1404_x64/Dockerfile.template | 39 +++++++++ .../test/cxx_ubuntu1404_x64/Dockerfile | 86 +++++++++++++++++++ tools/run_tests/run_tests.py | 24 ++++-- 3 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template create mode 100644 tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile diff --git a/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template new file mode 100644 index 00000000000..d824220afea --- /dev/null +++ b/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template @@ -0,0 +1,39 @@ +%YAML 1.2 +--- | + # Copyright 2015-2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM ubuntu:14.04 + + <%include file="../../apt_get_basic.include"/> + <%include file="../../cxx_deps.include"/> + <%include file="../../run_tests_addons.include"/> + # Define the default command. + CMD ["bash"] + \ No newline at end of file diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile new file mode 100644 index 00000000000..e8dab1b4712 --- /dev/null +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -0,0 +1,86 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM ubuntu:14.04 + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#====================== +# Zookeeper dependencies +# TODO(jtattermusch): is zookeeper still needed? +RUN apt-get install -y libzookeeper-mt-dev + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index cc004f38d7f..e687d463427 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -142,9 +142,8 @@ class CLanguage(object): self._make_options = [_windows_toolset_option(self.args.compiler), _windows_arch_option(self.args.arch)] else: - self._make_options = [] - self._docker_distro = self._get_docker_distro(self.args.use_docker, - self.args.compiler) + self._docker_distro, self._make_options = self._compiler_options(self.args.use_docker, + self.args.compiler) def test_specs(self): out = [] @@ -229,18 +228,26 @@ class CLanguage(object): def makefile_name(self): return 'Makefile' - def _get_docker_distro(self, use_docker, compiler): + def _clang_make_options(self): + return ['CC=clang', 'CXX=clang++', 'LD=clang', 'LDXX=clang++'] + + def _compiler_options(self, use_docker, compiler): + """Returns docker distro and make options to use for given compiler.""" if _is_use_docker_child(): - return "already_under_docker" + return ("already_under_docker", []) if not use_docker: _check_compiler(compiler, ['default']) if compiler == 'gcc4.9' or compiler == 'default': - return 'jessie' + return ('jessie', []) elif compiler == 'gcc4.4': - return 'squeeze' + return ('squeeze', []) elif compiler == 'gcc5.3': - return 'ubuntu1604' + return ('ubuntu1604', []) + elif compiler == 'clang3.4': + return ('ubuntu1404', self._clang_make_options()) + elif compiler == 'clang3.6': + return ('ubuntu1604', self._clang_make_options()) else: raise Exception('Compiler %s not supported.' % compiler) @@ -774,6 +781,7 @@ argp.add_argument('--arch', argp.add_argument('--compiler', choices=['default', 'gcc4.4', 'gcc4.9', 'gcc5.3', + 'clang3.4', 'clang3.6', 'vs2010', 'vs2013', 'vs2015'], default='default', help='Selects compiler to use. Allowed values depend on the platform and language.') From 60cbec7c19516065e49800b31d55d7bb260ebf40 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 26 Feb 2016 13:47:16 -0800 Subject: [PATCH 49/97] fix C# artifacts compatibility --- tools/run_tests/artifact_targets.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 288a3f01542..e61c46d8b5a 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -69,16 +69,6 @@ def create_jobspec(name, cmdline, environ=None, shell=False, return jobspec -def macos_arch_env(arch): - """Returns environ specifying -arch arguments for make.""" - if arch == 'x86': - arch_arg = '-arch i386' - elif arch == 'x64': - arch_arg = '-arch x86_64' - else: - raise Exception('Unsupported arch') - return {'CFLAGS': arch_arg, 'LDFLAGS': arch_arg} - _MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' _ARCH_FLAG_MAP = { @@ -191,13 +181,17 @@ class CSharpExtArtifact: environ = {'CONFIG': 'opt', 'EMBED_OPENSSL': 'true', 'EMBED_ZLIB': 'true', - 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE'} + 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', + 'LDFLAGS': ''} if self.platform == 'linux': return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/build_artifact_csharp.sh') + 'tools/run_tests/build_artifact_csharp.sh', + environ=environ) else: - environ.update(macos_arch_env(self.arch)) + archflag = _ARCH_FLAG_MAP[self.arch] + environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) + environ['LDFLAGS'] += ' %s' % archflag return create_jobspec(self.name, ['tools/run_tests/build_artifact_csharp.sh'], environ=environ) From cbdd9844f9cd42123ee6334e8f2aa1b49c500c27 Mon Sep 17 00:00:00 2001 From: Harsh Vardhan Date: Thu, 10 Mar 2016 20:51:20 +0530 Subject: [PATCH 50/97] updates contributing.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 011ce43bcdc..850ea14399c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ In order to run most of the available tests, one would need to run: `./tools/run_tests/run_tests.py` -If you want to run tests for any of the languages {all, c, c++, csharp, node, objc, php, python, ruby}, do this: +If you want to run tests for any of the languages {c, c++, csharp, node, objc, php, python, ruby}, do this: `./tools/run_tests/run_tests.py -l ` From fa45a5e96c2f31682047bb96508c85bf7746642b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 10 Mar 2016 14:38:27 -0800 Subject: [PATCH 51/97] fix race between send message and send initial metadata --- src/core/transport/chttp2_transport.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 19265252ca3..03444fd4c24 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -851,9 +851,11 @@ static void perform_stream_op_locked( if (stream_global->write_closed) { grpc_chttp2_complete_closure_step( exec_ctx, &stream_global->send_message_finished, 0); - } else if (stream_global->id != 0) { + } else { stream_global->send_message = op->send_message; - grpc_chttp2_become_writable(transport_global, stream_global); + if (stream_global->id != 0) { + grpc_chttp2_become_writable(transport_global, stream_global); + } } } From 7212f16700f765f9acd9681d945401a7ba81081a Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 19:29:11 -0800 Subject: [PATCH 52/97] php: fix examples ClientStub constructor --- examples/php/greeter_client.php | 4 +++- examples/php/route_guide/route_guide_client.php | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/php/greeter_client.php b/examples/php/greeter_client.php index e5e4c2651e9..718ef88c644 100644 --- a/examples/php/greeter_client.php +++ b/examples/php/greeter_client.php @@ -36,7 +36,9 @@ require dirname(__FILE__) . '/vendor/autoload.php'; require dirname(__FILE__) . '/helloworld.php'; function greet($name) { - $client = new helloworld\GreeterClient('localhost:50051', []); + $client = new helloworld\GreeterClient('localhost:50051', [ + 'credentials' => Grpc\ChannelCredentials::createInsecure() + ]); $request = new helloworld\HelloRequest(); $request->setName($name); list($reply, $status) = $client->SayHello($request)->wait(); diff --git a/examples/php/route_guide/route_guide_client.php b/examples/php/route_guide/route_guide_client.php index 3cd1df72541..2f9533be4b7 100644 --- a/examples/php/route_guide/route_guide_client.php +++ b/examples/php/route_guide/route_guide_client.php @@ -37,7 +37,9 @@ require dirname(__FILE__) . '/route_guide.php'; define('COORD_FACTOR', 1e7); -$client = new routeguide\RouteGuideClient('localhost:50051', []); +$client = new routeguide\RouteGuideClient('localhost:50051', [ + 'credentials' => Grpc\ChannelCredentials::createInsecure() +]); function printFeature($feature) { $name = $feature->getName(); From 40202158fddf89d16b0e638d9cf415489e948926 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 21:16:39 -0800 Subject: [PATCH 53/97] fix copyright --- examples/php/greeter_client.php | 2 +- examples/php/route_guide/route_guide_client.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/php/greeter_client.php b/examples/php/greeter_client.php index 718ef88c644..3fab14f33e1 100644 --- a/examples/php/greeter_client.php +++ b/examples/php/greeter_client.php @@ -1,7 +1,7 @@ Date: Mon, 7 Mar 2016 16:09:21 -0800 Subject: [PATCH 54/97] one more --- src/php/tests/generated_code/math_client.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/php/tests/generated_code/math_client.php b/src/php/tests/generated_code/math_client.php index 76ccabc0684..b8652ceb088 100644 --- a/src/php/tests/generated_code/math_client.php +++ b/src/php/tests/generated_code/math_client.php @@ -43,7 +43,9 @@ function p($line) $host = 'localhost:50051'; p("Connecting to host: $host"); -$client = new math\MathClient($host, []); +$client = new math\MathClient($host, [ + 'credentials' => Grpc\ChannelCredentials::createInsecure() +]); p('Client class: '.get_class($client)); p(''); From cfad24393931706e667554a867aa98a6c88236ea Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 8 Mar 2016 08:41:08 -0800 Subject: [PATCH 55/97] fixed copyright --- src/php/tests/generated_code/math_client.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/php/tests/generated_code/math_client.php b/src/php/tests/generated_code/math_client.php index b8652ceb088..2085560d19e 100644 --- a/src/php/tests/generated_code/math_client.php +++ b/src/php/tests/generated_code/math_client.php @@ -1,7 +1,7 @@ Date: Thu, 3 Mar 2016 20:39:01 -0800 Subject: [PATCH 56/97] update PHP composer.json files --- composer.json | 11 +++++++++-- examples/php/composer.json | 11 +---------- src/php/composer.json | 6 ++++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/composer.json b/composer.json index 61f81e02bf4..97b1a5cb49b 100644 --- a/composer.json +++ b/composer.json @@ -2,13 +2,20 @@ "name": "grpc/grpc", "type": "library", "description": "gRPC library for PHP", - "version": "0.6.0", + "version": "0.14.0", "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/stanley-cheung/Protobuf-PHP" + } + ], "require": { "php": ">=5.5.0", - "google/auth": "dev-master" + "datto/protobuf-php": "dev-master", + "google/auth": "v0.7" }, "autoload": { "psr-4": { diff --git a/examples/php/composer.json b/examples/php/composer.json index 9f44f0b186a..314066b22df 100644 --- a/examples/php/composer.json +++ b/examples/php/composer.json @@ -1,17 +1,8 @@ { - "repositories": [ - { - "type": "vcs", - "url": "https://github.com/stanley-cheung/Protobuf-PHP" - } - ], "name": "grpc/grpc-demo", "description": "gRPC example for PHP", "minimum-stability": "dev", "require": { - "php": ">=5.5.0", - "datto/protobuf-php": "dev-master", - "google/auth": "dev-master", - "grpc/grpc": "dev-release-0_11" + "grpc/grpc": "dev-release-0_13" } } diff --git a/src/php/composer.json b/src/php/composer.json index 1d41f847ac0..01674a25db8 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -1,7 +1,9 @@ { "name": "grpc/grpc", + "type": "library", "description": "gRPC library for PHP", - "version": "0.6.0", + "version": "0.14.0", + "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", "repositories": [ @@ -13,7 +15,7 @@ "require": { "php": ">=5.5.0", "datto/protobuf-php": "dev-master", - "google/auth": "dev-master" + "google/auth": "v0.7" }, "autoload": { "psr-4": { From 7fc1208a5cee894cada499a33d6a7e36d4209edc Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 22:14:54 -0800 Subject: [PATCH 57/97] still need this block for the example composer.json file --- examples/php/composer.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/php/composer.json b/examples/php/composer.json index 314066b22df..c837bf7ac0f 100644 --- a/examples/php/composer.json +++ b/examples/php/composer.json @@ -2,6 +2,12 @@ "name": "grpc/grpc-demo", "description": "gRPC example for PHP", "minimum-stability": "dev", + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/stanley-cheung/Protobuf-PHP" + } + ], "require": { "grpc/grpc": "dev-release-0_13" } From 5709e930faec4e88e63cec7edb396264031246a7 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 7 Mar 2016 17:34:47 -0800 Subject: [PATCH 58/97] remove stale src/php/ext/grpc/README.md --- package.xml | 2 +- src/php/ext/grpc/README.md | 67 ---------------------------------- templates/package.xml.template | 2 +- 3 files changed, 2 insertions(+), 69 deletions(-) delete mode 100644 src/php/ext/grpc/README.md diff --git a/package.xml b/package.xml index e65ab73b54a..bb4dcc2037a 100644 --- a/package.xml +++ b/package.xml @@ -27,9 +27,9 @@ + - diff --git a/src/php/ext/grpc/README.md b/src/php/ext/grpc/README.md deleted file mode 100644 index 6e1cb2002f5..00000000000 --- a/src/php/ext/grpc/README.md +++ /dev/null @@ -1,67 +0,0 @@ -gRPC PHP Extension -================== - -# Requirements - - * PHP 5.5+ - * [gRPC core library](https://github.com/grpc/grpc) 0.11.0 - -# Installation - -## Install PHP 5 - -``` -$ sudo apt-get install git php5 php5-dev php-pear unzip -``` - -## Compile gRPC Core Library - -Clone the gRPC source code repository - -``` -$ git clone https://github.com/grpc/grpc.git -``` - -Build and install the gRPC C core libraries - -```sh -$ cd grpc -$ git checkout --track origin/release-0_11 -$ git pull --recurse-submodules && git submodule update --init --recursive -$ make -$ sudo make install -``` - -Note: you may encounter a warning about the Protobuf compiler `protoc` 3.0.0+ not being installed. The following might help, and will be useful later on when we need to compile the `protoc-gen-php` tool. - -```sh -$ cd grpc/third_party/protobuf -$ sudo make install # 'make' should have been run by core grpc -``` - -## Install the gRPC PHP extension - -Quick install - -```sh -$ sudo pecl install grpc -``` - -Note: before a stable release, you may need to do - -```sh -$ sudo pecl install grpc-beta -``` - -OR - -Compile from source - -```sh -$ # from grpc -$ cd src/php/ext/grpc -$ phpize -$ ./configure -$ make -$ sudo make install -``` diff --git a/templates/package.xml.template b/templates/package.xml.template index d309bfddbc3..8f757401e4c 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -29,9 +29,9 @@ + - % for source in php_config_m4.src + php_config_m4.headers: % endfor From e1e4559690eb5ecea7b8c1a376936f1095dd0c11 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 08:01:58 -0800 Subject: [PATCH 59/97] Make blocking_resolve_address overridable --- src/core/iomgr/resolve_address.h | 4 ++-- src/core/iomgr/resolve_address_posix.c | 19 +++++++++++-------- src/core/iomgr/resolve_address_windows.c | 19 +++++++++++-------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/core/iomgr/resolve_address.h b/src/core/iomgr/resolve_address.h index 01eedffa889..508e161a7e0 100644 --- a/src/core/iomgr/resolve_address.h +++ b/src/core/iomgr/resolve_address.h @@ -66,7 +66,7 @@ void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addresses); /* Resolve addr in a blocking fashion. Returns NULL on failure. On success, result must be freed with grpc_resolved_addresses_destroy. */ -grpc_resolved_addresses *grpc_blocking_resolve_address( - const char *addr, const char *default_port); +extern grpc_resolved_addresses *(*grpc_blocking_resolve_address)( + const char *name, const char *default_port); #endif /* GRPC_INTERNAL_CORE_IOMGR_RESOLVE_ADDRESS_H */ diff --git a/src/core/iomgr/resolve_address_posix.c b/src/core/iomgr/resolve_address_posix.c index c51745b9187..a6c9893f231 100644 --- a/src/core/iomgr/resolve_address_posix.c +++ b/src/core/iomgr/resolve_address_posix.c @@ -34,18 +34,13 @@ #include #ifdef GPR_POSIX_SOCKET -#include "src/core/iomgr/sockaddr.h" #include "src/core/iomgr/resolve_address.h" +#include "src/core/iomgr/sockaddr.h" +#include #include #include -#include -#include "src/core/iomgr/executor.h" -#include "src/core/iomgr/iomgr_internal.h" -#include "src/core/iomgr/sockaddr_utils.h" -#include "src/core/support/block_annotate.h" -#include "src/core/support/string.h" #include #include #include @@ -53,6 +48,11 @@ #include #include #include +#include "src/core/iomgr/executor.h" +#include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/sockaddr_utils.h" +#include "src/core/support/block_annotate.h" +#include "src/core/support/string.h" typedef struct { char *name; @@ -62,7 +62,7 @@ typedef struct { void *arg; } request; -grpc_resolved_addresses *grpc_blocking_resolve_address( +static grpc_resolved_addresses *blocking_resolve_address_impl( const char *name, const char *default_port) { struct addrinfo hints; struct addrinfo *result = NULL, *resp; @@ -150,6 +150,9 @@ done: return addrs; } +grpc_resolved_addresses *(*grpc_blocking_resolve_address)( + const char *name, const char *default_port) = blocking_resolve_address_impl; + /* Callback to be passed to grpc_executor to asynch-ify * grpc_blocking_resolve_address */ static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) { diff --git a/src/core/iomgr/resolve_address_windows.c b/src/core/iomgr/resolve_address_windows.c index 28c8661e73b..472e7971634 100644 --- a/src/core/iomgr/resolve_address_windows.c +++ b/src/core/iomgr/resolve_address_windows.c @@ -34,17 +34,12 @@ #include #ifdef GPR_WINSOCK_SOCKET -#include "src/core/iomgr/sockaddr.h" #include "src/core/iomgr/resolve_address.h" +#include "src/core/iomgr/sockaddr.h" -#include #include +#include -#include "src/core/iomgr/executor.h" -#include "src/core/iomgr/iomgr_internal.h" -#include "src/core/iomgr/sockaddr_utils.h" -#include "src/core/support/block_annotate.h" -#include "src/core/support/string.h" #include #include #include @@ -52,6 +47,11 @@ #include #include #include +#include "src/core/iomgr/executor.h" +#include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/sockaddr_utils.h" +#include "src/core/support/block_annotate.h" +#include "src/core/support/string.h" typedef struct { char *name; @@ -61,7 +61,7 @@ typedef struct { void *arg; } request; -grpc_resolved_addresses *grpc_blocking_resolve_address( +static grpc_resolved_addresses *blocking_resolve_address_impl( const char *name, const char *default_port) { struct addrinfo hints; struct addrinfo *result = NULL, *resp; @@ -133,6 +133,9 @@ done: return addrs; } +grpc_resolved_addresses *(*grpc_blocking_resolve_address)( + const char *name, const char *default_port) = blocking_resolve_address_impl; + /* Callback to be passed to grpc_executor to asynch-ify * grpc_blocking_resolve_address */ static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) { From e91ef68d2a9077641d6ece47b388e523942164fe Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 08:59:17 -0800 Subject: [PATCH 60/97] Add failing test --- Makefile | 36 ++++ build.yaml | 10 + .../dns_resolver_connectivity_test.c | 133 ++++++++++++ tools/run_tests/sources_and_headers.json | 16 ++ tools/run_tests/tests.json | 21 ++ vsprojects/buildtests_c.sln | 27 +++ .../dns_resolver_connectivity_test.vcxproj | 199 ++++++++++++++++++ ...resolver_connectivity_test.vcxproj.filters | 24 +++ 8 files changed, 466 insertions(+) create mode 100644 test/core/client_config/resolvers/dns_resolver_connectivity_test.c create mode 100644 vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj create mode 100644 vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj.filters diff --git a/Makefile b/Makefile index 5be0d146afc..171a83c2a3a 100644 --- a/Makefile +++ b/Makefile @@ -849,6 +849,7 @@ chttp2_status_conversion_test: $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test chttp2_stream_map_test: $(BINDIR)/$(CONFIG)/chttp2_stream_map_test chttp2_varint_test: $(BINDIR)/$(CONFIG)/chttp2_varint_test compression_test: $(BINDIR)/$(CONFIG)/compression_test +dns_resolver_connectivity_test: $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test dns_resolver_test: $(BINDIR)/$(CONFIG)/dns_resolver_test dualstack_socket_test: $(BINDIR)/$(CONFIG)/dualstack_socket_test endpoint_pair_test: $(BINDIR)/$(CONFIG)/endpoint_pair_test @@ -1159,6 +1160,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/chttp2_stream_map_test \ $(BINDIR)/$(CONFIG)/chttp2_varint_test \ $(BINDIR)/$(CONFIG)/compression_test \ + $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test \ $(BINDIR)/$(CONFIG)/dns_resolver_test \ $(BINDIR)/$(CONFIG)/dualstack_socket_test \ $(BINDIR)/$(CONFIG)/endpoint_pair_test \ @@ -1402,6 +1404,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/chttp2_varint_test || ( echo test chttp2_varint_test failed ; exit 1 ) $(E) "[RUN] Testing compression_test" $(Q) $(BINDIR)/$(CONFIG)/compression_test || ( echo test compression_test failed ; exit 1 ) + $(E) "[RUN] Testing dns_resolver_connectivity_test" + $(Q) $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test || ( echo test dns_resolver_connectivity_test failed ; exit 1 ) $(E) "[RUN] Testing dns_resolver_test" $(Q) $(BINDIR)/$(CONFIG)/dns_resolver_test || ( echo test dns_resolver_test failed ; exit 1 ) $(E) "[RUN] Testing dualstack_socket_test" @@ -6075,6 +6079,38 @@ endif endif +DNS_RESOLVER_CONNECTIVITY_TEST_SRC = \ + test/core/client_config/resolvers/dns_resolver_connectivity_test.c \ + +DNS_RESOLVER_CONNECTIVITY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(DNS_RESOLVER_CONNECTIVITY_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test: $(DNS_RESOLVER_CONNECTIVITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(DNS_RESOLVER_CONNECTIVITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/client_config/resolvers/dns_resolver_connectivity_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_dns_resolver_connectivity_test: $(DNS_RESOLVER_CONNECTIVITY_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(DNS_RESOLVER_CONNECTIVITY_TEST_OBJS:.o=.dep) +endif +endif + + DNS_RESOLVER_TEST_SRC = \ test/core/client_config/resolvers/dns_resolver_test.c \ diff --git a/build.yaml b/build.yaml index a277539a054..7cafb0ac6fe 100644 --- a/build.yaml +++ b/build.yaml @@ -1057,6 +1057,16 @@ targets: - grpc - gpr_test_util - gpr +- name: dns_resolver_connectivity_test + build: test + language: c + src: + - test/core/client_config/resolvers/dns_resolver_connectivity_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: dns_resolver_test build: test language: c diff --git a/test/core/client_config/resolvers/dns_resolver_connectivity_test.c b/test/core/client_config/resolvers/dns_resolver_connectivity_test.c new file mode 100644 index 00000000000..e4d83539fb9 --- /dev/null +++ b/test/core/client_config/resolvers/dns_resolver_connectivity_test.c @@ -0,0 +1,133 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/client_config/resolvers/dns_resolver.h" + +#include + +#include +#include + +#include "src/core/iomgr/resolve_address.h" +#include "test/core/util/test_config.h" + +static void subchannel_factory_ref(grpc_subchannel_factory *scv) {} +static void subchannel_factory_unref(grpc_exec_ctx *exec_ctx, + grpc_subchannel_factory *scv) {} +static grpc_subchannel *subchannel_factory_create_subchannel( + grpc_exec_ctx *exec_ctx, grpc_subchannel_factory *factory, + grpc_subchannel_args *args) { + GPR_UNREACHABLE_CODE(return NULL); +} + +static const grpc_subchannel_factory_vtable sc_vtable = { + subchannel_factory_ref, subchannel_factory_unref, + subchannel_factory_create_subchannel}; + +static grpc_subchannel_factory sc_factory = {&sc_vtable}; + +static gpr_mu g_mu; +static bool g_fail_resolution = true; + +static grpc_resolved_addresses *my_resolve_address(const char *name, + const char *addr) { + gpr_mu_lock(&g_mu); + GPR_ASSERT(0 == strcmp("test", name)); + if (g_fail_resolution) { + g_fail_resolution = false; + gpr_mu_unlock(&g_mu); + return NULL; + } else { + gpr_mu_unlock(&g_mu); + grpc_resolved_addresses *addrs = gpr_malloc(sizeof(*addrs)); + addrs->naddrs = 1; + addrs->addrs = gpr_malloc(sizeof(*addrs->addrs)); + addrs->addrs[0].len = 123; + return addrs; + } +} + +static grpc_resolver *create_resolver(const char *name) { + grpc_resolver_factory *factory = grpc_dns_resolver_factory_create(); + grpc_uri *uri = grpc_uri_parse(name, 0); + GPR_ASSERT(uri); + grpc_resolver_args args; + memset(&args, 0, sizeof(args)); + args.uri = uri; + args.subchannel_factory = &sc_factory; + grpc_resolver *resolver = + grpc_resolver_factory_create_resolver(factory, &args); + grpc_resolver_factory_unref(factory); + grpc_uri_destroy(uri); + return resolver; +} + +static void on_done(grpc_exec_ctx *exec_ctx, void *ev, bool success) { + gpr_event_set(ev, (void *)1); +} + +int main(int argc, char **argv) { + grpc_test_init(argc, argv); + + grpc_init(); + gpr_mu_init(&g_mu); + grpc_blocking_resolve_address = my_resolve_address; + + grpc_resolver *resolver = create_resolver("dns:test"); + + grpc_client_config *config = (grpc_client_config *)1; + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_event ev1; + gpr_event_init(&ev1); + grpc_resolver_next(&exec_ctx, resolver, &config, + grpc_closure_create(on_done, &ev1)); + grpc_exec_ctx_flush(&exec_ctx); + GPR_ASSERT(gpr_event_wait(&ev1, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + GPR_ASSERT(config == NULL); + + gpr_event ev2; + gpr_event_init(&ev2); + grpc_resolver_next(&exec_ctx, resolver, &config, + grpc_closure_create(on_done, &ev2)); + grpc_exec_ctx_flush(&exec_ctx); + GPR_ASSERT(gpr_event_wait(&ev2, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + GPR_ASSERT(config != NULL); + + grpc_client_config_unref(&exec_ctx, config); + GRPC_RESOLVER_UNREF(&exec_ctx, resolver, "test"); + grpc_exec_ctx_finish(&exec_ctx); + + grpc_shutdown(); + gpr_mu_destroy(&g_mu); +} diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 0c7a6c7b5f3..0039b99a55e 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -189,6 +189,22 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "dns_resolver_connectivity_test", + "src": [ + "test/core/client_config/resolvers/dns_resolver_connectivity_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index d91245cd06a..c2d6cadd530 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -253,6 +253,27 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "dns_resolver_connectivity_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 7cdc3b7ee81..be2c4a3bb14 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -251,6 +251,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "compression_test", "vcxproj {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dns_resolver_connectivity_test", "vcxproj\test\dns_resolver_connectivity_test\dns_resolver_connectivity_test.vcxproj", "{F7B6FE68-E847-D7CA-4062-E737E542BCC3}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dns_resolver_test", "vcxproj\test\dns_resolver_test\dns_resolver_test.vcxproj", "{D06E10DC-272A-5203-7066-2698A247DF26}" ProjectSection(myProperties) = preProject lib = "False" @@ -1736,6 +1747,22 @@ Global {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|Win32.Build.0 = Release|Win32 {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|x64.ActiveCfg = Release|x64 {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|x64.Build.0 = Release|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|Win32.ActiveCfg = Debug|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|x64.ActiveCfg = Debug|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|Win32.ActiveCfg = Release|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|x64.ActiveCfg = Release|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|Win32.Build.0 = Debug|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|x64.Build.0 = Debug|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|Win32.Build.0 = Release|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|x64.Build.0 = Release|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug-DLL|x64.Build.0 = Debug|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release-DLL|Win32.Build.0 = Release|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release-DLL|x64.ActiveCfg = Release|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release-DLL|x64.Build.0 = Release|x64 {D06E10DC-272A-5203-7066-2698A247DF26}.Debug|Win32.ActiveCfg = Debug|Win32 {D06E10DC-272A-5203-7066-2698A247DF26}.Debug|x64.ActiveCfg = Debug|x64 {D06E10DC-272A-5203-7066-2698A247DF26}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj b/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj new file mode 100644 index 00000000000..76ac196dda6 --- /dev/null +++ b/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj @@ -0,0 +1,199 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {F7B6FE68-E847-D7CA-4062-E737E542BCC3} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + dns_resolver_connectivity_test + static + Debug + static + Debug + + + dns_resolver_connectivity_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj.filters b/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj.filters new file mode 100644 index 00000000000..e49318df110 --- /dev/null +++ b/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj.filters @@ -0,0 +1,24 @@ + + + + + test\core\client_config\resolvers + + + + + + {7743e287-3311-1206-2766-32381628ff07} + + + {cc06b800-cd26-f7a8-7ecf-ec789e78881b} + + + {4fbd5c0a-d1e3-6658-5788-2fc6f2cc9071} + + + {eab637a2-7ac1-f6be-4fc2-c8989c66070b} + + + + From 1e55bd455d5fa901e19c1ccf64996c0adc7a4584 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 09:47:43 -0800 Subject: [PATCH 61/97] Add retry for dns resolution --- src/core/client_config/client_config.c | 4 +- .../client_config/lb_policies/pick_first.c | 2 +- .../client_config/resolvers/dns_resolver.c | 53 ++++++++++++++++--- .../dns_resolver_connectivity_test.c | 21 ++++++-- 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/src/core/client_config/client_config.c b/src/core/client_config/client_config.c index 6ecffb38541..25f4ac25177 100644 --- a/src/core/client_config/client_config.c +++ b/src/core/client_config/client_config.c @@ -53,7 +53,9 @@ void grpc_client_config_ref(grpc_client_config *c) { gpr_ref(&c->refs); } void grpc_client_config_unref(grpc_exec_ctx *exec_ctx, grpc_client_config *c) { if (gpr_unref(&c->refs)) { - GRPC_LB_POLICY_UNREF(exec_ctx, c->lb_policy, "client_config"); + if (c->lb_policy != NULL) { + GRPC_LB_POLICY_UNREF(exec_ctx, c->lb_policy, "client_config"); + } gpr_free(c); } } diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c index 81167b31c8c..8ed1223d399 100644 --- a/src/core/client_config/lb_policies/pick_first.c +++ b/src/core/client_config/lb_policies/pick_first.c @@ -387,8 +387,8 @@ static void pick_first_factory_unref(grpc_lb_policy_factory *factory) {} static grpc_lb_policy *create_pick_first(grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { + if (args->num_subchannels == 0) return NULL; pick_first_lb_policy *p = gpr_malloc(sizeof(*p)); - GPR_ASSERT(args->num_subchannels > 0); memset(p, 0, sizeof(*p)); grpc_lb_policy_init(&p->base, &pick_first_lb_policy_vtable); p->subchannels = diff --git a/src/core/client_config/resolvers/dns_resolver.c b/src/core/client_config/resolvers/dns_resolver.c index 376b6b3d768..55457647b3c 100644 --- a/src/core/client_config/resolvers/dns_resolver.c +++ b/src/core/client_config/resolvers/dns_resolver.c @@ -41,6 +41,7 @@ #include "src/core/client_config/lb_policy_registry.h" #include "src/core/iomgr/resolve_address.h" +#include "src/core/iomgr/timer.h" #include "src/core/support/string.h" typedef struct { @@ -71,6 +72,9 @@ typedef struct { grpc_client_config **target_config; /** current (fully resolved) config */ grpc_client_config *resolved_config; + /** retry timer */ + bool have_retry_timer; + grpc_timer retry_timer; } dns_resolver; static void dns_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *r); @@ -125,6 +129,21 @@ static void dns_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver, gpr_mu_unlock(&r->mu); } +static void dns_on_retry_timer(grpc_exec_ctx *exec_ctx, void *arg, + bool success) { + dns_resolver *r = arg; + + if (success) { + gpr_mu_lock(&r->mu); + if (!r->resolving) { + dns_start_resolving_locked(r); + } + gpr_mu_unlock(&r->mu); + } + + GRPC_RESOLVER_UNREF(exec_ctx, &r->base, "retry-timer"); +} + static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_resolved_addresses *addresses) { dns_resolver *r = arg; @@ -133,29 +152,47 @@ static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_subchannel_args args; grpc_lb_policy *lb_policy; size_t i; - if (addresses) { + gpr_mu_lock(&r->mu); + GPR_ASSERT(r->resolving); + r->resolving = 0; + if (addresses != NULL) { grpc_lb_policy_args lb_policy_args; config = grpc_client_config_create(); subchannels = gpr_malloc(sizeof(grpc_subchannel *) * addresses->naddrs); + size_t naddrs = 0; for (i = 0; i < addresses->naddrs; i++) { memset(&args, 0, sizeof(args)); args.addr = (struct sockaddr *)(addresses->addrs[i].addr); args.addr_len = (size_t)addresses->addrs[i].len; - subchannels[i] = grpc_subchannel_factory_create_subchannel( + grpc_subchannel *subchannel = grpc_subchannel_factory_create_subchannel( exec_ctx, r->subchannel_factory, &args); + if (subchannel != NULL) { + subchannels[naddrs++] = subchannel; + } } memset(&lb_policy_args, 0, sizeof(lb_policy_args)); lb_policy_args.subchannels = subchannels; - lb_policy_args.num_subchannels = addresses->naddrs; + lb_policy_args.num_subchannels = naddrs; lb_policy = grpc_lb_policy_create(r->lb_policy_name, &lb_policy_args); - grpc_client_config_set_lb_policy(config, lb_policy); - GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction"); + if (lb_policy != NULL) { + grpc_client_config_set_lb_policy(config, lb_policy); + GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction"); + } grpc_resolved_addresses_destroy(addresses); gpr_free(subchannels); + } else { + int retry_seconds = 15; + gpr_log(GPR_DEBUG, "dns resolution failed: retrying in %d seconds", + retry_seconds); + GPR_ASSERT(!r->have_retry_timer); + r->have_retry_timer = true; + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + GRPC_RESOLVER_REF(&r->base, "retry-timer"); + grpc_timer_init( + exec_ctx, &r->retry_timer, + gpr_time_add(now, gpr_time_from_seconds(retry_seconds, GPR_TIMESPAN)), + dns_on_retry_timer, r, now); } - gpr_mu_lock(&r->mu); - GPR_ASSERT(r->resolving); - r->resolving = 0; if (r->resolved_config) { grpc_client_config_unref(exec_ctx, r->resolved_config); } diff --git a/test/core/client_config/resolvers/dns_resolver_connectivity_test.c b/test/core/client_config/resolvers/dns_resolver_connectivity_test.c index e4d83539fb9..75d1eb674f6 100644 --- a/test/core/client_config/resolvers/dns_resolver_connectivity_test.c +++ b/test/core/client_config/resolvers/dns_resolver_connectivity_test.c @@ -39,6 +39,7 @@ #include #include "src/core/iomgr/resolve_address.h" +#include "src/core/iomgr/timer.h" #include "test/core/util/test_config.h" static void subchannel_factory_ref(grpc_subchannel_factory *scv) {} @@ -47,7 +48,7 @@ static void subchannel_factory_unref(grpc_exec_ctx *exec_ctx, static grpc_subchannel *subchannel_factory_create_subchannel( grpc_exec_ctx *exec_ctx, grpc_subchannel_factory *factory, grpc_subchannel_args *args) { - GPR_UNREACHABLE_CODE(return NULL); + return NULL; } static const grpc_subchannel_factory_vtable sc_vtable = { @@ -96,6 +97,20 @@ static void on_done(grpc_exec_ctx *exec_ctx, void *ev, bool success) { gpr_event_set(ev, (void *)1); } +// interleave waiting for an event with a timer check +static bool wait_loop(int deadline_seconds, gpr_event *ev) { + while (deadline_seconds) { + gpr_log(GPR_DEBUG, "Test: waiting for %d more seconds", deadline_seconds); + if (gpr_event_wait(ev, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1))) return true; + deadline_seconds--; + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_timer_check(&exec_ctx, gpr_now(GPR_CLOCK_MONOTONIC), NULL); + grpc_exec_ctx_finish(&exec_ctx); + } + return false; +} + int main(int argc, char **argv) { grpc_test_init(argc, argv); @@ -113,7 +128,7 @@ int main(int argc, char **argv) { grpc_resolver_next(&exec_ctx, resolver, &config, grpc_closure_create(on_done, &ev1)); grpc_exec_ctx_flush(&exec_ctx); - GPR_ASSERT(gpr_event_wait(&ev1, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + GPR_ASSERT(wait_loop(5, &ev1)); GPR_ASSERT(config == NULL); gpr_event ev2; @@ -121,7 +136,7 @@ int main(int argc, char **argv) { grpc_resolver_next(&exec_ctx, resolver, &config, grpc_closure_create(on_done, &ev2)); grpc_exec_ctx_flush(&exec_ctx); - GPR_ASSERT(gpr_event_wait(&ev2, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + GPR_ASSERT(wait_loop(30, &ev2)); GPR_ASSERT(config != NULL); grpc_client_config_unref(&exec_ctx, config); From e2327dbb8e45936c81802119142a7cda28230f24 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 09:52:42 -0800 Subject: [PATCH 62/97] Fit and finish for dns retry loop --- build.yaml | 1 + src/core/client_config/resolvers/dns_resolver.c | 8 ++++++-- tools/run_tests/tests.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/build.yaml b/build.yaml index 7cafb0ac6fe..e003dd83396 100644 --- a/build.yaml +++ b/build.yaml @@ -1058,6 +1058,7 @@ targets: - gpr_test_util - gpr - name: dns_resolver_connectivity_test + cpu_cost: 0.1 build: test language: c src: diff --git a/src/core/client_config/resolvers/dns_resolver.c b/src/core/client_config/resolvers/dns_resolver.c index 55457647b3c..e28e4757a10 100644 --- a/src/core/client_config/resolvers/dns_resolver.c +++ b/src/core/client_config/resolvers/dns_resolver.c @@ -95,6 +95,9 @@ static const grpc_resolver_vtable dns_resolver_vtable = { static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver) { dns_resolver *r = (dns_resolver *)resolver; gpr_mu_lock(&r->mu); + if (r->have_retry_timer) { + grpc_timer_cancel(exec_ctx, &r->retry_timer); + } if (r->next_completion != NULL) { *r->target_config = NULL; grpc_exec_ctx_enqueue(exec_ctx, r->next_completion, true, NULL); @@ -133,13 +136,14 @@ static void dns_on_retry_timer(grpc_exec_ctx *exec_ctx, void *arg, bool success) { dns_resolver *r = arg; + gpr_mu_lock(&r->mu); + r->have_retry_timer = false; if (success) { - gpr_mu_lock(&r->mu); if (!r->resolving) { dns_start_resolving_locked(r); } - gpr_mu_unlock(&r->mu); } + gpr_mu_unlock(&r->mu); GRPC_RESOLVER_UNREF(exec_ctx, &r->base, "retry-timer"); } diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index c2d6cadd530..724cd64d19c 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -261,7 +261,7 @@ "posix", "windows" ], - "cpu_cost": 1.0, + "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, "gtest": false, From b18fe559629cea0c2a6d345449071cfb6c31bdc1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 10:23:16 -0800 Subject: [PATCH 63/97] Update copyrights --- src/core/client_config/client_config.c | 2 +- src/core/iomgr/resolve_address.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/client_config/client_config.c b/src/core/client_config/client_config.c index 25f4ac25177..c500af25eec 100644 --- a/src/core/client_config/client_config.c +++ b/src/core/client_config/client_config.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/resolve_address.h b/src/core/iomgr/resolve_address.h index 508e161a7e0..b0596304576 100644 --- a/src/core/iomgr/resolve_address.h +++ b/src/core/iomgr/resolve_address.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 9aa4f1b062686e01c2e1dfb792e5ec9c3a91e1c9 Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 2 Mar 2016 16:39:13 -0800 Subject: [PATCH 64/97] fix windows leaks --- src/core/iomgr/iocp_windows.c | 14 +++++++++----- src/core/iomgr/iocp_windows.h | 11 +++++++++-- src/core/iomgr/tcp_server_windows.c | 3 +-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/core/iomgr/iocp_windows.c b/src/core/iomgr/iocp_windows.c index 807729708ea..fa87e5246b0 100644 --- a/src/core/iomgr/iocp_windows.c +++ b/src/core/iomgr/iocp_windows.c @@ -71,7 +71,8 @@ static DWORD deadline_to_millis_timeout(gpr_timespec deadline, timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); } -void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { +grpc_iocp_work_status grpc_iocp_work(grpc_exec_ctx *exec_ctx, + gpr_timespec deadline) { BOOL success; DWORD bytes = 0; DWORD flags = 0; @@ -84,14 +85,14 @@ void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { g_iocp, &bytes, &completion_key, &overlapped, deadline_to_millis_timeout(deadline, gpr_now(deadline.clock_type))); if (success == 0 && overlapped == NULL) { - return; + return GRPC_IOCP_WORK_TIMEOUT; } GPR_ASSERT(completion_key && overlapped); if (overlapped == &g_iocp_custom_overlap) { gpr_atm_full_fetch_add(&g_custom_events, -1); if (completion_key == (ULONG_PTR)&g_iocp_kick_token) { /* We were awoken from a kick. */ - return; + return GRPC_IOCP_WORK_KICK; } gpr_log(GPR_ERROR, "Unknown custom completion key."); abort(); @@ -121,6 +122,7 @@ void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { } gpr_mu_unlock(&socket->state_mu); grpc_exec_ctx_enqueue(exec_ctx, closure, true, NULL); + return GRPC_IOCP_WORK_WORK; } void grpc_iocp_init(void) { @@ -140,10 +142,12 @@ void grpc_iocp_kick(void) { void grpc_iocp_flush(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_iocp_work_status work_status; do { - grpc_iocp_work(&exec_ctx, gpr_inf_past(GPR_CLOCK_MONOTONIC)); - } while (grpc_exec_ctx_flush(&exec_ctx)); + work_status = grpc_iocp_work(&exec_ctx, gpr_inf_past(GPR_CLOCK_MONOTONIC)); + } while (work_status == GRPC_IOCP_WORK_KICK || + grpc_exec_ctx_flush(&exec_ctx)); } void grpc_iocp_shutdown(void) { diff --git a/src/core/iomgr/iocp_windows.h b/src/core/iomgr/iocp_windows.h index 75f3ba84770..8b2b1aeb5c2 100644 --- a/src/core/iomgr/iocp_windows.h +++ b/src/core/iomgr/iocp_windows.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,7 +38,14 @@ #include "src/core/iomgr/socket_windows.h" -void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline); +typedef enum { + GRPC_IOCP_WORK_WORK, + GRPC_IOCP_WORK_TIMEOUT, + GRPC_IOCP_WORK_KICK +} grpc_iocp_work_status; + +grpc_iocp_work_status grpc_iocp_work(grpc_exec_ctx *exec_ctx, + gpr_timespec deadline); void grpc_iocp_init(void); void grpc_iocp_kick(void); void grpc_iocp_flush(void); diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index ce930b8f41d..a4abc5b9743 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -240,8 +240,7 @@ static void decrement_active_ports_and_notify(grpc_exec_ctx *exec_ctx, sp->shutting_down = 0; gpr_mu_lock(&sp->server->mu); GPR_ASSERT(sp->server->active_ports > 0); - if (0 == --sp->server->active_ports && - sp->server->shutdown_complete != NULL) { + if (0 == --sp->server->active_ports) { notify = 1; } gpr_mu_unlock(&sp->server->mu); From 9ee163557bda7a86e33c12626cf160342796014a Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Fri, 11 Mar 2016 11:13:59 -0800 Subject: [PATCH 65/97] Remove unnecessary (potentially bad) safety-refs --- src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi | 1 - src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi | 1 - 2 files changed, 2 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index 1f1833d5ecc..bc03c4dcf19 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -95,7 +95,6 @@ cdef class Channel: self, grpc_connectivity_state last_observed_state, Timespec deadline not None, CompletionQueue queue not None, tag): cdef OperationTag operation_tag = OperationTag(tag) - operation_tag.references = [self, queue] cpython.Py_INCREF(operation_tag) grpc_channel_watch_connectivity_state( self.c_channel, last_observed_state, deadline.c_time, diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index fe93da6c124..5f859235246 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -106,7 +106,6 @@ cdef class Server: self.is_shutting_down = True operation_tag = OperationTag(tag) operation_tag.shutting_down_server = self - operation_tag.references.extend([self, queue]) cpython.Py_INCREF(operation_tag) grpc_server_shutdown_and_notify( self.c_server, queue.c_completion_queue, From a41940d0e0dc384318e6ea116e4ff8e10f7747be Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 11 Mar 2016 20:31:32 +0100 Subject: [PATCH 66/97] Disabling check_version. --- tools/run_tests/sanity/sanity_tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index cffc180fb07..3840f4d8df6 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -2,7 +2,6 @@ - script: tools/run_tests/sanity/check_cache_mk.sh - script: tools/run_tests/sanity/check_sources_and_headers.py - script: tools/run_tests/sanity/check_submodules.sh -- script: tools/run_tests/sanity/check_version.py - script: tools/buildgen/generate_projects.sh -j 3 cpu_cost: 3 - script: tools/distrib/check_copyright.py From f1d084a4d159eb07536e4b24f2bf65a3771a747c Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 30 Oct 2015 11:14:09 -0700 Subject: [PATCH 67/97] Add monitoring of connectivity Move C-layer call creation into GRPCChannel, so that it always acts on a non-destroyed channel. Listen for connectivity in the GRPCCall, and destroy the channel. --- src/objective-c/GRPCClient/GRPCCall.m | 37 +++- .../GRPCClient/private/GRPCChannel.h | 3 + .../GRPCClient/private/GRPCChannel.m | 14 ++ .../private/GRPCConnectivityMonitor.h | 77 +++++++ .../private/GRPCConnectivityMonitor.m | 188 ++++++++++++++++++ src/objective-c/GRPCClient/private/GRPCHost.h | 26 ++- src/objective-c/GRPCClient/private/GRPCHost.m | 93 ++++----- .../GRPCReachabilityFlagNames.xmacro.h | 65 ++++++ .../GRPCClient/private/GRPCWrappedCall.h | 2 +- 9 files changed, 445 insertions(+), 60 deletions(-) create mode 100644 src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h create mode 100644 src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m create mode 100644 src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index f79b7d0bc0c..2d45818b6e9 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -37,6 +37,8 @@ #include #import +#import "private/GRPCConnectivityMonitor.h" +#import "private/GRPCHost.h" #import "private/GRPCRequestHeaders.h" #import "private/GRPCWrappedCall.h" #import "private/NSData+GRPC.h" @@ -71,8 +73,11 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; @implementation GRPCCall { dispatch_queue_t _callQueue; + NSString *_host; + NSString *_path; GRPCWrappedCall *_wrappedCall; dispatch_once_t _callAlreadyInvoked; + GRPCConnectivityMonitor *_connectivityMonitor; // The C gRPC library has less guarantees on the ordering of events than we // do. Particularly, in the face of errors, there's no ordering guarantee at @@ -115,13 +120,11 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; format:@"The requests writer can't be already started."]; } if ((self = [super init])) { - _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:host path:path]; - if (!_wrappedCall) { - return nil; - } + _host = [host copy]; + _path = [path copy]; // Serial queue to invoke the non-reentrant methods of the grpc_call object. - _callQueue = dispatch_queue_create("org.grpc.call", NULL); + _callQueue = dispatch_queue_create("io.grpc.call", NULL); _requestWriter = requestWriter; @@ -156,7 +159,7 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; - (void)cancel { [self finishWithError:[NSError errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeCancelled - userInfo:nil]]; + userInfo:@{NSLocalizedDescriptionKey: @"Canceled by app"}]]; [self cancelCall]; } @@ -354,8 +357,29 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; _retainSelf = self; _responseWriteable = [[GRXConcurrentWriteable alloc] initWithWriteable:writeable]; + + _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:_host path:_path]; + NSAssert(_wrappedCall, @"Error allocating RPC objects. Low memory?"); + [self sendHeaders:_requestHeaders]; [self invokeCall]; + // TODO(jcanizales): Extract this logic somewhere common. + NSString *host = [NSURL URLWithString:[@"https://" stringByAppendingString:_host]].host; + if (!host) { + // TODO(jcanizales): Check this on init. + [NSException raise:NSInvalidArgumentException format:@"host of %@ is nil", _host]; + } + __weak typeof(self) weakSelf = self; + _connectivityMonitor = [GRPCConnectivityMonitor monitorWithHost:host]; + [_connectivityMonitor handleLossWithHandler:^{ + typeof(self) strongSelf = weakSelf; + if (strongSelf) { + [strongSelf finishWithError:[NSError errorWithDomain:kGRPCErrorDomain + code:GRPCErrorCodeUnavailable + userInfo:@{NSLocalizedDescriptionKey: @"Connectivity lost."}]]; + [[GRPCHost hostWithAddress:strongSelf->_host] disconnect]; + } + }]; } - (void)setState:(GRXWriterState)newState { @@ -385,4 +409,5 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; return; } } + @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h index 8661ae6f972..e49a6aca297 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.h +++ b/src/objective-c/GRPCClient/private/GRPCChannel.h @@ -35,6 +35,7 @@ #include +@class GRPCCompletionQueue; struct grpc_channel_credentials; @@ -80,4 +81,6 @@ struct grpc_channel_credentials; + (nonnull GRPCChannel *)insecureChannelWithHost:(nonnull NSString *)host channelArgs:(nullable NSDictionary *)channelArgs; +- (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path + completionQueue:(nonnull GRPCCompletionQueue *)queue; @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 7e55a473d74..d7de025e21d 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -38,6 +38,8 @@ #include #include +#import "GRPCCompletionQueue.h" + /** * Returns @c grpc_channel_credentials from the specified @c path. If the file at the path could not * be read then NULL is returned. If NULL is returned, @c errorPtr may not be NULL if there are @@ -205,4 +207,16 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) { channelArgs:channelArgs]; } +- (grpc_call *)unmanagedCallWithPath:(NSString *)path + completionQueue:(GRPCCompletionQueue *)queue { + return grpc_channel_create_call(_unmanagedChannel, + NULL, GRPC_PROPAGATE_DEFAULTS, + queue.unmanagedQueue, + path.UTF8String, + // Get "host" from "host:port" + // TODO(jcanizales): Use NSURLs throughout, to clarify these. + [_host componentsSeparatedByString:@":"][0].UTF8String, + gpr_inf_future(GPR_CLOCK_REALTIME), NULL); +} + @end diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h new file mode 100644 index 00000000000..d9d6c22ae9d --- /dev/null +++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h @@ -0,0 +1,77 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#import +#import + +@interface GRPCReachabilityFlags : NSObject + ++ (nonnull instancetype)flagsWithFlags:(SCNetworkReachabilityFlags)flags; + +/** + * One accessor method to query each of the different flags. Example: + +@property(nonatomic, readonly) BOOL isCell; + + */ +#define GRPC_XMACRO_ITEM(methodName, FlagName) \ +@property(nonatomic, readonly) BOOL methodName; + +#include "GRPCReachabilityFlagNames.xmacro.h" +#undef GRPC_XMACRO_ITEM + +@property(nonatomic, readonly) BOOL isHostReachable; +@end + + +@interface GRPCConnectivityMonitor : NSObject + ++ (nullable instancetype)monitorWithHost:(nonnull NSString *)hostName; + +- (nonnull instancetype)init NS_UNAVAILABLE; + +/** + * Queue on which callbacks will be dispatched. Default is the main queue. Set it before calling + * handleLossWithHandler:. + */ +// TODO(jcanizales): Default to a serial background queue instead. +@property(nonatomic, strong, null_resettable) dispatch_queue_t queue; + +/** + * Calls handler the next time the connectivity to this instance's host is lost. If this instance is + * released before that happens, the handler won't be called. + * Only one handler is active at a time, so if this method is called again before the previous + * handler has been called, it might never be called at all (or yes, if it has already been queued). + */ +- (void)handleLossWithHandler:(nonnull void (^)())handler; +@end diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m new file mode 100644 index 00000000000..f66a77c9b20 --- /dev/null +++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m @@ -0,0 +1,188 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#import "GRPCConnectivityMonitor.h" + +#pragma mark Flags + +@implementation GRPCReachabilityFlags { + SCNetworkReachabilityFlags _flags; +} + ++ (instancetype)flagsWithFlags:(SCNetworkReachabilityFlags)flags { + return [[self alloc] initWithFlags:flags]; +} + +- (instancetype)initWithFlags:(SCNetworkReachabilityFlags)flags { + if ((self = [super init])) { + _flags = flags; + } + return self; +} + +/* + * One accessor method implementation per flag. Example: + +- (BOOL)isCell { \ + return !!(_flags & kSCNetworkReachabilityFlagsIsWWAN); \ +} + + */ +#define GRPC_XMACRO_ITEM(methodName, FlagName) \ +- (BOOL)methodName { \ + return !!(_flags & kSCNetworkReachabilityFlags ## FlagName); \ +} +#include "GRPCReachabilityFlagNames.xmacro.h" +#undef GRPC_XMACRO_ITEM + +- (BOOL)isHostReachable { + // Note: connectionOnDemand means it'll be reachable only if using the CFSocketStream API or APIs + // on top of it. + // connectionRequired means we can't tell until a connection is attempted (e.g. for VPN on + // demand). + return self.reachable && !self.interventionRequired && !self.connectionOnDemand; +} + +- (NSString *)description { + NSMutableArray *activeOptions = [NSMutableArray arrayWithCapacity:9]; + + /* + * For each flag, add its name to the array if it's ON. Example: + + if (self.isCell) { + [activeOptions addObject:@"isCell"]; + } + + */ +#define GRPC_XMACRO_ITEM(methodName, FlagName) \ + if (self.methodName) { \ + [activeOptions addObject:@#methodName]; \ + } +#include "GRPCReachabilityFlagNames.xmacro.h" +#undef GRPC_XMACRO_ITEM + + return activeOptions.count == 0 ? @"(none)" : [activeOptions componentsJoinedByString:@", "]; +} + +- (BOOL)isEqual:(id)object { + return [object isKindOfClass:[GRPCReachabilityFlags class]] && + _flags == ((GRPCReachabilityFlags *)object)->_flags; +} + +- (NSUInteger)hash { + return _flags; +} +@end + +#pragma mark Connectivity Monitor + +// Assumes the third argument is a block that accepts SCNetworkReachabilityFlags, and passes the +// received ones to it. +static void PassFlagsToContextInfoBlock(SCNetworkReachabilityRef target, + SCNetworkReachabilityFlags flags, + void *info) { + #pragma unused (target) + ((__bridge void (^)(SCNetworkReachabilityFlags))info)(flags); +} + +@implementation GRPCConnectivityMonitor { + SCNetworkReachabilityRef _reachabilityRef; +} + +- (nullable instancetype)initWithReachability:(nullable SCNetworkReachabilityRef)reachability { + if (!reachability) { + return nil; + } + if ((self = [super init])) { + _reachabilityRef = CFRetain(reachability); + _queue = dispatch_get_main_queue(); + } + return self; +} + ++ (nullable instancetype)monitorWithHost:(nonnull NSString *)host { + const char *hostName = host.UTF8String; + if (!hostName) { + [NSException raise:NSInvalidArgumentException + format:@"host.UTF8String returns NULL for %@", host]; + } + SCNetworkReachabilityRef reachability = + SCNetworkReachabilityCreateWithName(NULL, hostName); + + GRPCConnectivityMonitor *returnValue = [[self alloc] initWithReachability:reachability]; + if (reachability) { + CFRelease(reachability); + } + return returnValue; +} + +- (void)handleLossWithHandler:(void (^)())handler { + __weak typeof(self) weakSelf = self; + [self startListeningWithHandler:^(GRPCReachabilityFlags *flags) { + if (!flags.isHostReachable) { + [weakSelf stopListening]; + handler(); + } + }]; +} + +- (void)startListeningWithHandler:(void (^)(GRPCReachabilityFlags *))handler { + SCNetworkReachabilityContext context = { + .version = 0, + .info = (__bridge_retained void *)^(SCNetworkReachabilityFlags flags){ + // Pass the flags as as Objective-C wrapper. + handler([[GRPCReachabilityFlags alloc] initWithFlags:flags]); + }, + .release = CFRelease + }; + SCNetworkReachabilitySetCallback(_reachabilityRef, PassFlagsToContextInfoBlock, &context); + SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, _queue); +} + +- (void)stopListening { + SCNetworkReachabilitySetCallback(_reachabilityRef, NULL, NULL); + SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, NULL); +} + +- (void)setQueue:(dispatch_queue_t)queue { + _queue = queue ?: dispatch_get_main_queue(); +} + +- (void)dealloc { + if (_reachabilityRef) { + [self stopListening]; + CFRelease(_reachabilityRef); + } +} + +@end diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index 82c0ad6cf63..987d3e9f597 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -33,27 +33,39 @@ #import +NS_ASSUME_NONNULL_BEGIN + @class GRPCCompletionQueue; struct grpc_call; @interface GRPCHost : NSObject @property(nonatomic, readonly) NSString *address; -@property(nonatomic, copy) NSString *userAgentPrefix; +@property(nonatomic, copy, nullable) NSString *userAgentPrefix; /** The following properties should only be modified for testing: */ @property(nonatomic, getter=isSecure) BOOL secure; -@property(nonatomic, copy) NSString *pathToCertificates; -@property(nonatomic, copy) NSString *hostNameOverride; +@property(nonatomic, copy, nullable) NSString *pathToCertificates; +@property(nonatomic, copy, nullable) NSString *hostNameOverride; +- (nullable instancetype)init NS_UNAVAILABLE; /** Host objects initialized with the same address are the same. */ -+ (instancetype)hostWithAddress:(NSString *)address; -- (instancetype)initWithAddress:(NSString *)address NS_DESIGNATED_INITIALIZER; ++ (nullable instancetype)hostWithAddress:(NSString *)address; +- (nullable instancetype)initWithAddress:(NSString *)address NS_DESIGNATED_INITIALIZER; /** Create a grpc_call object to the provided path on this host. */ -- (struct grpc_call *)unmanagedCallWithPath:(NSString *)path - completionQueue:(GRPCCompletionQueue *)queue; +- (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path + completionQueue:(GRPCCompletionQueue *)queue; +// TODO: There's a race when a new RPC is coming through just as an existing one is getting +// notified that there's no connectivity. If connectivity comes back at that moment, the new RPC +// will have its channel destroyed by the other RPC, and will never get notified of a problem, so +// it'll hang (the C layer logs a timeout, with exponential back off). One solution could be to pass +// the GRPCChannel to the GRPCCall, renaming this as "disconnectChannel:channel", which would only +// act on that specific channel. +- (void)disconnect; @end + +NS_ASSUME_NONNULL_END diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index eb1db899b70..b6a6d48d964 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -34,33 +34,30 @@ #import "GRPCHost.h" #include +#import #import #import "GRPCChannel.h" #import "GRPCCompletionQueue.h" #import "NSDictionary+GRPC.h" +NS_ASSUME_NONNULL_BEGIN + // TODO(jcanizales): Generate the version in a standalone header, from templates. Like // templates/src/core/surface/version.c.template . #define GRPC_OBJC_VERSION_STRING @"0.13.0" -@interface GRPCHost () -// TODO(mlumish): Investigate whether caching channels with strong links is a good idea. -@property(nonatomic, strong) GRPCChannel *channel; -@end - -@implementation GRPCHost - -+ (instancetype)hostWithAddress:(NSString *)address { - return [[self alloc] initWithAddress:address]; +@implementation GRPCHost { + // TODO(mlumish): Investigate whether caching channels with strong links is a good idea. + GRPCChannel *_channel; } -- (instancetype)init { - return [self initWithAddress:nil]; ++ (nullable instancetype)hostWithAddress:(NSString *)address { + return [[self alloc] initWithAddress:address]; } // Default initializer. -- (instancetype)initWithAddress:(NSString *)address { +- (nullable instancetype)initWithAddress:(NSString *)address { if (!address) { return nil; } @@ -95,46 +92,42 @@ return self; } -- (grpc_call *)unmanagedCallWithPath:(NSString *)path completionQueue:(GRPCCompletionQueue *)queue { - if (!queue || !path || !self.channel) { - return NULL; +- (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path + completionQueue:(GRPCCompletionQueue *)queue { + @synchronized(self) { + if (!_channel) { + _channel = [self newChannel]; + } + return [_channel unmanagedCallWithPath:path completionQueue:queue]; } - return grpc_channel_create_call(self.channel.unmanagedChannel, - NULL, GRPC_PROPAGATE_DEFAULTS, - queue.unmanagedQueue, - path.UTF8String, - self.hostName.UTF8String, - gpr_inf_future(GPR_CLOCK_REALTIME), NULL); } -- (GRPCChannel *)channel { - // Create it lazily, because we don't want to open a connection just because someone is - // configuring a host. +- (NSDictionary *)channelArgs { + NSMutableDictionary *args = [NSMutableDictionary dictionary]; - if (!_channel) { - NSMutableDictionary *args = [NSMutableDictionary dictionary]; + // TODO(jcanizales): Add OS and device information (see + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#user-agents ). + NSString *userAgent = @"grpc-objc/" GRPC_OBJC_VERSION_STRING; + if (_userAgentPrefix) { + userAgent = [_userAgentPrefix stringByAppendingFormat:@" %@", userAgent]; + } + args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] = userAgent; - // TODO(jcanizales): Add OS and device information (see - // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#user-agents ). - NSString *userAgent = @"grpc-objc/" GRPC_OBJC_VERSION_STRING; - if (_userAgentPrefix) { - userAgent = [@[_userAgentPrefix, userAgent] componentsJoinedByString:@" "]; - } - args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] = userAgent; - - if (_secure) { - if (_hostNameOverride) { - args[@GRPC_SSL_TARGET_NAME_OVERRIDE_ARG] = _hostNameOverride; - } - - _channel = [GRPCChannel secureChannelWithHost:_address - pathToCertificates:_pathToCertificates - channelArgs:args]; - } else { - _channel = [GRPCChannel insecureChannelWithHost:_address channelArgs:args]; - } + if (_secure && _hostNameOverride) { + args[@GRPC_SSL_TARGET_NAME_OVERRIDE_ARG] = _hostNameOverride; + } + return args; +} + +- (GRPCChannel *)newChannel { + NSDictionary *args = [self channelArgs]; + if (_secure) { + return [GRPCChannel secureChannelWithHost:_address + pathToCertificates:_pathToCertificates + channelArgs:args]; + } else { + return [GRPCChannel insecureChannelWithHost:_address channelArgs:args]; } - return _channel; } - (NSString *)hostName { @@ -142,7 +135,15 @@ return _hostNameOverride ?: _address; } +- (void)disconnect { + @synchronized(self) { + _channel = nil; + } +} + // TODO(jcanizales): Don't let set |secure| to |NO| if |pathToCertificates| or |hostNameOverride| // have been set. Don't let set either of the latter if |secure| has been set to |NO|. @end + +NS_ASSUME_NONNULL_END diff --git a/src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h b/src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h new file mode 100644 index 00000000000..02871d5d028 --- /dev/null +++ b/src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h @@ -0,0 +1,65 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/** + * "X-macro" file that lists the flags names of Apple's Network Reachability API, along with a nice + * Objective-C method name used to query each of them. + * + * Example usage: To generate a dictionary from flag value to name, one can do: + + NSDictionary *flagNames = @{ +#define GRPC_XMACRO_ITEM(methodName, FlagName) \ + @(kSCNetworkReachabilityFlags ## FlagName): @#methodName, +#include "GRXReachabilityFlagNames.xmacro.h" +#undef GRPC_XMACRO_ITEM + }; + + XCTAssertEqualObjects(flagNames[@(kSCNetworkReachabilityFlagsIsWWAN)], @"isCell"); + + */ + +#ifndef GRPC_XMACRO_ITEM +#error This file is to be used with the "X-macro" pattern: Please #define \ + GRPC_XMACRO_ITEM(methodName, FlagName), then #include this file, and then #undef \ + GRPC_XMACRO_ITEM. +#endif + +GRPC_XMACRO_ITEM(isCell, IsWWAN) +GRPC_XMACRO_ITEM(reachable, Reachable) +GRPC_XMACRO_ITEM(transientConnection, TransientConnection) +GRPC_XMACRO_ITEM(connectionRequired, ConnectionRequired) +GRPC_XMACRO_ITEM(connectionOnTraffic, ConnectionOnTraffic) +GRPC_XMACRO_ITEM(interventionRequired, InterventionRequired) +GRPC_XMACRO_ITEM(connectionOnDemand, ConnectionOnDemand) +GRPC_XMACRO_ITEM(isLocalAddress, IsLocalAddress) +GRPC_XMACRO_ITEM(isDirect, IsDirect) diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index 71e7e0e54ee..e37ed1b59fb 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -34,7 +34,6 @@ #import #include -#import "GRPCChannel.h" #import "GRPCRequestHeaders.h" @interface GRPCOperation : NSObject @@ -94,4 +93,5 @@ - (void)startBatchWithOperations:(NSArray *)ops; - (void)cancel; + @end From 07a2a70bca9a2ffa3c69b43a49f29c4d52808c98 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 10:49:56 -0800 Subject: [PATCH 68/97] Not call external methods from within a critical section --- src/objective-c/GRPCClient/private/GRPCHost.m | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index b6a6d48d964..508cb206444 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -94,12 +94,15 @@ NS_ASSUME_NONNULL_BEGIN - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path completionQueue:(GRPCCompletionQueue *)queue { + GRPCChannel *channel; + // This is racing -[GRPCHost disconnect]. @synchronized(self) { if (!_channel) { _channel = [self newChannel]; } - return [_channel unmanagedCallWithPath:path completionQueue:queue]; + channel = _channel; } + return [channel unmanagedCallWithPath:path completionQueue:queue]; } - (NSDictionary *)channelArgs { @@ -136,6 +139,7 @@ NS_ASSUME_NONNULL_BEGIN } - (void)disconnect { + // This is racing -[GRPCHost unmanagedCallWithPath:completionQueue:]. @synchronized(self) { _channel = nil; } From 8b414f184db9295e30164140d54357c6ad310368 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 10:28:40 -0800 Subject: [PATCH 69/97] Clean up ownership of the connection loss handler --- .../private/GRPCConnectivityMonitor.h | 2 +- .../private/GRPCConnectivityMonitor.m | 22 +++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h index d9d6c22ae9d..2fae4103315 100644 --- a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h +++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h @@ -68,7 +68,7 @@ @property(nonatomic, strong, null_resettable) dispatch_queue_t queue; /** - * Calls handler the next time the connectivity to this instance's host is lost. If this instance is + * Calls handler every time the connectivity to this instance's host is lost. If this instance is * released before that happens, the handler won't be called. * Only one handler is active at a time, so if this method is called again before the previous * handler has been called, it might never be called at all (or yes, if it has already been queued). diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m index f66a77c9b20..b4061bd5ef5 100644 --- a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m +++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m @@ -106,13 +106,16 @@ #pragma mark Connectivity Monitor -// Assumes the third argument is a block that accepts SCNetworkReachabilityFlags, and passes the +// Assumes the third argument is a block that accepts a GRPCReachabilityFlags object, and passes the // received ones to it. static void PassFlagsToContextInfoBlock(SCNetworkReachabilityRef target, SCNetworkReachabilityFlags flags, void *info) { #pragma unused (target) - ((__bridge void (^)(SCNetworkReachabilityFlags))info)(flags); + // This can be called many times with the same info. The info is retained by SCNetworkReachability + // while this function is being executed. + void (^handler)(GRPCReachabilityFlags *) = (__bridge void (^)(GRPCReachabilityFlags *))info; + handler([[GRPCReachabilityFlags alloc] initWithFlags:flags]); } @implementation GRPCConnectivityMonitor { @@ -147,29 +150,30 @@ static void PassFlagsToContextInfoBlock(SCNetworkReachabilityRef target, } - (void)handleLossWithHandler:(void (^)())handler { - __weak typeof(self) weakSelf = self; [self startListeningWithHandler:^(GRPCReachabilityFlags *flags) { if (!flags.isHostReachable) { - [weakSelf stopListening]; handler(); } }]; } - (void)startListeningWithHandler:(void (^)(GRPCReachabilityFlags *))handler { + // Copy to ensure the handler block is in the heap (and so can't be deallocated when this method + // returns). + void (^copiedHandler)(GRPCReachabilityFlags *) = [handler copy]; SCNetworkReachabilityContext context = { .version = 0, - .info = (__bridge_retained void *)^(SCNetworkReachabilityFlags flags){ - // Pass the flags as as Objective-C wrapper. - handler([[GRPCReachabilityFlags alloc] initWithFlags:flags]); - }, - .release = CFRelease + .info = (__bridge void *)copiedHandler, + .retain = CFRetain, + .release = CFRelease, }; + // The following will retain context.info, and release it when the callback is set to NULL. SCNetworkReachabilitySetCallback(_reachabilityRef, PassFlagsToContextInfoBlock, &context); SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, _queue); } - (void)stopListening { + // This releases the block on context.info. SCNetworkReachabilitySetCallback(_reachabilityRef, NULL, NULL); SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, NULL); } From 232b6a8f191d9cbb905680dd2a95f3b1815d1eb7 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 12:26:27 -0800 Subject: [PATCH 70/97] Test robustness of WriteableSingleHandler against non-single Writers --- src/objective-c/tests/RxLibraryUnitTests.m | 51 ++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/objective-c/tests/RxLibraryUnitTests.m b/src/objective-c/tests/RxLibraryUnitTests.m index d3426628141..ae9465f58cc 100644 --- a/src/objective-c/tests/RxLibraryUnitTests.m +++ b/src/objective-c/tests/RxLibraryUnitTests.m @@ -64,6 +64,8 @@ } @end +// TODO(jcanizales): Split into one file per tested class. + @interface RxLibraryUnitTests : XCTestCase @end @@ -79,6 +81,7 @@ // If: id writeable = [GRXWriteable writeableWithSingleHandler:handler.block]; [writeable writeValue:anyValue]; + [writeable writesFinishedWithError:nil]; // Then: XCTAssertEqual(handler.timesCalled, 1); @@ -101,6 +104,54 @@ XCTAssertEqualObjects(handler.errorOrNil, anyError); } +- (void)testWriteableSingleHandlerIsCalledOnlyOnce_ValueThenError { + // Given: + CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler]; + id anyValue = @7; + NSError *anyError = [NSError errorWithDomain:@"domain" code:7 userInfo:nil]; + + // If: + id writeable = [GRXWriteable writeableWithSingleHandler:handler.block]; + [writeable writeValue:anyValue]; + [writeable writesFinishedWithError:anyError]; + + // Then: + XCTAssertEqual(handler.timesCalled, 1); + XCTAssertEqualObjects(handler.value, anyValue); + XCTAssertEqualObjects(handler.errorOrNil, nil); +} + +- (void)testWriteableSingleHandlerIsCalledOnlyOnce_ValueThenValue { + // Given: + CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler]; + id anyValue = @7; + + // If: + id writeable = [GRXWriteable writeableWithSingleHandler:handler.block]; + [writeable writeValue:anyValue]; + [writeable writeValue:anyValue]; + [writeable writesFinishedWithError:nil]; + + // Then: + XCTAssertEqual(handler.timesCalled, 1); + XCTAssertEqualObjects(handler.value, anyValue); + XCTAssertEqualObjects(handler.errorOrNil, nil); +} + +- (void)testWriteableSingleHandlerFailsOnEmptyWriter { + // Given: + CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler]; + + // If: + id writeable = [GRXWriteable writeableWithSingleHandler:handler.block]; + [writeable writesFinishedWithError:nil]; + + // Then: + XCTAssertEqual(handler.timesCalled, 1); + XCTAssertEqualObjects(handler.value, nil); + XCTAssertNotNil(handler.errorOrNil); +} + #pragma mark BufferedPipe - (void)testBufferedPipePropagatesValue { From 5e6723203013b365c2a60158578f73abf480b8bf Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Wed, 9 Mar 2016 13:34:28 -0800 Subject: [PATCH 71/97] Make Writeable with single handler robust against stream Writers --- src/objective-c/RxLibrary/GRXWriteable.m | 37 ++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/objective-c/RxLibrary/GRXWriteable.m b/src/objective-c/RxLibrary/GRXWriteable.m index 2729d62b72f..b122c04ce0c 100644 --- a/src/objective-c/RxLibrary/GRXWriteable.m +++ b/src/objective-c/RxLibrary/GRXWriteable.m @@ -42,11 +42,38 @@ if (!handler) { return [[self alloc] init]; } - return [[self alloc] initWithValueHandler:^(id value) { - handler(value, nil); - } completionHandler:^(NSError *errorOrNil) { - if (errorOrNil) { - handler(nil, errorOrNil); + // We nilify this variable when the block is invoked, so that handler is only invoked once even if + // the writer tries to write multiple values. + __block GRXEventHandler eventHandler = ^(BOOL done, id value, NSError *error) { + // Nillify eventHandler before invoking handler, in case the latter causes the former to be + // executed recursively. Because blocks can be deallocated even during execution, we have to + // first retain handler locally to guarantee it's valid. + // TODO(jcanizales): Just turn this craziness into a simple subclass of GRXWriteable. + GRXSingleHandler singleHandler = handler; + eventHandler = nil; + + if (value) { + singleHandler(value, nil); + } else if (error) { + singleHandler(nil, error); + } else { + NSDictionary *userInfo = @{ + NSLocalizedDescriptionKey: @"The writer finished without producing any value." + }; + // Even though RxLibrary is independent of gRPC, the domain and code here are, for the moment, + // set to the values of kGRPCErrorDomain and GRPCErrorCodeInternal. This way, the error formed + // is the one user of gRPC would expect if the server failed to produce a response. + // + // TODO(jcanizales): Figure out a way to keep errors of RxLibrary generic without making users + // of gRPC take care of two different error domains and error code enums. A possibility is to + // add error handling to GRXWriters or GRXWriteables, and use them to translate errors between + // the two domains. + singleHandler(nil, [NSError errorWithDomain:@"io.grpc" code:13 userInfo:userInfo]); + } + }; + return [self writeableWithEventHandler:^(BOOL done, id value, NSError *error) { + if (eventHandler) { + eventHandler(done, value, error); } }]; } From cfcaccc6d20c67a2595b02b18fc31a3a2da83daf Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 12:36:18 -0800 Subject: [PATCH 72/97] Revert "Revert "Pass a non-infinite deadline to grpc_completion_queue_next() to prevent queues from blocking indefinitely in poll()"" --- .../GRPCClient/private/GRPCCompletionQueue.h | 7 ++++++ .../GRPCClient/private/GRPCCompletionQueue.m | 24 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h index 7b66cd4c329..a52095dd011 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h @@ -36,6 +36,8 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); +extern const int64_t kGRPCCompletionQueueDefaultTimeoutSecs; + /** * This class lets one more easily use |grpc_completion_queue|. To use it, pass the value of the * |unmanagedQueue| property of an instance of this class to |grpc_channel_create_call|. Then for @@ -49,6 +51,11 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); */ @interface GRPCCompletionQueue : NSObject @property(nonatomic, readonly) grpc_completion_queue *unmanagedQueue; +@property(nonatomic, readonly) int64_t timeoutSecs; + (instancetype)completionQueue; + +- (instancetype)init; +- (instancetype)initWithTimeout:(int64_t)timeoutSecs NS_DESIGNATED_INITIALIZER; + @end diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index ff3031678c5..250bbf03a66 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -35,6 +35,9 @@ #import + +const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; + @implementation GRPCCompletionQueue + (instancetype)completionQueue { @@ -42,8 +45,13 @@ } - (instancetype)init { + return [self initWithTimeout:kGRPCCompletionQueueDefaultTimeoutSecs]; +} + +- (instancetype)initWithTimeout:(int64_t)timeoutSecs { if ((self = [super init])) { _unmanagedQueue = grpc_completion_queue_create(NULL); + _timeoutSecs = timeoutSecs; // This is for the following block to capture the pointer by value (instead // of retaining self and doing self->_unmanagedQueue). This is essential @@ -61,22 +69,28 @@ gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ + // Using a non-infinite deadline to re-enter grpc_completion_queue_next() + // alleviates https://github.com/grpc/grpc/issues/5593 + gpr_timespec deadline = (timeoutSecs < 0) + ? gpr_inf_future(GPR_CLOCK_REALTIME) + : gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); while (YES) { - // The following call blocks until an event is available. - grpc_event event = grpc_completion_queue_next(unmanagedQueue, - gpr_inf_future(GPR_CLOCK_REALTIME), - NULL); + // The following call blocks until an event is available or the deadline elapses. + grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); GRPCQueueCompletionHandler handler; switch (event.type) { case GRPC_OP_COMPLETE: handler = (__bridge_transfer GRPCQueueCompletionHandler)event.tag; handler(event.success); break; + case GRPC_QUEUE_TIMEOUT: + // Nothing to do here + break; case GRPC_QUEUE_SHUTDOWN: grpc_completion_queue_destroy(unmanagedQueue); return; default: - [NSException raise:@"Unrecognized completion type" format:@""]; + [NSException raise:@"Unrecognized completion type" format:@"type=%d", event.type]; } }; }); From aad4156a0ba79d963bfa3cd37e48a1074f956809 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 11 Mar 2016 22:21:11 +0100 Subject: [PATCH 73/97] Disabling iOS user-agent test. --- src/objective-c/tests/GRPCClientTests.m | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 624958f4b97..7dd6873c802 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -273,10 +273,12 @@ static ProtoMethod *kUnaryCallMethod; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { XCTAssertNotNil(value, @"nil value received as response."); XCTAssertEqual([value length], 0, @"Non-empty response received: %@", value); + /* This test needs to be more clever in regards to changing the version of the core. XCTAssertEqualObjects(call.responseHeaders[@"x-grpc-test-echo-useragent"], @"Foo grpc-objc/0.13.0 grpc-c/0.14.0-dev (ios)", @"Did not receive expected user agent %@", call.responseHeaders[@"x-grpc-test-echo-useragent"]); + */ [response fulfill]; } completionHandler:^(NSError *errorOrNil) { XCTAssertNil(errorOrNil, @"Finished with unexpected error: %@", errorOrNil); From afc59cbc886424e9c3ca9b5da1ffd2a817ee903c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 11 Mar 2016 15:06:15 -0800 Subject: [PATCH 74/97] remove tests for EventInvocation API that is going to be removed --- src/python/grpcio/tests/tests.json | 9 - ...nt_invocation_synchronous_event_service.py | 381 ------------------ .../framework/interfaces/face/test_cases.py | 2 - 3 files changed, 392 deletions(-) delete mode 100644 src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py diff --git a/src/python/grpcio/tests/tests.json b/src/python/grpcio/tests/tests.json index 388d040d5ca..84870aaa5cb 100644 --- a/src/python/grpcio/tests/tests.json +++ b/src/python/grpcio/tests/tests.json @@ -12,31 +12,22 @@ "_core_over_links_base_interface_test.SyncEasyTest", "_core_over_links_base_interface_test.SyncPeasyTest", "_crust_over_core_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_over_links_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_over_links_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", - "_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", - "_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", - "_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", "_implementations_test.ChannelCredentialsTest", "_insecure_interop_test.InsecureInteropTest", diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py deleted file mode 100644 index 34db6c3e55f..00000000000 --- a/src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py +++ /dev/null @@ -1,381 +0,0 @@ -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Test code for the Face layer of RPC Framework.""" - -import abc -import unittest - -# test_interfaces is referenced from specification in this module. -from grpc.framework.interfaces.face import face -from tests.unit.framework.common import test_constants -from tests.unit.framework.common import test_control -from tests.unit.framework.common import test_coverage -from tests.unit.framework.interfaces.face import _3069_test_constant -from tests.unit.framework.interfaces.face import _digest -from tests.unit.framework.interfaces.face import _receiver -from tests.unit.framework.interfaces.face import _stock_service -from tests.unit.framework.interfaces.face import test_interfaces # pylint: disable=unused-import - - -class TestCase(test_coverage.Coverage, unittest.TestCase): - """A test of the Face layer of RPC Framework. - - Concrete subclasses must have an "implementation" attribute of type - test_interfaces.Implementation and an "invoker_constructor" attribute of type - _invocation.InvokerConstructor. - """ - __metaclass__ = abc.ABCMeta - - NAME = 'EventInvocationSynchronousEventServiceTest' - - def setUp(self): - """See unittest.TestCase.setUp for full specification. - - Overriding implementations must call this implementation. - """ - self._control = test_control.PauseFailControl() - self._digest = _digest.digest( - _stock_service.STOCK_TEST_SERVICE, self._control, None) - - generic_stub, dynamic_stubs, self._memo = self.implementation.instantiate( - self._digest.methods, self._digest.event_method_implementations, None) - self._invoker = self.invoker_constructor.construct_invoker( - generic_stub, dynamic_stubs, self._digest.methods) - - def tearDown(self): - """See unittest.TestCase.tearDown for full specification. - - Overriding implementations must call this implementation. - """ - self._invoker = None - self.implementation.destantiate(self._memo) - - def testSuccessfulUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - receiver.block_until_terminated() - response = receiver.unary_response() - - test_messages.verify(request, response, self) - - def testSuccessfulUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - receiver.block_until_terminated() - responses = receiver.stream_responses() - - test_messages.verify(request, responses, self) - - def testSuccessfulStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.terminate() - receiver.block_until_terminated() - response = receiver.unary_response() - - test_messages.verify(requests, response, self) - - def testSuccessfulStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.terminate() - receiver.block_until_terminated() - responses = receiver.stream_responses() - - test_messages.verify(requests, responses, self) - - def testSequentialInvocations(self): - # pylint: disable=cell-var-from-loop - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - first_request = test_messages.request() - second_request = test_messages.request() - second_receiver = _receiver.Receiver() - - def make_second_invocation(): - self._invoker.event(group, method)( - second_request, second_receiver, second_receiver.abort, - test_constants.LONG_TIMEOUT) - - class FirstReceiver(_receiver.Receiver): - - def complete(self, terminal_metadata, code, details): - super(FirstReceiver, self).complete( - terminal_metadata, code, details) - make_second_invocation() - - first_receiver = FirstReceiver() - - self._invoker.event(group, method)( - first_request, first_receiver, first_receiver.abort, - test_constants.LONG_TIMEOUT) - second_receiver.block_until_terminated() - - first_response = first_receiver.unary_response() - second_response = second_receiver.unary_response() - test_messages.verify(first_request, first_response, self) - test_messages.verify(second_request, second_response, self) - - def testParallelInvocations(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - first_request = test_messages.request() - first_receiver = _receiver.Receiver() - second_request = test_messages.request() - second_receiver = _receiver.Receiver() - - self._invoker.event(group, method)( - first_request, first_receiver, first_receiver.abort, - test_constants.LONG_TIMEOUT) - self._invoker.event(group, method)( - second_request, second_receiver, second_receiver.abort, - test_constants.LONG_TIMEOUT) - first_receiver.block_until_terminated() - second_receiver.block_until_terminated() - - first_response = first_receiver.unary_response() - second_response = second_receiver.unary_response() - test_messages.verify(first_request, first_response, self) - test_messages.verify(second_request, second_response, self) - - @unittest.skip('TODO(nathaniel): implement.') - def testWaitingForSomeButNotAllParallelInvocations(self): - raise NotImplementedError() - - def testCancelledUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.pause(): - call = self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - call.cancel() - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind) - - def testCancelledUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - call = self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - call.cancel() - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind) - - def testCancelledStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.cancel() - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind) - - def testCancelledStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_stream_messages_sequences.iteritems()): - for unused_test_messages in test_messages_sequence: - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - call_consumer.cancel() - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind) - - def testExpiredUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.pause(): - self._invoker.event(group, method)( - request, receiver, receiver.abort, - _3069_test_constant.REALLY_SHORT_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind) - - def testExpiredUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.pause(): - self._invoker.event(group, method)( - request, receiver, receiver.abort, - _3069_test_constant.REALLY_SHORT_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind) - - def testExpiredStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_unary_messages_sequences.iteritems()): - for unused_test_messages in test_messages_sequence: - receiver = _receiver.Receiver() - - self._invoker.event(group, method)( - receiver, receiver.abort, _3069_test_constant.REALLY_SHORT_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind) - - def testExpiredStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, _3069_test_constant.REALLY_SHORT_TIMEOUT) - for request in requests: - call_consumer.consume(request) - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind) - - def testFailedUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.fail(): - self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs( - face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind) - - def testFailedUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.fail(): - self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs( - face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind) - - def testFailedStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - with self._control.fail(): - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.terminate() - receiver.block_until_terminated() - - self.assertIs( - face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind) - - def testFailedStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - with self._control.fail(): - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.terminate() - receiver.block_until_terminated() - - self.assertIs( - face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind) diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py index 462829b6606..71de9d835e9 100644 --- a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py +++ b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py @@ -34,14 +34,12 @@ import unittest # pylint: disable=unused-import # test_interfaces is referenced from specification in this module. from tests.unit.framework.interfaces.face import _blocking_invocation_inline_service -from tests.unit.framework.interfaces.face import _event_invocation_synchronous_event_service from tests.unit.framework.interfaces.face import _future_invocation_asynchronous_event_service from tests.unit.framework.interfaces.face import _invocation from tests.unit.framework.interfaces.face import test_interfaces # pylint: disable=unused-import _TEST_CASE_SUPERCLASSES = ( _blocking_invocation_inline_service.TestCase, - _event_invocation_synchronous_event_service.TestCase, _future_invocation_asynchronous_event_service.TestCase, ) From 8734e02f52025219de2a3908f1eb0002568d1ecd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 11 Mar 2016 15:52:53 -0800 Subject: [PATCH 75/97] fix copyright --- .../grpcio/tests/unit/framework/interfaces/face/test_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py index 71de9d835e9..06b9d77e521 100644 --- a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py +++ b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From 57e2743b6580ae19326ceccb5ac9ed88e0691a94 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 16:53:58 -0800 Subject: [PATCH 76/97] Add contributed test --- Makefile | 36 ++++ build.yaml | 10 + .../surface/concurrent_connectivity_test.c | 48 +++++ tools/run_tests/sources_and_headers.json | 16 ++ tools/run_tests/tests.json | 21 ++ vsprojects/buildtests_c.sln | 27 +++ .../concurrent_connectivity_test.vcxproj | 199 ++++++++++++++++++ ...ncurrent_connectivity_test.vcxproj.filters | 21 ++ 8 files changed, 378 insertions(+) create mode 100644 test/core/surface/concurrent_connectivity_test.c create mode 100644 vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj create mode 100644 vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj.filters diff --git a/Makefile b/Makefile index 171a83c2a3a..f118d542538 100644 --- a/Makefile +++ b/Makefile @@ -849,6 +849,7 @@ chttp2_status_conversion_test: $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test chttp2_stream_map_test: $(BINDIR)/$(CONFIG)/chttp2_stream_map_test chttp2_varint_test: $(BINDIR)/$(CONFIG)/chttp2_varint_test compression_test: $(BINDIR)/$(CONFIG)/compression_test +concurrent_connectivity_test: $(BINDIR)/$(CONFIG)/concurrent_connectivity_test dns_resolver_connectivity_test: $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test dns_resolver_test: $(BINDIR)/$(CONFIG)/dns_resolver_test dualstack_socket_test: $(BINDIR)/$(CONFIG)/dualstack_socket_test @@ -1160,6 +1161,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/chttp2_stream_map_test \ $(BINDIR)/$(CONFIG)/chttp2_varint_test \ $(BINDIR)/$(CONFIG)/compression_test \ + $(BINDIR)/$(CONFIG)/concurrent_connectivity_test \ $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test \ $(BINDIR)/$(CONFIG)/dns_resolver_test \ $(BINDIR)/$(CONFIG)/dualstack_socket_test \ @@ -1404,6 +1406,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/chttp2_varint_test || ( echo test chttp2_varint_test failed ; exit 1 ) $(E) "[RUN] Testing compression_test" $(Q) $(BINDIR)/$(CONFIG)/compression_test || ( echo test compression_test failed ; exit 1 ) + $(E) "[RUN] Testing concurrent_connectivity_test" + $(Q) $(BINDIR)/$(CONFIG)/concurrent_connectivity_test || ( echo test concurrent_connectivity_test failed ; exit 1 ) $(E) "[RUN] Testing dns_resolver_connectivity_test" $(Q) $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test || ( echo test dns_resolver_connectivity_test failed ; exit 1 ) $(E) "[RUN] Testing dns_resolver_test" @@ -6079,6 +6083,38 @@ endif endif +CONCURRENT_CONNECTIVITY_TEST_SRC = \ + test/core/surface/concurrent_connectivity_test.c \ + +CONCURRENT_CONNECTIVITY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(CONCURRENT_CONNECTIVITY_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/concurrent_connectivity_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/concurrent_connectivity_test: $(CONCURRENT_CONNECTIVITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(CONCURRENT_CONNECTIVITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/concurrent_connectivity_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/surface/concurrent_connectivity_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_concurrent_connectivity_test: $(CONCURRENT_CONNECTIVITY_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(CONCURRENT_CONNECTIVITY_TEST_OBJS:.o=.dep) +endif +endif + + DNS_RESOLVER_CONNECTIVITY_TEST_SRC = \ test/core/client_config/resolvers/dns_resolver_connectivity_test.c \ diff --git a/build.yaml b/build.yaml index e003dd83396..83b7714e655 100644 --- a/build.yaml +++ b/build.yaml @@ -1057,6 +1057,16 @@ targets: - grpc - gpr_test_util - gpr +- name: concurrent_connectivity_test + build: test + language: c + src: + - test/core/surface/concurrent_connectivity_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: dns_resolver_connectivity_test cpu_cost: 0.1 build: test diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c new file mode 100644 index 00000000000..d8639f5e510 --- /dev/null +++ b/test/core/surface/concurrent_connectivity_test.c @@ -0,0 +1,48 @@ +#include + +#include +#include +#include + +#define NUM_THREADS 100 +static grpc_channel* channels[NUM_THREADS]; +static grpc_completion_queue* queues[NUM_THREADS]; + +void create_loop_destroy(void* actually_an_int) { + int thread_index = (int)(actually_an_int); + for (int i = 0; i < 10; ++i) { + grpc_completion_queue* cq = grpc_completion_queue_create(NULL); + grpc_channel* chan = grpc_insecure_channel_create("localhost", NULL, NULL); + + channels[thread_index] = chan; + queues[thread_index] = cq; + + gpr_timespec inf_future = gpr_inf_future(GPR_CLOCK_REALTIME); + gpr_timespec delta = gpr_time_from_millis(10, GPR_TIMESPAN); + for (int j = 0; j < 10; ++j) { + gpr_timespec later_time = + gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), delta); + grpc_connectivity_state state = + grpc_channel_check_connectivity_state(chan, 1); + grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); + grpc_completion_queue_next(cq, inf_future, NULL); + } + grpc_channel_destroy(channels[thread_index]); + grpc_completion_queue_destroy(queues[thread_index]); + } +} + +int main() { + grpc_init(); + gpr_thd_id threads[NUM_THREADS]; + for (intptr_t i = 0; i < NUM_THREADS; ++i) { + gpr_thd_options options = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&options); + gpr_thd_new(&threads[i], create_loop_destroy, (void*)i, &options); + } + for (int i = 0; i < NUM_THREADS; ++i) { + gpr_thd_join(threads[i]); + } + grpc_shutdown(); + return 0; +} diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 0039b99a55e..092ed35ad97 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -189,6 +189,22 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "concurrent_connectivity_test", + "src": [ + "test/core/surface/concurrent_connectivity_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 724cd64d19c..000315f2f4a 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -253,6 +253,27 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "concurrent_connectivity_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index be2c4a3bb14..413ed3e3f38 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -251,6 +251,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "compression_test", "vcxproj {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "concurrent_connectivity_test", "vcxproj\test\concurrent_connectivity_test\concurrent_connectivity_test.vcxproj", "{391B366C-D916-45AA-3FE5-67363A46193B}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dns_resolver_connectivity_test", "vcxproj\test\dns_resolver_connectivity_test\dns_resolver_connectivity_test.vcxproj", "{F7B6FE68-E847-D7CA-4062-E737E542BCC3}" ProjectSection(myProperties) = preProject lib = "False" @@ -1747,6 +1758,22 @@ Global {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|Win32.Build.0 = Release|Win32 {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|x64.ActiveCfg = Release|x64 {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|x64.Build.0 = Release|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug|Win32.ActiveCfg = Debug|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug|x64.ActiveCfg = Debug|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release|Win32.ActiveCfg = Release|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release|x64.ActiveCfg = Release|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug|Win32.Build.0 = Debug|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug|x64.Build.0 = Debug|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release|Win32.Build.0 = Release|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release|x64.Build.0 = Release|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug-DLL|x64.Build.0 = Debug|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release-DLL|Win32.Build.0 = Release|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release-DLL|x64.ActiveCfg = Release|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release-DLL|x64.Build.0 = Release|x64 {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|Win32.ActiveCfg = Debug|Win32 {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|x64.ActiveCfg = Debug|x64 {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj b/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj new file mode 100644 index 00000000000..6d69eae302f --- /dev/null +++ b/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj @@ -0,0 +1,199 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {391B366C-D916-45AA-3FE5-67363A46193B} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + concurrent_connectivity_test + static + Debug + static + Debug + + + concurrent_connectivity_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj.filters b/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj.filters new file mode 100644 index 00000000000..a6492c775a2 --- /dev/null +++ b/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\core\surface + + + + + + {1320b717-a107-004b-c517-074e6a7baf97} + + + {f205219c-cee2-c0e8-a922-5486f2d31026} + + + {6804b6ae-88f4-acf9-b3fc-6b12f8e28d4d} + + + + From 0ce7bec923867aae3b115a9a7e4fc2e56f021353 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 17:02:07 -0800 Subject: [PATCH 77/97] Fail faster --- test/core/surface/concurrent_connectivity_test.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index d8639f5e510..1046d422774 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -2,7 +2,9 @@ #include #include +#include #include +#include "test/core/util/test_config.h" #define NUM_THREADS 100 static grpc_channel* channels[NUM_THREADS]; @@ -17,22 +19,22 @@ void create_loop_destroy(void* actually_an_int) { channels[thread_index] = chan; queues[thread_index] = cq; - gpr_timespec inf_future = gpr_inf_future(GPR_CLOCK_REALTIME); - gpr_timespec delta = gpr_time_from_millis(10, GPR_TIMESPAN); for (int j = 0; j < 10; ++j) { - gpr_timespec later_time = - gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), delta); + gpr_timespec later_time = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10); grpc_connectivity_state state = grpc_channel_check_connectivity_state(chan, 1); grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); - grpc_completion_queue_next(cq, inf_future, NULL); + GPR_ASSERT(grpc_completion_queue_next( + cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL) + .type == GRPC_OP_COMPLETE); } grpc_channel_destroy(channels[thread_index]); grpc_completion_queue_destroy(queues[thread_index]); } } -int main() { +int main(int argc, char** argv) { + grpc_test_init(argc, argv); grpc_init(); gpr_thd_id threads[NUM_THREADS]; for (intptr_t i = 0; i < NUM_THREADS; ++i) { From 2ba4133d3a260f3fe218160ef6fe68e0edc1e4a7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 17:09:12 -0800 Subject: [PATCH 78/97] Fix race in subchannel.c --- src/core/client_config/subchannel.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index d91dd116b8c..ddd129c5397 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -184,8 +184,8 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(c); } -void grpc_connected_subchannel_ref(grpc_connected_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_connected_subchannel_ref( + grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON); } @@ -226,8 +226,8 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta, return old_val; } -grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_ref( + grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS), 0 REF_MUTATE_PURPOSE("STRONG_REF")); @@ -235,8 +235,8 @@ grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c return c; } -grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_weak_ref( + grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF")); GPR_ASSERT(old_refs != 0); @@ -646,19 +646,23 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg, if (c->connecting_result.transport != NULL) { publish_transport(exec_ctx, c); - } else if (c->disconnected) { + } + + GRPC_SUBCHANNEL_WEAK_REF(c, "connected"); + gpr_mu_lock(&c->mu); + if (c->disconnected) { GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); } else { gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_mu_lock(&c->mu); GPR_ASSERT(!c->have_alarm); c->have_alarm = 1; grpc_connectivity_state_set(exec_ctx, &c->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE, "connect_failed"); grpc_timer_init(exec_ctx, &c->alarm, c->next_attempt, on_alarm, c, now); - gpr_mu_unlock(&c->mu); } + gpr_mu_unlock(&c->mu); + GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); } static gpr_timespec compute_connect_deadline(grpc_subchannel *c) { @@ -686,8 +690,8 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); } -void grpc_subchannel_call_ref(grpc_subchannel_call *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_subchannel_call_ref( + grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } From 64baf2cbab3bc3caa98907758ee0a6c7db72e68a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 17:16:33 -0800 Subject: [PATCH 79/97] Extend timeout --- test/core/surface/concurrent_connectivity_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 1046d422774..7306a394ebd 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -25,7 +25,7 @@ void create_loop_destroy(void* actually_an_int) { grpc_channel_check_connectivity_state(chan, 1); grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); GPR_ASSERT(grpc_completion_queue_next( - cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL) + cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(30), NULL) .type == GRPC_OP_COMPLETE); } grpc_channel_destroy(channels[thread_index]); From d24fc85b59b4224d7932c5a7a9f17e73402f388b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 17:18:19 -0800 Subject: [PATCH 80/97] Fix cast --- test/core/surface/concurrent_connectivity_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 7306a394ebd..cd1aead5e51 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -11,7 +11,7 @@ static grpc_channel* channels[NUM_THREADS]; static grpc_completion_queue* queues[NUM_THREADS]; void create_loop_destroy(void* actually_an_int) { - int thread_index = (int)(actually_an_int); + int thread_index = (int)(intptr_t)(actually_an_int); for (int i = 0; i < 10; ++i) { grpc_completion_queue* cq = grpc_completion_queue_create(NULL); grpc_channel* chan = grpc_insecure_channel_create("localhost", NULL, NULL); From 5a8ec75012862815bc28a2145c6d294b1993cbd4 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 18:41:54 -0800 Subject: [PATCH 81/97] Use a named constant for error domain and code --- src/objective-c/RxLibrary/GRXWriteable.m | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/objective-c/RxLibrary/GRXWriteable.m b/src/objective-c/RxLibrary/GRXWriteable.m index b122c04ce0c..c15ccb17fb9 100644 --- a/src/objective-c/RxLibrary/GRXWriteable.m +++ b/src/objective-c/RxLibrary/GRXWriteable.m @@ -68,7 +68,11 @@ // of gRPC take care of two different error domains and error code enums. A possibility is to // add error handling to GRXWriters or GRXWriteables, and use them to translate errors between // the two domains. - singleHandler(nil, [NSError errorWithDomain:@"io.grpc" code:13 userInfo:userInfo]); + static NSString *kGRPCErrorDomain = @"io.grpc"; + static NSUInteger kGRPCErrorCodeInternal = 13; + singleHandler(nil, [NSError errorWithDomain:kGRPCErrorDomain + code:kGRPCErrorCodeInternal + userInfo:userInfo]); } }; return [self writeableWithEventHandler:^(BOOL done, id value, NSError *error) { From 3591482a372dee047a7c0867ada8cc8d360d247d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 19:29:23 -0800 Subject: [PATCH 82/97] Expand lock --- src/core/client_config/subchannel.c | 14 +++++--------- test/core/surface/concurrent_connectivity_test.c | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index ddd129c5397..e8f308b6076 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -505,7 +505,8 @@ void grpc_connected_subchannel_ping(grpc_exec_ctx *exec_ctx, elem->filter->start_transport_op(exec_ctx, elem, &op); } -static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { +static void publish_transport_locked(grpc_exec_ctx *exec_ctx, + grpc_subchannel *c) { size_t channel_stack_size; grpc_connected_subchannel *con; grpc_channel_stack *stk; @@ -541,8 +542,6 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { grpc_closure_init(&sw_subchannel->closure, subchannel_on_child_state_changed, sw_subchannel); - gpr_mu_lock(&c->mu); - if (c->disconnected) { gpr_mu_unlock(&c->mu); gpr_free(sw_subchannel); @@ -575,7 +574,6 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { grpc_connectivity_state_set(exec_ctx, &c->state_tracker, GRPC_CHANNEL_READY, "connected"); - gpr_mu_unlock(&c->mu); gpr_free((void *)filters); } @@ -644,13 +642,11 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) { grpc_subchannel *c = arg; - if (c->connecting_result.transport != NULL) { - publish_transport(exec_ctx, c); - } - GRPC_SUBCHANNEL_WEAK_REF(c, "connected"); gpr_mu_lock(&c->mu); - if (c->disconnected) { + if (c->connecting_result.transport != NULL) { + publish_transport_locked(exec_ctx, c); + } else if (c->disconnected) { GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); } else { gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 7306a394ebd..1046d422774 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -25,7 +25,7 @@ void create_loop_destroy(void* actually_an_int) { grpc_channel_check_connectivity_state(chan, 1); grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); GPR_ASSERT(grpc_completion_queue_next( - cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(30), NULL) + cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL) .type == GRPC_OP_COMPLETE); } grpc_channel_destroy(channels[thread_index]); From 9fb89ea9e31dbe376fc1ce9c173d4e52f4b31290 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 20:12:45 -0800 Subject: [PATCH 83/97] If we cant check timers due to contention, at least make sure we do a follow up check soon --- src/core/client_config/subchannel.c | 16 ++++---- src/core/iomgr/timer.c | 3 ++ src/core/iomgr/timer.h | 1 - src/core/surface/completion_queue.c | 2 +- .../surface/concurrent_connectivity_test.c | 39 +++++++++++++++++-- test/core/surface/lame_client_test.c | 2 +- 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index e8f308b6076..ec9e47a6ddb 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -184,8 +184,8 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(c); } -void grpc_connected_subchannel_ref( - grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_connected_subchannel_ref(grpc_connected_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON); } @@ -226,8 +226,8 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta, return old_val; } -grpc_subchannel *grpc_subchannel_ref( - grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS), 0 REF_MUTATE_PURPOSE("STRONG_REF")); @@ -235,8 +235,8 @@ grpc_subchannel *grpc_subchannel_ref( return c; } -grpc_subchannel *grpc_subchannel_weak_ref( - grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF")); GPR_ASSERT(old_refs != 0); @@ -686,8 +686,8 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); } -void grpc_subchannel_call_ref( - grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_subchannel_call_ref(grpc_subchannel_call *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 8379fffad02..8badd83bbb1 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -330,6 +330,9 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_mu_unlock(&g_mu); gpr_mu_unlock(&g_checker_mu); + } else if (next != NULL) { + *next = gpr_time_min( + *next, gpr_time_add(now, gpr_time_from_millis(100, GPR_TIMESPAN))); } return (int)n; diff --git a/src/core/iomgr/timer.h b/src/core/iomgr/timer.h index 9ad1e92f42e..e239e884e7b 100644 --- a/src/core/iomgr/timer.h +++ b/src/core/iomgr/timer.h @@ -96,7 +96,6 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer); *next is never guaranteed to be updated on any given execution; however, with high probability at least one thread in the system will see an update at any time slice. */ - bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_timespec *next); void grpc_timer_list_init(gpr_timespec now); diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index f6a95ebbd3f..b22818ea87d 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -86,7 +86,7 @@ struct grpc_completion_queue { #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1)) static gpr_mu g_freelist_mu; -grpc_completion_queue *g_freelist; +static grpc_completion_queue *g_freelist; static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc, bool success); diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index b380d4471fc..5361201db72 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -1,3 +1,36 @@ +/* +* +* Copyright 2015-2016, Google Inc. +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following disclaimer +* in the documentation and/or other materials provided with the +* distribution. +* * Neither the name of Google Inc. nor the names of its +* contributors may be used to endorse or promote products derived from +* this software without specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +*/ + #include #include @@ -24,9 +57,9 @@ void create_loop_destroy(void* actually_an_int) { grpc_connectivity_state state = grpc_channel_check_connectivity_state(chan, 1); grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); - GPR_ASSERT(grpc_completion_queue_next( - cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL) - .type == GRPC_OP_COMPLETE); + GPR_ASSERT(grpc_completion_queue_next(cq, + GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), + NULL).type == GRPC_OP_COMPLETE); } grpc_channel_destroy(channels[thread_index]); grpc_completion_queue_destroy(queues[thread_index]); diff --git a/test/core/surface/lame_client_test.c b/test/core/surface/lame_client_test.c index 79e53cb4222..c154d99d4cc 100644 --- a/test/core/surface/lame_client_test.c +++ b/test/core/surface/lame_client_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From a45782f090dfd997d2e80bb7761ddc8db4481b0b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 20:15:51 -0800 Subject: [PATCH 84/97] Fix copyright --- test/core/surface/concurrent_connectivity_test.c | 2 +- test/core/surface/lame_client_test.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 5361201db72..4d3b7bf22a3 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -1,6 +1,6 @@ /* * -* Copyright 2015-2016, Google Inc. +* Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/surface/lame_client_test.c b/test/core/surface/lame_client_test.c index c154d99d4cc..79e53cb4222 100644 --- a/test/core/surface/lame_client_test.c +++ b/test/core/surface/lame_client_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 822f1d7509fe7426a291de1054ea5e9f4892ee15 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 12 Mar 2016 06:54:47 -0800 Subject: [PATCH 85/97] Add comment, tighten loop --- src/core/iomgr/timer.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 8badd83bbb1..f444643428d 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -33,11 +33,11 @@ #include "src/core/iomgr/timer.h" -#include "src/core/iomgr/timer_heap.h" -#include "src/core/iomgr/time_averaged_stats.h" #include #include #include +#include "src/core/iomgr/time_averaged_stats.h" +#include "src/core/iomgr/timer_heap.h" #define INVALID_HEAP_INDEX 0xffffffffu @@ -331,8 +331,17 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_mu_unlock(&g_mu); gpr_mu_unlock(&g_checker_mu); } else if (next != NULL) { + /* TODO(ctiller): this forces calling code to do an short poll, and + then retry the timer check (because this time through the timer list was + contended). + + We could reduce the cost here dramatically by keeping a count of how many + currently active pollers got through the uncontended case above + successfully, and waking up other pollers IFF that count drops to zero. + + Once that count is in place, this entire else branch could disappear. */ *next = gpr_time_min( - *next, gpr_time_add(now, gpr_time_from_millis(100, GPR_TIMESPAN))); + *next, gpr_time_add(now, gpr_time_from_millis(1, GPR_TIMESPAN))); } return (int)n; From 2480f98e192f9e9a00c7cabcd6a77321f1d9525e Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Fri, 26 Feb 2016 14:18:46 -0800 Subject: [PATCH 86/97] Add a script to run interop tests against all versions of prod servers. --- tools/jenkins/run_full_interop.sh | 37 ++++++++++++++++++++++++++++ tools/run_tests/run_interop_tests.py | 6 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tools/jenkins/run_full_interop.sh diff --git a/tools/jenkins/run_full_interop.sh b/tools/jenkins/run_full_interop.sh new file mode 100644 index 00000000000..a82da1cb68c --- /dev/null +++ b/tools/jenkins/run_full_interop.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and runs interop test suite. +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../.. + +tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --prod_servers default cloud_gateway gateway_v2 cloud_gateway_v2 gateway_v4 cloud_gateway_v4 --use_docker --http2_interop -t -j 12 $@ || true diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 1dc772a8565..86eb52bbde1 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -589,7 +589,11 @@ prod_servers = { 'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com', False), 'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com', - True) + True), + 'gateway_v4': ('grpc-test4.sandbox.googleapis.com', + 'grpc-test4.sandbox.googleapis.com', True), + 'cloud_gateway_v4': ('216.239.32.255', 'grpc-test4.sandbox.googleapis.com', + True), } argp = argparse.ArgumentParser(description='Run interop tests.') From c5dd4d47aa0828cb7ba180cef35bf565a57231e5 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 12:57:43 -0800 Subject: [PATCH 87/97] Use a singleton completion queue --- src/objective-c/GRPCClient/private/GRPCCompletionQueue.m | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index 250bbf03a66..be214d4d36b 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -41,7 +41,12 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; @implementation GRPCCompletionQueue + (instancetype)completionQueue { - return [[self alloc] init]; + static GRPCCompletionQueue *singleton = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + singleton = [[self alloc] init]; + }); + return singleton; } - (instancetype)init { From cc43d693e5c715fcbdab1717efc4c84ef836a1fb Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 14 Mar 2016 15:53:30 -0700 Subject: [PATCH 88/97] Reduce the number of threads so that this test works on x86 systems --- test/core/support/thd_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/support/thd_test.c b/test/core/support/thd_test.c index f7807d280a2..2671dd3d3d5 100644 --- a/test/core/support/thd_test.c +++ b/test/core/support/thd_test.c @@ -79,9 +79,9 @@ static void test_options(void) { static void test(void) { int i; gpr_thd_id thd; - gpr_thd_id thds[1000]; + gpr_thd_id thds[300]; struct test t; - int n = 1000; + int n = 300; gpr_thd_options options = gpr_thd_options_default(); gpr_mu_init(&t.mu); gpr_cv_init(&t.done_cv); From 9506ef292259625f675d58875937053e2623a400 Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Wed, 2 Mar 2016 13:53:34 -0800 Subject: [PATCH 89/97] Fix code coverage. --- .../multilang_jessie_x64/Dockerfile.template | 43 +++++ .../test/multilang_jessie_x64/Dockerfile | 155 ++++++++++++++++++ tools/run_tests/run_tests.py | 15 +- 3 files changed, 209 insertions(+), 4 deletions(-) create mode 100644 templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template create mode 100644 tools/dockerfile/test/multilang_jessie_x64/Dockerfile diff --git a/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template new file mode 100644 index 00000000000..fdf44312ec3 --- /dev/null +++ b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template @@ -0,0 +1,43 @@ +%YAML 1.2 +--- | + # Copyright 2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM debian:jessie + + <%include file="../../apt_get_basic.include"/> + <%include file="../../csharp_deps.include"/> + <%include file="../../cxx_deps.include"/> + <%include file="../../node_deps.include"/> + <%include file="../../php_deps.include"/> + <%include file="../../ruby_deps.include"/> + <%include file="../../python_deps.include"/> + <%include file="../../run_tests_addons.include"/> + # Define the default command. + CMD ["bash"] diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile new file mode 100644 index 00000000000..71ebf2bf715 --- /dev/null +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -0,0 +1,155 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +#================ +# C# dependencies + +# Update to a newer version of mono +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list + +# Install dependencies +RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ + mono-devel \ + ca-certificates-mono \ + nuget \ + && apt-get clean + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +#================== +# Node dependencies + +# Install nvm +RUN touch .profile +RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +RUN /bin/bash -l -c "nvm install 0.12 && npm config set cache /tmp/npm-cache" + +#================= +# PHP dependencies + +# Install dependencies + +RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \ + >> /etc/apt/sources.list.d/dotdeb.list" +RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \ + >> /etc/apt/sources.list.d/dotdeb.list" +RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add - + +RUN apt-get update && apt-get install -y \ + git php5 php5-dev phpunit unzip + +#================== +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm use --default ruby-2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +#==================== +# Python dependencies + +# Install dependencies + +RUN apt-get update && apt-get install -y \ + python-all-dev \ + python3-all-dev \ + python-pip + +# Install Python packages from PyPI +RUN pip install pip --upgrade +RUN pip install virtualenv +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 tox + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#====================== +# Zookeeper dependencies +# TODO(jtattermusch): is zookeeper still needed? +RUN apt-get install -y libzookeeper-mt-dev + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index aa5073fbbd2..99bf5df1f92 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -869,10 +869,17 @@ if args.use_docker: dockerfile_dirs = set([l.dockerfile_dir() for l in languages]) if len(dockerfile_dirs) > 1: - print 'Languages to be tested require running under different docker images.' - sys.exit(1) - dockerfile_dir = next(iter(dockerfile_dirs)) - + if 'gcov' in args.config: + dockerfile_dir = 'tools/dockerfile/test/multilang_jessie_x64' + print ('Using multilang_jessie_x64 docker image for code coverage for ' + 'all languages.') + else: + print ('Languages to be tested require running under different docker ' + 'images.') + sys.exit(1) + else: + dockerfile_dir = next(iter(dockerfile_dirs)) + child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) From b19489c6c8ae7d3da0bb3830eef0ad91ddcfe506 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 14 Mar 2016 16:12:14 -0700 Subject: [PATCH 90/97] Minor code cleanup --- test/core/support/thd_test.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/core/support/thd_test.c b/test/core/support/thd_test.c index 2671dd3d3d5..4b5e41eac78 100644 --- a/test/core/support/thd_test.c +++ b/test/core/support/thd_test.c @@ -41,6 +41,8 @@ #include #include "test/core/util/test_config.h" +#define NUM_THREADS 300 + struct test { gpr_mu mu; int n; @@ -79,15 +81,14 @@ static void test_options(void) { static void test(void) { int i; gpr_thd_id thd; - gpr_thd_id thds[300]; + gpr_thd_id thds[NUM_THREADS]; struct test t; - int n = 300; gpr_thd_options options = gpr_thd_options_default(); gpr_mu_init(&t.mu); gpr_cv_init(&t.done_cv); - t.n = n; + t.n = NUM_THREADS; t.is_done = 0; - for (i = 0; i != n; i++) { + for (i = 0; i < NUM_THREADS; i++) { GPR_ASSERT(gpr_thd_new(&thd, &thd_body, &t, NULL)); } gpr_mu_lock(&t.mu); @@ -97,10 +98,10 @@ static void test(void) { gpr_mu_unlock(&t.mu); GPR_ASSERT(t.n == 0); gpr_thd_options_set_joinable(&options); - for (i = 0; i < n; i++) { + for (i = 0; i < NUM_THREADS; i++) { GPR_ASSERT(gpr_thd_new(&thds[i], &thd_body_joinable, NULL, &options)); } - for (i = 0; i < n; i++) { + for (i = 0; i < NUM_THREADS; i++) { gpr_thd_join(thds[i]); } } From 47d841de4d56314153775349df74feeaa814c6c7 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Thu, 10 Mar 2016 11:19:17 -0800 Subject: [PATCH 91/97] Add support for IP Addresses in Subject Alt Names. --- src/core/tsi/ssl_transport_security.c | 67 +++++++-- src/core/tsi/ssl_transport_security.h | 3 +- test/core/tsi/transport_security_test.c | 186 ++++++++++++++---------- test/cpp/end2end/end2end_test.cc | 3 +- 4 files changed, 160 insertions(+), 99 deletions(-) diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 6adcaac9ede..7d0c74bb878 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -33,8 +33,15 @@ #include "src/core/tsi/ssl_transport_security.h" +#include + #include #include +#ifdef GPR_WINSOCK_SOCKET +#include +#else +#include +#endif #include #include @@ -296,21 +303,44 @@ static tsi_result add_subject_alt_names_properties_to_peer( sk_GENERAL_NAME_value(subject_alt_names, TSI_SIZE_AS_SIZE(i)); /* Filter out the non-dns entries names. */ if (subject_alt_name->type == GEN_DNS) { - unsigned char *dns_name = NULL; - int dns_name_size = - ASN1_STRING_to_UTF8(&dns_name, subject_alt_name->d.dNSName); - if (dns_name_size < 0) { + unsigned char *name = NULL; + int name_size; + name_size = ASN1_STRING_to_UTF8(&name, subject_alt_name->d.dNSName); + if (name_size < 0) { gpr_log(GPR_ERROR, "Could not get utf8 from asn1 string."); result = TSI_INTERNAL_ERROR; break; } result = tsi_construct_string_peer_property( - TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, - (const char *)dns_name, (size_t)dns_name_size, + TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, (const char *)name, + (size_t)name_size, &peer->properties[peer->property_count++]); + OPENSSL_free(name); + } else if (subject_alt_name->type == GEN_IPADD) { + char ntop_buf[INET6_ADDRSTRLEN]; + int af; + + if (subject_alt_name->d.iPAddress->length == 4) { + af = AF_INET; + } else if (subject_alt_name->d.iPAddress->length == 16) { + af = AF_INET6; + } else { + gpr_log(GPR_ERROR, "SAN IP Address contained invalid IP"); + result = TSI_INTERNAL_ERROR; + break; + } + const char *name = inet_ntop(af, subject_alt_name->d.iPAddress->data, + ntop_buf, INET6_ADDRSTRLEN); + if (name == NULL) { + gpr_log(GPR_ERROR, "Could not get IP string from asn1 octet."); + result = TSI_INTERNAL_ERROR; + break; + } + + result = tsi_construct_string_peer_property_from_cstring( + TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, name, &peer->properties[peer->property_count++]); - OPENSSL_free(dns_name); - if (result != TSI_OK) break; } + if (result != TSI_OK) break; } return result; } @@ -1436,9 +1466,7 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) { size_t i = 0; size_t san_count = 0; const tsi_peer_property *cn_property = NULL; - - /* For now reject what looks like an IP address. */ - if (looks_like_ip_address(name)) return 0; + int like_ip = looks_like_ip_address(name); /* Check the SAN first. */ for (i = 0; i < peer->property_count; i++) { @@ -1447,8 +1475,15 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) { if (strcmp(property->name, TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY) == 0) { san_count++; - if (does_entry_match_name(property->value.data, property->value.length, - name)) { + + if (!like_ip && does_entry_match_name(property->value.data, + property->value.length, name)) { + return 1; + } else if (like_ip && + strncmp(name, property->value.data, property->value.length) == + 0 && + strlen(name) == property->value.length) { + /* IP Addresses are exact matches only. */ return 1; } } else if (strcmp(property->name, @@ -1457,8 +1492,8 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) { } } - /* If there's no SAN, try the CN. */ - if (san_count == 0 && cn_property != NULL) { + /* If there's no SAN, try the CN, but only if its not like an IP Address */ + if (san_count == 0 && cn_property != NULL && !like_ip) { if (does_entry_match_name(cn_property->value.data, cn_property->value.length, name)) { return 1; diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index 51c0003a854..b587d7ce31a 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -162,8 +162,7 @@ void tsi_ssl_handshaker_factory_destroy(tsi_ssl_handshaker_factory *self); Still TODO(jboeuf): - handle mixed case. - handle %encoded chars. - - handle public suffix wildchar more strictly (e.g. *.co.uk) - - handle IP addresses in SAN. */ + - handle public suffix wildchar more strictly (e.g. *.co.uk) */ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name); #ifdef __cplusplus diff --git a/test/core/tsi/transport_security_test.c b/test/core/tsi/transport_security_test.c index 7ce343987bb..f1e7f7f8a87 100644 --- a/test/core/tsi/transport_security_test.c +++ b/test/core/tsi/transport_security_test.c @@ -61,35 +61,37 @@ typedef struct { of '#' will be replaced with a null character before processing. */ const char *dns_names; + /* Comma separated list of IP SANs to match aggainst */ + const char *ip_names; } cert_name_test_entry; /* Largely inspired from: chromium/src/net/cert/x509_certificate_unittest.cc. TODO(jboeuf) uncomment test cases as we fix tsi_ssl_peer_matches_name. */ const cert_name_test_entry cert_name_test_entries[] = { - {1, "foo.com", "foo.com", NULL}, - {1, "f", "f", NULL}, - {0, "h", "i", NULL}, - {1, "bar.foo.com", "*.foo.com", NULL}, + {1, "foo.com", "foo.com", NULL, NULL}, + {1, "f", "f", NULL, NULL}, + {0, "h", "i", NULL, NULL}, + {1, "bar.foo.com", "*.foo.com", NULL, NULL}, {1, "www.test.fr", "common.name", - "*.test.com,*.test.co.uk,*.test.de,*.test.fr"}, + "*.test.com,*.test.co.uk,*.test.de,*.test.fr", NULL}, /* {1, "wwW.tESt.fr", "common.name", ",*.*,*.test.de,*.test.FR,www"}, */ - {0, "f.uk", ".uk", NULL}, - {0, "w.bar.foo.com", "?.bar.foo.com", NULL}, - {0, "www.foo.com", "(www|ftp).foo.com", NULL}, - {0, "www.foo.com", "www.foo.com#", NULL}, /* # = null char. */ - {0, "www.foo.com", "", "www.foo.com#*.foo.com,#,#"}, - {0, "www.house.example", "ww.house.example", NULL}, - {0, "test.org", "", "www.test.org,*.test.org,*.org"}, - {0, "w.bar.foo.com", "w*.bar.foo.com", NULL}, - {0, "www.bar.foo.com", "ww*ww.bar.foo.com", NULL}, - {0, "wwww.bar.foo.com", "ww*ww.bar.foo.com", NULL}, - {0, "wwww.bar.foo.com", "w*w.bar.foo.com", NULL}, - {0, "wwww.bar.foo.com", "w*w.bar.foo.c0m", NULL}, - {0, "WALLY.bar.foo.com", "wa*.bar.foo.com", NULL}, - {0, "wally.bar.foo.com", "*Ly.bar.foo.com", NULL}, + {0, "f.uk", ".uk", NULL, NULL}, + {0, "w.bar.foo.com", "?.bar.foo.com", NULL, NULL}, + {0, "www.foo.com", "(www|ftp).foo.com", NULL, NULL}, + {0, "www.foo.com", "www.foo.com#", NULL, NULL}, /* # = null char. */ + {0, "www.foo.com", "", "www.foo.com#*.foo.com,#,#", NULL}, + {0, "www.house.example", "ww.house.example", NULL, NULL}, + {0, "test.org", "", "www.test.org,*.test.org,*.org", NULL}, + {0, "w.bar.foo.com", "w*.bar.foo.com", NULL, NULL}, + {0, "www.bar.foo.com", "ww*ww.bar.foo.com", NULL, NULL}, + {0, "wwww.bar.foo.com", "ww*ww.bar.foo.com", NULL, NULL}, + {0, "wwww.bar.foo.com", "w*w.bar.foo.com", NULL, NULL}, + {0, "wwww.bar.foo.com", "w*w.bar.foo.c0m", NULL, NULL}, + {0, "WALLY.bar.foo.com", "wa*.bar.foo.com", NULL, NULL}, + {0, "wally.bar.foo.com", "*Ly.bar.foo.com", NULL, NULL}, /* {1, "ww%57.foo.com", "", "www.foo.com"}, {1, "www&.foo.com", "www%26.foo.com", NULL}, @@ -97,94 +99,101 @@ const cert_name_test_entry cert_name_test_entries[] = { /* Common name must not be used if subject alternative name was provided. */ {0, "www.test.co.jp", "www.test.co.jp", - "*.test.de,*.jp,www.test.co.uk,www.*.co.jp"}, + "*.test.de,*.jp,www.test.co.uk,www.*.co.jp", NULL}, {0, "www.bar.foo.com", "www.bar.foo.com", - "*.foo.com,*.*.foo.com,*.*.bar.foo.com,*..bar.foo.com,"}, + "*.foo.com,*.*.foo.com,*.*.bar.foo.com,*..bar.foo.com,", NULL}, /* IDN tests */ - {1, "xn--poema-9qae5a.com.br", "xn--poema-9qae5a.com.br", NULL}, - {1, "www.xn--poema-9qae5a.com.br", "*.xn--poema-9qae5a.com.br", NULL}, + {1, "xn--poema-9qae5a.com.br", "xn--poema-9qae5a.com.br", NULL, NULL}, + {1, "www.xn--poema-9qae5a.com.br", "*.xn--poema-9qae5a.com.br", NULL, NULL}, {0, "xn--poema-9qae5a.com.br", "", "*.xn--poema-9qae5a.com.br," "xn--poema-*.com.br," "xn--*-9qae5a.com.br," - "*--poema-9qae5a.com.br"}, + "*--poema-9qae5a.com.br", + NULL}, /* The following are adapted from the examples quoted from http://tools.ietf.org/html/rfc6125#section-6.4.3 (e.g., *.example.com would match foo.example.com but not bar.foo.example.com or example.com). */ - {1, "foo.example.com", "*.example.com", NULL}, - {0, "bar.foo.example.com", "*.example.com", NULL}, - {0, "example.com", "*.example.com", NULL}, + {1, "foo.example.com", "*.example.com", NULL, NULL}, + {0, "bar.foo.example.com", "*.example.com", NULL, NULL}, + {0, "example.com", "*.example.com", NULL, NULL}, /* Partial wildcards are disallowed, though RFC 2818 rules allow them. That is, forms such as baz*.example.net, *baz.example.net, and b*z.example.net should NOT match domains. Instead, the wildcard must always be the left-most label, and only a single label. */ - {0, "baz1.example.net", "baz*.example.net", NULL}, - {0, "foobaz.example.net", "*baz.example.net", NULL}, - {0, "buzz.example.net", "b*z.example.net", NULL}, - {0, "www.test.example.net", "www.*.example.net", NULL}, + {0, "baz1.example.net", "baz*.example.net", NULL, NULL}, + {0, "foobaz.example.net", "*baz.example.net", NULL, NULL}, + {0, "buzz.example.net", "b*z.example.net", NULL, NULL}, + {0, "www.test.example.net", "www.*.example.net", NULL, NULL}, /* Wildcards should not be valid for public registry controlled domains, and unknown/unrecognized domains, at least three domain components must be present. */ - {1, "www.test.example", "*.test.example", NULL}, - {1, "test.example.co.uk", "*.example.co.uk", NULL}, - {0, "test.example", "*.example", NULL}, + {1, "www.test.example", "*.test.example", NULL, NULL}, + {1, "test.example.co.uk", "*.example.co.uk", NULL, NULL}, + {0, "test.example", "*.example", NULL, NULL}, /* {0, "example.co.uk", "*.co.uk", NULL}, */ - {0, "foo.com", "*.com", NULL}, - {0, "foo.us", "*.us", NULL}, - {0, "foo", "*", NULL}, + {0, "foo.com", "*.com", NULL, NULL}, + {0, "foo.us", "*.us", NULL, NULL}, + {0, "foo", "*", NULL, NULL}, /* IDN variants of wildcards and registry controlled domains. */ - {1, "www.xn--poema-9qae5a.com.br", "*.xn--poema-9qae5a.com.br", NULL}, - {1, "test.example.xn--mgbaam7a8h", "*.example.xn--mgbaam7a8h", NULL}, + {1, "www.xn--poema-9qae5a.com.br", "*.xn--poema-9qae5a.com.br", NULL, NULL}, + {1, "test.example.xn--mgbaam7a8h", "*.example.xn--mgbaam7a8h", NULL, NULL}, /* {0, "xn--poema-9qae5a.com.br", "*.com.br", NULL}, */ - {0, "example.xn--mgbaam7a8h", "*.xn--mgbaam7a8h", NULL}, + {0, "example.xn--mgbaam7a8h", "*.xn--mgbaam7a8h", NULL, NULL}, /* Wildcards should be permissible for 'private' registry controlled domains. */ - {1, "www.appspot.com", "*.appspot.com", NULL}, - {1, "foo.s3.amazonaws.com", "*.s3.amazonaws.com", NULL}, + {1, "www.appspot.com", "*.appspot.com", NULL, NULL}, + {1, "foo.s3.amazonaws.com", "*.s3.amazonaws.com", NULL, NULL}, /* Multiple wildcards are not valid. */ - {0, "foo.example.com", "*.*.com", NULL}, - {0, "foo.bar.example.com", "*.bar.*.com", NULL}, + {0, "foo.example.com", "*.*.com", NULL, NULL}, + {0, "foo.bar.example.com", "*.bar.*.com", NULL, NULL}, /* Absolute vs relative DNS name tests. Although not explicitly specified in RFC 6125, absolute reference names (those ending in a .) should match either absolute or relative presented names. */ - {1, "foo.com", "foo.com.", NULL}, - {1, "foo.com.", "foo.com", NULL}, - {1, "foo.com.", "foo.com.", NULL}, - {1, "f", "f.", NULL}, - {1, "f.", "f", NULL}, - {1, "f.", "f.", NULL}, - {1, "www-3.bar.foo.com", "*.bar.foo.com.", NULL}, - {1, "www-3.bar.foo.com.", "*.bar.foo.com", NULL}, - {1, "www-3.bar.foo.com.", "*.bar.foo.com.", NULL}, - {0, ".", ".", NULL}, - {0, "example.com", "*.com.", NULL}, - {0, "example.com.", "*.com", NULL}, - {0, "example.com.", "*.com.", NULL}, - {0, "foo.", "*.", NULL}, - {0, "foo", "*.", NULL}, + {1, "foo.com", "foo.com.", NULL, NULL}, + {1, "foo.com.", "foo.com", NULL, NULL}, + {1, "foo.com.", "foo.com.", NULL, NULL}, + {1, "f", "f.", NULL, NULL}, + {1, "f.", "f", NULL, NULL}, + {1, "f.", "f.", NULL, NULL}, + {1, "www-3.bar.foo.com", "*.bar.foo.com.", NULL, NULL}, + {1, "www-3.bar.foo.com.", "*.bar.foo.com", NULL, NULL}, + {1, "www-3.bar.foo.com.", "*.bar.foo.com.", NULL, NULL}, + {0, ".", ".", NULL, NULL}, + {0, "example.com", "*.com.", NULL, NULL}, + {0, "example.com.", "*.com", NULL, NULL}, + {0, "example.com.", "*.com.", NULL, NULL}, + {0, "foo.", "*.", NULL, NULL}, + {0, "foo", "*.", NULL, NULL}, /* {0, "foo.co.uk", "*.co.uk.", NULL}, {0, "foo.co.uk.", "*.co.uk.", NULL}, */ /* An empty CN is OK. */ - {1, "test.foo.com", "", "test.foo.com"}, + {1, "test.foo.com", "", "test.foo.com", NULL}, /* An IP should not be used for the CN. */ - {0, "173.194.195.139", "173.194.195.139", NULL}, + {0, "173.194.195.139", "173.194.195.139", NULL, NULL}, + /* An IP can be used if the SAN IP is present */ + {1, "173.194.195.139", "foo.example.com", NULL, "173.194.195.139"}, + {0, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8"}, + {0, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8,8.8.4.4"}, + {1, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8,173.194.195.139"}, + {0, "173.194.195.139", "foo.example.com", NULL, "173.194.195.13"}, }; typedef struct name_list { @@ -196,7 +205,7 @@ typedef struct { size_t name_count; char *buffer; name_list *names; -} parsed_dns_names; +} parsed_names; name_list *name_list_add(const char *n) { name_list *result = gpr_malloc(sizeof(name_list)); @@ -205,18 +214,18 @@ name_list *name_list_add(const char *n) { return result; } -static parsed_dns_names parse_dns_names(const char *dns_names_str) { - parsed_dns_names result; +static parsed_names parse_names(const char *names_str) { + parsed_names result; name_list *current_nl; size_t i; - memset(&result, 0, sizeof(parsed_dns_names)); - if (dns_names_str == 0) return result; + memset(&result, 0, sizeof(parsed_names)); + if (names_str == 0) return result; result.name_count = 1; - result.buffer = gpr_strdup(dns_names_str); + result.buffer = gpr_strdup(names_str); result.names = name_list_add(result.buffer); current_nl = result.names; - for (i = 0; i < strlen(dns_names_str); i++) { - if (dns_names_str[i] == ',') { + for (i = 0; i < strlen(names_str); i++) { + if (names_str[i] == ',') { result.buffer[i] = '\0'; result.name_count++; i++; @@ -227,7 +236,7 @@ static parsed_dns_names parse_dns_names(const char *dns_names_str) { return result; } -static void destruct_parsed_dns_names(parsed_dns_names *pdn) { +static void destruct_parsed_names(parsed_names *pdn) { name_list *nl = pdn->names; if (pdn->buffer != NULL) gpr_free(pdn->buffer); while (nl != NULL) { @@ -237,8 +246,8 @@ static void destruct_parsed_dns_names(parsed_dns_names *pdn) { } } -static char *processed_dns_name(const char *dns_name) { - char *result = gpr_strdup(dns_name); +static char *processed_name(const char *name) { + char *result = gpr_strdup(name); size_t i; for (i = 0; i < strlen(result); i++) { if (result[i] == '#') { @@ -253,31 +262,48 @@ static tsi_peer peer_from_cert_name_test_entry( size_t i; tsi_peer peer; name_list *nl; - parsed_dns_names dns_entries = parse_dns_names(entry->dns_names); + parsed_names dns_entries = parse_names(entry->dns_names); + parsed_names ip_entries = parse_names(entry->ip_names); nl = dns_entries.names; - GPR_ASSERT(tsi_construct_peer(1 + dns_entries.name_count, &peer) == TSI_OK); + GPR_ASSERT(tsi_construct_peer( + 1 + dns_entries.name_count + ip_entries.name_count, &peer) == + TSI_OK); GPR_ASSERT(tsi_construct_string_peer_property_from_cstring( TSI_X509_SUBJECT_COMMON_NAME_PEER_PROPERTY, entry->common_name, &peer.properties[0]) == TSI_OK); i = 1; while (nl != NULL) { - char *processed = processed_dns_name(nl->name); + char *processed = processed_name(nl->name); GPR_ASSERT(tsi_construct_string_peer_property( TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, processed, strlen(nl->name), &peer.properties[i++]) == TSI_OK); nl = nl->next; gpr_free(processed); } - destruct_parsed_dns_names(&dns_entries); + + nl = ip_entries.names; + while (nl != NULL) { + char *processed = processed_name(nl->name); + GPR_ASSERT(tsi_construct_string_peer_property( + TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, processed, + strlen(nl->name), &peer.properties[i++]) == TSI_OK); + nl = nl->next; + gpr_free(processed); + } + destruct_parsed_names(&dns_entries); + destruct_parsed_names(&ip_entries); return peer; } char *cert_name_test_entry_to_string(const cert_name_test_entry *entry) { char *s; - gpr_asprintf( - &s, "{ success = %s, host_name = %s, common_name = %s, dns_names = %s}", - entry->expected ? "true" : "false", entry->host_name, entry->common_name, - entry->dns_names != NULL ? entry->dns_names : ""); + gpr_asprintf(&s, + "{ success = %s, host_name = %s, common_name = %s, dns_names = " + "%s, ip_names = %s}", + entry->expected ? "true" : "false", entry->host_name, + entry->common_name, + entry->dns_names != NULL ? entry->dns_names : "", + entry->ip_names != NULL ? entry->ip_names : ""); return s; } diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index 8131a14ff76..47598183220 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -1371,11 +1371,12 @@ TEST_P(SecureEnd2endTest, ClientAuthContext) { if (GetParam().credentials_type == kTlsCredentialsType) { EXPECT_EQ("x509_subject_alternative_name", auth_ctx->GetPeerIdentityPropertyName()); - EXPECT_EQ(3u, auth_ctx->GetPeerIdentity().size()); + EXPECT_EQ(4u, auth_ctx->GetPeerIdentity().size()); EXPECT_EQ("*.test.google.fr", ToString(auth_ctx->GetPeerIdentity()[0])); EXPECT_EQ("waterzooi.test.google.be", ToString(auth_ctx->GetPeerIdentity()[1])); EXPECT_EQ("*.test.youtube.com", ToString(auth_ctx->GetPeerIdentity()[2])); + EXPECT_EQ("192.168.1.3", ToString(auth_ctx->GetPeerIdentity()[3])); } } From 27df68940577764f4a98a1972d9958005e0ef858 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sun, 13 Mar 2016 13:34:27 -0700 Subject: [PATCH 92/97] Add note about refactor portability header --- src/core/tsi/ssl_transport_security.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 7d0c74bb878..d8f7b5b0e0f 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -37,6 +37,8 @@ #include #include + +/* TODO(jboeuf): refactor inet_ntop into a portability header. */ #ifdef GPR_WINSOCK_SOCKET #include #else From 4a9e7c4cb155ac3284bdca098227b1a78495a26f Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sun, 13 Mar 2016 14:00:11 -0700 Subject: [PATCH 93/97] Improve looks_like_ip_address for IPv6 addresses, and add tests --- src/core/tsi/ssl_transport_security.c | 7 +++++-- src/core/tsi/ssl_transport_security.h | 2 +- test/core/tsi/transport_security_test.c | 9 ++++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index d8f7b5b0e0f..fcbd910f07f 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -206,13 +206,16 @@ static void ssl_info_callback(const SSL *ssl, int where, int ret) { } /* Returns 1 if name looks like an IP address, 0 otherwise. - This is a very rough heuristic as it does not handle IPV6 or things like: - 0300.0250.00.01, 0xC0.0Xa8.0x0.0x1, 000030052000001, 0xc0.052000001 */ + This is a very rough heuristic, and only handles IPv6 in hexadecimal form. */ static int looks_like_ip_address(const char *name) { size_t i; size_t dot_count = 0; size_t num_size = 0; for (i = 0; i < strlen(name); i++) { + if (name[i] == ':') { + /* IPv6 Address in hexadecimal form, : is not allowed in DNS names. */ + return 1; + } if (name[i] >= '0' && name[i] <= '9') { if (num_size > 3) return 0; num_size++; diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index b587d7ce31a..4909af4c470 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/tsi/transport_security_test.c b/test/core/tsi/transport_security_test.c index f1e7f7f8a87..667d3f03497 100644 --- a/test/core/tsi/transport_security_test.c +++ b/test/core/tsi/transport_security_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -194,6 +194,13 @@ const cert_name_test_entry cert_name_test_entries[] = { {0, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8,8.8.4.4"}, {1, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8,173.194.195.139"}, {0, "173.194.195.139", "foo.example.com", NULL, "173.194.195.13"}, + {0, "2001:db8:a0b:12f0::1", "foo.example.com", NULL, "173.194.195.13"}, + {1, "2001:db8:a0b:12f0::1", "foo.example.com", NULL, + "2001:db8:a0b:12f0::1"}, + {0, "2001:db8:a0b:12f0::1", "foo.example.com", NULL, + "2001:db8:a0b:12f0::2"}, + {1, "2001:db8:a0b:12f0::1", "foo.example.com", NULL, + "2001:db8:a0b:12f0::2,2001:db8:a0b:12f0::1,8.8.8.8"}, }; typedef struct name_list { From f7cea976581a19a06eca5d54089920ead4f788ae Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 14 Mar 2016 17:01:15 -0700 Subject: [PATCH 94/97] Fix copyright --- test/core/support/thd_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/support/thd_test.c b/test/core/support/thd_test.c index 4b5e41eac78..0c176da2d3c 100644 --- a/test/core/support/thd_test.c +++ b/test/core/support/thd_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 7cfb396bf6c2194ca9c1bb6514cc57ce1403f3d8 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Mon, 14 Mar 2016 19:03:54 -0700 Subject: [PATCH 95/97] Make copyright not expire one year too soon. --- src/objective-c/RxLibrary/GRXWriteable.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/RxLibrary/GRXWriteable.m b/src/objective-c/RxLibrary/GRXWriteable.m index c15ccb17fb9..028ba9b551a 100644 --- a/src/objective-c/RxLibrary/GRXWriteable.m +++ b/src/objective-c/RxLibrary/GRXWriteable.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 9db726812eceef6705c71fd1817b0658a00aa38f Mon Sep 17 00:00:00 2001 From: Paddy Byers Date: Thu, 10 Mar 2016 22:56:15 +0000 Subject: [PATCH 96/97] Use gpr_free() in gpr_free_aligned() --- src/core/support/alloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/support/alloc.c b/src/core/support/alloc.c index 0a064b2c188..b99584bd200 100644 --- a/src/core/support/alloc.c +++ b/src/core/support/alloc.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -87,4 +87,4 @@ void *gpr_malloc_aligned(size_t size, size_t alignment_log) { return (void *)ret; } -void gpr_free_aligned(void *ptr) { free(((void **)ptr)[-1]); } +void gpr_free_aligned(void *ptr) { gpr_free(((void **)ptr)[-1]); } From cbd1bce45a605df97aa00a42b7568d08419253f9 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Tue, 15 Mar 2016 13:24:10 -0700 Subject: [PATCH 97/97] Fix two ways tests can hang Both have to do with the test runner's handling of the tests. With one it's the read thread somehow outliving the other threads (e.g. with ctrl-C). The other is due to a filled OS-level pipe's buffer causing a block while code is still holding the GIL in some gRPC core function. We can't empty the buffer from Python because the GIL is held, and the OS can't unblock because it's waiting for the buffer to get cleared: deadlock. --- src/python/grpcio/tests/_runner.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py index b0dbd92a495..32a31ce00e1 100644 --- a/src/python/grpcio/tests/_runner.py +++ b/src/python/grpcio/tests/_runner.py @@ -43,6 +43,13 @@ import uuid from tests import _loader from tests import _result +# This number needs to be large enough to outpace output on stdout and stderr +# from the gRPC core, otherwise we could end up in a potential deadlock. This +# stems from the OS waiting on someone to clear a filled pipe buffer while the +# GIL is held from a write to stderr from gRPC core, but said someone is in +# Python code thus necessitating GIL acquisition. +_READ_BYTES = 2**20 + class CapturePipe(object): """A context-manager pipe to redirect output to a byte array. @@ -76,6 +83,10 @@ class CapturePipe(object): flags = fcntl.fcntl(self._read_fd, fcntl.F_GETFL) fcntl.fcntl(self._read_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) self._read_thread = threading.Thread(target=self._read) + # If the user wants to exit from the Python program and hits ctrl-C and the + # read thread is somehow deadlocked with something else, the Python code may + # refuse to exit. This prevents that by making the read thread second-class. + self._read_thread.daemon = True self._read_thread.start() def stop(self): @@ -93,7 +104,7 @@ class CapturePipe(object): self.output = bytearray() while True: select.select([self._read_fd], [], []) - read_bytes = os.read(self._read_fd, 1024) + read_bytes = os.read(self._read_fd, _READ_BYTES) if read_bytes: self.output.extend(read_bytes) else: