From e5e0131c8d4e7d8b99bedbfe7902aadf5eace3bf Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 17 Feb 2016 13:06:01 -0800 Subject: [PATCH 001/160] Add more information to precompiled name --- src/python/grpcio/precompiled.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio/precompiled.py b/src/python/grpcio/precompiled.py index 05c651b506f..6d9a4b85c26 100644 --- a/src/python/grpcio/precompiled.py +++ b/src/python/grpcio/precompiled.py @@ -31,6 +31,7 @@ import os import platform import shutil import sys +import sysconfig import setuptools @@ -51,9 +52,15 @@ USE_PRECOMPILED_BINARIES = bool(int(os.environ.get( def _tagged_ext_name(base): uname = platform.uname() - tags = '-'.join((grpc_version.VERSION, uname[0], uname[4])) - flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4' - return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor) + tags = ( + grpc_version.VERSION, + 'py{}'.format(sysconfig.get_python_version()), + uname[0], + uname[4], + ) + ucs = 'ucs{}'.format(sysconfig.get_config_var('Py_UNICODE_SIZE')) + return '{base}-{tags}-{ucs}'.format( + base=base, tags='-'.join(tags), ucs=ucs) class BuildTaggedExt(setuptools.Command): From c0a97a9cfe257138d293774868b4157151d64339 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 22 Feb 2016 17:51:02 -0800 Subject: [PATCH 002/160] Update Python doc with pip version notes --- src/python/grpcio/README.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index 698c760ebe2..bb6d99c4b15 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -23,11 +23,15 @@ Else system wide (on Ubuntu)... $ sudo pip install grpcio +n.b. On Windows and on Mac OS X one *must* have a recent release of :code:`pip` +to retrieve the proper wheel from PyPI. Be sure to upgrade to the latest +version! + From Source ~~~~~~~~~~~ Building from source requires that you have the Python headers (usually a -package named `python-dev`). +package named :code:`python-dev`). :: From fe8ad714d452b27a66058e111b9832ab08c1036a Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 22 Feb 2016 17:51:02 -0800 Subject: [PATCH 003/160] Update Python doc w.r.t. Windows availability --- src/python/grpcio/README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index bb6d99c4b15..d491311886b 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -6,7 +6,7 @@ Package for gRPC Python. Installation ------------ -gRPC Python is available for Linux and Mac OS X running Python 2.7. +gRPC Python is available for Linux, Mac OS X, and Windows running Python 2.7. From PyPI ~~~~~~~~~ @@ -40,5 +40,5 @@ package named :code:`python-dev`). $ cd $REPO_ROOT $ pip install . -Note that `$REPO_ROOT` can be assigned to whatever directory name floats your -fancy. +Note that :code:`$REPO_ROOT` can be assigned to whatever directory name floats +your fancy. From 3ee1f9b7784b680b03d28d92efdffae9b7ad476d Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 22 Feb 2016 18:37:18 -0800 Subject: [PATCH 004/160] Update Python docgen --- setup.py | 1 + src/python/grpcio/commands.py | 3 +-- tools/distrib/python/docgen.py | 12 +++++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index cfb578e2158..cf5ab4920fb 100644 --- a/setup.py +++ b/setup.py @@ -154,6 +154,7 @@ INSTALL_REQUIRES = ( SETUP_REQUIRES = ( 'sphinx>=1.3', + 'sphinx_rtd_theme>=0.1.8' ) + INSTALL_REQUIRES COMMAND_CLASS = { diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index aa29c728f25..99ba41b614f 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -119,8 +119,7 @@ class SphinxDocumentation(setuptools.Command): import sphinx import sphinx.apidoc metadata = self.distribution.metadata - src_dir = os.path.join( - PYTHON_STEM, self.distribution.package_dir[''], 'grpc') + src_dir = os.path.join(PYTHON_STEM, 'grpc') sys.path.append(src_dir) sphinx.apidoc.main([ '', '--force', '--full', '-H', metadata.name, '-A', metadata.author, diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index 4ac8f9c64a8..72c65ad14a8 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -51,29 +51,35 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..', '..')) CONFIG = args.config -SETUP_PATH = os.path.join(PROJECT_ROOT, 'src/python/grpcio/setup.py') -DOC_PATH = os.path.join(PROJECT_ROOT, 'src/python/grpcio/doc/build') +SETUP_PATH = os.path.join(PROJECT_ROOT, 'setup.py') +REQUIREMENTS_PATH = os.path.join(PROJECT_ROOT, 'requirements.txt') +DOC_PATH = os.path.join(PROJECT_ROOT, 'doc/build') INCLUDE_PATH = os.path.join(PROJECT_ROOT, 'include') LIBRARY_PATH = os.path.join(PROJECT_ROOT, 'libs/{}'.format(CONFIG)) VIRTUALENV_DIR = os.path.join(SCRIPT_DIR, 'distrib_virtualenv') VIRTUALENV_PYTHON_PATH = os.path.join(VIRTUALENV_DIR, 'bin', 'python') +VIRTUALENV_PIP_PATH = os.path.join(VIRTUALENV_DIR, 'bin', 'pip') environment = os.environ.copy() environment.update({ 'CONFIG': CONFIG, 'CFLAGS': '-I{}'.format(INCLUDE_PATH), 'LDFLAGS': '-L{}'.format(LIBRARY_PATH), - 'LD_LIBRARY_PATH': LIBRARY_PATH + 'LD_LIBRARY_PATH': LIBRARY_PATH, + 'GRPC_PYTHON_BUILD_WITH_CYTHON': '1', }) subprocess_arguments_list = [ {'args': ['make'], 'cwd': PROJECT_ROOT}, {'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment}, + {'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH], + 'env': environment}, {'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'build'], 'env': environment}, {'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'doc'], 'env': environment}, ] for subprocess_arguments in subprocess_arguments_list: + print('Running command: {}'.format(subprocess_arguments['args'])) subprocess.check_call(**subprocess_arguments) if args.submit: From 071aff284f0ba700e1296688ed2b52dddfce051e Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Feb 2016 18:31:27 -0800 Subject: [PATCH 005/160] Accept instance name as argument --- tools/gce/create_linux_worker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh index 2a9e77ab173..399e3ec4e43 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_worker.sh @@ -37,7 +37,7 @@ cd $(dirname $0) CLOUD_PROJECT=grpc-testing ZONE=us-central1-a -INSTANCE_NAME=grpc-jenkins-worker1 +INSTANCE_NAME="${1:-grpc-jenkins-worker1}" gcloud compute instances create $INSTANCE_NAME \ --project="$CLOUD_PROJECT" \ From 02eda41d215d2e49f5626e0ecd2c2d16e9c9a1f2 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 25 Feb 2016 19:43:57 -0800 Subject: [PATCH 006/160] Do stream operations in a more idiomatic way: Write, WritesDone, Read until fail, Finish --- test/cpp/qps/driver.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 1c7fdf87960..18a9ae4d256 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -351,9 +351,11 @@ std::unique_ptr RunScenario( gpr_log(GPR_INFO, "Finishing"); for (auto server = &servers[0]; server != &servers[num_servers]; server++) { GPR_ASSERT(server->stream->Write(server_mark)); + GPR_ASSERT(server->stream->WritesDone()); } for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Write(client_mark)); + GPR_ASSERT(client->stream->WritesDone()); } for (auto server = &servers[0]; server != &servers[num_servers]; server++) { GPR_ASSERT(server->stream->Read(&server_status)); @@ -361,6 +363,7 @@ std::unique_ptr RunScenario( result->server_resources.emplace_back( stats.time_elapsed(), stats.time_user(), stats.time_system(), server_status.cores()); + GPR_ASSERT(!server->stream->Read(&server_status)); } for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Read(&client_status)); @@ -368,14 +371,13 @@ std::unique_ptr RunScenario( result->latencies.MergeProto(stats.latencies()); result->client_resources.emplace_back( stats.time_elapsed(), stats.time_user(), stats.time_system(), -1); + GPR_ASSERT(!client->stream->Read(&client_status)); } for (auto client = &clients[0]; client != &clients[num_clients]; client++) { - GPR_ASSERT(client->stream->WritesDone()); GPR_ASSERT(client->stream->Finish().ok()); } for (auto server = &servers[0]; server != &servers[num_servers]; server++) { - GPR_ASSERT(server->stream->WritesDone()); GPR_ASSERT(server->stream->Finish().ok()); } delete[] clients; From e7042b5ddf4f72dbba79e53621f28ba7d3cd9df4 Mon Sep 17 00:00:00 2001 From: vjpai Date: Thu, 25 Feb 2016 20:28:05 -0800 Subject: [PATCH 007/160] Simplify Mark processing code --- test/cpp/qps/client.h | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 2dc83f0f292..6a822e35c78 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -123,15 +123,13 @@ class Client { if (reset) { Histogram* to_merge = new Histogram[threads_.size()]; for (size_t i = 0; i < threads_.size(); i++) { - threads_[i]->BeginSwap(&to_merge[i]); - } - std::unique_ptr timer(new UsageTimer); - timer_.swap(timer); - for (size_t i = 0; i < threads_.size(); i++) { - threads_[i]->EndSwap(); + threads_[i]->Swap(&to_merge[i]); latencies.Merge(to_merge[i]); } delete[] to_merge; + + std::unique_ptr timer(new UsageTimer); + timer_.swap(timer); timer_result = timer->Mark(); } else { // merge snapshots of each thread histogram @@ -227,7 +225,6 @@ class Client { public: Thread(Client* client, size_t idx) : done_(false), - new_stats_(nullptr), client_(client), idx_(idx), impl_(&Thread::ThreadFunc, this) {} @@ -240,16 +237,9 @@ class Client { impl_.join(); } - void BeginSwap(Histogram* n) { + void Swap(Histogram* n) { std::lock_guard g(mu_); - new_stats_ = n; - } - - void EndSwap() { - std::unique_lock g(mu_); - while (new_stats_ != nullptr) { - cv_.wait(g); - }; + n->Swap(&histogram_); } void MergeStatsInto(Histogram* hist) { @@ -263,10 +253,11 @@ class Client { void ThreadFunc() { for (;;) { + // lock since the thread should only be doing one thing at a time + std::lock_guard g(mu_); // run the loop body const bool thread_still_ok = client_->ThreadFunc(&histogram_, idx_); - // lock, see if we're done - std::lock_guard g(mu_); + // see if we're done if (!thread_still_ok) { gpr_log(GPR_ERROR, "Finishing client thread due to RPC error"); done_ = true; @@ -274,19 +265,11 @@ class Client { if (done_) { return; } - // check if we're resetting stats, swap out the histogram if so - if (new_stats_) { - new_stats_->Swap(&histogram_); - new_stats_ = nullptr; - cv_.notify_one(); - } } } std::mutex mu_; - std::condition_variable cv_; bool done_; - Histogram* new_stats_; Histogram histogram_; Client* client_; const size_t idx_; From 762ed4368afa513ef468eac4cf16e4c91ff9b61a Mon Sep 17 00:00:00 2001 From: Chris Bacon Date: Fri, 26 Feb 2016 12:44:10 +0000 Subject: [PATCH 008/160] Support for coreclr Use .ToLowerInvariant, which is supported by coreclr --- src/csharp/Grpc.Core/Metadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Metadata.cs b/src/csharp/Grpc.Core/Metadata.cs index aa22f840d6c..52cef96f405 100644 --- a/src/csharp/Grpc.Core/Metadata.cs +++ b/src/csharp/Grpc.Core/Metadata.cs @@ -323,7 +323,7 @@ namespace Grpc.Core private static string NormalizeKey(string key) { - var normalized = GrpcPreconditions.CheckNotNull(key, "key").ToLower(CultureInfo.InvariantCulture); + var normalized = GrpcPreconditions.CheckNotNull(key, "key").ToLowerInvariant(); GrpcPreconditions.CheckArgument(ValidKeyRegex.IsMatch(normalized), "Metadata entry key not valid. Keys can only contain lowercase alphanumeric characters, underscores and hyphens."); return normalized; From 60a7e3336df1554891bae57b3f8d6e7304f6e039 Mon Sep 17 00:00:00 2001 From: vjpai Date: Fri, 26 Feb 2016 10:30:17 -0800 Subject: [PATCH 009/160] Finish clients fully before finishing servers --- test/cpp/qps/driver.cc | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 18a9ae4d256..bc8780f74d5 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -348,23 +348,11 @@ std::unique_ptr RunScenario( std::unique_ptr result(new ScenarioResult); result->client_config = result_client_config; result->server_config = result_server_config; - gpr_log(GPR_INFO, "Finishing"); - for (auto server = &servers[0]; server != &servers[num_servers]; server++) { - GPR_ASSERT(server->stream->Write(server_mark)); - GPR_ASSERT(server->stream->WritesDone()); - } + gpr_log(GPR_INFO, "Finishing clients"); for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Write(client_mark)); GPR_ASSERT(client->stream->WritesDone()); } - for (auto server = &servers[0]; server != &servers[num_servers]; server++) { - GPR_ASSERT(server->stream->Read(&server_status)); - const auto& stats = server_status.stats(); - result->server_resources.emplace_back( - stats.time_elapsed(), stats.time_user(), stats.time_system(), - server_status.cores()); - GPR_ASSERT(!server->stream->Read(&server_status)); - } for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Read(&client_status)); const auto& stats = client_status.stats(); @@ -373,14 +361,28 @@ std::unique_ptr RunScenario( stats.time_elapsed(), stats.time_user(), stats.time_system(), -1); GPR_ASSERT(!client->stream->Read(&client_status)); } - for (auto client = &clients[0]; client != &clients[num_clients]; client++) { GPR_ASSERT(client->stream->Finish().ok()); } + delete[] clients; + + gpr_log(GPR_INFO, "Finishing servers"); + for (auto server = &servers[0]; server != &servers[num_servers]; server++) { + GPR_ASSERT(server->stream->Write(server_mark)); + GPR_ASSERT(server->stream->WritesDone()); + } + for (auto server = &servers[0]; server != &servers[num_servers]; server++) { + GPR_ASSERT(server->stream->Read(&server_status)); + const auto& stats = server_status.stats(); + result->server_resources.emplace_back( + stats.time_elapsed(), stats.time_user(), stats.time_system(), + server_status.cores()); + GPR_ASSERT(!server->stream->Read(&server_status)); + } for (auto server = &servers[0]; server != &servers[num_servers]; server++) { GPR_ASSERT(server->stream->Finish().ok()); } - delete[] clients; + delete[] servers; return result; } From ef3528e2a19f96a17434fcc6a672e1279ea4c262 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 26 Feb 2016 13:25:49 -0800 Subject: [PATCH 010/160] Make client properly report when message deserialization fails --- src/node/src/client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/src/client.js b/src/node/src/client.js index c65dd736503..d6aa999502c 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -336,7 +336,7 @@ function makeUnaryRequestFunction(method, serialize, deserialize) { } } if (status.code !== grpc.status.OK) { - error = new Error(response.status.details); + error = new Error(status.details); error.code = status.code; error.metadata = status.metadata; callback(error); From c47c9497fa6518205a598c2bf74517df09541fed Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 26 Feb 2016 14:39:59 -0800 Subject: [PATCH 011/160] clang-format --- test/cpp/qps/client.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 6a822e35c78..92e77eed9b2 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -253,7 +253,7 @@ class Client { void ThreadFunc() { for (;;) { - // lock since the thread should only be doing one thing at a time + // lock since the thread should only be doing one thing at a time std::lock_guard g(mu_); // run the loop body const bool thread_still_ok = client_->ThreadFunc(&histogram_, idx_); From 6156f0962efdd065c53f550293db7bb1aaa3355b Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Sun, 28 Feb 2016 19:14:07 -0800 Subject: [PATCH 012/160] Simplify PHP installation for release 0.13 --- build.yaml | 27 + config.m4 | 599 ++++++++++++++++++++ package.xml | 964 +++++++++++++++++++++++++++++++++ src/php/README.md | 32 -- templates/config.m4.template | 59 ++ templates/package.xml.template | 161 ++++++ tools/distrib/python/docgen.py | 2 +- 7 files changed, 1811 insertions(+), 33 deletions(-) create mode 100644 config.m4 create mode 100644 package.xml create mode 100644 templates/config.m4.template create mode 100644 templates/package.xml.template diff --git a/build.yaml b/build.yaml index 23853a8f83d..68912fc07ea 100644 --- a/build.yaml +++ b/build.yaml @@ -2788,6 +2788,33 @@ node_modules: - src/node/ext/server.cc - src/node/ext/server_credentials.cc - src/node/ext/timeval.cc +php_config_m4: + deps: + - grpc + - gpr + - boringssl + headers: + - src/php/ext/grpc/byte_buffer.h + - src/php/ext/grpc/call.h + - src/php/ext/grpc/call_credentials.h + - src/php/ext/grpc/channel.h + - src/php/ext/grpc/channel_credentials.h + - src/php/ext/grpc/completion_queue.h + - src/php/ext/grpc/php_grpc.h + - src/php/ext/grpc/server.h + - src/php/ext/grpc/server_credentials.h + - src/php/ext/grpc/timeval.h + src: + - src/php/ext/grpc/byte_buffer.c + - src/php/ext/grpc/call.c + - src/php/ext/grpc/call_credentials.c + - src/php/ext/grpc/channel.c + - src/php/ext/grpc/channel_credentials.c + - src/php/ext/grpc/completion_queue.c + - src/php/ext/grpc/php_grpc.c + - src/php/ext/grpc/server.c + - src/php/ext/grpc/server_credentials.c + - src/php/ext/grpc/timeval.c python_dependencies: deps: - grpc diff --git a/config.m4 b/config.m4 new file mode 100644 index 00000000000..ab614d19ae5 --- /dev/null +++ b/config.m4 @@ -0,0 +1,599 @@ +PHP_ARG_ENABLE(grpc, whether to enable grpc support, +[ --enable-grpc Enable grpc support]) + +if test "$PHP_GRPC" != "no"; then + dnl Write more examples of tests here... + + dnl # --with-grpc -> add include path + PHP_ADD_INCLUDE(../../grpc/include) + PHP_ADD_INCLUDE(../../grpc/src/php/ext/grpc) + PHP_ADD_INCLUDE(../../grpc/third_party/boringssl/include) + + LIBS="-lpthread $LIBS" + + GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" + PHP_ADD_LIBRARY(pthread) + + PHP_ADD_LIBRARY(dl,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(dl) + + case $host in + *darwin*) ;; + *) + PHP_ADD_LIBRARY(rt,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(rt) + ;; + esac + + PHP_NEW_EXTENSION(grpc, + src/php/ext/grpc/byte_buffer.c \ + src/php/ext/grpc/call.c \ + src/php/ext/grpc/call_credentials.c \ + src/php/ext/grpc/channel.c \ + src/php/ext/grpc/channel_credentials.c \ + src/php/ext/grpc/completion_queue.c \ + src/php/ext/grpc/php_grpc.c \ + src/php/ext/grpc/server.c \ + src/php/ext/grpc/server_credentials.c \ + src/php/ext/grpc/timeval.c \ + src/core/profiling/basic_timers.c \ + src/core/profiling/stap_timers.c \ + src/core/support/alloc.c \ + src/core/support/avl.c \ + src/core/support/cmdline.c \ + src/core/support/cpu_iphone.c \ + src/core/support/cpu_linux.c \ + src/core/support/cpu_posix.c \ + src/core/support/cpu_windows.c \ + src/core/support/env_linux.c \ + src/core/support/env_posix.c \ + src/core/support/env_win32.c \ + src/core/support/file.c \ + src/core/support/file_posix.c \ + src/core/support/file_win32.c \ + src/core/support/histogram.c \ + src/core/support/host_port.c \ + src/core/support/log.c \ + src/core/support/log_android.c \ + src/core/support/log_linux.c \ + src/core/support/log_posix.c \ + src/core/support/log_win32.c \ + src/core/support/murmur_hash.c \ + src/core/support/slice.c \ + src/core/support/slice_buffer.c \ + src/core/support/stack_lockfree.c \ + src/core/support/string.c \ + src/core/support/string_posix.c \ + src/core/support/string_win32.c \ + src/core/support/subprocess_posix.c \ + src/core/support/subprocess_windows.c \ + src/core/support/sync.c \ + src/core/support/sync_posix.c \ + src/core/support/sync_win32.c \ + src/core/support/thd.c \ + src/core/support/thd_posix.c \ + src/core/support/thd_win32.c \ + src/core/support/time.c \ + src/core/support/time_posix.c \ + src/core/support/time_precise.c \ + src/core/support/time_win32.c \ + src/core/support/tls_pthread.c \ + src/core/support/wrap_memcpy.c \ + src/core/census/grpc_context.c \ + src/core/census/grpc_filter.c \ + src/core/channel/channel_args.c \ + src/core/channel/channel_stack.c \ + src/core/channel/client_channel.c \ + src/core/channel/client_uchannel.c \ + src/core/channel/compress_filter.c \ + src/core/channel/connected_channel.c \ + src/core/channel/http_client_filter.c \ + src/core/channel/http_server_filter.c \ + src/core/channel/subchannel_call_holder.c \ + src/core/client_config/client_config.c \ + src/core/client_config/connector.c \ + src/core/client_config/default_initial_connect_string.c \ + src/core/client_config/initial_connect_string.c \ + src/core/client_config/lb_policies/pick_first.c \ + src/core/client_config/lb_policies/round_robin.c \ + src/core/client_config/lb_policy.c \ + src/core/client_config/lb_policy_factory.c \ + src/core/client_config/lb_policy_registry.c \ + src/core/client_config/resolver.c \ + src/core/client_config/resolver_factory.c \ + src/core/client_config/resolver_registry.c \ + src/core/client_config/resolvers/dns_resolver.c \ + src/core/client_config/resolvers/sockaddr_resolver.c \ + src/core/client_config/subchannel.c \ + src/core/client_config/subchannel_factory.c \ + src/core/client_config/subchannel_index.c \ + src/core/client_config/uri_parser.c \ + src/core/compression/algorithm.c \ + src/core/compression/message_compress.c \ + src/core/debug/trace.c \ + src/core/httpcli/format_request.c \ + src/core/httpcli/httpcli.c \ + src/core/httpcli/parser.c \ + src/core/iomgr/closure.c \ + src/core/iomgr/endpoint.c \ + src/core/iomgr/endpoint_pair_posix.c \ + src/core/iomgr/endpoint_pair_windows.c \ + src/core/iomgr/exec_ctx.c \ + src/core/iomgr/executor.c \ + src/core/iomgr/fd_posix.c \ + src/core/iomgr/iocp_windows.c \ + src/core/iomgr/iomgr.c \ + src/core/iomgr/iomgr_posix.c \ + src/core/iomgr/iomgr_windows.c \ + src/core/iomgr/pollset_multipoller_with_epoll.c \ + src/core/iomgr/pollset_multipoller_with_poll_posix.c \ + src/core/iomgr/pollset_posix.c \ + src/core/iomgr/pollset_set_posix.c \ + src/core/iomgr/pollset_set_windows.c \ + src/core/iomgr/pollset_windows.c \ + src/core/iomgr/resolve_address_posix.c \ + src/core/iomgr/resolve_address_windows.c \ + src/core/iomgr/sockaddr_utils.c \ + src/core/iomgr/socket_utils_common_posix.c \ + src/core/iomgr/socket_utils_linux.c \ + src/core/iomgr/socket_utils_posix.c \ + src/core/iomgr/socket_windows.c \ + src/core/iomgr/tcp_client_posix.c \ + src/core/iomgr/tcp_client_windows.c \ + src/core/iomgr/tcp_posix.c \ + src/core/iomgr/tcp_server_posix.c \ + src/core/iomgr/tcp_server_windows.c \ + src/core/iomgr/tcp_windows.c \ + src/core/iomgr/time_averaged_stats.c \ + src/core/iomgr/timer.c \ + src/core/iomgr/timer_heap.c \ + src/core/iomgr/udp_server.c \ + src/core/iomgr/wakeup_fd_eventfd.c \ + src/core/iomgr/wakeup_fd_nospecial.c \ + src/core/iomgr/wakeup_fd_pipe.c \ + src/core/iomgr/wakeup_fd_posix.c \ + src/core/iomgr/workqueue_posix.c \ + src/core/iomgr/workqueue_windows.c \ + src/core/json/json.c \ + src/core/json/json_reader.c \ + src/core/json/json_string.c \ + src/core/json/json_writer.c \ + src/core/surface/alarm.c \ + src/core/surface/api_trace.c \ + src/core/surface/byte_buffer.c \ + src/core/surface/byte_buffer_reader.c \ + src/core/surface/call.c \ + src/core/surface/call_details.c \ + src/core/surface/call_log_batch.c \ + src/core/surface/channel.c \ + src/core/surface/channel_connectivity.c \ + src/core/surface/channel_create.c \ + src/core/surface/channel_ping.c \ + src/core/surface/completion_queue.c \ + src/core/surface/event_string.c \ + src/core/surface/init.c \ + src/core/surface/lame_client.c \ + src/core/surface/metadata_array.c \ + src/core/surface/server.c \ + src/core/surface/server_chttp2.c \ + src/core/surface/server_create.c \ + src/core/surface/validate_metadata.c \ + src/core/surface/version.c \ + src/core/transport/byte_stream.c \ + src/core/transport/chttp2/alpn.c \ + src/core/transport/chttp2/bin_encoder.c \ + src/core/transport/chttp2/frame_data.c \ + src/core/transport/chttp2/frame_goaway.c \ + src/core/transport/chttp2/frame_ping.c \ + src/core/transport/chttp2/frame_rst_stream.c \ + src/core/transport/chttp2/frame_settings.c \ + src/core/transport/chttp2/frame_window_update.c \ + src/core/transport/chttp2/hpack_encoder.c \ + src/core/transport/chttp2/hpack_parser.c \ + src/core/transport/chttp2/hpack_table.c \ + src/core/transport/chttp2/huffsyms.c \ + src/core/transport/chttp2/incoming_metadata.c \ + src/core/transport/chttp2/parsing.c \ + src/core/transport/chttp2/status_conversion.c \ + src/core/transport/chttp2/stream_lists.c \ + src/core/transport/chttp2/stream_map.c \ + src/core/transport/chttp2/timeout_encoding.c \ + src/core/transport/chttp2/varint.c \ + src/core/transport/chttp2/writing.c \ + src/core/transport/chttp2_transport.c \ + src/core/transport/connectivity_state.c \ + src/core/transport/metadata.c \ + src/core/transport/metadata_batch.c \ + src/core/transport/static_metadata.c \ + src/core/transport/transport.c \ + src/core/transport/transport_op_string.c \ + src/core/httpcli/httpcli_security_connector.c \ + src/core/security/base64.c \ + src/core/security/client_auth_filter.c \ + src/core/security/credentials.c \ + src/core/security/credentials_metadata.c \ + src/core/security/credentials_posix.c \ + src/core/security/credentials_win32.c \ + src/core/security/google_default_credentials.c \ + src/core/security/handshake.c \ + src/core/security/json_token.c \ + src/core/security/jwt_verifier.c \ + src/core/security/secure_endpoint.c \ + src/core/security/security_connector.c \ + src/core/security/security_context.c \ + src/core/security/server_auth_filter.c \ + src/core/security/server_secure_chttp2.c \ + src/core/surface/init_secure.c \ + src/core/surface/secure_channel_create.c \ + src/core/tsi/fake_transport_security.c \ + src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security.c \ + src/core/census/context.c \ + src/core/census/initialize.c \ + src/core/census/mlog.c \ + src/core/census/operation.c \ + src/core/census/placeholders.c \ + src/core/census/tracing.c \ + src/boringssl/err_data.c \ + third_party/boringssl/crypto/aes/aes.c \ + third_party/boringssl/crypto/aes/mode_wrappers.c \ + third_party/boringssl/crypto/asn1/a_bitstr.c \ + third_party/boringssl/crypto/asn1/a_bool.c \ + third_party/boringssl/crypto/asn1/a_bytes.c \ + third_party/boringssl/crypto/asn1/a_d2i_fp.c \ + third_party/boringssl/crypto/asn1/a_dup.c \ + third_party/boringssl/crypto/asn1/a_enum.c \ + third_party/boringssl/crypto/asn1/a_gentm.c \ + third_party/boringssl/crypto/asn1/a_i2d_fp.c \ + third_party/boringssl/crypto/asn1/a_int.c \ + third_party/boringssl/crypto/asn1/a_mbstr.c \ + third_party/boringssl/crypto/asn1/a_object.c \ + third_party/boringssl/crypto/asn1/a_octet.c \ + third_party/boringssl/crypto/asn1/a_print.c \ + third_party/boringssl/crypto/asn1/a_strnid.c \ + third_party/boringssl/crypto/asn1/a_time.c \ + third_party/boringssl/crypto/asn1/a_type.c \ + third_party/boringssl/crypto/asn1/a_utctm.c \ + third_party/boringssl/crypto/asn1/a_utf8.c \ + third_party/boringssl/crypto/asn1/asn1_lib.c \ + third_party/boringssl/crypto/asn1/asn1_par.c \ + third_party/boringssl/crypto/asn1/asn_pack.c \ + third_party/boringssl/crypto/asn1/bio_asn1.c \ + third_party/boringssl/crypto/asn1/bio_ndef.c \ + third_party/boringssl/crypto/asn1/f_enum.c \ + third_party/boringssl/crypto/asn1/f_int.c \ + third_party/boringssl/crypto/asn1/f_string.c \ + third_party/boringssl/crypto/asn1/t_bitst.c \ + third_party/boringssl/crypto/asn1/t_pkey.c \ + third_party/boringssl/crypto/asn1/tasn_dec.c \ + third_party/boringssl/crypto/asn1/tasn_enc.c \ + third_party/boringssl/crypto/asn1/tasn_fre.c \ + third_party/boringssl/crypto/asn1/tasn_new.c \ + third_party/boringssl/crypto/asn1/tasn_prn.c \ + third_party/boringssl/crypto/asn1/tasn_typ.c \ + third_party/boringssl/crypto/asn1/tasn_utl.c \ + third_party/boringssl/crypto/asn1/x_bignum.c \ + third_party/boringssl/crypto/asn1/x_long.c \ + third_party/boringssl/crypto/base64/base64.c \ + third_party/boringssl/crypto/bio/bio.c \ + third_party/boringssl/crypto/bio/bio_mem.c \ + third_party/boringssl/crypto/bio/buffer.c \ + third_party/boringssl/crypto/bio/connect.c \ + third_party/boringssl/crypto/bio/fd.c \ + third_party/boringssl/crypto/bio/file.c \ + third_party/boringssl/crypto/bio/hexdump.c \ + third_party/boringssl/crypto/bio/pair.c \ + third_party/boringssl/crypto/bio/printf.c \ + third_party/boringssl/crypto/bio/socket.c \ + third_party/boringssl/crypto/bio/socket_helper.c \ + third_party/boringssl/crypto/bn/add.c \ + third_party/boringssl/crypto/bn/asm/x86_64-gcc.c \ + third_party/boringssl/crypto/bn/bn.c \ + third_party/boringssl/crypto/bn/bn_asn1.c \ + third_party/boringssl/crypto/bn/cmp.c \ + third_party/boringssl/crypto/bn/convert.c \ + third_party/boringssl/crypto/bn/ctx.c \ + third_party/boringssl/crypto/bn/div.c \ + third_party/boringssl/crypto/bn/exponentiation.c \ + third_party/boringssl/crypto/bn/gcd.c \ + third_party/boringssl/crypto/bn/generic.c \ + third_party/boringssl/crypto/bn/kronecker.c \ + third_party/boringssl/crypto/bn/montgomery.c \ + third_party/boringssl/crypto/bn/mul.c \ + third_party/boringssl/crypto/bn/prime.c \ + third_party/boringssl/crypto/bn/random.c \ + third_party/boringssl/crypto/bn/rsaz_exp.c \ + third_party/boringssl/crypto/bn/shift.c \ + third_party/boringssl/crypto/bn/sqrt.c \ + third_party/boringssl/crypto/buf/buf.c \ + third_party/boringssl/crypto/bytestring/ber.c \ + third_party/boringssl/crypto/bytestring/cbb.c \ + third_party/boringssl/crypto/bytestring/cbs.c \ + third_party/boringssl/crypto/chacha/chacha_generic.c \ + third_party/boringssl/crypto/chacha/chacha_vec.c \ + third_party/boringssl/crypto/cipher/aead.c \ + third_party/boringssl/crypto/cipher/cipher.c \ + third_party/boringssl/crypto/cipher/derive_key.c \ + third_party/boringssl/crypto/cipher/e_aes.c \ + third_party/boringssl/crypto/cipher/e_chacha20poly1305.c \ + third_party/boringssl/crypto/cipher/e_des.c \ + third_party/boringssl/crypto/cipher/e_null.c \ + third_party/boringssl/crypto/cipher/e_rc2.c \ + third_party/boringssl/crypto/cipher/e_rc4.c \ + third_party/boringssl/crypto/cipher/e_ssl3.c \ + third_party/boringssl/crypto/cipher/e_tls.c \ + third_party/boringssl/crypto/cipher/tls_cbc.c \ + third_party/boringssl/crypto/cmac/cmac.c \ + third_party/boringssl/crypto/conf/conf.c \ + third_party/boringssl/crypto/cpu-arm.c \ + third_party/boringssl/crypto/cpu-intel.c \ + third_party/boringssl/crypto/crypto.c \ + third_party/boringssl/crypto/curve25519/curve25519.c \ + third_party/boringssl/crypto/des/des.c \ + third_party/boringssl/crypto/dh/check.c \ + third_party/boringssl/crypto/dh/dh.c \ + third_party/boringssl/crypto/dh/dh_asn1.c \ + third_party/boringssl/crypto/dh/params.c \ + third_party/boringssl/crypto/digest/digest.c \ + third_party/boringssl/crypto/digest/digests.c \ + third_party/boringssl/crypto/directory_posix.c \ + third_party/boringssl/crypto/directory_win.c \ + third_party/boringssl/crypto/dsa/dsa.c \ + third_party/boringssl/crypto/dsa/dsa_asn1.c \ + third_party/boringssl/crypto/ec/ec.c \ + third_party/boringssl/crypto/ec/ec_asn1.c \ + third_party/boringssl/crypto/ec/ec_key.c \ + third_party/boringssl/crypto/ec/ec_montgomery.c \ + third_party/boringssl/crypto/ec/oct.c \ + third_party/boringssl/crypto/ec/p224-64.c \ + third_party/boringssl/crypto/ec/p256-64.c \ + third_party/boringssl/crypto/ec/p256-x86_64.c \ + third_party/boringssl/crypto/ec/simple.c \ + third_party/boringssl/crypto/ec/util-64.c \ + third_party/boringssl/crypto/ec/wnaf.c \ + third_party/boringssl/crypto/ecdh/ecdh.c \ + third_party/boringssl/crypto/ecdsa/ecdsa.c \ + third_party/boringssl/crypto/ecdsa/ecdsa_asn1.c \ + third_party/boringssl/crypto/engine/engine.c \ + third_party/boringssl/crypto/err/err.c \ + third_party/boringssl/crypto/evp/algorithm.c \ + third_party/boringssl/crypto/evp/digestsign.c \ + third_party/boringssl/crypto/evp/evp.c \ + third_party/boringssl/crypto/evp/evp_asn1.c \ + third_party/boringssl/crypto/evp/evp_ctx.c \ + third_party/boringssl/crypto/evp/p_dsa_asn1.c \ + third_party/boringssl/crypto/evp/p_ec.c \ + third_party/boringssl/crypto/evp/p_ec_asn1.c \ + third_party/boringssl/crypto/evp/p_rsa.c \ + third_party/boringssl/crypto/evp/p_rsa_asn1.c \ + third_party/boringssl/crypto/evp/pbkdf.c \ + third_party/boringssl/crypto/evp/sign.c \ + third_party/boringssl/crypto/ex_data.c \ + third_party/boringssl/crypto/hkdf/hkdf.c \ + third_party/boringssl/crypto/hmac/hmac.c \ + third_party/boringssl/crypto/lhash/lhash.c \ + third_party/boringssl/crypto/md4/md4.c \ + third_party/boringssl/crypto/md5/md5.c \ + third_party/boringssl/crypto/mem.c \ + third_party/boringssl/crypto/modes/cbc.c \ + third_party/boringssl/crypto/modes/cfb.c \ + third_party/boringssl/crypto/modes/ctr.c \ + third_party/boringssl/crypto/modes/gcm.c \ + third_party/boringssl/crypto/modes/ofb.c \ + third_party/boringssl/crypto/obj/obj.c \ + third_party/boringssl/crypto/obj/obj_xref.c \ + third_party/boringssl/crypto/pem/pem_all.c \ + third_party/boringssl/crypto/pem/pem_info.c \ + third_party/boringssl/crypto/pem/pem_lib.c \ + third_party/boringssl/crypto/pem/pem_oth.c \ + third_party/boringssl/crypto/pem/pem_pk8.c \ + third_party/boringssl/crypto/pem/pem_pkey.c \ + third_party/boringssl/crypto/pem/pem_x509.c \ + third_party/boringssl/crypto/pem/pem_xaux.c \ + third_party/boringssl/crypto/pkcs8/p5_pbe.c \ + third_party/boringssl/crypto/pkcs8/p5_pbev2.c \ + third_party/boringssl/crypto/pkcs8/p8_pkey.c \ + third_party/boringssl/crypto/pkcs8/pkcs8.c \ + third_party/boringssl/crypto/poly1305/poly1305.c \ + third_party/boringssl/crypto/poly1305/poly1305_arm.c \ + third_party/boringssl/crypto/poly1305/poly1305_vec.c \ + third_party/boringssl/crypto/rand/rand.c \ + third_party/boringssl/crypto/rand/urandom.c \ + third_party/boringssl/crypto/rand/windows.c \ + third_party/boringssl/crypto/rc4/rc4.c \ + third_party/boringssl/crypto/refcount_c11.c \ + third_party/boringssl/crypto/refcount_lock.c \ + third_party/boringssl/crypto/rsa/blinding.c \ + third_party/boringssl/crypto/rsa/padding.c \ + third_party/boringssl/crypto/rsa/rsa.c \ + third_party/boringssl/crypto/rsa/rsa_asn1.c \ + third_party/boringssl/crypto/rsa/rsa_impl.c \ + third_party/boringssl/crypto/sha/sha1.c \ + third_party/boringssl/crypto/sha/sha256.c \ + third_party/boringssl/crypto/sha/sha512.c \ + third_party/boringssl/crypto/stack/stack.c \ + third_party/boringssl/crypto/thread.c \ + third_party/boringssl/crypto/thread_none.c \ + third_party/boringssl/crypto/thread_pthread.c \ + third_party/boringssl/crypto/thread_win.c \ + third_party/boringssl/crypto/time_support.c \ + third_party/boringssl/crypto/x509/a_digest.c \ + third_party/boringssl/crypto/x509/a_sign.c \ + third_party/boringssl/crypto/x509/a_strex.c \ + third_party/boringssl/crypto/x509/a_verify.c \ + third_party/boringssl/crypto/x509/asn1_gen.c \ + third_party/boringssl/crypto/x509/by_dir.c \ + third_party/boringssl/crypto/x509/by_file.c \ + third_party/boringssl/crypto/x509/i2d_pr.c \ + third_party/boringssl/crypto/x509/pkcs7.c \ + third_party/boringssl/crypto/x509/t_crl.c \ + third_party/boringssl/crypto/x509/t_req.c \ + third_party/boringssl/crypto/x509/t_x509.c \ + third_party/boringssl/crypto/x509/t_x509a.c \ + third_party/boringssl/crypto/x509/x509.c \ + third_party/boringssl/crypto/x509/x509_att.c \ + third_party/boringssl/crypto/x509/x509_cmp.c \ + third_party/boringssl/crypto/x509/x509_d2.c \ + third_party/boringssl/crypto/x509/x509_def.c \ + third_party/boringssl/crypto/x509/x509_ext.c \ + third_party/boringssl/crypto/x509/x509_lu.c \ + third_party/boringssl/crypto/x509/x509_obj.c \ + third_party/boringssl/crypto/x509/x509_r2x.c \ + third_party/boringssl/crypto/x509/x509_req.c \ + third_party/boringssl/crypto/x509/x509_set.c \ + third_party/boringssl/crypto/x509/x509_trs.c \ + third_party/boringssl/crypto/x509/x509_txt.c \ + third_party/boringssl/crypto/x509/x509_v3.c \ + third_party/boringssl/crypto/x509/x509_vfy.c \ + third_party/boringssl/crypto/x509/x509_vpm.c \ + third_party/boringssl/crypto/x509/x509cset.c \ + third_party/boringssl/crypto/x509/x509name.c \ + third_party/boringssl/crypto/x509/x509rset.c \ + third_party/boringssl/crypto/x509/x509spki.c \ + third_party/boringssl/crypto/x509/x509type.c \ + third_party/boringssl/crypto/x509/x_algor.c \ + third_party/boringssl/crypto/x509/x_all.c \ + third_party/boringssl/crypto/x509/x_attrib.c \ + third_party/boringssl/crypto/x509/x_crl.c \ + third_party/boringssl/crypto/x509/x_exten.c \ + third_party/boringssl/crypto/x509/x_info.c \ + third_party/boringssl/crypto/x509/x_name.c \ + third_party/boringssl/crypto/x509/x_pkey.c \ + third_party/boringssl/crypto/x509/x_pubkey.c \ + third_party/boringssl/crypto/x509/x_req.c \ + third_party/boringssl/crypto/x509/x_sig.c \ + third_party/boringssl/crypto/x509/x_spki.c \ + third_party/boringssl/crypto/x509/x_val.c \ + third_party/boringssl/crypto/x509/x_x509.c \ + third_party/boringssl/crypto/x509/x_x509a.c \ + third_party/boringssl/crypto/x509v3/pcy_cache.c \ + third_party/boringssl/crypto/x509v3/pcy_data.c \ + third_party/boringssl/crypto/x509v3/pcy_lib.c \ + third_party/boringssl/crypto/x509v3/pcy_map.c \ + third_party/boringssl/crypto/x509v3/pcy_node.c \ + third_party/boringssl/crypto/x509v3/pcy_tree.c \ + third_party/boringssl/crypto/x509v3/v3_akey.c \ + third_party/boringssl/crypto/x509v3/v3_akeya.c \ + third_party/boringssl/crypto/x509v3/v3_alt.c \ + third_party/boringssl/crypto/x509v3/v3_bcons.c \ + third_party/boringssl/crypto/x509v3/v3_bitst.c \ + third_party/boringssl/crypto/x509v3/v3_conf.c \ + third_party/boringssl/crypto/x509v3/v3_cpols.c \ + third_party/boringssl/crypto/x509v3/v3_crld.c \ + third_party/boringssl/crypto/x509v3/v3_enum.c \ + third_party/boringssl/crypto/x509v3/v3_extku.c \ + third_party/boringssl/crypto/x509v3/v3_genn.c \ + third_party/boringssl/crypto/x509v3/v3_ia5.c \ + third_party/boringssl/crypto/x509v3/v3_info.c \ + third_party/boringssl/crypto/x509v3/v3_int.c \ + third_party/boringssl/crypto/x509v3/v3_lib.c \ + third_party/boringssl/crypto/x509v3/v3_ncons.c \ + third_party/boringssl/crypto/x509v3/v3_pci.c \ + third_party/boringssl/crypto/x509v3/v3_pcia.c \ + third_party/boringssl/crypto/x509v3/v3_pcons.c \ + third_party/boringssl/crypto/x509v3/v3_pku.c \ + third_party/boringssl/crypto/x509v3/v3_pmaps.c \ + third_party/boringssl/crypto/x509v3/v3_prn.c \ + third_party/boringssl/crypto/x509v3/v3_purp.c \ + third_party/boringssl/crypto/x509v3/v3_skey.c \ + third_party/boringssl/crypto/x509v3/v3_sxnet.c \ + third_party/boringssl/crypto/x509v3/v3_utl.c \ + third_party/boringssl/ssl/custom_extensions.c \ + third_party/boringssl/ssl/d1_both.c \ + third_party/boringssl/ssl/d1_clnt.c \ + third_party/boringssl/ssl/d1_lib.c \ + third_party/boringssl/ssl/d1_meth.c \ + third_party/boringssl/ssl/d1_pkt.c \ + third_party/boringssl/ssl/d1_srtp.c \ + third_party/boringssl/ssl/d1_srvr.c \ + third_party/boringssl/ssl/dtls_record.c \ + third_party/boringssl/ssl/pqueue/pqueue.c \ + third_party/boringssl/ssl/s3_both.c \ + third_party/boringssl/ssl/s3_clnt.c \ + third_party/boringssl/ssl/s3_enc.c \ + third_party/boringssl/ssl/s3_lib.c \ + third_party/boringssl/ssl/s3_meth.c \ + third_party/boringssl/ssl/s3_pkt.c \ + third_party/boringssl/ssl/s3_srvr.c \ + third_party/boringssl/ssl/ssl_aead_ctx.c \ + third_party/boringssl/ssl/ssl_asn1.c \ + third_party/boringssl/ssl/ssl_buffer.c \ + third_party/boringssl/ssl/ssl_cert.c \ + third_party/boringssl/ssl/ssl_cipher.c \ + third_party/boringssl/ssl/ssl_file.c \ + third_party/boringssl/ssl/ssl_lib.c \ + third_party/boringssl/ssl/ssl_rsa.c \ + third_party/boringssl/ssl/ssl_session.c \ + third_party/boringssl/ssl/ssl_stat.c \ + third_party/boringssl/ssl/t1_enc.c \ + third_party/boringssl/ssl/t1_lib.c \ + third_party/boringssl/ssl/tls_record.c \ + , $ext_shared, , -Wall -Werror \ + -Wno-parentheses-equality -Wno-unused-value -std=c11 \ + -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN \ + -D_HAS_EXCEPTIONS=0 -DNOMINMAX) + + PHP_ADD_BUILD_DIR($ext_builddir/src/php/ext/grpc) + + PHP_ADD_BUILD_DIR($ext_builddir/src/boringssl) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/census) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/channel) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_config) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_config/lb_policies) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_config/resolvers) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/compression) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/debug) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/httpcli) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/iomgr) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/json) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/profiling) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/security) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/support) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/surface) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/transport) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/transport/chttp2) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/aes) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/asn1) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/base64) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bio) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bn) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bn/asm) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/buf) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bytestring) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/chacha) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/cipher) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/cmac) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/conf) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/curve25519) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/des) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/dh) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/digest) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/dsa) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/ec) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/ecdh) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/ecdsa) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/engine) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/err) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/evp) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/hkdf) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/hmac) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/lhash) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/md4) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/md5) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/modes) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/obj) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/pem) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/pkcs8) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/poly1305) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/rand) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/rc4) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/rsa) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/sha) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/stack) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/x509) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/x509v3) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/ssl) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/ssl/pqueue) +fi diff --git a/package.xml b/package.xml new file mode 100644 index 00000000000..68d6dfb9402 --- /dev/null +++ b/package.xml @@ -0,0 +1,964 @@ + + + grpc + pecl.php.net + A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. + Remote Procedure Calls (RPCs) provide a useful abstraction for building distributed applications and services. The libraries in this repository provide a concrete implementation of the gRPC protocol, layered over HTTP/2. These libraries enable communication between clients and servers using any combination of the supported languages. + + Stanley Cheung + stanleycheung + grpc-packages@google.com + yes + + 2016-02-24 + + + 0.8.0 + 0.8.0 + + + beta + beta + + BSD + +- Simplify gRPC PHP installation #4517 +- Wrap gRPC core library version 0.13 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5.5.0 + + + 1.4.0 + + + + grpc + + + + + 0.5.0 + 0.5.0 + + + alpha + alpha + + 2015-06-16 + BSD + +First alpha release + + + + + 0.5.1 + 0.5.1 + + + alpha + alpha + + 2015-07-09 + BSD + +Update to wrap gRPC C Core version 0.10.0 + + + + + 0.6.0 + 0.6.0 + + + beta + beta + + 2015-09-24 + BSD + +- support per message compression disable +- expose per-call host override option +- expose connectivity API +- expose channel target and call peer +- add user-agent +- update to wrap gRPC C core library beta version 0.11.0 + + + + + 0.6.1 + 0.6.0 + + + beta + beta + + 2015-10-21 + BSD + +- fixed undefined constant fatal error when run with apache/nginx #2275 + + + + + 0.7.0 + 0.7.0 + + + beta + beta + + 2016-01-13 + BSD + +- Breaking change to Credentials class (removed) #3765 +- Replaced by ChannelCredentials and CallCredentials class #3765 +- New plugin based metadata auth API #4394 +- Explicit ChannelCredentials::createInsecure() call + + + + + 0.8.0 + 0.8.0 + + + beta + beta + + 2016-02-24 + BSD + +- Simplify gRPC PHP installation #4517 +- Wrap gRPC core library version 0.13 + + + + diff --git a/src/php/README.md b/src/php/README.md index b1823b92261..08d0b413c95 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -33,44 +33,12 @@ $ sudo mv phpunit.phar /usr/local/bin/phpunit ## Quick Install -**Linux (Debian):** - -Add [Debian jessie-backports][] to your `sources.list` file. Example: - -```sh -echo "deb http://http.debian.net/debian jessie-backports main" | \ -sudo tee -a /etc/apt/sources.list -``` - -Install the gRPC Debian package - -```sh -sudo apt-get update -sudo apt-get install libgrpc-dev -``` - Install the gRPC PHP extension ```sh sudo pecl install grpc-beta ``` -**Mac OS X:** - -Install [homebrew][]. Example: - -```sh -ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -``` - -Install the gRPC core library and the PHP extension in one step - -```sh -$ curl -fsSL https://goo.gl/getgrpc | bash -s php -``` - -This will download and run the [gRPC install script][] and compile the gRPC PHP extension. - ## Build from Source diff --git a/templates/config.m4.template b/templates/config.m4.template new file mode 100644 index 00000000000..5847d456f51 --- /dev/null +++ b/templates/config.m4.template @@ -0,0 +1,59 @@ +%YAML 1.2 +--- | + PHP_ARG_ENABLE(grpc, whether to enable grpc support, + [ --enable-grpc Enable grpc support]) + + if test "$PHP_GRPC" != "no"; then + dnl Write more examples of tests here... + + dnl # --with-grpc -> add include path + PHP_ADD_INCLUDE(../../grpc/include) + PHP_ADD_INCLUDE(../../grpc/src/php/ext/grpc) + PHP_ADD_INCLUDE(../../grpc/third_party/boringssl/include) + + LIBS="-lpthread $LIBS" + + GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" + PHP_ADD_LIBRARY(pthread) + + PHP_ADD_LIBRARY(dl,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(dl) + + case $host in + *darwin*) ;; + *) + PHP_ADD_LIBRARY(rt,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(rt) + ;; + esac + + PHP_NEW_EXTENSION(grpc, + % for source in php_config_m4.src: + ${source} ${"\\"} + % endfor + % for lib in libs: + % if lib.name in php_config_m4.get('deps', []): + % for source in lib.src: + ${source} ${"\\"} + % endfor + % endif + % endfor + , $ext_shared, , -Wall -Werror ${"\\"} + -Wno-parentheses-equality -Wno-unused-value -std=c11 ${"\\"} + -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN ${"\\"} + -D_HAS_EXCEPTIONS=0 -DNOMINMAX) + + PHP_ADD_BUILD_DIR($ext_builddir/src/php/ext/grpc) + <% + dirs = {} + for lib in libs: + if lib.name in php_config_m4.get('deps', []): + for source in lib.src: + dirs[source[:source.rfind('/')]] = 1 + dirs = dirs.keys() + dirs.sort() + %> + % for dir in dirs: + PHP_ADD_BUILD_DIR($ext_builddir/${dir}) + % endfor + fi diff --git a/templates/package.xml.template b/templates/package.xml.template new file mode 100644 index 00000000000..058f5064715 --- /dev/null +++ b/templates/package.xml.template @@ -0,0 +1,161 @@ +%YAML 1.2 +--- | + + + grpc + pecl.php.net + A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. + Remote Procedure Calls (RPCs) provide a useful abstraction for building distributed applications and services. The libraries in this repository provide a concrete implementation of the gRPC protocol, layered over HTTP/2. These libraries enable communication between clients and servers using any combination of the supported languages. + + Stanley Cheung + stanleycheung + grpc-packages@google.com + yes + + 2016-02-24 + + + 0.8.0 + 0.8.0 + + + beta + beta + + BSD + + - Simplify gRPC PHP installation #4517 + - Wrap gRPC core library version 0.13 + + + + + + + + % for source in php_config_m4.src + php_config_m4.headers: + + % endfor + % for lib in libs: + % if lib.name in php_config_m4.get('deps', []): + % for source in lib.get('public_headers', []) + lib.headers + lib.src: + + % endfor + % endif + % endfor + + + + + + 5.5.0 + + + 1.4.0 + + + + grpc + + + + + 0.5.0 + 0.5.0 + + + alpha + alpha + + 2015-06-16 + BSD + + First alpha release + + + + + 0.5.1 + 0.5.1 + + + alpha + alpha + + 2015-07-09 + BSD + + Update to wrap gRPC C Core version 0.10.0 + + + + + 0.6.0 + 0.6.0 + + + beta + beta + + 2015-09-24 + BSD + + - support per message compression disable + - expose per-call host override option + - expose connectivity API + - expose channel target and call peer + - add user-agent + - update to wrap gRPC C core library beta version 0.11.0 + + + + + 0.6.1 + 0.6.0 + + + beta + beta + + 2015-10-21 + BSD + + - fixed undefined constant fatal error when run with apache/nginx #2275 + + + + + 0.7.0 + 0.7.0 + + + beta + beta + + 2016-01-13 + BSD + + - Breaking change to Credentials class (removed) #3765 + - Replaced by ChannelCredentials and CallCredentials class #3765 + - New plugin based metadata auth API #4394 + - Explicit ChannelCredentials::createInsecure() call + + + + + 0.8.0 + 0.8.0 + + + beta + beta + + 2016-02-24 + BSD + + - Simplify gRPC PHP installation #4517 + - Wrap gRPC core library version 0.13 + + + + diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index 72c65ad14a8..161d83e3b7f 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From 494f3128331adcf601903b00ea62b767d99f84c2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 1 Mar 2016 13:55:42 -0800 Subject: [PATCH 013/160] fix reporting for multiple test runs --- tools/run_tests/jobset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index adf178bb3c3..a3b246dc084 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -384,7 +384,8 @@ class Jobset(object): self._travis, self._add_env) self._running.add(job) - self.resultset[job.GetSpec().shortname] = [] + if not self.resultset.has_key(job.GetSpec().shortname): + self.resultset[job.GetSpec().shortname] = [] return True def reap(self): From 7d91dc3181e94bef7cf85bf8885cb8c3f3d0ff62 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 1 Mar 2016 12:33:20 -0800 Subject: [PATCH 014/160] fix #4427 --- src/csharp/Grpc.Core/Internal/AsyncCallServer.cs | 10 ---------- .../Grpc.IntegrationTesting/InteropClientServerTest.cs | 2 -- 2 files changed, 12 deletions(-) diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs index b72cbd795f6..9380c0d0ea9 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs @@ -193,16 +193,6 @@ namespace Grpc.Core.Internal lock (myLock) { finished = true; - - if (cancelled) - { - // Once we cancel, we don't have to care that much - // about reads and writes. - - // TODO(jtattermusch): is this still necessary? - Cancel(); - } - ReleaseResourcesIfPossible(); } // TODO(jtattermusch): handle error diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs index 18168f99704..5facb87971e 100644 --- a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs @@ -140,14 +140,12 @@ namespace Grpc.IntegrationTesting } [Test] - [Ignore("TODO: see #4427")] public async Task StatusCodeAndMessage() { await InteropClient.RunStatusCodeAndMessageAsync(client); } [Test] - [Ignore("TODO: see #4427")] public void UnimplementedMethod() { InteropClient.RunUnimplementedMethod(UnimplementedService.NewClient(channel)); From 06c98fa49a6a92d230932b79a153de03a2f16db9 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 1 Mar 2016 14:45:03 -0800 Subject: [PATCH 015/160] fix copyright --- src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs index 5facb87971e..0d12c4168c9 100644 --- a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs @@ -1,6 +1,6 @@ #region Copyright notice and license -// Copyright 2015, Google Inc. +// Copyright 2015-2016, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without From 6c016efa34113916810cf16a0a0981c637204f50 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 1 Mar 2016 16:27:53 -0800 Subject: [PATCH 016/160] ServerTryCancel was not actually respecting the API since it could be an arbitrary amount of time between when the cancel is tried and actually observable. --- test/cpp/end2end/test_service_impl.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc index 66d11d0dfce..7c3e514effa 100644 --- a/test/cpp/end2end/test_service_impl.cc +++ b/test/cpp/end2end/test_service_impl.cc @@ -326,7 +326,11 @@ void TestServiceImpl::ServerTryCancel(ServerContext* context) { EXPECT_FALSE(context->IsCancelled()); context->TryCancel(); gpr_log(GPR_INFO, "Server called TryCancel() to cancel the request"); - EXPECT_TRUE(context->IsCancelled()); + // Now wait until it's really canceled + while (!context->IsCancelled()) { + gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_micros(1000, GPR_TIMESPAN))); + } } } // namespace testing From 9c691c59da630832e8b5962ff2e51e8e890e359e Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 1 Mar 2016 16:45:40 -0800 Subject: [PATCH 017/160] updated templates/README.md --- templates/README.md | 103 ++++++++++++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/templates/README.md b/templates/README.md index 6740972cfb6..a27c44da3c1 100644 --- a/templates/README.md +++ b/templates/README.md @@ -6,12 +6,12 @@ was going to single handedly cover all of our usage cases. So instead we decided to work the following way: -* A build.json file at the root is the source of truth for listing all of the -target and files needed to build grpc and its tests, as well as basic system -dependencies description. +* A `build.yaml` file at the root is the source of truth for listing all the +targets and files needed to build grpc and its tests, as well as a basic system +for dependency description. * Each project file (Makefile, Visual Studio project files, Bazel's BUILD) is -a plain-text template that uses the build.json file to generate the final +a plain-text template that uses the `build.yaml` file to generate the final output file. This way we can maintain as many project system as we see fit, without having @@ -20,64 +20,77 @@ Only the structure of the project file is relevant to the template. The actual list of source code and targets isn't. We currently have template files for GNU Make, Visual Studio 2010 to 2015, -and Bazel. In the future, we would like to expand to generating gyp or cmake +and [Bazel](http://bazel.io). In the future, we would like to expand to +generating [gyp](https://gyp.gsrc.io/) or [cmake](https://cmake.org) project files (or potentially both), XCode project files, and an Android.mk file to be able to compile gRPC using Android's NDK. We'll gladly accept contribution that'd create additional project files using that system. -# Structure of build.json +# Structure of `build.yaml` -The build.json file has the following structure: +The `build.yaml` file has the following structure: ``` -{ - "settings": { ... }, # global settings, such as version number - "filegroups": [ ... ], # groups of file that is automatically expanded - "libs": [ ... ], # list of libraries to build - "targets": [ ... ], # list of targets to build -} +settings: # global settings, such as version number + ... +filegroups: # groups of files that are automatically expanded + ... +libs: # list of libraries to build + ... +target: # list of targets to build + ... ``` The `filegroups` are helpful to re-use a subset of files in multiple targets. One `filegroups` entry has the following structure: ``` -{ - "name": "arbitrary string", # the name of the filegroup - "public_headers": [ ... ], # list of public headers defined in that filegroup - "headers": [ ... ], # list of headers defined in that filegroup - "src": [ ... ], # list of source files defined in that filegroup -} +- name: "arbitrary string", # the name of the filegroup + public_headers: # list of public headers defined in that filegroup + - ... + headers: # list of headers defined in that filegroup + - ... + src: # list of source files defined in that filegroup + - ... ``` -The `libs` array contains the list of all the libraries we describe. Some may be +The `libs` collection contains the list of all the libraries we describe. Some may be helper libraries for the tests. Some may be installable libraries. Some may be helper libraries for installable binaries. The `targets` array contains the list of all the binary targets we describe. Some may be installable binaries. -One `libs` or `targets` entry has the following structure: +One `libs` or `targets` entry has the following structure (see below for +details): ``` -{ - "name": "arbitrary string", # the name of the library - "build": "build type", # in which situation we want that library to be - # built and potentially installed - "language": "...", # the language tag; "c" or "c++" - "public_headers": [ ... ], # list of public headers to install - "headers": [ ... ], # list of headers used by that target - "src": [ ... ], # list of files to compile - "secure": "...", # "yes", "no" or "check" - "baselib": boolean, # this is a low level library that has system - # dependencies - "vs_project_guid: "...", # Visual Studio's unique guid for that project - "filegroups": [ ... ], # list of filegroups to merge to that project - # note that this will be expanded automatically - "deps": [ ... ], # list of libraries this target depends on -} +name: "arbitrary string", # the name of the library +build: "build type", # in which situation we want that library to be + # built and potentially installed (see below). +language: "...", # the language tag; "c" or "c++" +public_headers: # list of public headers to install +headers: # list of headers used by that target +src: # list of files to compile +secure: boolean, # see below +baselib: boolean, # this is a low level library that has system + # dependencies +vs_project_guid: '{...}', # Visual Studio's unique guid for that project +filegroups: # list of filegroups to merge to that project + # note that this will be expanded automatically +deps: # list of libraries this target depends on +deps_linkage: "..." # "static" or "dynamic". Used by the Makefile only to + # determine the way dependencies are linkned. Defaults + # to "dynamic". +dll: "..." # see below. +dll_def: "..." # Visual Studio's dll definition file. +vs_props: # List of property sheets to attach to that project. +vs_config_type: "..." # DynamicLibrary/StaticLibrary. Used only when + # creating a library. Specifies if we're building a + # static library or a dll. Use in conjunction with `dll_def`. +vs_packages: # List of nuget packages this project depends on. ``` ## The `"build"` tag @@ -86,8 +99,9 @@ Currently, the "`build`" tag have these meanings: * `"all"`: library to build on `"make all"`, and install on the system. * `"protoc"`: a protoc plugin to build on `"make all"` and install on the system. -* `"priviate"`: a library to only build for tests. +* `"private"`: a library to only build for tests. * `"test"`: a test binary to run on `"make test"`. +* `"tool"`: a binary to be built upon `"make tools"`. All of the targets should always be present in the generated project file, if possible and applicable. But the build tag is what should group the targets @@ -111,6 +125,18 @@ should merge OpenSSL, protobuf or zlib inside that library. That effect depends on the `"language"` tag. OpenSSL and zlib are for `"c"` libraries, while protobuf is for `"c++"` ones. +## The `"dll"` tag + +Used only by Visual Studio's project files. "true" means the project will be +built with both static and dynamic runtimes. "false" means it'll only be built +with static runtime. "only" means it'll only be built with the dll runtime. + +## The `"dll_def"` tag + +Specifies the visual studio's dll definition file. When creating a DLL, you +sometimes (not always) need a def file (see grpc.def). + + # The template system We're currently using the [mako templates](http://www.makotemplates.org/) @@ -137,3 +163,4 @@ The structure of a plugin is simple. The plugin must defined the function `mako_plugin` that takes a Python dictionary. That dictionary represents the current state of the build.json contents. The plugin can alter it to whatever feature it needs to add. + From 3db3c63a41ec8245f8b4fcd274086eae6db04543 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 1 Mar 2016 17:03:18 -0800 Subject: [PATCH 018/160] Addressed comments --- templates/README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/templates/README.md b/templates/README.md index a27c44da3c1..283a0f5e7be 100644 --- a/templates/README.md +++ b/templates/README.md @@ -11,19 +11,19 @@ targets and files needed to build grpc and its tests, as well as a basic system for dependency description. * Each project file (Makefile, Visual Studio project files, Bazel's BUILD) is -a plain-text template that uses the `build.yaml` file to generate the final -output file. +a [YAML](http://yaml.org) file used by the `build.yaml` file to generate the +final output file. This way we can maintain as many project system as we see fit, without having to manually maintain them when we add or remove new code to the repository. Only the structure of the project file is relevant to the template. The actual list of source code and targets isn't. -We currently have template files for GNU Make, Visual Studio 2010 to 2015, -and [Bazel](http://bazel.io). In the future, we would like to expand to -generating [gyp](https://gyp.gsrc.io/) or [cmake](https://cmake.org) -project files (or potentially both), XCode project files, and an Android.mk -file to be able to compile gRPC using Android's NDK. +We currently have template files for GNU Make, Visual Studio 2013, +[Bazel](http://bazel.io) and [gyp](https://gyp.gsrc.io/). In the future, we +would like to expand to also generate [cmake](https://cmake.org) +project files, XCode project files, and an Android.mk file allowing to compile +gRPC using Android's NDK. We'll gladly accept contribution that'd create additional project files using that system. @@ -163,4 +163,3 @@ The structure of a plugin is simple. The plugin must defined the function `mako_plugin` that takes a Python dictionary. That dictionary represents the current state of the build.json contents. The plugin can alter it to whatever feature it needs to add. - From a66225957532c1847bc215aa9dbca537ae943526 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 1 Mar 2016 17:16:25 -0800 Subject: [PATCH 019/160] Added comment for gyp --- templates/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/README.md b/templates/README.md index 283a0f5e7be..eedc6e9c09f 100644 --- a/templates/README.md +++ b/templates/README.md @@ -20,7 +20,8 @@ Only the structure of the project file is relevant to the template. The actual list of source code and targets isn't. We currently have template files for GNU Make, Visual Studio 2013, -[Bazel](http://bazel.io) and [gyp](https://gyp.gsrc.io/). In the future, we +[Bazel](http://bazel.io) and [gyp](https://gyp.gsrc.io/) (albeit only for +Node.js). In the future, we would like to expand to also generate [cmake](https://cmake.org) project files, XCode project files, and an Android.mk file allowing to compile gRPC using Android's NDK. From 8e19f61d6230bfb938a82532d5da4c79824501df Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 1 Mar 2016 21:41:13 -0800 Subject: [PATCH 020/160] Fix esan detected race in subchannel state --- src/core/client_config/subchannel.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index bec06bf4144..d91dd116b8c 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -395,7 +395,6 @@ void grpc_subchannel_notify_on_state_change( grpc_exec_ctx *exec_ctx, grpc_subchannel *c, grpc_pollset_set *interested_parties, grpc_connectivity_state *state, grpc_closure *notify) { - int do_connect = 0; external_state_watcher *w; if (state == NULL) { @@ -425,17 +424,13 @@ void grpc_subchannel_notify_on_state_change( w->next->prev = w->prev->next = w; if (grpc_connectivity_state_notify_on_state_change( exec_ctx, &c->state_tracker, state, &w->closure)) { - do_connect = 1; c->connecting = 1; /* released by connection */ GRPC_SUBCHANNEL_WEAK_REF(c, "connecting"); + start_connect(exec_ctx, c); } gpr_mu_unlock(&c->mu); } - - if (do_connect) { - start_connect(exec_ctx, c); - } } void grpc_connected_subchannel_process_transport_op( @@ -635,11 +630,12 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) { if (c->disconnected) { iomgr_success = 0; } - gpr_mu_unlock(&c->mu); if (iomgr_success) { update_reconnect_parameters(c); continue_connect(exec_ctx, c); + gpr_mu_unlock(&c->mu); } else { + gpr_mu_unlock(&c->mu); GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); } } From 18720fff2f2bd79542753837401fd24359c3d5f8 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 2 Mar 2016 11:59:42 -0800 Subject: [PATCH 021/160] Maintain correct queue invariants against core --- .../_cython/_cygrpc/completion_queue.pxd.pxi | 7 ++-- .../_cython/_cygrpc/completion_queue.pyx.pxi | 40 +++++++++---------- .../grpcio/grpc/_cython/_cygrpc/grpc.pxi | 2 + 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi index 757f1245e85..305475c0060 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -31,8 +31,9 @@ cdef class CompletionQueue: cdef grpc_completion_queue *c_completion_queue - cdef object poll_condition - cdef bint is_polling + cdef object pluck_condition + cdef int num_plucking + cdef int num_polling cdef bint is_shutting_down cdef bint is_shutdown diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index b299dfee22f..c139147114b 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -39,8 +39,9 @@ cdef class CompletionQueue: self.c_completion_queue = grpc_completion_queue_create(NULL) self.is_shutting_down = False self.is_shutdown = False - self.poll_condition = threading.Condition() - self.is_polling = False + self.pluck_condition = threading.Condition() + self.num_plucking = 0 + self.num_polling = 0 cdef _interpret_event(self, grpc_event event): cdef OperationTag tag = None @@ -87,19 +88,15 @@ cdef class CompletionQueue: c_deadline = deadline.c_time cdef grpc_event event - # Poll within a critical section - # TODO(atash) consider making queue polling contention a hard error to - # enable easier bug discovery - with self.poll_condition: - while self.is_polling: - self.poll_condition.wait(float(deadline) - time.time()) - self.is_polling = True + # Poll within a critical section to detect contention + with self.pluck_condition: + assert self.num_plucking == 0, 'cannot simultaneously pluck and poll' + self.num_polling += 1 with nogil: event = grpc_completion_queue_next( self.c_completion_queue, c_deadline, NULL) - with self.poll_condition: - self.is_polling = False - self.poll_condition.notify() + with self.pluck_condition: + self.num_polling -= 1 return self._interpret_event(event) def pluck(self, OperationTag tag, Timespec deadline=None): @@ -111,19 +108,18 @@ cdef class CompletionQueue: c_deadline = deadline.c_time cdef grpc_event event - # Poll within a critical section - # TODO(atash) consider making queue polling contention a hard error to - # enable easier bug discovery - with self.poll_condition: - while self.is_polling: - self.poll_condition.wait(float(deadline) - time.time()) - self.is_polling = True + # Pluck within a critical section to detect contention + with self.pluck_condition: + assert self.num_polling == 0, 'cannot simultaneously pluck and poll' + assert self.num_plucking < GRPC_MAX_COMPLETION_QUEUE_PLUCKERS, ( + 'cannot pluck more than {} times simultaneously'.format( + GRPC_MAX_COMPLETION_QUEUE_PLUCKERS)) + self.num_plucking += 1 with nogil: event = grpc_completion_queue_pluck( self.c_completion_queue, tag, c_deadline, NULL) - with self.poll_condition: - self.is_polling = False - self.poll_condition.notify() + with self.pluck_condition: + self.num_plucking -= 1 return self._interpret_event(event) def shutdown(self): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 800d0ea2f6f..dbf0045710e 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -138,6 +138,8 @@ cdef extern from "grpc/_cython/loader.h": const int GRPC_WRITE_NO_COMPRESS const int GRPC_WRITE_USED_MASK + const int GRPC_MAX_COMPLETION_QUEUE_PLUCKERS + ctypedef struct grpc_completion_queue: # We don't care about the internals (and in fact don't know them) pass From bd50f305a3af3c45aadb2b7eb7f4752475f6d820 Mon Sep 17 00:00:00 2001 From: makdharma Date: Wed, 2 Mar 2016 13:49:51 -0800 Subject: [PATCH 022/160] Update reconnect_interop_client.cc --- test/cpp/interop/reconnect_interop_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1f6b352db17..3ad733e1114 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -30,7 +30,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ - +// Test description at doc/connection-backoff-interop-test-description.md #include #include From 38a560b6ba45bb8db16fac609a72a6c235469841 Mon Sep 17 00:00:00 2001 From: makdharma Date: Wed, 2 Mar 2016 13:50:58 -0800 Subject: [PATCH 023/160] Update reconnect_interop_server.cc --- test/cpp/interop/reconnect_interop_server.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 3602b8c2b05..785f9c7ad5f 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -30,6 +30,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ + +// Test description at doc/connection-backoff-interop-test-description.md #include #include From 7bee07555af388cd080fd8c15e7c8ee4725974f9 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 2 Mar 2016 14:38:07 -0800 Subject: [PATCH 024/160] Add troubleshooting section to package description --- src/python/grpcio/README.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index 698c760ebe2..f3e962c1971 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -38,3 +38,17 @@ package named `python-dev`). Note that `$REPO_ROOT` can be assigned to whatever directory name floats your fancy. + +Troubleshooting +~~~~~~~~~~~~~~~ + +Help, I ... + +* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install + grpc!** + + This is likely because :code:`pip` doesn't own the offending dependency, + which in turn is likely because your operating system's package manager owns + it. You'll need to force the installation of the dependency: + + :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY` From 42dab364a337666003a17d72b4dcad0c4568587a Mon Sep 17 00:00:00 2001 From: Greg Haines Date: Wed, 2 Mar 2016 15:04:41 -0800 Subject: [PATCH 025/160] Pass a non-infinite deadline to grpc_completion_queue_next() to prevent queues from blocking indefinitely in poll(). --- .../GRPCClient/private/GRPCCompletionQueue.h | 7 ++++++ .../GRPCClient/private/GRPCCompletionQueue.m | 25 +++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h index fe3b8f39d12..03fd2e0d955 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h @@ -36,6 +36,8 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); +extern const int64_t kGRPCCompletionQueueDefaultTimeoutSecs; + /** * This class lets one more easily use |grpc_completion_queue|. To use it, pass the value of the * |unmanagedQueue| property of an instance of this class to |grpc_channel_create_call|. Then for @@ -49,6 +51,11 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); */ @interface GRPCCompletionQueue : NSObject @property(nonatomic, readonly) grpc_completion_queue *unmanagedQueue; +@property(nonatomic, readonly) int64_t timeoutSecs; + (instancetype)completionQueue; + +- (instancetype)init; +- (instancetype)initWithTimeout:(int64_t)timeoutSecs NS_DESIGNATED_INITIALIZER; + @end diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index ea2b01ee1d7..8163236cc4a 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -35,6 +35,9 @@ #import + +const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; + @implementation GRPCCompletionQueue + (instancetype)completionQueue { @@ -42,8 +45,13 @@ } - (instancetype)init { + return [self initWithTimeout:kGRPCCompletionQueueDefaultTimeoutSecs]; +} + +- (instancetype)initWithTimeout:(int64_t)timeoutSecs { if ((self = [super init])) { _unmanagedQueue = grpc_completion_queue_create(NULL); + _timeoutSecs = timeoutSecs; // This is for the following block to capture the pointer by value (instead // of retaining self and doing self->_unmanagedQueue). This is essential @@ -52,6 +60,7 @@ // anymore (i.e. on self dealloc). So the block would never end if it // retained self. grpc_completion_queue *unmanagedQueue = _unmanagedQueue; + int64_t lTimeoutSecs = _timeoutSecs; // Start a loop on a concurrent queue to read events from the completion // queue and dispatch each. @@ -61,22 +70,24 @@ gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ + gpr_timespec deadline = gpr_time_from_seconds(lTimeoutSecs, GPR_CLOCK_REALTIME); while (YES) { - // The following call blocks until an event is available. - grpc_event event = grpc_completion_queue_next(unmanagedQueue, - gpr_inf_future(GPR_CLOCK_REALTIME), - NULL); + // The following call blocks until an event is available or the deadline elapses. + grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); GRPCQueueCompletionHandler handler; switch (event.type) { - case GRPC_OP_COMPLETE: + case GRPC_OP_COMPLETE: // Falling through deliberately + case GRPC_QUEUE_TIMEOUT: handler = (__bridge_transfer GRPCQueueCompletionHandler)event.tag; - handler(event.success); + if (handler) { + handler(event.success); + } break; case GRPC_QUEUE_SHUTDOWN: grpc_completion_queue_destroy(unmanagedQueue); return; default: - [NSException raise:@"Unrecognized completion type" format:@""]; + [NSException raise:@"Unrecognized completion type" format:@"type=%d", event.type]; } }; }); From c6611efb67e8f5eae0451963e98df1890dcdb3d5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 2 Mar 2016 17:43:09 -0800 Subject: [PATCH 026/160] Revert "Update reconnect_interop_client.cc" --- test/cpp/interop/reconnect_interop_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 3ad733e1114..1f6b352db17 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -30,7 +30,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ -// Test description at doc/connection-backoff-interop-test-description.md + #include #include From 072ebaa1537673e100b0a027d3935252da14fccb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 1 Mar 2016 18:33:12 -0800 Subject: [PATCH 027/160] make python test suites run in parallel --- src/python/grpcio/tests/_runner.py | 11 +++- src/python/grpcio/tests/tests.json | 62 +++++++++++++++++++ .../grpcio/tests/unit/_sanity/__init__.py | 30 +++++++++ .../grpcio/tests/unit/_sanity/_sanity_test.py | 53 ++++++++++++++++ tools/run_tests/build_python.sh | 2 + tools/run_tests/run_python.sh | 7 ++- tools/run_tests/run_tests.py | 26 +++++--- 7 files changed, 181 insertions(+), 10 deletions(-) create mode 100644 src/python/grpcio/tests/tests.json create mode 100644 src/python/grpcio/tests/unit/_sanity/__init__.py create mode 100644 src/python/grpcio/tests/unit/_sanity/_sanity_test.py diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py index 4f1ddb57fcf..38a5432e791 100644 --- a/src/python/grpcio/tests/_runner.py +++ b/src/python/grpcio/tests/_runner.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -143,10 +143,17 @@ class Runner(object): def run(self, suite): """See setuptools' test_runner setup argument for information.""" + # only run test cases with id starting with given prefix + testcase_filter = os.getenv('GPRC_PYTHON_TESTRUNNER_FILTER') + filtered_cases = [] + for case in _loader.iterate_suite_cases(suite): + if not testcase_filter or case.id().startswith(testcase_filter): + filtered_cases.append(case) + # Ensure that every test case has no collision with any other test case in # the augmented results. augmented_cases = [AugmentedCase(case, uuid.uuid4()) - for case in _loader.iterate_suite_cases(suite)] + for case in filtered_cases] case_id_by_case = dict((augmented_case.case, augmented_case.id) for augmented_case in augmented_cases) result_out = StringIO.StringIO() diff --git a/src/python/grpcio/tests/tests.json b/src/python/grpcio/tests/tests.json new file mode 100644 index 00000000000..388d040d5ca --- /dev/null +++ b/src/python/grpcio/tests/tests.json @@ -0,0 +1,62 @@ +[ + "_base_interface_test.AsyncEasyTest", + "_base_interface_test.AsyncPeasyTest", + "_base_interface_test.SyncEasyTest", + "_base_interface_test.SyncPeasyTest", + "_beta_features_test.BetaFeaturesTest", + "_beta_features_test.ContextManagementAndLifecycleTest", + "_channel_test.ChannelTest", + "_connectivity_channel_test.ChannelConnectivityTest", + "_core_over_links_base_interface_test.AsyncEasyTest", + "_core_over_links_base_interface_test.AsyncPeasyTest", + "_core_over_links_base_interface_test.SyncEasyTest", + "_core_over_links_base_interface_test.SyncPeasyTest", + "_crust_over_core_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_over_links_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_over_links_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", + "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", + "_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", + "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", + "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", + "_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", + "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", + "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", + "_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", + "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", + "_implementations_test.ChannelCredentialsTest", + "_insecure_interop_test.InsecureInteropTest", + "_intermediary_low_test.CancellationTest", + "_intermediary_low_test.EchoTest", + "_intermediary_low_test.ExpirationTest", + "_intermediary_low_test.LonelyClientTest", + "_later_test.LaterTest", + "_logging_pool_test.LoggingPoolTest", + "_lonely_invocation_link_test.LonelyInvocationLinkTest", + "_low_test.HangingServerShutdown", + "_low_test.InsecureServerInsecureClient", + "_not_found_test.NotFoundTest", + "_sanity_test.Sanity", + "_secure_interop_test.SecureInteropTest", + "_transmission_test.RoundTripTest", + "_transmission_test.TransmissionTest", + "_utilities_test.ChannelConnectivityTest", + "beta_python_plugin_test.PythonPluginTest", + "cygrpc_test.InsecureServerInsecureClient", + "cygrpc_test.SecureServerSecureClient", + "cygrpc_test.TypeSmokeTest" +] \ No newline at end of file diff --git a/src/python/grpcio/tests/unit/_sanity/__init__.py b/src/python/grpcio/tests/unit/_sanity/__init__.py new file mode 100644 index 00000000000..2f88fa04122 --- /dev/null +++ b/src/python/grpcio/tests/unit/_sanity/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + diff --git a/src/python/grpcio/tests/unit/_sanity/_sanity_test.py b/src/python/grpcio/tests/unit/_sanity/_sanity_test.py new file mode 100644 index 00000000000..0a5a715c0e1 --- /dev/null +++ b/src/python/grpcio/tests/unit/_sanity/_sanity_test.py @@ -0,0 +1,53 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +import unittest + +import tests + + +class Sanity(unittest.TestCase): + + def testTestsJsonUpToDate(self): + """Autodiscovers all test suites and checks that tests.json is up to date""" + loader = tests.Loader() + loader.loadTestsFromNames(['tests']) + test_suite_names = [ + test_case_class.id().rsplit('.', 1)[0] + for test_case_class in tests._loader.iterate_suite_cases(loader.suite)] + test_suite_names = sorted(set(test_suite_names)) + + with open('src/python/grpcio/tests/tests.json') as tests_json_file: + tests_json = json.load(tests_json_file) + self.assertListEqual(test_suite_names, tests_json) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index e0fcbb602d4..365abb4d863 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -45,3 +45,5 @@ export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 tox --notest $ROOT/.tox/py27/bin/python $ROOT/setup.py build +$ROOT/.tox/py27/bin/python $ROOT/setup.py build_py +$ROOT/.tox/py27/bin/python $ROOT/setup.py gather --test diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index ffe9c12af1a..beb747a6169 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -42,7 +42,12 @@ export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 -tox +if [ "$CONFIG" = "gcov" ] +then + tox +else + $ROOT/.tox/py27/bin/python $ROOT/setup.py test +fi mkdir -p $ROOT/reports rm -rf $ROOT/reports/python-coverage diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 08a5ff0e8fa..c55d1fbe633 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -353,15 +353,27 @@ class PythonLanguage(object): _check_compiler(self.args.compiler, ['default']) def test_specs(self): + # load list of known test suites + with open('src/python/grpcio/tests/tests.json') as tests_json_file: + tests_json = json.load(tests_json_file) environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) environment['PYVER'] = '2.7' - return [self.config.job_spec( - ['tools/run_tests/run_python.sh'], - None, - environ=environment, - shortname='py.test', - timeout_seconds=15*60 - )] + if self.config.build_config != 'gcov': + return [self.config.job_spec( + ['tools/run_tests/run_python.sh'], + None, + environ=dict(environment.items() + + [('GPRC_PYTHON_TESTRUNNER_FILTER', suite_name)]), + shortname='py.test.%s' % suite_name, + timeout_seconds=5*60) + for suite_name in tests_json] + else: + return [self.config.job_spec(['tools/run_tests/run_python.sh'], + None, + environ=environment, + shortname='py.test.coverage', + timeout_seconds=15*60)] + def pre_build_steps(self): return [] From 7d757ca29dc413777a9648cc2db9246235438d43 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 2 Mar 2016 19:44:16 -0800 Subject: [PATCH 028/160] Ensure that no #includes are inside of a namespace. --- test/cpp/qps/limit_cores.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/limit_cores.cc b/test/cpp/qps/limit_cores.cc index fad9a323afd..28264d031a8 100644 --- a/test/cpp/qps/limit_cores.cc +++ b/test/cpp/qps/limit_cores.cc @@ -37,14 +37,16 @@ #include #include -namespace grpc { -namespace testing { #ifdef GPR_CPU_LINUX #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include + +namespace grpc { +namespace testing { + int LimitCores(const int* cores, int cores_size) { const int num_cores = gpr_cpu_num_cores(); int cores_set = 0; From 2a8c28037061b94c02eb05fb532e268709af88e5 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 2 Mar 2016 20:46:54 -0800 Subject: [PATCH 029/160] sanity --- test/cpp/interop/reconnect_interop_client.cc | 12 ++++++------ test/cpp/interop/reconnect_interop_server.cc | 12 ++++++------ test/cpp/qps/limit_cores.cc | 1 - 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1f6b352db17..c668edaceb0 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,16 +34,16 @@ #include #include -#include -#include #include #include #include -#include "test/cpp/util/create_test_channel.h" -#include "test/cpp/util/test_config.h" -#include "src/proto/grpc/testing/test.grpc.pb.h" +#include +#include #include "src/proto/grpc/testing/empty.grpc.pb.h" #include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/test.grpc.pb.h" +#include "test/cpp/util/create_test_channel.h" +#include "test/cpp/util/test_config.h" DEFINE_int32(server_control_port, 0, "Server port for control rpcs."); DEFINE_int32(server_retry_port, 0, "Server port for testing reconnection."); diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 785f9c7ad5f..1f9147d0efa 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -30,7 +30,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ - + // Test description at doc/connection-backoff-interop-test-description.md #include @@ -42,17 +42,17 @@ #include #include -#include -#include #include #include #include +#include +#include -#include "test/core/util/reconnect_server.h" -#include "test/cpp/util/test_config.h" -#include "src/proto/grpc/testing/test.grpc.pb.h" #include "src/proto/grpc/testing/empty.grpc.pb.h" #include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/test.grpc.pb.h" +#include "test/core/util/reconnect_server.h" +#include "test/cpp/util/test_config.h" DEFINE_int32(control_port, 0, "Server port for controlling the server."); DEFINE_int32(retry_port, 0, diff --git a/test/cpp/qps/limit_cores.cc b/test/cpp/qps/limit_cores.cc index 28264d031a8..1fb2d628f6d 100644 --- a/test/cpp/qps/limit_cores.cc +++ b/test/cpp/qps/limit_cores.cc @@ -37,7 +37,6 @@ #include #include - #ifdef GPR_CPU_LINUX #ifndef _GNU_SOURCE #define _GNU_SOURCE From 98990726a02829c29674f143ab6e02df5415514d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 2 Mar 2016 22:04:00 -0800 Subject: [PATCH 030/160] Revert "Update reconnect_interop_server.cc" --- test/cpp/interop/reconnect_interop_server.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 785f9c7ad5f..3602b8c2b05 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -30,8 +30,6 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ - -// Test description at doc/connection-backoff-interop-test-description.md #include #include From 0cb803d9ca4286601e9e6a3240cfa3488b662b7c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 2 Mar 2016 22:17:24 -0800 Subject: [PATCH 031/160] Always ref writable streams We suffered a bug whereby doing a follow-up write to another write could resurrect a deleted stream, causing all sorts of crash. Fix: when a stream becomes writable (vs when we start writing) take a ref on the stream, and only relinquish it once we're done writing. --- grpc.def | 1 + include/grpc/impl/codegen/sync.h | 4 ++ src/core/iomgr/iomgr.c | 16 +++++++ src/core/iomgr/iomgr_internal.h | 6 ++- src/core/support/sync.c | 7 ++- src/core/transport/chttp2/internal.h | 18 +++++--- src/core/transport/chttp2/parsing.c | 6 +-- src/core/transport/chttp2/stream_lists.c | 38 ++++++++-------- src/core/transport/chttp2/writing.c | 37 ++++++---------- src/core/transport/chttp2_transport.c | 43 +++++++++++-------- src/core/transport/metadata.c | 8 ++++ src/core/transport/transport.c | 2 +- .../grpcio/grpc/_cython/imports.generated.c | 2 + .../grpcio/grpc/_cython/imports.generated.h | 3 ++ src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 + src/ruby/ext/grpc/rb_grpc_imports.generated.h | 3 ++ test/cpp/interop/reconnect_interop_client.cc | 2 +- 17 files changed, 127 insertions(+), 71 deletions(-) diff --git a/grpc.def b/grpc.def index bd0bc85a7c3..f81aa1b05a6 100644 --- a/grpc.def +++ b/grpc.def @@ -182,6 +182,7 @@ EXPORTS gpr_event_wait gpr_ref_init gpr_ref + gpr_ref_non_zero gpr_refn gpr_unref gpr_stats_init diff --git a/include/grpc/impl/codegen/sync.h b/include/grpc/impl/codegen/sync.h index d2f19d37d66..6fd7d64b299 100644 --- a/include/grpc/impl/codegen/sync.h +++ b/include/grpc/impl/codegen/sync.h @@ -182,6 +182,10 @@ GPRAPI void gpr_ref_init(gpr_refcount *r, int n); /* Increment the reference count *r. Requires *r initialized. */ GPRAPI void gpr_ref(gpr_refcount *r); +/* Increment the reference count *r. Requires *r initialized. + Crashes if refcount is zero */ +GPRAPI void gpr_ref_non_zero(gpr_refcount *r); + /* Increment the reference count *r by n. Requires *r initialized, n > 0. */ GPRAPI void gpr_refn(gpr_refcount *r, int n); diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c index 04580150f3a..9c89c2c08a4 100644 --- a/src/core/iomgr/iomgr.c +++ b/src/core/iomgr/iomgr.c @@ -41,9 +41,11 @@ #include #include #include +#include #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/timer.h" +#include "src/core/support/env.h" #include "src/core/support/string.h" static gpr_mu g_mu; @@ -116,6 +118,9 @@ void grpc_iomgr_shutdown(void) { "memory leaks are likely", count_objects()); dump_objects("LEAKED"); + if (grpc_iomgr_abort_on_leaks()) { + abort(); + } } break; } @@ -154,3 +159,14 @@ void grpc_iomgr_unregister_object(grpc_iomgr_object *obj) { gpr_mu_unlock(&g_mu); gpr_free(obj->name); } + +bool grpc_iomgr_abort_on_leaks(void) { + char *env = gpr_getenv("GRPC_ABORT_ON_LEAKS"); + if (env == NULL) return false; + static const char *truthy[] = {"yes", "Yes", "YES", "true", + "True", "TRUE", "1"}; + for (size_t i = 0; i < GPR_ARRAY_SIZE(truthy); i++) { + if (0 == strcmp(env, truthy[i])) return true; + } + return false; +} diff --git a/src/core/iomgr/iomgr_internal.h b/src/core/iomgr/iomgr_internal.h index e372c18e8a0..ac2c46ebe62 100644 --- a/src/core/iomgr/iomgr_internal.h +++ b/src/core/iomgr/iomgr_internal.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,6 +34,8 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H #define GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H +#include + #include "src/core/iomgr/iomgr.h" #include @@ -55,4 +57,6 @@ void grpc_iomgr_platform_flush(void); /** tear down all platform specific global iomgr structures */ void grpc_iomgr_platform_shutdown(void); +bool grpc_iomgr_abort_on_leaks(void); + #endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H */ diff --git a/src/core/support/sync.c b/src/core/support/sync.c index d368422d9ea..69e3e39c5ce 100644 --- a/src/core/support/sync.c +++ b/src/core/support/sync.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -98,6 +98,11 @@ void gpr_ref_init(gpr_refcount *r, int n) { gpr_atm_rel_store(&r->count, n); } void gpr_ref(gpr_refcount *r) { gpr_atm_no_barrier_fetch_add(&r->count, 1); } +void gpr_ref_non_zero(gpr_refcount *r) { + gpr_atm prior = gpr_atm_no_barrier_fetch_add(&r->count, 1); + GPR_ASSERT(prior > 0); +} + void gpr_refn(gpr_refcount *r, int n) { gpr_atm_no_barrier_fetch_add(&r->count, n); } diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h index d76d31be23f..891aad6ef28 100644 --- a/src/core/transport/chttp2/internal.h +++ b/src/core/transport/chttp2/internal.h @@ -417,7 +417,7 @@ typedef struct { /** HTTP2 stream id for this stream, or zero if one has not been assigned */ uint32_t id; uint8_t fetching; - uint8_t sent_initial_metadata; + bool sent_initial_metadata; uint8_t sent_message; uint8_t sent_trailing_metadata; uint8_t read_closed; @@ -509,7 +509,7 @@ void grpc_chttp2_publish_reads(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *global, grpc_chttp2_transport_parsing *parsing); -void grpc_chttp2_list_add_writable_stream( +bool grpc_chttp2_list_add_writable_stream( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global); /** Get a writable stream @@ -519,14 +519,13 @@ int grpc_chttp2_list_pop_writable_stream( grpc_chttp2_transport_writing *transport_writing, grpc_chttp2_stream_global **stream_global, grpc_chttp2_stream_writing **stream_writing); -void grpc_chttp2_list_remove_writable_stream( +bool grpc_chttp2_list_remove_writable_stream( grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); + grpc_chttp2_stream_global *stream_global) GRPC_MUST_USE_RESULT; -/* returns 1 if stream added, 0 if it was already present */ -int grpc_chttp2_list_add_writing_stream( +void grpc_chttp2_list_add_writing_stream( grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing) GRPC_MUST_USE_RESULT; + grpc_chttp2_stream_writing *stream_writing); int grpc_chttp2_list_have_writing_streams( grpc_chttp2_transport_writing *transport_writing); int grpc_chttp2_list_pop_writing_stream( @@ -770,4 +769,9 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *parsing, const uint8_t *opaque_8bytes); +/** add a ref to the stream and add it to the writable list; + ref will be dropped in writing.c */ +void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global); + #endif diff --git a/src/core/transport/chttp2/parsing.c b/src/core/transport/chttp2/parsing.c index 8fdebd7f139..0516f39fa9e 100644 --- a/src/core/transport/chttp2/parsing.c +++ b/src/core/transport/chttp2/parsing.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -149,7 +149,7 @@ void grpc_chttp2_publish_reads( if (was_zero && !is_zero) { while (grpc_chttp2_list_pop_stalled_by_transport(transport_global, &stream_global)) { - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -178,7 +178,7 @@ void grpc_chttp2_publish_reads( outgoing_window); is_zero = stream_global->outgoing_window <= 0; if (was_zero && !is_zero) { - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } stream_global->max_recv_bytes -= (uint32_t)GPR_MIN( diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/transport/chttp2/stream_lists.c index b284c788183..60fe735cfce 100644 --- a/src/core/transport/chttp2/stream_lists.c +++ b/src/core/transport/chttp2/stream_lists.c @@ -100,11 +100,14 @@ static void stream_list_remove(grpc_chttp2_transport *t, grpc_chttp2_stream *s, } } -static void stream_list_maybe_remove(grpc_chttp2_transport *t, +static bool stream_list_maybe_remove(grpc_chttp2_transport *t, grpc_chttp2_stream *s, grpc_chttp2_stream_list_id id) { if (s->included[id]) { stream_list_remove(t, s, id); + return true; + } else { + return false; } } @@ -125,23 +128,24 @@ static void stream_list_add_tail(grpc_chttp2_transport *t, s->included[id] = 1; } -static int stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, - grpc_chttp2_stream_list_id id) { +static bool stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, + grpc_chttp2_stream_list_id id) { if (s->included[id]) { - return 0; + return false; } stream_list_add_tail(t, s, id); - return 1; + return true; } /* wrappers for specializations */ -void grpc_chttp2_list_add_writable_stream( +bool grpc_chttp2_list_add_writable_stream( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { GPR_ASSERT(stream_global->id != 0); - stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), GRPC_CHTTP2_LIST_WRITABLE); + return stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), + STREAM_FROM_GLOBAL(stream_global), + GRPC_CHTTP2_LIST_WRITABLE); } int grpc_chttp2_list_pop_writable_stream( @@ -159,20 +163,20 @@ int grpc_chttp2_list_pop_writable_stream( return r; } -void grpc_chttp2_list_remove_writable_stream( +bool grpc_chttp2_list_remove_writable_stream( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { - stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_WRITABLE); + return stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global), + STREAM_FROM_GLOBAL(stream_global), + GRPC_CHTTP2_LIST_WRITABLE); } -int grpc_chttp2_list_add_writing_stream( +void grpc_chttp2_list_add_writing_stream( grpc_chttp2_transport_writing *transport_writing, grpc_chttp2_stream_writing *stream_writing) { - return stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), - STREAM_FROM_WRITING(stream_writing), - GRPC_CHTTP2_LIST_WRITING); + GPR_ASSERT(stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), + STREAM_FROM_WRITING(stream_writing), + GRPC_CHTTP2_LIST_WRITING)); } int grpc_chttp2_list_have_writing_streams( @@ -332,7 +336,7 @@ void grpc_chttp2_list_flush_writing_stalled_by_transport( while (stream_list_pop(transport, &stream, GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT)) { if (is_window_available) { - grpc_chttp2_list_add_writable_stream(&transport->global, &stream->global); + grpc_chttp2_become_writable(&transport->global, &stream->global); } else { grpc_chttp2_list_add_stalled_by_transport(transport_writing, &stream->writing); diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c index 356fd8174a7..107725cbc79 100644 --- a/src/core/transport/chttp2/writing.c +++ b/src/core/transport/chttp2/writing.c @@ -83,7 +83,8 @@ int grpc_chttp2_unlocking_check_writes( (according to available window sizes) and add to the output buffer */ while (grpc_chttp2_list_pop_writable_stream( transport_global, transport_writing, &stream_global, &stream_writing)) { - uint8_t sent_initial_metadata; + bool sent_initial_metadata = stream_writing->sent_initial_metadata; + bool become_writable = false; stream_writing->id = stream_global->id; stream_writing->read_closed = stream_global->read_closed; @@ -92,16 +93,12 @@ int grpc_chttp2_unlocking_check_writes( outgoing_window, stream_global, outgoing_window); - sent_initial_metadata = stream_writing->sent_initial_metadata; if (!sent_initial_metadata && stream_global->send_initial_metadata) { stream_writing->send_initial_metadata = stream_global->send_initial_metadata; stream_global->send_initial_metadata = NULL; - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); - } - sent_initial_metadata = 1; + become_writable = true; + sent_initial_metadata = true; } if (sent_initial_metadata) { if (stream_global->send_message != NULL) { @@ -128,10 +125,7 @@ int grpc_chttp2_unlocking_check_writes( stream_writing->flow_controlled_buffer.length > 0) && stream_writing->outgoing_window > 0) { if (transport_writing->outgoing_window > 0) { - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); - } + become_writable = true; } else { grpc_chttp2_list_add_stalled_by_transport(transport_writing, stream_writing); @@ -141,10 +135,7 @@ int grpc_chttp2_unlocking_check_writes( stream_writing->send_trailing_metadata = stream_global->send_trailing_metadata; stream_global->send_trailing_metadata = NULL; - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); - } + become_writable = true; } } @@ -153,10 +144,13 @@ int grpc_chttp2_unlocking_check_writes( GRPC_CHTTP2_FLOW_MOVE_STREAM("write", transport_global, stream_writing, announce_window, stream_global, unannounced_incoming_window_for_writing); - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); - } + become_writable = true; + } + + if (become_writable) { + grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing); + } else { + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing"); } } @@ -310,10 +304,7 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx, (stream_writing->send_message && !stream_writing->fetching)) && stream_writing->outgoing_window > 0) { if (transport_writing->outgoing_window > 0) { - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - /* do nothing - already reffed */ - } + grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing); } else { grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing, stream_writing); diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index b9f511e9460..a1ca78d4c4f 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -142,7 +142,7 @@ static void incoming_byte_stream_update_flow_control( static void fail_pending_writes(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream_global *stream_global); -/* +/******************************************************************************* * CONSTRUCTION/DESTRUCTION/REFCOUNTING */ @@ -521,7 +521,6 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->global.id) == NULL); } - grpc_chttp2_list_remove_writable_stream(&t->global, &s->global); grpc_chttp2_list_remove_unannounced_incoming_window_available(&t->global, &s->global); grpc_chttp2_list_remove_stalled_by_transport(&t->global, &s->global); @@ -583,7 +582,7 @@ grpc_chttp2_stream_parsing *grpc_chttp2_parsing_accept_stream( return &accepting->parsing; } -/* +/******************************************************************************* * LOCK MANAGEMENT */ @@ -611,10 +610,18 @@ static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { GPR_TIMER_END("unlock", 0); } -/* +/******************************************************************************* * OUTPUT PROCESSING */ +void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global) { + if (!TRANSPORT_FROM_GLOBAL(transport_global)->closed && + grpc_chttp2_list_add_writable_stream(transport_global, stream_global)) { + GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); + } +} + static void push_setting(grpc_chttp2_transport *t, grpc_chttp2_setting_id id, uint32_t value) { const grpc_chttp2_setting_parameters *sp = @@ -732,7 +739,7 @@ static void maybe_start_some_streams( stream_global->id, STREAM_FROM_GLOBAL(stream_global)); stream_global->in_stream_map = 1; transport_global->concurrent_stream_count++; - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } /* cancel out streams that will never be started */ while (transport_global->next_stream_id >= MAX_CLIENT_STREAM_ID && @@ -821,7 +828,7 @@ static void perform_stream_op_locked( maybe_start_some_streams(exec_ctx, transport_global); } else { GPR_ASSERT(stream_global->id != 0); - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } else { grpc_chttp2_complete_closure_step( @@ -838,7 +845,7 @@ static void perform_stream_op_locked( exec_ctx, &stream_global->send_message_finished, 0); } else if (stream_global->id != 0) { stream_global->send_message = op->send_message; - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -858,7 +865,7 @@ static void perform_stream_op_locked( } else if (stream_global->id != 0) { /* TODO(ctiller): check if there's flow control for any outstanding bytes before going writable */ - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -999,7 +1006,7 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, } } -/* +/******************************************************************************* * INPUT PROCESSING */ @@ -1064,7 +1071,6 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (!s) { s = grpc_chttp2_stream_map_delete(&t->new_stream_map, id); } - grpc_chttp2_list_remove_writable_stream(&t->global, &s->global); GPR_ASSERT(s); s->global.in_stream_map = 0; if (t->parsing.incoming_stream == &s->parsing) { @@ -1080,6 +1086,9 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (grpc_chttp2_unregister_stream(t, s) && t->global.sent_goaway) { close_transport_locked(exec_ctx, t); } + if (grpc_chttp2_list_remove_writable_stream(&t->global, &s->global)) { + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &s->global, "chttp2_writing"); + } new_stream_count = grpc_chttp2_stream_map_size(&t->parsing_stream_map) + grpc_chttp2_stream_map_size(&t->new_stream_map); @@ -1331,7 +1340,7 @@ static void update_global_window(void *args, uint32_t id, void *stream) { is_zero = stream_global->outgoing_window <= 0; if (was_zero && !is_zero) { - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -1426,7 +1435,7 @@ static void recv_data(grpc_exec_ctx *exec_ctx, void *tp, bool success) { GPR_TIMER_END("recv_data", 0); } -/* +/******************************************************************************* * CALLBACK LOOP */ @@ -1440,7 +1449,7 @@ static void connectivity_state_set( state, reason); } -/* +/******************************************************************************* * POLLSET STUFF */ @@ -1468,7 +1477,7 @@ static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt, unlock(exec_ctx, t); } -/* +/******************************************************************************* * BYTE STREAM */ @@ -1508,7 +1517,7 @@ static void incoming_byte_stream_update_flow_control( add_max_recv_bytes); grpc_chttp2_list_add_unannounced_incoming_window_available(transport_global, stream_global); - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -1623,7 +1632,7 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( return incoming_byte_stream; } -/* +/******************************************************************************* * TRACING */ @@ -1709,7 +1718,7 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase, gpr_free(prefix); } -/* +/******************************************************************************* * INTEGRATION GLUE */ diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c index 14912af7df1..807ae071a30 100644 --- a/src/core/transport/metadata.c +++ b/src/core/transport/metadata.c @@ -43,11 +43,13 @@ #include #include #include + #include "src/core/profiling/timers.h" #include "src/core/support/murmur_hash.h" #include "src/core/support/string.h" #include "src/core/transport/chttp2/bin_encoder.h" #include "src/core/transport/static_metadata.h" +#include "src/core/iomgr/iomgr_internal.h" /* There are two kinds of mdelem and mdstr instances. * Static instances are declared in static_metadata.{h,c} and @@ -227,6 +229,9 @@ void grpc_mdctx_global_shutdown(void) { if (shard->count != 0) { gpr_log(GPR_DEBUG, "WARNING: %d metadata elements were leaked", shard->count); + if (grpc_iomgr_abort_on_leaks()) { + abort(); + } } gpr_free(shard->elems); } @@ -237,6 +242,9 @@ void grpc_mdctx_global_shutdown(void) { if (shard->count != 0) { gpr_log(GPR_DEBUG, "WARNING: %d metadata strings were leaked", shard->count); + if (grpc_iomgr_abort_on_leaks()) { + abort(); + } } gpr_free(shard->strs); } diff --git a/src/core/transport/transport.c b/src/core/transport/transport.c index 6e154b629ab..3b555fa9338 100644 --- a/src/core/transport/transport.c +++ b/src/core/transport/transport.c @@ -45,7 +45,7 @@ void grpc_stream_ref(grpc_stream_refcount *refcount, const char *reason) { #else void grpc_stream_ref(grpc_stream_refcount *refcount) { #endif - gpr_ref(&refcount->refs); + gpr_ref_non_zero(&refcount->refs); } #ifdef GRPC_STREAM_REFCOUNT_DEBUG diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c index 4b1860ce8c9..8bd6ae6372b 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.c +++ b/src/python/grpcio/grpc/_cython/imports.generated.c @@ -220,6 +220,7 @@ gpr_event_get_type gpr_event_get_import; gpr_event_wait_type gpr_event_wait_import; gpr_ref_init_type gpr_ref_init_import; gpr_ref_type gpr_ref_import; +gpr_ref_non_zero_type gpr_ref_non_zero_import; gpr_refn_type gpr_refn_import; gpr_unref_type gpr_unref_import; gpr_stats_init_type gpr_stats_init_import; @@ -485,6 +486,7 @@ void pygrpc_load_imports(HMODULE library) { gpr_event_wait_import = (gpr_event_wait_type) GetProcAddress(library, "gpr_event_wait"); gpr_ref_init_import = (gpr_ref_init_type) GetProcAddress(library, "gpr_ref_init"); gpr_ref_import = (gpr_ref_type) GetProcAddress(library, "gpr_ref"); + gpr_ref_non_zero_import = (gpr_ref_non_zero_type) GetProcAddress(library, "gpr_ref_non_zero"); gpr_refn_import = (gpr_refn_type) GetProcAddress(library, "gpr_refn"); gpr_unref_import = (gpr_unref_type) GetProcAddress(library, "gpr_unref"); gpr_stats_init_import = (gpr_stats_init_type) GetProcAddress(library, "gpr_stats_init"); diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h index a395dce7d6a..b70dcccd17a 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.h +++ b/src/python/grpcio/grpc/_cython/imports.generated.h @@ -610,6 +610,9 @@ extern gpr_ref_init_type gpr_ref_init_import; typedef void(*gpr_ref_type)(gpr_refcount *r); extern gpr_ref_type gpr_ref_import; #define gpr_ref gpr_ref_import +typedef void(*gpr_ref_non_zero_type)(gpr_refcount *r); +extern gpr_ref_non_zero_type gpr_ref_non_zero_import; +#define gpr_ref_non_zero gpr_ref_non_zero_import typedef void(*gpr_refn_type)(gpr_refcount *r, int n); extern gpr_refn_type gpr_refn_import; #define gpr_refn gpr_refn_import diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 1af34d97fbc..56db4ec686b 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -220,6 +220,7 @@ gpr_event_get_type gpr_event_get_import; gpr_event_wait_type gpr_event_wait_import; gpr_ref_init_type gpr_ref_init_import; gpr_ref_type gpr_ref_import; +gpr_ref_non_zero_type gpr_ref_non_zero_import; gpr_refn_type gpr_refn_import; gpr_unref_type gpr_unref_import; gpr_stats_init_type gpr_stats_init_import; @@ -481,6 +482,7 @@ void grpc_rb_load_imports(HMODULE library) { gpr_event_wait_import = (gpr_event_wait_type) GetProcAddress(library, "gpr_event_wait"); gpr_ref_init_import = (gpr_ref_init_type) GetProcAddress(library, "gpr_ref_init"); gpr_ref_import = (gpr_ref_type) GetProcAddress(library, "gpr_ref"); + gpr_ref_non_zero_import = (gpr_ref_non_zero_type) GetProcAddress(library, "gpr_ref_non_zero"); gpr_refn_import = (gpr_refn_type) GetProcAddress(library, "gpr_refn"); gpr_unref_import = (gpr_unref_type) GetProcAddress(library, "gpr_unref"); gpr_stats_init_import = (gpr_stats_init_type) GetProcAddress(library, "gpr_stats_init"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 38aabfaca8f..b972f60fc35 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -610,6 +610,9 @@ extern gpr_ref_init_type gpr_ref_init_import; typedef void(*gpr_ref_type)(gpr_refcount *r); extern gpr_ref_type gpr_ref_import; #define gpr_ref gpr_ref_import +typedef void(*gpr_ref_non_zero_type)(gpr_refcount *r); +extern gpr_ref_non_zero_type gpr_ref_non_zero_import; +#define gpr_ref_non_zero gpr_ref_non_zero_import typedef void(*gpr_refn_type)(gpr_refcount *r, int n); extern gpr_refn_type gpr_refn_import; #define gpr_refn gpr_refn_import diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1f6b352db17..79a60cc8602 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 91bd627de4fb1ab1b8c31a915d4a91ccdcf72bbe Mon Sep 17 00:00:00 2001 From: Greg Haines Date: Wed, 2 Mar 2016 23:05:46 -0800 Subject: [PATCH 032/160] Feedback from @jcanizales and @vjpai --- .../GRPCClient/private/GRPCCompletionQueue.m | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index 8163236cc4a..ffbb14374d1 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -60,7 +60,6 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; // anymore (i.e. on self dealloc). So the block would never end if it // retained self. grpc_completion_queue *unmanagedQueue = _unmanagedQueue; - int64_t lTimeoutSecs = _timeoutSecs; // Start a loop on a concurrent queue to read events from the completion // queue and dispatch each. @@ -70,18 +69,18 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ - gpr_timespec deadline = gpr_time_from_seconds(lTimeoutSecs, GPR_CLOCK_REALTIME); + gpr_timespec deadline = gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); while (YES) { // The following call blocks until an event is available or the deadline elapses. grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); GRPCQueueCompletionHandler handler; switch (event.type) { - case GRPC_OP_COMPLETE: // Falling through deliberately - case GRPC_QUEUE_TIMEOUT: + case GRPC_OP_COMPLETE: handler = (__bridge_transfer GRPCQueueCompletionHandler)event.tag; - if (handler) { - handler(event.success); - } + handler(event.success); + break; + case GRPC_QUEUE_TIMEOUT: + // Nothing to do here break; case GRPC_QUEUE_SHUTDOWN: grpc_completion_queue_destroy(unmanagedQueue); From d7f12e30a0050b3fecb72fe7ad558b4d837e140f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 3 Mar 2016 10:08:31 -0800 Subject: [PATCH 033/160] Fix accept_stream being called post-channel deletion - Have the server clear the accept_stream callback prior to destroying the channel (required a small transport op protocol change) - Have the transport not enact transport ops until parsing is completed (prevents accept_stream from disappearing mid-parse) --- src/core/channel/client_channel.c | 2 +- src/core/channel/client_uchannel.c | 2 +- src/core/surface/server.c | 14 +++++++-- src/core/transport/chttp2/internal.h | 3 ++ src/core/transport/chttp2_transport.c | 43 +++++++++++++++++++-------- src/core/transport/transport.h | 6 ++-- 6 files changed, 51 insertions(+), 19 deletions(-) diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index eeac3c146c3..d4ba9508185 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -251,7 +251,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL); - GPR_ASSERT(op->set_accept_stream == NULL); + GPR_ASSERT(op->set_accept_stream == false); if (op->bind_pollset != NULL) { grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, op->bind_pollset); diff --git a/src/core/channel/client_uchannel.c b/src/core/channel/client_uchannel.c index b1e7155773f..83fcc3a87f5 100644 --- a/src/core/channel/client_uchannel.c +++ b/src/core/channel/client_uchannel.c @@ -107,7 +107,7 @@ static void cuc_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL); - GPR_ASSERT(op->set_accept_stream == NULL); + GPR_ASSERT(op->set_accept_stream == false); GPR_ASSERT(op->bind_pollset == NULL); if (op->on_connectivity_state_change != NULL) { diff --git a/src/core/surface/server.c b/src/core/surface/server.c index fb5e0d4b9e7..5b13d4ba526 100644 --- a/src/core/surface/server.c +++ b/src/core/surface/server.c @@ -407,8 +407,15 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand) { maybe_finish_shutdown(exec_ctx, chand->server); chand->finish_destroy_channel_closure.cb = finish_destroy_channel; chand->finish_destroy_channel_closure.cb_arg = chand; - grpc_exec_ctx_enqueue(exec_ctx, &chand->finish_destroy_channel_closure, true, - NULL); + + grpc_transport_op op; + memset(&op, 0, sizeof(op)); + op.set_accept_stream = true; + op.on_consumed = &chand->finish_destroy_channel_closure; + grpc_channel_next_op(exec_ctx, + grpc_channel_stack_element( + grpc_channel_get_channel_stack(chand->channel), 0), + &op); } static void finish_start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_server *server, @@ -971,7 +978,8 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s, GRPC_CHANNEL_INTERNAL_REF(channel, "connectivity"); memset(&op, 0, sizeof(op)); - op.set_accept_stream = accept_stream; + op.set_accept_stream = true; + op.set_accept_stream_fn = accept_stream; op.set_accept_stream_user_data = chand; op.on_connectivity_state_change = &chand->channel_connectivity_changed; op.connectivity_state = &chand->connectivity_state; diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h index 891aad6ef28..b720d1ab3e5 100644 --- a/src/core/transport/chttp2/internal.h +++ b/src/core/transport/chttp2/internal.h @@ -358,6 +358,9 @@ struct grpc_chttp2_transport { /** connectivity tracking */ grpc_connectivity_state_tracker state_tracker; } channel_callback; + + /** Transport op to be applied post-parsing */ + grpc_transport_op *post_parsing_op; }; typedef struct { diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index a1ca78d4c4f..369ff0ad7f6 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -951,12 +951,10 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, unlock(exec_ctx, t); } -static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_transport_op *op) { - grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; - int close_transport = 0; - - lock(t); +static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_transport_op *op) { + bool close_transport = false; grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL); @@ -975,8 +973,8 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, close_transport = !grpc_chttp2_has_streams(t); } - if (op->set_accept_stream != NULL) { - t->channel_callback.accept_stream = op->set_accept_stream; + if (op->set_accept_stream) { + t->channel_callback.accept_stream = op->set_accept_stream_fn; t->channel_callback.accept_stream_user_data = op->set_accept_stream_user_data; } @@ -997,15 +995,29 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, close_transport_locked(exec_ctx, t); } - unlock(exec_ctx, t); - if (close_transport) { - lock(t); close_transport_locked(exec_ctx, t); - unlock(exec_ctx, t); } } +static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_transport_op *op) { + grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; + + lock(t); + + /* Let's be overly cautious: don't change any state while we're parsing */ + if (t->parsing_active) { + GPR_ASSERT(t->post_parsing_op == NULL); + t->post_parsing_op = gpr_malloc(sizeof(*op)); + memcpy(t->post_parsing_op, op, sizeof(*op)); + } else { + perform_transport_op_locked(exec_ctx, t, op); + } + + unlock(exec_ctx, t); +} + /******************************************************************************* * INPUT PROCESSING */ @@ -1401,6 +1413,13 @@ static void recv_data(grpc_exec_ctx *exec_ctx, void *tp, bool success) { /* handle higher level things */ grpc_chttp2_publish_reads(exec_ctx, transport_global, transport_parsing); t->parsing_active = 0; + /* handle delayed transport ops (if there is one) */ + if (t->post_parsing_op) { + grpc_transport_op *op = t->post_parsing_op; + t->post_parsing_op = NULL; + perform_transport_op_locked(exec_ctx, t, op); + gpr_free(op); + } /* if a stream is in the stream map, and gets cancelled, we need to ensure * we are not parsing before continuing the cancellation to keep things in * a sane state */ diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h index 8902c5d2f65..7a007ef777b 100644 --- a/src/core/transport/transport.h +++ b/src/core/transport/transport.h @@ -139,8 +139,10 @@ typedef struct grpc_transport_op { gpr_slice *goaway_message; /** set the callback for accepting new streams; this is a permanent callback, unlike the other one-shot closures */ - void (*set_accept_stream)(grpc_exec_ctx *exec_ctx, void *user_data, - grpc_transport *transport, const void *server_data); + bool set_accept_stream; + void (*set_accept_stream_fn)(grpc_exec_ctx *exec_ctx, void *user_data, + grpc_transport *transport, + const void *server_data); void *set_accept_stream_user_data; /** add this transport to a pollset */ grpc_pollset *bind_pollset; From 093ff5db8b4d9fa9965b505fc8e33316c3daf2bb Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 3 Mar 2016 12:18:27 -0800 Subject: [PATCH 034/160] Fix copyright --- test/cpp/interop/reconnect_interop_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1f6b352db17..79a60cc8602 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 66a6d014a4b2afec45906ff803e31dd89e82e22a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 3 Mar 2016 12:35:08 -0800 Subject: [PATCH 035/160] run build_ext in build_python step --- tools/run_tests/build_python.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 365abb4d863..f120fc7ed69 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -46,4 +46,5 @@ tox --notest $ROOT/.tox/py27/bin/python $ROOT/setup.py build $ROOT/.tox/py27/bin/python $ROOT/setup.py build_py +$ROOT/.tox/py27/bin/python $ROOT/setup.py build_ext --inplace $ROOT/.tox/py27/bin/python $ROOT/setup.py gather --test From ebf1fcff7cf3ba0ad515fccab5dcf0ab75c5bd31 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 13:53:49 -0800 Subject: [PATCH 036/160] Update base INSTALL to markdown and remove outdated content --- CONTRIBUTING.md | 2 +- INSTALL | 225 ------------------------------ INSTALL.md | 45 ++++++ README.md | 2 +- examples/cpp/README.md | 2 +- examples/cpp/cpptutorial.md | 2 +- examples/cpp/helloworld/README.md | 2 +- 7 files changed, 50 insertions(+), 230 deletions(-) delete mode 100644 INSTALL create mode 100644 INSTALL.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9423c46547a..03d2e276a82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ In order to protect both you and ourselves, you will need to sign the ### Technical requirements You will need several tools to work with this repository. In addition to all of -the packages described in the [INSTALL](INSTALL) file, you will also need +the packages described in the [INSTALL](INSTALL.md) file, you will also need python, and the mako template renderer. To install the latter, using pip, one should simply be able to do `pip install mako`. diff --git a/INSTALL b/INSTALL deleted file mode 100644 index a0df57dcd35..00000000000 --- a/INSTALL +++ /dev/null @@ -1,225 +0,0 @@ -These instructions only cover building grpc C and C++ libraries under -typical unix systems. If you need more information, please try grpc's -wiki pages: - - https://github.com/google/grpc/wiki - - -************************* -* If you are in a hurry * -************************* - -On Linux (Debian): - - Note: you will need to add the Debian 'jessie-backports' distribution to your sources - file first. - - Add the following line to your `/etc/apt/sources.list` file: - - deb http://http.debian.net/debian jessie-backports main - - Install the gRPC library: - - $ [sudo] apt-get install libgrpc-dev - -OR - - $ git clone https://github.com/grpc/grpc.git - $ cd grpc - $ git submodule update --init - $ make - $ [sudo] make install - -You don't need anything else than GNU Make, gcc and autotools. Under a Debian -or Ubuntu system, this should boil down to the following packages: - - $ [sudo] apt-get install build-essential autoconf libtool - -Building the python wrapper requires the following: - - $ [sudo] apt-get install python-all-dev python-virtualenv - -If you want to install in a different directory than the default /usr/lib, you can -override it on the command line: - - $ [sudo] make install prefix=/opt - - -******************************* -* More detailled instructions * -******************************* - -Setting up dependencies -======================= - -Dependencies to compile the libraries -------------------------------------- - -grpc libraries have few external dependencies. If you need to compile and -install them, they are present in the third_party directory if you have -cloned the github repository recursively. If you didn't clone recursively, -you can still get them later by running the following command: - - $ git submodule update --init - -Note that the Makefile makes it much easier for you to compile from sources -if you were to clone recursively our git repository: it will automatically -compile zlib and OpenSSL, which are core requirements for grpc. Note this -creates grpc libraries that will have zlib and OpenSSL built-in inside of them, -which significantly increases the libraries' size. - -In order to decrease that size, you can manually install zlib and OpenSSL on -your system, so that the Makefile can use them instead. - -Under a Debian or Ubuntu system, one can acquire the development package -for zlib this way: - - # apt-get install zlib1g-dev - -To the best of our knowledge, no distribution has an OpenSSL package that -supports ALPN yet, so you would still have to depend on installing from source -for that particular dependency if you want to reduce the libraries' size. - -The recommended version of OpenSSL that provides ALPN support is available -at this URL: - - https://www.openssl.org/source/openssl-1.0.2.tar.gz - - -Dependencies to compile and run the tests ------------------------------------------ - -Compiling and running grpc plain-C tests dont't require any more dependency. - - -Compiling and running grpc C++ tests depend on protobuf 3.0.0, gtest and -gflags. Although gflags is provided in third_party, you will need to manually -install that dependency on your system to run these tests. - -Under a Debian or Ubuntu system, you can install the gtests and gflags packages -using apt-get: - - # apt-get install libgflags-dev libgtest-dev - -However, protobuf 3.0.0 isn't in a debian package yet, but the Makefile will -automatically try and compile the one present in third_party if you cloned the -repository recursively, and that it detects your system is lacking it. - -Compiling and installing protobuf 3.0.0 requires a few more dependencies in -itself, notably the autoconf suite. If you have apt-get, you can install -these dependencies this way: - - # apt-get install autoconf libtool - -If you want to run the tests using one of the sanitized configurations, you -will need clang and its instrumented libc++: - - # apt-get install clang libc++-dev - -Mac-specific notes: -------------------- - -For a Mac system, git is not available by default. You will first need to -install Xcode from the Mac AppStore and then run the following command from a -terminal: - - $ sudo xcode-select --install - -You should also install "port" following the instructions at -https://www.macports.org . This will reside in /opt/local/bin/port for -most Mac installations. Do the "git submodule" command listed above. - -Then execute the following for all the needed build dependencies - - $ sudo /opt/local/bin/port install autoconf automake libtool gflags cmake - $ mkdir ~/gtest-svn - $ svn checkout http://googletest.googlecode.com/svn/trunk/ gtest-svn - $ mkdir mybuild - $ cd mybuild - $ cmake ../gtest-svn - $ make - $ make gtest.a gtest_main.a - $ sudo cp libgtest.a libgtest_main.a /opt/local/lib - $ sudo mkdir /opt/local/include/gtest - $ sudo cp -pr ../gtest-svn/include/gtest /opt/local/include/gtest - -We will also need to make openssl and install it appropriately - - $ cd - $ cd third_party/openssl - $ ./config - $ sudo make install - $ cd ../../ - -If you are going to make changes and need to regenerate the projects file, -you will need to install certain modules for python. - - $ sudo easy_install simplejson mako - -Mingw-specific notes: ---------------------- - -While gRPC compiles properly under mingw, some more preparation work is needed. -The recommendation is to use msys2. The installation instructions are available -at that address: http://msys2.github.io/ - -Once this is installed, make sure you are using the following: MinGW-w64 Win64. -You'll be required to install a few more packages: - - $ pacman -S make mingw-w64-x86_64-gcc mingw-w64-x86_64-zlib autoconf automake libtool - -Please also install OpenSSL from that website: - - http://slproweb.com/products/Win32OpenSSL.html - -The package Win64 OpenSSL v1.0.2a should do. At that point you should be able -to compile gRPC with the following: - - $ export LDFLAGS="-L/mingw64/lib -L/c/OpenSSL-Win64" - $ export CPPFLAGS="-I/mingw64/include -I/c/OpenSSL-Win64/include" - $ make - -A word on OpenSSL ------------------ - -Secure HTTP2 requires the TLS extension ALPN (see rfc 7301 and -http://http2.github.io/http2-spec/ section 3.3). Our HTTP2 implementation -relies on OpenSSL's implementation. OpenSSL 1.0.2 is the first released version -of OpenSSL that has ALPN support, and this explains our dependency on it. - -Note that the Makefile supports compiling only the unsecure elements of grpc, -and if you do not have OpenSSL and do not want it, you can still proceed -with installing only the elements you require. However, we strongly recommend -the use of encryption for all network traffic, and discourage the use of grpc -without TLS. - - -Compiling -========= - -If you have all the dependencies mentioned above, you should simply be able -to go ahead and run "make" to compile grpc's C and C++ libraries: - - $ make - - -Testing -======= - -To build and run the tests, you can run the command: - - $ make test - -If you want to be able to run them in parallel, and get better output, you can -also use the python tool we have written: - - $ ./tools/run_tests/run_tests.py - - -Installing -========== - -Once everything is compiled, you should be able to install grpc C and C++ -libraries and headers: - - # make install diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 00000000000..aa809f23c8f --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,45 @@ +#If you are in a hurry + +For language-specific installation instructions, please refer to these +documents + + * [C++](examples/cpp) + * [C#](src/csharp): NuGet package `Grpc` + * [Go](https://github.com/grpc/grpc-go): `go get google.golang.org/grpc` + * [Java](https://github.com/grpc/grpc-java) + * [Node](src/node): `npm install grpc` + * [Objective-C](src/objective-c) + * [PHP](src/php): `pecl install grpc-beta` + * [Python](src/python/grpcio): `pip install grpcio` + * [Ruby](src/ruby): `gem install grpc` + + +#Pre-requisites + +##Linux + +```sh + $ [sudo] apt-get install build-essential autoconf libtool +``` + +##Mac OSX + +For a Mac system, git is not available by default. You will first need to +install Xcode from the Mac AppStore and then run the following command from a +terminal: + +```sh + $ [sudo] xcode-select --install +``` + +#Build from Source + +This is for compiling just the gRPC C Core library. + +```sh + $ git clone https://github.com/grpc/grpc.git + $ cd grpc + $ git submodule update --init + $ make + $ [sudo] make install +``` diff --git a/README.md b/README.md index 033e09b91b7..abb4905392c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You can find more detailed documentation and examples in the [doc](doc) and [exa #Installation -See [grpc/INSTALL](INSTALL) for installation instructions for various platforms. +See [INSTALL](INSTALL.md) for installation instructions for various platforms. #Repository Structure & Status diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 85c495099b7..979ba55dbb5 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -2,7 +2,7 @@ ## Installation -To install gRPC on your system, follow the instructions [here](../../INSTALL). +To install gRPC on your system, follow the instructions [here](../../INSTALL.md). ## Hello C++ gRPC! diff --git a/examples/cpp/cpptutorial.md b/examples/cpp/cpptutorial.md index cd1cddb1115..ef9ca99c0fe 100644 --- a/examples/cpp/cpptutorial.md +++ b/examples/cpp/cpptutorial.md @@ -91,7 +91,7 @@ message Point { Next we need to generate the gRPC client and server interfaces from our .proto service definition. We do this using the protocol buffer compiler `protoc` with a special gRPC C++ plugin. -For simplicity, we've provided a [makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL) first): +For simplicity, we've provided a [makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL.md) first): ```shell $ make route_guide.grpc.pb.cc route_guide.pb.cc diff --git a/examples/cpp/helloworld/README.md b/examples/cpp/helloworld/README.md index 90f3d6b1475..8e11f7cdf3e 100644 --- a/examples/cpp/helloworld/README.md +++ b/examples/cpp/helloworld/README.md @@ -2,7 +2,7 @@ ### Install gRPC Make sure you have installed gRPC on your system. Follow the instructions here: -[https://github.com/grpc/grpc/blob/master/INSTALL](../../../INSTALL). +[https://github.com/grpc/grpc/blob/master/INSTALL](../../../INSTALL.md). ### Get the tutorial source code From 13ee2f2df3ded81d096a1440a0f273eeece6a2cc Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 3 Mar 2016 13:57:32 -0800 Subject: [PATCH 037/160] Properly integrate async API with server-side cancellations. There is a comment above IsCancelled that says when it is ok to use this. --- .../grpc++/impl/codegen/completion_queue.h | 1 + include/grpc++/impl/codegen/server_context.h | 3 + src/cpp/server/server_context.cc | 26 ++- test/cpp/end2end/async_end2end_test.cc | 218 +++++++++++++----- 4 files changed, 187 insertions(+), 61 deletions(-) diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h index 102831e1c9b..928ab2db317 100644 --- a/include/grpc++/impl/codegen/completion_queue.h +++ b/include/grpc++/impl/codegen/completion_queue.h @@ -184,6 +184,7 @@ class CompletionQueue : private GrpcLibrary { bool Pluck(CompletionQueueTag* tag); /// Performs a single polling pluck on \a tag. + /// \warning Must not be mixed with calls to \a Next. void TryPluck(CompletionQueueTag* tag); grpc_completion_queue* cq_; // owned diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h index ad08b8210d6..91ebe574b14 100644 --- a/include/grpc++/impl/codegen/server_context.h +++ b/include/grpc++/impl/codegen/server_context.h @@ -103,6 +103,9 @@ class ServerContext { void AddInitialMetadata(const grpc::string& key, const grpc::string& value); void AddTrailingMetadata(const grpc::string& key, const grpc::string& value); + // IsCancelled is always safe to call when using sync API + // When using async API, it is only safe to call IsCancelled after + // the AsyncNotifyWhenDone tag has been delivered bool IsCancelled() const; // Cancel the Call from the server. This is a best-effort API and depending on diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index e205a1969b3..eb49b210379 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -62,7 +62,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void FillOps(grpc_op* ops, size_t* nops) GRPC_OVERRIDE; bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE; - bool CheckCancelled(CompletionQueue* cq); + bool CheckCancelled(CompletionQueue* cq) { + cq->TryPluck(this); + return CheckCancelledNoPluck(); + } + bool CheckCancelledAsync() { return CheckCancelledNoPluck(); } void set_tag(void* tag) { has_tag_ = true; @@ -72,6 +76,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void Unref(); private: + bool CheckCancelledNoPluck() { + grpc::lock_guard g(mu_); + return finalized_ ? (cancelled_ != 0) : false; + } + bool has_tag_; void* tag_; grpc::mutex mu_; @@ -88,12 +97,6 @@ void ServerContext::CompletionOp::Unref() { } } -bool ServerContext::CompletionOp::CheckCancelled(CompletionQueue* cq) { - cq->TryPluck(this); - grpc::lock_guard g(mu_); - return finalized_ ? cancelled_ != 0 : false; -} - void ServerContext::CompletionOp::FillOps(grpc_op* ops, size_t* nops) { ops->op = GRPC_OP_RECV_CLOSE_ON_SERVER; ops->data.recv_close_on_server.cancelled = &cancelled_; @@ -182,7 +185,14 @@ void ServerContext::TryCancel() const { } bool ServerContext::IsCancelled() const { - return completion_op_ && completion_op_->CheckCancelled(cq_); + if (has_notify_when_done_tag_) { + // when using async API, but the result is only valid + // if the tag has already been delivered at the completion queue + return completion_op_ && completion_op_->CheckCancelledAsync(); + } else { + // when using sync API + return completion_op_ && completion_op_->CheckCancelled(cq_); + } } void ServerContext::set_compression_level(grpc_compression_level level) { diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 9ca3bf98f85..09e973f28d5 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -68,6 +68,7 @@ namespace testing { namespace { void* tag(int i) { return (void*)(intptr_t)i; } +int detag(void* p) { return static_cast(reinterpret_cast(p)); } #ifdef GPR_POSIX_SOCKET static int maybe_assert_non_blocking_poll(struct pollfd* pfds, nfds_t nfds, @@ -111,30 +112,35 @@ class Verifier { return *this; } + int Next(CompletionQueue* cq, bool ignore_ok) { + bool ok; + void* got_tag; + if (spin_) { + for (;;) { + auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); + if (r == CompletionQueue::TIMEOUT) continue; + if (r == CompletionQueue::GOT_EVENT) break; + gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); + abort(); + } + } else { + EXPECT_TRUE(cq->Next(&got_tag, &ok)); + } + auto it = expectations_.find(got_tag); + EXPECT_TRUE(it != expectations_.end()); + if (!ignore_ok) { + EXPECT_EQ(it->second, ok); + } + expectations_.erase(it); + return detag(got_tag); + } + void Verify(CompletionQueue* cq) { Verify(cq, false); } void Verify(CompletionQueue* cq, bool ignore_ok) { GPR_ASSERT(!expectations_.empty()); while (!expectations_.empty()) { - bool ok; - void* got_tag; - if (spin_) { - for (;;) { - auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); - if (r == CompletionQueue::TIMEOUT) continue; - if (r == CompletionQueue::GOT_EVENT) break; - gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); - abort(); - } - } else { - EXPECT_TRUE(cq->Next(&got_tag, &ok)); - } - auto it = expectations_.find(got_tag); - EXPECT_TRUE(it != expectations_.end()); - if (!ignore_ok) { - EXPECT_EQ(it->second, ok); - } - expectations_.erase(it); + Next(cq, ignore_ok); } } void Verify(CompletionQueue* cq, @@ -793,7 +799,8 @@ TEST_P(AsyncEnd2endTest, UnimplementedRpc) { } // This class is for testing scenarios where RPCs are cancelled on the server -// by calling ServerContext::TryCancel() +// by calling ServerContext::TryCancel(). Server uses AsyncNotifyWhenDone +// API to check for cancellation class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { protected: typedef enum { @@ -803,13 +810,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { CANCEL_AFTER_PROCESSING } ServerTryCancelRequestPhase; - void ServerTryCancel(ServerContext* context) { - EXPECT_FALSE(context->IsCancelled()); - context->TryCancel(); - gpr_log(GPR_INFO, "Server called TryCancel()"); - EXPECT_TRUE(context->IsCancelled()); - } - // Helper for testing client-streaming RPCs which are cancelled on the server. // Depending on the value of server_try_cancel parameter, this will test one // of the following three scenarios: @@ -843,6 +843,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of 'RequestStream' calls // and receive the 'RequestStream' call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestRequestStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -858,9 +859,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_server_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // Since cancellation is done before server reads any results, we know // for sure that all cq results will return false from this point forward @@ -868,22 +872,39 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + &ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while reading the // requests from the client. Since the cancellation can happen at anytime, // some of the cq results (i.e those until cancellation) might be true but // its non deterministic. So better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } // Server reads 3 messages (tags 6, 7 and 8) + // But if want_done_tag is true, we might also see tag 11 for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { srv_stream.Read(&recv_request, tag(tag_idx)); - Verifier(GetParam()) - .Expect(tag_idx, expected_server_cq_result) - .Verify(cq_.get(), ignore_cq_result); + // Note that we'll add something to the verifier and verify that + // something was seen, but it might be tag 11 and not what we + // just added + int got_tag = verif.Expect(tag_idx, expected_server_cq_result) + .Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); + } } if (server_try_cancel_thd != NULL) { @@ -892,7 +913,15 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); + } + + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; } // The RPC has been cancelled at this point for sure (i.e irrespective of @@ -945,6 +974,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { Verifier(GetParam()).Expect(1, true).Verify(cq_.get()); // On the server, request to be notified of 'ResponseStream' calls and // receive the call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -952,9 +982,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -962,24 +995,41 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + &ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while writing responses // to the client. Since the cancellation can happen at anytime, some of // the cq results (i.e those until cancellation) might be true but it is // non deterministic. So better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } // Server sends three messages (tags 3, 4 and 5) + // But if want_done tag is true, we might also see tag 11 for (int tag_idx = 3; tag_idx <= 5; tag_idx++) { send_response.set_message("Pong " + std::to_string(tag_idx)); srv_stream.Write(send_response, tag(tag_idx)); - Verifier(GetParam()) - .Expect(tag_idx, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + // Note that we'll add something to the verifier and verify that + // something was seen, but it might be tag 11 and not what we + // just added + int got_tag = verif.Expect(tag_idx, expected_cq_result) + .Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); + } } if (server_try_cancel_thd != NULL) { @@ -988,13 +1038,21 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); // Client reads may fail bacause it is notified that the stream is // cancelled. ignore_cq_result = true; } + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + } + // Client attemts to read the three messages from the server for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { cli_stream->Read(&recv_response, tag(tag_idx)); @@ -1052,6 +1110,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of the 'BidiStream' call and // receive the call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestBidiStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -1063,9 +1122,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -1073,42 +1135,84 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + &ServerContext::TryCancel, &srv_ctx); // Since server is going to cancel the RPC in a parallel thread, some of // the cq results (i.e those until the cancellation) might be true. Since // that number is non-deterministic, it is better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } + int got_tag; srv_stream.Read(&recv_request, tag(4)); - Verifier(GetParam()) - .Expect(4, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(4, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 4) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 4); + } send_response.set_message("Pong"); srv_stream.Write(send_response, tag(5)); - Verifier(GetParam()) - .Expect(5, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(5, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 5) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 5); + } cli_stream->Read(&recv_response, tag(6)); - Verifier(GetParam()) - .Expect(6, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(6, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 6) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 6); + } // This is expected to succeed in all cases cli_stream->WritesDone(tag(7)); - Verifier(GetParam()).Expect(7, true).Verify(cq_.get()); + verif.Expect(7, true); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 7) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 7); + } // This is expected to fail in all cases i.e for all values of // server_try_cancel. This is because at this point, either there are no // more msgs from the client (because client called WritesDone) or the RPC // is cancelled on the server srv_stream.Read(&recv_request, tag(8)); - Verifier(GetParam()).Expect(8, false).Verify(cq_.get()); + verif.Expect(8, false); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 8) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 8); + } if (server_try_cancel_thd != NULL) { server_try_cancel_thd->join(); @@ -1116,7 +1220,15 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); + } + + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; } // The RPC has been cancelled at this point for sure (i.e irrespective of From 2e729387f7943971de2591d0db09a0a87026cfd7 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 3 Mar 2016 14:22:12 -0800 Subject: [PATCH 038/160] clang-format --- test/cpp/end2end/async_end2end_test.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 09e973f28d5..30028dd4aa6 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -876,8 +876,8 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { auto verif = Verifier(GetParam()); if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while reading the // requests from the client. Since the cancellation can happen at anytime, // some of the cq results (i.e those until cancellation) might be true but @@ -897,7 +897,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // something was seen, but it might be tag 11 and not what we // just added int got_tag = verif.Expect(tag_idx, expected_server_cq_result) - .Next(cq_.get(), ignore_cq_result); + .Next(cq_.get(), ignore_cq_result); GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); if (got_tag == 11) { EXPECT_TRUE(srv_ctx.IsCancelled()); @@ -999,8 +999,8 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { auto verif = Verifier(GetParam()); if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while writing responses // to the client. Since the cancellation can happen at anytime, some of @@ -1022,7 +1022,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // something was seen, but it might be tag 11 and not what we // just added int got_tag = verif.Expect(tag_idx, expected_cq_result) - .Next(cq_.get(), ignore_cq_result); + .Next(cq_.get(), ignore_cq_result); GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); if (got_tag == 11) { EXPECT_TRUE(srv_ctx.IsCancelled()); @@ -1139,8 +1139,8 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { auto verif = Verifier(GetParam()); if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Since server is going to cancel the RPC in a parallel thread, some of // the cq results (i.e those until the cancellation) might be true. Since From 1279bd7934e3c5522b0b6ee6dffcb5b944bf752d Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 14:49:03 -0800 Subject: [PATCH 039/160] bit of text change --- INSTALL.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index aa809f23c8f..d9411db021d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,7 +1,7 @@ #If you are in a hurry -For language-specific installation instructions, please refer to these -documents +For language-specific installation instructions for gRPC runtime, please +refer to these documents * [C++](examples/cpp) * [C#](src/csharp): NuGet package `Grpc` @@ -34,7 +34,8 @@ terminal: #Build from Source -This is for compiling just the gRPC C Core library. +For developers who are interested to contribute, here is how to compile the +gRPC C Core library. ```sh $ git clone https://github.com/grpc/grpc.git From 6dd3c707c2dd921288a82ec9813d41b567ff07b6 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 13:53:49 -0800 Subject: [PATCH 040/160] Update base INSTALL to markdown and remove outdated content --- CONTRIBUTING.md | 2 +- INSTALL | 217 ------------------------------ INSTALL.md | 45 +++++++ README.md | 2 +- examples/cpp/README.md | 2 +- examples/cpp/cpptutorial.md | 2 +- examples/cpp/helloworld/README.md | 2 +- 7 files changed, 50 insertions(+), 222 deletions(-) delete mode 100644 INSTALL create mode 100644 INSTALL.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9423c46547a..03d2e276a82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ In order to protect both you and ourselves, you will need to sign the ### Technical requirements You will need several tools to work with this repository. In addition to all of -the packages described in the [INSTALL](INSTALL) file, you will also need +the packages described in the [INSTALL](INSTALL.md) file, you will also need python, and the mako template renderer. To install the latter, using pip, one should simply be able to do `pip install mako`. diff --git a/INSTALL b/INSTALL deleted file mode 100644 index e33f8970a95..00000000000 --- a/INSTALL +++ /dev/null @@ -1,217 +0,0 @@ -These instructions only cover building grpc C and C++ libraries under -typical unix systems. If you need more information, please try grpc's -wiki pages: - - https://github.com/google/grpc/wiki - - -************************* -* If you are in a hurry * -************************* - -On Linux (Debian): - - Note: you will need to add the Debian 'jessie-backports' distribution to your sources - file first. - - Add the following line to your `/etc/apt/sources.list` file: - - deb http://http.debian.net/debian jessie-backports main - - Install the gRPC library: - - $ [sudo] apt-get install libgrpc-dev - -OR - - $ git clone https://github.com/grpc/grpc.git - $ cd grpc - $ git submodule update --init - $ make - $ [sudo] make install - -You don't need anything else than GNU Make, gcc and autotools. Under a Debian -or Ubuntu system, this should boil down to the following packages: - - $ [sudo] apt-get install build-essential autoconf libtool - -Building the python wrapper requires the following: - - $ [sudo] apt-get install python-all-dev python-virtualenv - -If you want to install in a different directory than the default /usr/lib, you can -override it on the command line: - - $ [sudo] make install prefix=/opt - - -******************************* -* More detailled instructions * -******************************* - -Setting up dependencies -======================= - -Dependencies to compile the libraries -------------------------------------- - -grpc libraries have few external dependencies. If you need to compile and -install them, they are present in the third_party directory if you have -cloned the github repository recursively. If you didn't clone recursively, -you can still get them later by running the following command: - - $ git submodule update --init - -Note that the Makefile makes it much easier for you to compile from sources -if you were to clone recursively our git repository: it will automatically -compile zlib and OpenSSL, which are core requirements for grpc. Note this -creates grpc libraries that will have zlib and OpenSSL built-in inside of them, -which significantly increases the libraries' size. - -In order to decrease that size, you can manually install zlib and OpenSSL on -your system, so that the Makefile can use them instead. - -Under a Debian or Ubuntu system, one can acquire the development package -for zlib this way: - - # apt-get install zlib1g-dev - -To the best of our knowledge, no distribution has an OpenSSL package that -supports ALPN yet, so you would still have to depend on installing from source -for that particular dependency if you want to reduce the libraries' size. - -The recommended version of OpenSSL that provides ALPN support is available -at this URL: - - https://www.openssl.org/source/openssl-1.0.2.tar.gz - - -Dependencies to compile and run the tests ------------------------------------------ - -Compiling and running grpc plain-C tests dont't require any more dependency. - - -Compiling and running grpc C++ tests depend on protobuf 3.0.0, gtest and -gflags. Although gflags is provided in third_party, you will need to manually -install that dependency on your system to run these tests. - -Under a Debian or Ubuntu system, you can install the gtests and gflags packages -using apt-get: - - # apt-get install libgflags-dev libgtest-dev - -However, protobuf 3.0.0 isn't in a debian package yet, but the Makefile will -automatically try and compile the one present in third_party if you cloned the -repository recursively, and that it detects your system is lacking it. - -Compiling and installing protobuf 3.0.0 requires a few more dependencies in -itself, notably the autoconf suite. If you have apt-get, you can install -these dependencies this way: - - # apt-get install autoconf libtool - -If you want to run the tests using one of the sanitized configurations, you -will need clang and its instrumented libc++: - - # apt-get install clang libc++-dev - -Mac-specific notes: -------------------- - -For a Mac system, git is not available by default. You will first need to -install Xcode from the Mac AppStore and then run the following command from a -terminal: - - $ sudo xcode-select --install - -You should also install "port" following the instructions at -https://www.macports.org . This will reside in /opt/local/bin/port for -most Mac installations. Do the "git submodule" command listed above. - -Then execute the following for all the needed build dependencies - - $ sudo /opt/local/bin/port install autoconf automake libtool gflags cmake - $ mkdir ~/gtest-svn - $ svn checkout http://googletest.googlecode.com/svn/trunk/ gtest-svn - $ mkdir mybuild - $ cd mybuild - $ cmake ../gtest-svn - $ make - $ make gtest.a gtest_main.a - $ sudo cp libgtest.a libgtest_main.a /opt/local/lib - $ sudo mkdir /opt/local/include/gtest - $ sudo cp -pr ../gtest-svn/include/gtest /opt/local/include/gtest - -If you are going to make changes and need to regenerate the projects file, -you will need to install certain modules for python. - - $ sudo easy_install simplejson mako - -Mingw-specific notes: ---------------------- - -While gRPC compiles properly under mingw, some more preparation work is needed. -The recommendation is to use msys2. The installation instructions are available -at that address: http://msys2.github.io/ - -Once this is installed, make sure you are using the following: MinGW-w64 Win64. -You'll be required to install a few more packages: - - $ pacman -S make mingw-w64-x86_64-gcc mingw-w64-x86_64-zlib autoconf automake libtool - -Please also install OpenSSL from that website: - - http://slproweb.com/products/Win32OpenSSL.html - -The package Win64 OpenSSL v1.0.2a should do. At that point you should be able -to compile gRPC with the following: - - $ export LDFLAGS="-L/mingw64/lib -L/c/OpenSSL-Win64" - $ export CPPFLAGS="-I/mingw64/include -I/c/OpenSSL-Win64/include" - $ make - -A word on OpenSSL ------------------ - -Secure HTTP2 requires the TLS extension ALPN (see rfc 7301 and -http://http2.github.io/http2-spec/ section 3.3). Our HTTP2 implementation -relies on OpenSSL's implementation. OpenSSL 1.0.2 is the first released version -of OpenSSL that has ALPN support, and this explains our dependency on it. - -Note that the Makefile supports compiling only the unsecure elements of grpc, -and if you do not have OpenSSL and do not want it, you can still proceed -with installing only the elements you require. However, we strongly recommend -the use of encryption for all network traffic, and discourage the use of grpc -without TLS. - - -Compiling -========= - -If you have all the dependencies mentioned above, you should simply be able -to go ahead and run "make" to compile grpc's C and C++ libraries: - - $ make - - -Testing -======= - -To build and run the tests, you can run the command: - - $ make test - -If you want to be able to run them in parallel, and get better output, you can -also use the python tool we have written: - - $ ./tools/run_tests/run_tests.py - - -Installing -========== - -Once everything is compiled, you should be able to install grpc C and C++ -libraries and headers: - - # make install diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 00000000000..aa809f23c8f --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,45 @@ +#If you are in a hurry + +For language-specific installation instructions, please refer to these +documents + + * [C++](examples/cpp) + * [C#](src/csharp): NuGet package `Grpc` + * [Go](https://github.com/grpc/grpc-go): `go get google.golang.org/grpc` + * [Java](https://github.com/grpc/grpc-java) + * [Node](src/node): `npm install grpc` + * [Objective-C](src/objective-c) + * [PHP](src/php): `pecl install grpc-beta` + * [Python](src/python/grpcio): `pip install grpcio` + * [Ruby](src/ruby): `gem install grpc` + + +#Pre-requisites + +##Linux + +```sh + $ [sudo] apt-get install build-essential autoconf libtool +``` + +##Mac OSX + +For a Mac system, git is not available by default. You will first need to +install Xcode from the Mac AppStore and then run the following command from a +terminal: + +```sh + $ [sudo] xcode-select --install +``` + +#Build from Source + +This is for compiling just the gRPC C Core library. + +```sh + $ git clone https://github.com/grpc/grpc.git + $ cd grpc + $ git submodule update --init + $ make + $ [sudo] make install +``` diff --git a/README.md b/README.md index 033e09b91b7..abb4905392c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You can find more detailed documentation and examples in the [doc](doc) and [exa #Installation -See [grpc/INSTALL](INSTALL) for installation instructions for various platforms. +See [INSTALL](INSTALL.md) for installation instructions for various platforms. #Repository Structure & Status diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 85c495099b7..979ba55dbb5 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -2,7 +2,7 @@ ## Installation -To install gRPC on your system, follow the instructions [here](../../INSTALL). +To install gRPC on your system, follow the instructions [here](../../INSTALL.md). ## Hello C++ gRPC! diff --git a/examples/cpp/cpptutorial.md b/examples/cpp/cpptutorial.md index cd1cddb1115..ef9ca99c0fe 100644 --- a/examples/cpp/cpptutorial.md +++ b/examples/cpp/cpptutorial.md @@ -91,7 +91,7 @@ message Point { Next we need to generate the gRPC client and server interfaces from our .proto service definition. We do this using the protocol buffer compiler `protoc` with a special gRPC C++ plugin. -For simplicity, we've provided a [makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL) first): +For simplicity, we've provided a [makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL.md) first): ```shell $ make route_guide.grpc.pb.cc route_guide.pb.cc diff --git a/examples/cpp/helloworld/README.md b/examples/cpp/helloworld/README.md index 90f3d6b1475..8e11f7cdf3e 100644 --- a/examples/cpp/helloworld/README.md +++ b/examples/cpp/helloworld/README.md @@ -2,7 +2,7 @@ ### Install gRPC Make sure you have installed gRPC on your system. Follow the instructions here: -[https://github.com/grpc/grpc/blob/master/INSTALL](../../../INSTALL). +[https://github.com/grpc/grpc/blob/master/INSTALL](../../../INSTALL.md). ### Get the tutorial source code From a22a50d61795c38522fe8b032603c63fcef7f93f Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 14:49:03 -0800 Subject: [PATCH 041/160] bit of text change --- INSTALL.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index aa809f23c8f..d9411db021d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,7 +1,7 @@ #If you are in a hurry -For language-specific installation instructions, please refer to these -documents +For language-specific installation instructions for gRPC runtime, please +refer to these documents * [C++](examples/cpp) * [C#](src/csharp): NuGet package `Grpc` @@ -34,7 +34,8 @@ terminal: #Build from Source -This is for compiling just the gRPC C Core library. +For developers who are interested to contribute, here is how to compile the +gRPC C Core library. ```sh $ git clone https://github.com/grpc/grpc.git From 19f703dbb7805b95bdc64d166a4e5fb36e2dd192 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 3 Mar 2016 15:33:29 -0800 Subject: [PATCH 042/160] increase timeout for interop tests --- tools/run_tests/run_interop_tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index df3ab90a839..1dc772a8565 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -60,6 +60,8 @@ _SKIP_COMPRESSION = ['large_compressed_unary', _SKIP_ADVANCED = ['custom_metadata', 'status_code_and_message', 'unimplemented_method'] +_TEST_TIMEOUT = 3*60 + class CXXLanguage: def __init__(self): @@ -459,7 +461,7 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name, environ=environ, shortname='%s:%s:%s:%s' % (suite_name, server_host_name, language, test_case), - timeout_seconds=90, + timeout_seconds=_TEST_TIMEOUT, flake_retries=5 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, kill_handler=_job_kill_handler) @@ -495,7 +497,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, environ=environ, shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name, test_case), - timeout_seconds=90, + timeout_seconds=_TEST_TIMEOUT, flake_retries=5 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, kill_handler=_job_kill_handler) From dbf47fabd406b51e8d3d03c5eb06c5fe6f7d0d5d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 3 Mar 2016 16:25:06 -0800 Subject: [PATCH 043/160] Better comments. --- test/cpp/end2end/async_end2end_test.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 30028dd4aa6..dc8c2bb6e5b 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -107,11 +107,14 @@ class PollingOverrider { class Verifier { public: explicit Verifier(bool spin) : spin_(spin) {} + // Expect sets the expected ok value for a specific tag Verifier& Expect(int i, bool expect_ok) { expectations_[tag(i)] = expect_ok; return *this; } + // Next waits for 1 async tag to complete, checks its + // expectations, and returns the tag int Next(CompletionQueue* cq, bool ignore_ok) { bool ok; void* got_tag; @@ -135,14 +138,19 @@ class Verifier { return detag(got_tag); } + // Verify keeps calling Next until all currently set + // expected tags are complete void Verify(CompletionQueue* cq) { Verify(cq, false); } + // This version of Verify allows optionally ignoring the + // outcome of the expectation void Verify(CompletionQueue* cq, bool ignore_ok) { GPR_ASSERT(!expectations_.empty()); while (!expectations_.empty()) { Next(cq, ignore_ok); } } + // This version of Verify stops after a certain deadline void Verify(CompletionQueue* cq, std::chrono::system_clock::time_point deadline) { if (expectations_.empty()) { From 79a0f0611c6235fbf636c3e754860e6de28b4e3c Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 3 Mar 2016 17:18:48 -0800 Subject: [PATCH 044/160] Properly delete Node OpenSSL headers after downloading them --- tools/run_tests/build_node.bat | 10 +++++++++- tools/run_tests/pre_build_node.bat | 9 +-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tools/run_tests/build_node.bat b/tools/run_tests/build_node.bat index 6896bc1d1bb..3ddd0e73cf7 100644 --- a/tools/run_tests/build_node.bat +++ b/tools/run_tests/build_node.bat @@ -27,4 +27,12 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -npm install --build-from-source \ No newline at end of file +call npm install --build-from-source + +@rem delete the redundant openssl headers +for /f "delims=v" %%v in ('node --version') do ( + rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q +) + +@rem rebuild, because it probably failed the first time +call npm install --build-from-source \ No newline at end of file diff --git a/tools/run_tests/pre_build_node.bat b/tools/run_tests/pre_build_node.bat index 6e7cbe5d420..ffb4a09f15a 100644 --- a/tools/run_tests/pre_build_node.bat +++ b/tools/run_tests/pre_build_node.bat @@ -28,12 +28,5 @@ @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @rem Expire cache after 1 week -npm update --cache-min 604800 +call npm update --cache-min 604800 -npm install node-gyp-install -.\node_modules\.bin\node-gyp-install.cmd - -@rem delete the redundant openssl headers -for /f "delims=v" %%v in ('node --version') do ( - rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q -) \ No newline at end of file From 4913ea06eb4093a2229b3009dedb02828af1edc2 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 3 Mar 2016 17:22:53 -0800 Subject: [PATCH 045/160] Ensure node and npm are in the path when running tests --- tools/run_tests/build_node.bat | 4 ++++ tools/run_tests/pre_build_node.bat | 2 ++ tools/run_tests/run_node.bat | 1 + 3 files changed, 7 insertions(+) diff --git a/tools/run_tests/build_node.bat b/tools/run_tests/build_node.bat index 3ddd0e73cf7..886af0610ff 100644 --- a/tools/run_tests/build_node.bat +++ b/tools/run_tests/build_node.bat @@ -27,6 +27,10 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm + +del /f /q BUILD || rmdir build /s /q + call npm install --build-from-source @rem delete the redundant openssl headers diff --git a/tools/run_tests/pre_build_node.bat b/tools/run_tests/pre_build_node.bat index ffb4a09f15a..a29456f9ed9 100644 --- a/tools/run_tests/pre_build_node.bat +++ b/tools/run_tests/pre_build_node.bat @@ -27,6 +27,8 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm + @rem Expire cache after 1 week call npm update --cache-min 604800 diff --git a/tools/run_tests/run_node.bat b/tools/run_tests/run_node.bat index 41777363568..0987fbee559 100644 --- a/tools/run_tests/run_node.bat +++ b/tools/run_tests/run_node.bat @@ -27,6 +27,7 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm set JUNIT_REPORT_PATH=src\node\report.xml set JUNIT_REPORT_STACK=1 .\node_modules\.bin\mocha.cmd --reporter mocha-jenkins-reporter --timeout 8000 src\node\test \ No newline at end of file From f2f7d57c01037993220132817a2897777288f3ca Mon Sep 17 00:00:00 2001 From: Dan Born Date: Thu, 3 Mar 2016 17:26:12 -0800 Subject: [PATCH 046/160] Allow use of alternative credential types. --- src/proto/grpc/testing/echo_messages.proto | 1 + test/cpp/end2end/end2end_test.cc | 33 ++++++++++++++-------- test/cpp/end2end/test_service_impl.cc | 10 ++++--- test/cpp/util/test_credentials_provider.h | 5 +++- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/proto/grpc/testing/echo_messages.proto b/src/proto/grpc/testing/echo_messages.proto index d05a35548d1..5ce0a1fd642 100644 --- a/src/proto/grpc/testing/echo_messages.proto +++ b/src/proto/grpc/testing/echo_messages.proto @@ -42,6 +42,7 @@ message RequestParams { bool echo_peer = 7; string expected_client_identity = 8; // will force check_auth_context. bool skip_cancelled_check = 9; + string expected_transport_security_type = 10; } message EchoRequest { diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index dc2c4f6426f..8131a14ff76 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -59,6 +59,7 @@ using grpc::testing::EchoRequest; using grpc::testing::EchoResponse; +using grpc::testing::kTlsCredentialsType; using std::chrono::system_clock; namespace grpc { @@ -1194,6 +1195,8 @@ TEST_P(SecureEnd2endTest, BlockingAuthMetadataPluginAndProcessorSuccess) { request.mutable_param()->set_echo_metadata(true); request.mutable_param()->set_expected_client_identity( TestAuthMetadataProcessor::kGoodGuy); + request.mutable_param()->set_expected_transport_security_type( + GetParam().credentials_type); Status s = stub_->Echo(&context, request, &response); EXPECT_EQ(request.message(), response.message()); @@ -1301,6 +1304,8 @@ TEST_P(SecureEnd2endTest, NonBlockingAuthMetadataPluginAndProcessorSuccess) { request.mutable_param()->set_echo_metadata(true); request.mutable_param()->set_expected_client_identity( TestAuthMetadataProcessor::kGoodGuy); + request.mutable_param()->set_expected_transport_security_type( + GetParam().credentials_type); Status s = stub_->Echo(&context, request, &response); EXPECT_EQ(request.message(), response.message()); @@ -1349,25 +1354,29 @@ TEST_P(SecureEnd2endTest, ClientAuthContext) { EchoRequest request; EchoResponse response; request.set_message("Hello"); - request.mutable_param()->set_check_auth_context(true); - + request.mutable_param()->set_check_auth_context(GetParam().credentials_type == + kTlsCredentialsType); + request.mutable_param()->set_expected_transport_security_type( + GetParam().credentials_type); ClientContext context; Status s = stub_->Echo(&context, request, &response); EXPECT_EQ(response.message(), request.message()); EXPECT_TRUE(s.ok()); std::shared_ptr auth_ctx = context.auth_context(); - std::vector ssl = + std::vector tst = auth_ctx->FindPropertyValues("transport_security_type"); - EXPECT_EQ(1u, ssl.size()); - EXPECT_EQ("ssl", ToString(ssl[0])); - EXPECT_EQ("x509_subject_alternative_name", - auth_ctx->GetPeerIdentityPropertyName()); - EXPECT_EQ(3u, auth_ctx->GetPeerIdentity().size()); - EXPECT_EQ("*.test.google.fr", ToString(auth_ctx->GetPeerIdentity()[0])); - EXPECT_EQ("waterzooi.test.google.be", - ToString(auth_ctx->GetPeerIdentity()[1])); - EXPECT_EQ("*.test.youtube.com", ToString(auth_ctx->GetPeerIdentity()[2])); + EXPECT_EQ(1u, tst.size()); + EXPECT_EQ(GetParam().credentials_type, ToString(tst[0])); + if (GetParam().credentials_type == kTlsCredentialsType) { + EXPECT_EQ("x509_subject_alternative_name", + auth_ctx->GetPeerIdentityPropertyName()); + EXPECT_EQ(3u, auth_ctx->GetPeerIdentity().size()); + EXPECT_EQ("*.test.google.fr", ToString(auth_ctx->GetPeerIdentity()[0])); + EXPECT_EQ("waterzooi.test.google.be", + ToString(auth_ctx->GetPeerIdentity()[1])); + EXPECT_EQ("*.test.youtube.com", ToString(auth_ctx->GetPeerIdentity()[2])); + } } std::vector CreateTestScenarios(bool use_proxy, diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc index 7c3e514effa..698d4678c87 100644 --- a/test/cpp/end2end/test_service_impl.cc +++ b/test/cpp/end2end/test_service_impl.cc @@ -63,13 +63,14 @@ void MaybeEchoDeadline(ServerContext* context, const EchoRequest* request, } void CheckServerAuthContext(const ServerContext* context, + const grpc::string& expected_transport_security_type, const grpc::string& expected_client_identity) { std::shared_ptr auth_ctx = context->auth_context(); - std::vector ssl = + std::vector tst = auth_ctx->FindPropertyValues("transport_security_type"); - EXPECT_EQ(1u, ssl.size()); - EXPECT_EQ("ssl", ToString(ssl[0])); - if (expected_client_identity.length() == 0) { + EXPECT_EQ(1u, tst.size()); + EXPECT_EQ(expected_transport_security_type, ToString(tst[0])); + if (expected_client_identity.empty()) { EXPECT_TRUE(auth_ctx->GetPeerIdentityPropertyName().empty()); EXPECT_TRUE(auth_ctx->GetPeerIdentity().empty()); EXPECT_FALSE(auth_ctx->IsPeerAuthenticated()); @@ -139,6 +140,7 @@ Status TestServiceImpl::Echo(ServerContext* context, const EchoRequest* request, (request->param().expected_client_identity().length() > 0 || request->param().check_auth_context())) { CheckServerAuthContext(context, + request->param().expected_transport_security_type(), request->param().expected_client_identity()); } if (request->has_param() && request->param().response_message_length() > 0) { diff --git a/test/cpp/util/test_credentials_provider.h b/test/cpp/util/test_credentials_provider.h index 50fadb53a24..1fb311e556e 100644 --- a/test/cpp/util/test_credentials_provider.h +++ b/test/cpp/util/test_credentials_provider.h @@ -44,7 +44,10 @@ namespace grpc { namespace testing { const char kInsecureCredentialsType[] = "INSECURE_CREDENTIALS"; -const char kTlsCredentialsType[] = "TLS_CREDENTIALS"; + +// For real credentials, like tls/ssl, this name should match the AuthContext +// property "transport_security_type". +const char kTlsCredentialsType[] = "ssl"; // Provide test credentials of a particular type. class CredentialTypeProvider { From 51221f1e7e65231539614f9b020c331f234663a5 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Thu, 3 Mar 2016 17:44:50 -0800 Subject: [PATCH 047/160] clang reformat. --- test/cpp/end2end/test_service_impl.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc index 698d4678c87..fe29c4afe9e 100644 --- a/test/cpp/end2end/test_service_impl.cc +++ b/test/cpp/end2end/test_service_impl.cc @@ -62,9 +62,10 @@ void MaybeEchoDeadline(ServerContext* context, const EchoRequest* request, } } -void CheckServerAuthContext(const ServerContext* context, - const grpc::string& expected_transport_security_type, - const grpc::string& expected_client_identity) { +void CheckServerAuthContext( + const ServerContext* context, + const grpc::string& expected_transport_security_type, + const grpc::string& expected_client_identity) { std::shared_ptr auth_ctx = context->auth_context(); std::vector tst = auth_ctx->FindPropertyValues("transport_security_type"); From 77fae2784ed7f894dc557ee2004364f2bb1b6b74 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 19:39:56 -0800 Subject: [PATCH 048/160] fix broken links --- doc/interop-test-descriptions.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md index e618e967ee0..3beb1d11f4a 100644 --- a/doc/interop-test-descriptions.md +++ b/doc/interop-test-descriptions.md @@ -2,9 +2,8 @@ Interoperability Test Case Descriptions ======================================= Client and server use -[test.proto](https://github.com/grpc/grpc/blob/master/test/proto/test.proto) -and the [gRPC over HTTP/2 v2 -protocol](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md). +[test.proto](../src/proto/grpc/testing/test.proto) +and the [gRPC over HTTP/2 v2 protocol](./PROTOCOL-HTTP2.md). Client ------ From 3f5c5c7e18f6b09d85e39f310f214e90316cc0ef Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 19:46:13 -0800 Subject: [PATCH 049/160] fix broken link --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 84ec80057e1..cd85417ca20 100644 --- a/examples/README.md +++ b/examples/README.md @@ -447,4 +447,4 @@ $ greeter_client ## Read more! - You can find links to language-specific tutorials, examples, and other docs in each language's [quick start](#quickstart). -- [gRPC Authentication Support](doc/grpc-auth-support.md) introduces authentication support in gRPC with supported mechanisms and examples. +- [gRPC Authentication Support](http://www.grpc.io/docs/guides/auth.html) introduces authentication support in gRPC with supported mechanisms and examples. From 5821a444c703757d4bacdf2d58a3045c6cfbb838 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 19:46:58 -0800 Subject: [PATCH 050/160] remove stray grpc_common reference --- examples/node/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/node/README.md b/examples/node/README.md index 7a2bc9794f3..c1ef6b05ab0 100644 --- a/examples/node/README.md +++ b/examples/node/README.md @@ -20,7 +20,7 @@ TRY IT! - Run the server ```sh - $ # from this directory (grpc_common/node). + $ # from this directory $ node ./greeter_server.js & ``` From f9946d579bc5b258261177b5ff87df22e7d49a07 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 20:06:20 -0800 Subject: [PATCH 051/160] remove old stuff from tools/README.md --- tools/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/README.md b/tools/README.md index a0c41eb79ff..df4c6ef7d25 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,7 +1,5 @@ buildgen: template renderer for our build system. -distpackages: script to generate debian packages. - distrib: scripts to distribute language-specific packages. dockerfile: Docker files to test gRPC. @@ -12,6 +10,4 @@ gce: scripts to help setup testing infrastructure on GCE. jenkins: support for running tests on Jenkins. -profile_analyzer: pretty printer for gRPC profiling data. - run_tests: scripts to run gRPC tests in parallel. From 93744fccb784c2cab8a9f008348f1e87cc1293e5 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 20:41:38 -0800 Subject: [PATCH 052/160] update PHP composer.json files --- composer.json | 11 +++++++++-- src/php/composer.json | 6 ++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index 61f81e02bf4..9ba45fd3a36 100644 --- a/composer.json +++ b/composer.json @@ -2,13 +2,20 @@ "name": "grpc/grpc", "type": "library", "description": "gRPC library for PHP", - "version": "0.6.0", + "version": "0.13.0", "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/stanley-cheung/Protobuf-PHP" + } + ], "require": { "php": ">=5.5.0", - "google/auth": "dev-master" + "datto/protobuf-php": "dev-master", + "google/auth": "v0.7" }, "autoload": { "psr-4": { diff --git a/src/php/composer.json b/src/php/composer.json index 1d41f847ac0..59f846fdcc6 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -1,7 +1,9 @@ { "name": "grpc/grpc", + "type": "library", "description": "gRPC library for PHP", - "version": "0.6.0", + "version": "0.13.0", + "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", "repositories": [ @@ -13,7 +15,7 @@ "require": { "php": ">=5.5.0", "datto/protobuf-php": "dev-master", - "google/auth": "dev-master" + "google/auth": "v0.7" }, "autoload": { "psr-4": { From 389297dedfe384246d4831a4fc33e7af7b0200e5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 3 Mar 2016 21:02:54 -0800 Subject: [PATCH 053/160] Document some things --- src/core/transport/transport.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h index 7a007ef777b..5a90bd6d387 100644 --- a/src/core/transport/transport.h +++ b/src/core/transport/transport.h @@ -123,7 +123,8 @@ typedef struct grpc_transport_stream_op { /** Transport op: a set of operations to perform on a transport as a whole */ typedef struct grpc_transport_op { - /** called when processing of this op is done */ + /** Called when processing of this op is done. + Only one transport_op is allowed to be outstanding at any time. */ grpc_closure *on_consumed; /** connectivity monitoring - set connectivity_state to NULL to unsubscribe */ grpc_closure *on_connectivity_state_change; @@ -138,7 +139,9 @@ typedef struct grpc_transport_op { grpc_status_code goaway_status; gpr_slice *goaway_message; /** set the callback for accepting new streams; - this is a permanent callback, unlike the other one-shot closures */ + this is a permanent callback, unlike the other one-shot closures. + If true, the callback is set to set_accept_stream_fn, with its + user_data argument set to set_accept_stream_user_data */ bool set_accept_stream; void (*set_accept_stream_fn)(grpc_exec_ctx *exec_ctx, void *user_data, grpc_transport *transport, From 1bd9ea407fd673f6c795a524c6454bc5db339a85 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 3 Mar 2016 21:09:31 -0800 Subject: [PATCH 054/160] Refine condition --- src/core/transport/chttp2_transport.c | 5 +++-- src/core/transport/transport.h | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 369ff0ad7f6..cf1dbfa0e59 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -1006,8 +1006,9 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, lock(t); - /* Let's be overly cautious: don't change any state while we're parsing */ - if (t->parsing_active) { + /* If there's a set_accept_stream ensure that we're not parsing + to avoid changing things out from underneath */ + if (t->parsing_active && t->set_accept_stream) { GPR_ASSERT(t->post_parsing_op == NULL); t->post_parsing_op = gpr_malloc(sizeof(*op)); memcpy(t->post_parsing_op, op, sizeof(*op)); diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h index 5a90bd6d387..ed6e121c9cb 100644 --- a/src/core/transport/transport.h +++ b/src/core/transport/transport.h @@ -123,8 +123,7 @@ typedef struct grpc_transport_stream_op { /** Transport op: a set of operations to perform on a transport as a whole */ typedef struct grpc_transport_op { - /** Called when processing of this op is done. - Only one transport_op is allowed to be outstanding at any time. */ + /** Called when processing of this op is done. */ grpc_closure *on_consumed; /** connectivity monitoring - set connectivity_state to NULL to unsubscribe */ grpc_closure *on_connectivity_state_change; From 687bf6295f5e3a02f249aced69868e497c1bcc30 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 3 Mar 2016 21:18:48 -0800 Subject: [PATCH 055/160] Fix typo --- src/core/transport/chttp2_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index cf1dbfa0e59..643e3feeb97 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -1008,7 +1008,7 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, /* If there's a set_accept_stream ensure that we're not parsing to avoid changing things out from underneath */ - if (t->parsing_active && t->set_accept_stream) { + if (t->parsing_active && op->set_accept_stream) { GPR_ASSERT(t->post_parsing_op == NULL); t->post_parsing_op = gpr_malloc(sizeof(*op)); memcpy(t->post_parsing_op, op, sizeof(*op)); From af22087a43ffe7e69cb269419f046635adfc46e6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 10:00:31 -0800 Subject: [PATCH 056/160] Fix leak: dont become writable if writes are closed --- src/core/transport/chttp2_transport.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 643e3feeb97..ce54f7e6e82 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -617,6 +617,7 @@ static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { if (!TRANSPORT_FROM_GLOBAL(transport_global)->closed && + !stream_global->write_closed && grpc_chttp2_list_add_writable_stream(transport_global, stream_global)) { GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); } From ad23ae196702cc3889a8c04a21273aa6a43aad75 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 10:50:56 -0800 Subject: [PATCH 057/160] Unref writable streams on transport close to avoid leaks --- src/core/transport/chttp2_transport.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index ce54f7e6e82..19265252ca3 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -432,6 +432,14 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, if (t->ep) { allow_endpoint_shutdown_locked(exec_ctx, t); } + + /* flush writable stream list to avoid dangling references */ + grpc_chttp2_stream_global *stream_global; + grpc_chttp2_stream_writing *stream_writing; + while (grpc_chttp2_list_pop_writable_stream( + &t->global, &t->writing, &stream_global, &stream_writing)) { + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing"); + } } } @@ -617,7 +625,6 @@ static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { if (!TRANSPORT_FROM_GLOBAL(transport_global)->closed && - !stream_global->write_closed && grpc_chttp2_list_add_writable_stream(transport_global, stream_global)) { GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); } From 34fcf43d374d312290d05408fbd554c3138a1d7d Mon Sep 17 00:00:00 2001 From: vjpai Date: Fri, 4 Mar 2016 11:02:05 -0800 Subject: [PATCH 058/160] Restart workers in each scenario --- tools/jenkins/run_performance.sh | 33 ++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index fbc078330f7..abad97358cc 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -42,15 +42,10 @@ config=opt make CONFIG=$config qps_worker qps_driver -j8 -bins/$config/qps_worker -driver_port 10000 & -PID1=$! -bins/$config/qps_worker -driver_port 10010 & -PID2=$! - # # Put a timeout on these tests # -((sleep 900; kill $$ && killall qps_worker && rm -f /tmp/qps-test.$$ )&) +((sleep 900; killall qps_worker && rm -f /tmp/qps-test.$$ && kill $$)&) export QPS_WORKERS="localhost:10000,localhost:10010" @@ -71,38 +66,52 @@ halfcores=`expr $cores / 2` for secure in true false; do # Scenario 1: generic async streaming ping-pong (contentionless latency) + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - + bins/$config/qps_driver --quit=true + # Scenario 2: generic async streaming "unconstrained" (QPS) + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 \ --server_core_limit=$halfcores --client_core_limit=0 |& tee /tmp/qps-test.$$ + bins/$config/qps_driver --quit=true # Scenario 2b: QPS with a single server core + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 --server_core_limit=1 --client_core_limit=0 + bins/$config/qps_driver --quit=true # Scenario 2c: protobuf-based QPS + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --simple_req_size=0 --simple_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 \ --server_core_limit=$halfcores --client_core_limit=0 + bins/$config/qps_driver --quit=true # Scenario 3: Latency at sub-peak load (all clients equally loaded) for loadfactor in 0.7; do + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -110,27 +119,31 @@ for secure in true false; do --num_servers=1 --num_clients=0 --poisson_load=`awk -v lf=$loadfactor \ '$5 == "QPS:" {print int(lf * $6); exit}' /tmp/qps-test.$$` \ --server_core_limit=$halfcores --client_core_limit=0 + bins/$config/qps_driver --quit=true done rm /tmp/qps-test.$$ # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM). + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 + bins/$config/qps_driver --quit=true # Scenario 5: Sync unary ping-pong with protobufs + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=UNARY --client_type=SYNC_CLIENT \ --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \ --secure_test=$secure --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - + bins/$config/qps_driver --quit=true done -bins/$config/qps_driver --quit=true - wait From 2d9476898b119ead1045eb31894bcd10176344fd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 11:05:42 -0800 Subject: [PATCH 059/160] Revert "Ensure that no #includes are inside of a namespace." --- test/cpp/interop/reconnect_interop_client.cc | 10 +++++----- test/cpp/interop/reconnect_interop_server.cc | 12 +++++------- test/cpp/qps/limit_cores.cc | 7 +++---- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index c668edaceb0..79a60cc8602 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -34,16 +34,16 @@ #include #include +#include +#include #include #include #include -#include -#include -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" -#include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/util/create_test_channel.h" #include "test/cpp/util/test_config.h" +#include "src/proto/grpc/testing/test.grpc.pb.h" +#include "src/proto/grpc/testing/empty.grpc.pb.h" +#include "src/proto/grpc/testing/messages.grpc.pb.h" DEFINE_int32(server_control_port, 0, "Server port for control rpcs."); DEFINE_int32(server_retry_port, 0, "Server port for testing reconnection."); diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 1f9147d0efa..3602b8c2b05 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -31,8 +31,6 @@ * */ -// Test description at doc/connection-backoff-interop-test-description.md - #include #include @@ -42,17 +40,17 @@ #include #include +#include +#include #include #include #include -#include -#include -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" -#include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/core/util/reconnect_server.h" #include "test/cpp/util/test_config.h" +#include "src/proto/grpc/testing/test.grpc.pb.h" +#include "src/proto/grpc/testing/empty.grpc.pb.h" +#include "src/proto/grpc/testing/messages.grpc.pb.h" DEFINE_int32(control_port, 0, "Server port for controlling the server."); DEFINE_int32(retry_port, 0, diff --git a/test/cpp/qps/limit_cores.cc b/test/cpp/qps/limit_cores.cc index 1fb2d628f6d..fad9a323afd 100644 --- a/test/cpp/qps/limit_cores.cc +++ b/test/cpp/qps/limit_cores.cc @@ -37,15 +37,14 @@ #include #include +namespace grpc { +namespace testing { + #ifdef GPR_CPU_LINUX #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include - -namespace grpc { -namespace testing { - int LimitCores(const int* cores, int cores_size) { const int num_cores = gpr_cpu_num_cores(); int cores_set = 0; From 180f6dfc8d16f16c607f0b5e51f1efe1596f3212 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 4 Mar 2016 11:23:48 -0800 Subject: [PATCH 060/160] Keep changing the ports each scenario also --- tools/jenkins/run_performance.sh | 58 ++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index abad97358cc..b06fce7acb9 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -47,8 +47,6 @@ make CONFIG=$config qps_worker qps_driver -j8 # ((sleep 900; killall qps_worker && rm -f /tmp/qps-test.$$ && kill $$)&) -export QPS_WORKERS="localhost:10000,localhost:10010" - # big is the size in bytes of large messages (0 is the size otherwise) big=65536 @@ -64,10 +62,18 @@ deep=100 cores=`grep -c ^processor /proc/cpuinfo` halfcores=`expr $cores / 2` +# +# Keep changing the ports +port1=10000 +port2=10100 + for secure in true false; do # Scenario 1: generic async streaming ping-pong (contentionless latency) - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -75,10 +81,13 @@ for secure in true false; do --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 bins/$config/qps_driver --quit=true - + # Scenario 2: generic async streaming "unconstrained" (QPS) - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -88,8 +97,11 @@ for secure in true false; do bins/$config/qps_driver --quit=true # Scenario 2b: QPS with a single server core - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -98,8 +110,11 @@ for secure in true false; do bins/$config/qps_driver --quit=true # Scenario 2c: protobuf-based QPS - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --simple_req_size=0 --simple_resp_size=0 \ @@ -110,8 +125,11 @@ for secure in true false; do # Scenario 3: Latency at sub-peak load (all clients equally loaded) for loadfactor in 0.7; do - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -125,8 +143,11 @@ for secure in true false; do rm /tmp/qps-test.$$ # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM). - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \ @@ -136,8 +157,11 @@ for secure in true false; do bins/$config/qps_driver --quit=true # Scenario 5: Sync unary ping-pong with protobufs - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=UNARY --client_type=SYNC_CLIENT \ --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \ From edd96e4926373644bac1c169431c213d361bed21 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 4 Mar 2016 14:32:50 -0500 Subject: [PATCH 061/160] Revert "Properly integrate async API with server-side cancellations." --- .../grpc++/impl/codegen/completion_queue.h | 1 - include/grpc++/impl/codegen/server_context.h | 3 - src/cpp/server/server_context.cc | 26 +- test/cpp/end2end/async_end2end_test.cc | 232 +++++------------- 4 files changed, 64 insertions(+), 198 deletions(-) diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h index 928ab2db317..102831e1c9b 100644 --- a/include/grpc++/impl/codegen/completion_queue.h +++ b/include/grpc++/impl/codegen/completion_queue.h @@ -184,7 +184,6 @@ class CompletionQueue : private GrpcLibrary { bool Pluck(CompletionQueueTag* tag); /// Performs a single polling pluck on \a tag. - /// \warning Must not be mixed with calls to \a Next. void TryPluck(CompletionQueueTag* tag); grpc_completion_queue* cq_; // owned diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h index 91ebe574b14..ad08b8210d6 100644 --- a/include/grpc++/impl/codegen/server_context.h +++ b/include/grpc++/impl/codegen/server_context.h @@ -103,9 +103,6 @@ class ServerContext { void AddInitialMetadata(const grpc::string& key, const grpc::string& value); void AddTrailingMetadata(const grpc::string& key, const grpc::string& value); - // IsCancelled is always safe to call when using sync API - // When using async API, it is only safe to call IsCancelled after - // the AsyncNotifyWhenDone tag has been delivered bool IsCancelled() const; // Cancel the Call from the server. This is a best-effort API and depending on diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index eb49b210379..e205a1969b3 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -62,11 +62,7 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void FillOps(grpc_op* ops, size_t* nops) GRPC_OVERRIDE; bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE; - bool CheckCancelled(CompletionQueue* cq) { - cq->TryPluck(this); - return CheckCancelledNoPluck(); - } - bool CheckCancelledAsync() { return CheckCancelledNoPluck(); } + bool CheckCancelled(CompletionQueue* cq); void set_tag(void* tag) { has_tag_ = true; @@ -76,11 +72,6 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void Unref(); private: - bool CheckCancelledNoPluck() { - grpc::lock_guard g(mu_); - return finalized_ ? (cancelled_ != 0) : false; - } - bool has_tag_; void* tag_; grpc::mutex mu_; @@ -97,6 +88,12 @@ void ServerContext::CompletionOp::Unref() { } } +bool ServerContext::CompletionOp::CheckCancelled(CompletionQueue* cq) { + cq->TryPluck(this); + grpc::lock_guard g(mu_); + return finalized_ ? cancelled_ != 0 : false; +} + void ServerContext::CompletionOp::FillOps(grpc_op* ops, size_t* nops) { ops->op = GRPC_OP_RECV_CLOSE_ON_SERVER; ops->data.recv_close_on_server.cancelled = &cancelled_; @@ -185,14 +182,7 @@ void ServerContext::TryCancel() const { } bool ServerContext::IsCancelled() const { - if (has_notify_when_done_tag_) { - // when using async API, but the result is only valid - // if the tag has already been delivered at the completion queue - return completion_op_ && completion_op_->CheckCancelledAsync(); - } else { - // when using sync API - return completion_op_ && completion_op_->CheckCancelled(cq_); - } + return completion_op_ && completion_op_->CheckCancelled(cq_); } void ServerContext::set_compression_level(grpc_compression_level level) { diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index dc8c2bb6e5b..9ca3bf98f85 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -68,7 +68,6 @@ namespace testing { namespace { void* tag(int i) { return (void*)(intptr_t)i; } -int detag(void* p) { return static_cast(reinterpret_cast(p)); } #ifdef GPR_POSIX_SOCKET static int maybe_assert_non_blocking_poll(struct pollfd* pfds, nfds_t nfds, @@ -107,50 +106,37 @@ class PollingOverrider { class Verifier { public: explicit Verifier(bool spin) : spin_(spin) {} - // Expect sets the expected ok value for a specific tag Verifier& Expect(int i, bool expect_ok) { expectations_[tag(i)] = expect_ok; return *this; } - // Next waits for 1 async tag to complete, checks its - // expectations, and returns the tag - int Next(CompletionQueue* cq, bool ignore_ok) { - bool ok; - void* got_tag; - if (spin_) { - for (;;) { - auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); - if (r == CompletionQueue::TIMEOUT) continue; - if (r == CompletionQueue::GOT_EVENT) break; - gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); - abort(); - } - } else { - EXPECT_TRUE(cq->Next(&got_tag, &ok)); - } - auto it = expectations_.find(got_tag); - EXPECT_TRUE(it != expectations_.end()); - if (!ignore_ok) { - EXPECT_EQ(it->second, ok); - } - expectations_.erase(it); - return detag(got_tag); - } - - // Verify keeps calling Next until all currently set - // expected tags are complete void Verify(CompletionQueue* cq) { Verify(cq, false); } - // This version of Verify allows optionally ignoring the - // outcome of the expectation void Verify(CompletionQueue* cq, bool ignore_ok) { GPR_ASSERT(!expectations_.empty()); while (!expectations_.empty()) { - Next(cq, ignore_ok); + bool ok; + void* got_tag; + if (spin_) { + for (;;) { + auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); + if (r == CompletionQueue::TIMEOUT) continue; + if (r == CompletionQueue::GOT_EVENT) break; + gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); + abort(); + } + } else { + EXPECT_TRUE(cq->Next(&got_tag, &ok)); + } + auto it = expectations_.find(got_tag); + EXPECT_TRUE(it != expectations_.end()); + if (!ignore_ok) { + EXPECT_EQ(it->second, ok); + } + expectations_.erase(it); } } - // This version of Verify stops after a certain deadline void Verify(CompletionQueue* cq, std::chrono::system_clock::time_point deadline) { if (expectations_.empty()) { @@ -807,8 +793,7 @@ TEST_P(AsyncEnd2endTest, UnimplementedRpc) { } // This class is for testing scenarios where RPCs are cancelled on the server -// by calling ServerContext::TryCancel(). Server uses AsyncNotifyWhenDone -// API to check for cancellation +// by calling ServerContext::TryCancel() class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { protected: typedef enum { @@ -818,6 +803,13 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { CANCEL_AFTER_PROCESSING } ServerTryCancelRequestPhase; + void ServerTryCancel(ServerContext* context) { + EXPECT_FALSE(context->IsCancelled()); + context->TryCancel(); + gpr_log(GPR_INFO, "Server called TryCancel()"); + EXPECT_TRUE(context->IsCancelled()); + } + // Helper for testing client-streaming RPCs which are cancelled on the server. // Depending on the value of server_try_cancel parameter, this will test one // of the following three scenarios: @@ -851,7 +843,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of 'RequestStream' calls // and receive the 'RequestStream' call just made by the client - srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestRequestStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -867,12 +858,9 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_server_cq_result = true; bool ignore_cq_result = false; - bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - srv_ctx.TryCancel(); - Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); + ServerTryCancel(&srv_ctx); // Since cancellation is done before server reads any results, we know // for sure that all cq results will return false from this point forward @@ -880,39 +868,22 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; - - auto verif = Verifier(GetParam()); - if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = - new std::thread(&ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = new std::thread( + &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); // Server will cancel the RPC in a parallel thread while reading the // requests from the client. Since the cancellation can happen at anytime, // some of the cq results (i.e those until cancellation) might be true but // its non deterministic. So better to ignore the cq results ignore_cq_result = true; - // Expect that we might possibly see the done tag that - // indicates cancellation completion in this case - want_done_tag = true; - verif.Expect(11, true); } // Server reads 3 messages (tags 6, 7 and 8) - // But if want_done_tag is true, we might also see tag 11 for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { srv_stream.Read(&recv_request, tag(tag_idx)); - // Note that we'll add something to the verifier and verify that - // something was seen, but it might be tag 11 and not what we - // just added - int got_tag = verif.Expect(tag_idx, expected_server_cq_result) - .Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); - } + Verifier(GetParam()) + .Expect(tag_idx, expected_server_cq_result) + .Verify(cq_.get(), ignore_cq_result); } if (server_try_cancel_thd != NULL) { @@ -921,15 +892,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - srv_ctx.TryCancel(); - want_done_tag = true; - verif.Expect(11, true); - } - - if (want_done_tag) { - verif.Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; + ServerTryCancel(&srv_ctx); } // The RPC has been cancelled at this point for sure (i.e irrespective of @@ -982,7 +945,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { Verifier(GetParam()).Expect(1, true).Verify(cq_.get()); // On the server, request to be notified of 'ResponseStream' calls and // receive the call just made by the client - srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -990,12 +952,9 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; - bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - srv_ctx.TryCancel(); - Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); + ServerTryCancel(&srv_ctx); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -1003,41 +962,24 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; - - auto verif = Verifier(GetParam()); - if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = - new std::thread(&ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = new std::thread( + &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); // Server will cancel the RPC in a parallel thread while writing responses // to the client. Since the cancellation can happen at anytime, some of // the cq results (i.e those until cancellation) might be true but it is // non deterministic. So better to ignore the cq results ignore_cq_result = true; - // Expect that we might possibly see the done tag that - // indicates cancellation completion in this case - want_done_tag = true; - verif.Expect(11, true); } // Server sends three messages (tags 3, 4 and 5) - // But if want_done tag is true, we might also see tag 11 for (int tag_idx = 3; tag_idx <= 5; tag_idx++) { send_response.set_message("Pong " + std::to_string(tag_idx)); srv_stream.Write(send_response, tag(tag_idx)); - // Note that we'll add something to the verifier and verify that - // something was seen, but it might be tag 11 and not what we - // just added - int got_tag = verif.Expect(tag_idx, expected_cq_result) - .Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); - } + Verifier(GetParam()) + .Expect(tag_idx, expected_cq_result) + .Verify(cq_.get(), ignore_cq_result); } if (server_try_cancel_thd != NULL) { @@ -1046,21 +988,13 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - srv_ctx.TryCancel(); - want_done_tag = true; - verif.Expect(11, true); + ServerTryCancel(&srv_ctx); // Client reads may fail bacause it is notified that the stream is // cancelled. ignore_cq_result = true; } - if (want_done_tag) { - verif.Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - } - // Client attemts to read the three messages from the server for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { cli_stream->Read(&recv_response, tag(tag_idx)); @@ -1118,7 +1052,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of the 'BidiStream' call and // receive the call just made by the client - srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestBidiStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -1130,12 +1063,9 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; - bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - srv_ctx.TryCancel(); - Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); + ServerTryCancel(&srv_ctx); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -1143,84 +1073,42 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; - - auto verif = Verifier(GetParam()); - if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = - new std::thread(&ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = new std::thread( + &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); // Since server is going to cancel the RPC in a parallel thread, some of // the cq results (i.e those until the cancellation) might be true. Since // that number is non-deterministic, it is better to ignore the cq results ignore_cq_result = true; - // Expect that we might possibly see the done tag that - // indicates cancellation completion in this case - want_done_tag = true; - verif.Expect(11, true); } - int got_tag; srv_stream.Read(&recv_request, tag(4)); - verif.Expect(4, expected_cq_result); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 4) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 4); - } + Verifier(GetParam()) + .Expect(4, expected_cq_result) + .Verify(cq_.get(), ignore_cq_result); send_response.set_message("Pong"); srv_stream.Write(send_response, tag(5)); - verif.Expect(5, expected_cq_result); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 5) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 5); - } + Verifier(GetParam()) + .Expect(5, expected_cq_result) + .Verify(cq_.get(), ignore_cq_result); cli_stream->Read(&recv_response, tag(6)); - verif.Expect(6, expected_cq_result); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 6) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 6); - } + Verifier(GetParam()) + .Expect(6, expected_cq_result) + .Verify(cq_.get(), ignore_cq_result); // This is expected to succeed in all cases cli_stream->WritesDone(tag(7)); - verif.Expect(7, true); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 7) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 7); - } + Verifier(GetParam()).Expect(7, true).Verify(cq_.get()); // This is expected to fail in all cases i.e for all values of // server_try_cancel. This is because at this point, either there are no // more msgs from the client (because client called WritesDone) or the RPC // is cancelled on the server srv_stream.Read(&recv_request, tag(8)); - verif.Expect(8, false); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 8) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 8); - } + Verifier(GetParam()).Expect(8, false).Verify(cq_.get()); if (server_try_cancel_thd != NULL) { server_try_cancel_thd->join(); @@ -1228,15 +1116,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - srv_ctx.TryCancel(); - want_done_tag = true; - verif.Expect(11, true); - } - - if (want_done_tag) { - verif.Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; + ServerTryCancel(&srv_ctx); } // The RPC has been cancelled at this point for sure (i.e irrespective of From 4cbf32ff66b92162da3553e254ea7a8ca0d82057 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Fri, 4 Mar 2016 12:21:05 -0800 Subject: [PATCH 062/160] mention how to get protoc compiler in base INSTALL.md --- INSTALL.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index d9411db021d..ee4bc2b73d1 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -32,6 +32,16 @@ terminal: $ [sudo] xcode-select --install ``` +##Protoc + +By default gRPC uses [protocol buffers](https://github.com/google/protobuf), +you will need the `protoc` compiler to generate stub server and client code. + +If you compile from source, see below, the Makefile will automatically try +and compile the one present in third_party if you cloned the repository +recursively, and that it detects your system is lacking it. + + #Build from Source For developers who are interested to contribute, here is how to compile the From 60d38cbc1d4c3d5a460ab3e718e7d9acc34c3fba Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Fri, 4 Mar 2016 13:05:04 -0800 Subject: [PATCH 063/160] Simplify Node Windows tests slightly Simplify command that removes OpenSSL headers and remove now-extraneous post-test script. --- tools/run_tests/build_node.bat | 2 +- tools/run_tests/post_test_node.bat | 30 ------------------------------ tools/run_tests/run_tests.py | 5 +---- 3 files changed, 2 insertions(+), 35 deletions(-) delete mode 100644 tools/run_tests/post_test_node.bat diff --git a/tools/run_tests/build_node.bat b/tools/run_tests/build_node.bat index 886af0610ff..82e82083486 100644 --- a/tools/run_tests/build_node.bat +++ b/tools/run_tests/build_node.bat @@ -35,7 +35,7 @@ call npm install --build-from-source @rem delete the redundant openssl headers for /f "delims=v" %%v in ('node --version') do ( - rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q + rmdir "%USERPROFILE%\.node-gyp\%%v\include\node\openssl" /S /Q ) @rem rebuild, because it probably failed the first time diff --git a/tools/run_tests/post_test_node.bat b/tools/run_tests/post_test_node.bat deleted file mode 100644 index 1a2a5491fa7..00000000000 --- a/tools/run_tests/post_test_node.bat +++ /dev/null @@ -1,30 +0,0 @@ -@rem Copyright 2016, Google Inc. -@rem All rights reserved. -@rem -@rem Redistribution and use in source and binary forms, with or without -@rem modification, are permitted provided that the following conditions are -@rem met: -@rem -@rem * Redistributions of source code must retain the above copyright -@rem notice, this list of conditions and the following disclaimer. -@rem * Redistributions in binary form must reproduce the above -@rem copyright notice, this list of conditions and the following disclaimer -@rem in the documentation and/or other materials provided with the -@rem distribution. -@rem * Neither the name of Google Inc. nor the names of its -@rem contributors may be used to endorse or promote products derived from -@rem this software without specific prior written permission. -@rem -@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -rmdir node_modules /S /Q \ No newline at end of file diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index c55d1fbe633..cc004f38d7f 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -290,10 +290,7 @@ class NodeLanguage(object): return [['tools/run_tests/build_node.sh', self.node_version]] def post_tests_steps(self): - if self.platform == 'windows': - return [['tools\\run_tests\\post_test_node.bat']] - else: - return [] + return [] def makefile_name(self): return 'Makefile' From aaf66a9981a658b62be005ac5e04d704545ab2da Mon Sep 17 00:00:00 2001 From: Greg Haines Date: Fri, 4 Mar 2016 13:10:26 -0800 Subject: [PATCH 064/160] Add comments and feature flag. --- src/objective-c/GRPCClient/private/GRPCCompletionQueue.m | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index ffbb14374d1..b7d5af67d9b 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -69,7 +69,11 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ - gpr_timespec deadline = gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); + // Using a non-infinite deadline to re-enter grpc_completion_queue_next() + // alleviates https://github.com/grpc/grpc/issues/5593 + gpr_timespec deadline = (timeoutSecs < 0) + ? gpr_inf_future(GPR_CLOCK_REALTIME) + : gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); while (YES) { // The following call blocks until an event is available or the deadline elapses. grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); From 11ce4ffc75fef5a790ae55285457d787cbb50cea Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Fri, 4 Mar 2016 13:47:32 -0800 Subject: [PATCH 065/160] update min node version --- src/node/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/README.md b/src/node/README.md index b46b9862432..3501b54a665 100644 --- a/src/node/README.md +++ b/src/node/README.md @@ -5,7 +5,7 @@ Beta ## PREREQUISITES -- `node`: This requires `node` to be installed. If you instead have the `nodejs` executable on Debian, you should install the [`nodejs-legacy`](https://packages.debian.org/sid/nodejs-legacy) package. +- `node`: This requires `node` to be installed, version `0.12` or above. If you instead have the `nodejs` executable on Debian, you should install the [`nodejs-legacy`](https://packages.debian.org/sid/nodejs-legacy) package. ## INSTALLATION From 459edd97123b522de40e03302eb8741841aad8ec Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 4 Mar 2016 14:32:11 -0800 Subject: [PATCH 066/160] Revert "Pass a non-infinite deadline to grpc_completion_queue_next() to prevent queues from blocking indefinitely in poll()" --- .../GRPCClient/private/GRPCCompletionQueue.h | 7 ------ .../GRPCClient/private/GRPCCompletionQueue.m | 24 ++++--------------- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h index 03fd2e0d955..fe3b8f39d12 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h @@ -36,8 +36,6 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); -extern const int64_t kGRPCCompletionQueueDefaultTimeoutSecs; - /** * This class lets one more easily use |grpc_completion_queue|. To use it, pass the value of the * |unmanagedQueue| property of an instance of this class to |grpc_channel_create_call|. Then for @@ -51,11 +49,6 @@ extern const int64_t kGRPCCompletionQueueDefaultTimeoutSecs; */ @interface GRPCCompletionQueue : NSObject @property(nonatomic, readonly) grpc_completion_queue *unmanagedQueue; -@property(nonatomic, readonly) int64_t timeoutSecs; + (instancetype)completionQueue; - -- (instancetype)init; -- (instancetype)initWithTimeout:(int64_t)timeoutSecs NS_DESIGNATED_INITIALIZER; - @end diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index b7d5af67d9b..ea2b01ee1d7 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -35,9 +35,6 @@ #import - -const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; - @implementation GRPCCompletionQueue + (instancetype)completionQueue { @@ -45,13 +42,8 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; } - (instancetype)init { - return [self initWithTimeout:kGRPCCompletionQueueDefaultTimeoutSecs]; -} - -- (instancetype)initWithTimeout:(int64_t)timeoutSecs { if ((self = [super init])) { _unmanagedQueue = grpc_completion_queue_create(NULL); - _timeoutSecs = timeoutSecs; // This is for the following block to capture the pointer by value (instead // of retaining self and doing self->_unmanagedQueue). This is essential @@ -69,28 +61,22 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ - // Using a non-infinite deadline to re-enter grpc_completion_queue_next() - // alleviates https://github.com/grpc/grpc/issues/5593 - gpr_timespec deadline = (timeoutSecs < 0) - ? gpr_inf_future(GPR_CLOCK_REALTIME) - : gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); while (YES) { - // The following call blocks until an event is available or the deadline elapses. - grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); + // The following call blocks until an event is available. + grpc_event event = grpc_completion_queue_next(unmanagedQueue, + gpr_inf_future(GPR_CLOCK_REALTIME), + NULL); GRPCQueueCompletionHandler handler; switch (event.type) { case GRPC_OP_COMPLETE: handler = (__bridge_transfer GRPCQueueCompletionHandler)event.tag; handler(event.success); break; - case GRPC_QUEUE_TIMEOUT: - // Nothing to do here - break; case GRPC_QUEUE_SHUTDOWN: grpc_completion_queue_destroy(unmanagedQueue); return; default: - [NSException raise:@"Unrecognized completion type" format:@"type=%d", event.type]; + [NSException raise:@"Unrecognized completion type" format:@""]; } }; }); From f5dca1089639ab124e9cbc0db6893f6f0ada0fc6 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 4 Mar 2016 14:45:39 -0800 Subject: [PATCH 067/160] Revert "Improve perf smoke test stability" --- tools/jenkins/run_performance.sh | 59 ++++++-------------------------- 1 file changed, 11 insertions(+), 48 deletions(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index b06fce7acb9..fbc078330f7 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -42,10 +42,17 @@ config=opt make CONFIG=$config qps_worker qps_driver -j8 +bins/$config/qps_worker -driver_port 10000 & +PID1=$! +bins/$config/qps_worker -driver_port 10010 & +PID2=$! + # # Put a timeout on these tests # -((sleep 900; killall qps_worker && rm -f /tmp/qps-test.$$ && kill $$)&) +((sleep 900; kill $$ && killall qps_worker && rm -f /tmp/qps-test.$$ )&) + +export QPS_WORKERS="localhost:10000,localhost:10010" # big is the size in bytes of large messages (0 is the size otherwise) big=65536 @@ -62,74 +69,40 @@ deep=100 cores=`grep -c ^processor /proc/cpuinfo` halfcores=`expr $cores / 2` -# -# Keep changing the ports -port1=10000 -port2=10100 - for secure in true false; do # Scenario 1: generic async streaming ping-pong (contentionless latency) - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true # Scenario 2: generic async streaming "unconstrained" (QPS) - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 \ --server_core_limit=$halfcores --client_core_limit=0 |& tee /tmp/qps-test.$$ - bins/$config/qps_driver --quit=true # Scenario 2b: QPS with a single server core - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 --server_core_limit=1 --client_core_limit=0 - bins/$config/qps_driver --quit=true # Scenario 2c: protobuf-based QPS - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --simple_req_size=0 --simple_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true # Scenario 3: Latency at sub-peak load (all clients equally loaded) for loadfactor in 0.7; do - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -137,37 +110,27 @@ for secure in true false; do --num_servers=1 --num_clients=0 --poisson_load=`awk -v lf=$loadfactor \ '$5 == "QPS:" {print int(lf * $6); exit}' /tmp/qps-test.$$` \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true done rm /tmp/qps-test.$$ # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM). - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true # Scenario 5: Sync unary ping-pong with protobufs - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=UNARY --client_type=SYNC_CLIENT \ --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \ --secure_test=$secure --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true + done +bins/$config/qps_driver --quit=true + wait From b3320fc7c387c0105aeb13911288ec33e9f9470e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 14:52:06 -0800 Subject: [PATCH 068/160] Fix race in poll() based pollset It was possible for entries in watchers[] to be deleted by another thread before we called begin_poll. I'd like to do something nicer than this eventually... but that'll be easier once we've got the polling loop cleaned up (which is currently WIP) --- src/core/iomgr/pollset_multipoller_with_poll_posix.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 4dddfff2302..0a447ba96c1 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -122,6 +122,7 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( } else { h->fds[fd_count++] = h->fds[i]; watchers[pfd_count].fd = h->fds[i]; + GRPC_FD_REF(watchers[pfd_count].fd, "multipoller_start"); pfds[pfd_count].fd = h->fds[i]->fd; pfds[pfd_count].revents = 0; pfd_count++; @@ -135,8 +136,10 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( gpr_mu_unlock(&pollset->mu); for (i = 2; i < pfd_count; i++) { - pfds[i].events = (short)grpc_fd_begin_poll(watchers[i].fd, pollset, worker, + grpc_fd *fd = watchers[i].fd; + pfds[i].events = (short)grpc_fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, &watchers[i]); + GRPC_FD_UNREF(fd, "multipoller_start"); } /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid From 9adecb06e0bc66ffc98aafa087f946ee42473c01 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 4 Mar 2016 14:54:10 -0800 Subject: [PATCH 069/160] Fix race between parsing messages and receiving status in Node client --- src/node/interop/interop_client.js | 3 + src/node/src/client.js | 114 +++++++++++++++++++++-------- src/node/test/surface_test.js | 8 ++ 3 files changed, 95 insertions(+), 30 deletions(-) diff --git a/src/node/interop/interop_client.js b/src/node/interop/interop_client.js index db383e4d000..5602011a8e0 100644 --- a/src/node/interop/interop_client.js +++ b/src/node/interop/interop_client.js @@ -290,6 +290,7 @@ function timeoutOnSleepingServer(client, done) { call.write({ payload: {body: zeroBuffer(27182)} }); + call.on('data', function() {}); call.on('error', function(error) { assert(error.code === grpc.status.DEADLINE_EXCEEDED || @@ -336,6 +337,7 @@ function customMetadata(client, done) { ['test_initial_metadata_value']); done(); }); + stream.on('data', function() {}); stream.on('status', function(status) { var echo_trailer = status.metadata.get(ECHO_TRAILING_KEY); assert(echo_trailer.length > 0); @@ -361,6 +363,7 @@ function statusCodeAndMessage(client, done) { done(); }); var duplex = client.fullDuplexCall(); + duplex.on('data', function() {}); duplex.on('status', function(status) { assert(status); assert.strictEqual(status.code, 2); diff --git a/src/node/src/client.js b/src/node/src/client.js index c65dd736503..9acf51bd98b 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -131,8 +131,68 @@ function ClientReadableStream(call, deserialize) { this.finished = false; this.reading = false; this.deserialize = common.wrapIgnoreNull(deserialize); + /* Status generated from reading messages from the server. Overrides the + * status from the server if not OK */ + this.read_status = null; + /* Status received from the server. */ + this.received_status = null; } +/** + * Called when all messages from the server have been processed. The status + * parameter indicates that the call should end with that status. status + * defaults to OK if not provided. + * @param {Object!} status The status that the call should end with + */ +function _readsDone(status) { + /* jshint validthis: true */ + if (!status) { + status = {code: grpc.status.OK, details: 'OK'}; + } + this.finished = true; + this.read_status = status; + this._emitStatusIfDone(); +} + +ClientReadableStream.prototype._readsDone = _readsDone; + +/** + * Called to indicate that we have received a status from the server. + */ +function _receiveStatus(status) { + /* jshint validthis: true */ + this.received_status = status; + this._emitStatusIfDone(); +} + +ClientReadableStream.prototype._receiveStatus = _receiveStatus; + +/** + * If we have both processed all incoming messages and received the status from + * the server, emit the status. Otherwise, do nothing. + */ +function _emitStatusIfDone() { + /* jshint validthis: true */ + var status; + if (this.read_status && this.received_status) { + if (this.read_status.code !== grpc.status.OK) { + status = this.read_status; + } else { + status = this.received_status; + } + this.emit('status', status); + if (status.code !== grpc.status.OK) { + var error = new Error(status.details); + error.code = status.code; + error.metadata = status.metadata; + this.emit('error', error); + return; + } + } +} + +ClientReadableStream.prototype._emitStatusIfDone = _emitStatusIfDone; + /** * Read the next object from the stream. * @access private @@ -150,6 +210,7 @@ function _read(size) { if (err) { // Something has gone wrong. Stop reading and wait for status self.finished = true; + self._readsDone(); return; } var data = event.read; @@ -157,8 +218,11 @@ function _read(size) { try { deserialized = self.deserialize(data); } catch (e) { - self.call.cancelWithStatus(grpc.status.INTERNAL, - 'Failed to parse server response'); + self._readsDone({code: grpc.status.INTERNAL, + details: 'Failed to parse server response'}); + } + if (data === null) { + self._readsDone(); } if (self.push(deserialized) && data !== null) { var read_batch = {}; @@ -198,6 +262,11 @@ function ClientDuplexStream(call, serialize, deserialize) { this.serialize = common.wrapIgnoreNull(serialize); this.deserialize = common.wrapIgnoreNull(deserialize); this.call = call; + /* Status generated from reading messages from the server. Overrides the + * status from the server if not OK */ + this.read_status = null; + /* Status received from the server. */ + this.received_status = null; this.on('finish', function() { var batch = {}; batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; @@ -205,6 +274,9 @@ function ClientDuplexStream(call, serialize, deserialize) { }); } +ClientDuplexStream.prototype._readsDone = _readsDone; +ClientDuplexStream.prototype._receiveStatus = _receiveStatus; +ClientDuplexStream.prototype._emitStatusIfDone = _emitStatusIfDone; ClientDuplexStream.prototype._read = _read; ClientDuplexStream.prototype._write = _write; @@ -487,22 +559,13 @@ function makeServerStreamRequestFunction(method, serialize, deserialize) { var status_batch = {}; status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; call.startBatch(status_batch, function(err, response) { - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - stream.emit('status', response.status); - if (response.status.code !== grpc.status.OK) { - var error = new Error(response.status.details); - error.code = response.status.code; - error.metadata = response.status.metadata; - stream.emit('error', error); + if (err) { + stream.emit('error', err); return; - } else { - if (err) { - // Got a batch error, but OK status. Something went wrong - stream.emit('error', err); - return; - } } + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + stream._receiveStatus(response.status); }); return stream; } @@ -552,22 +615,13 @@ function makeBidiStreamRequestFunction(method, serialize, deserialize) { var status_batch = {}; status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; call.startBatch(status_batch, function(err, response) { - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - stream.emit('status', response.status); - if (response.status.code !== grpc.status.OK) { - var error = new Error(response.status.details); - error.code = response.status.code; - error.metadata = response.status.metadata; - stream.emit('error', error); + if (err) { + stream.emit('error', err); return; - } else { - if (err) { - // Got a batch error, but OK status. Something went wrong - stream.emit('error', err); - return; - } } + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + stream._receiveStatus(response.status); }); return stream; } diff --git a/src/node/test/surface_test.js b/src/node/test/surface_test.js index 530f1f77494..8a232d6fc44 100644 --- a/src/node/test/surface_test.js +++ b/src/node/test/surface_test.js @@ -1000,6 +1000,7 @@ describe('Call propagation', function() { proxy_impl.serverStream = function(parent) { var child = client.serverStream(parent.request, null, {parent: parent}); + child.on('data', function() {}); child.on('error', function(err) { assert(err); assert.strictEqual(err.code, grpc.status.CANCELLED); @@ -1013,6 +1014,7 @@ describe('Call propagation', function() { var proxy_client = new Client('localhost:' + proxy_port, grpc.credentials.createInsecure()); call = proxy_client.serverStream({}); + call.on('data', function() {}); call.on('error', function(err) { done(); }); @@ -1022,6 +1024,7 @@ describe('Call propagation', function() { var call; proxy_impl.bidiStream = function(parent) { var child = client.bidiStream(null, {parent: parent}); + child.on('data', function() {}); child.on('error', function(err) { assert(err); assert.strictEqual(err.code, grpc.status.CANCELLED); @@ -1035,6 +1038,7 @@ describe('Call propagation', function() { var proxy_client = new Client('localhost:' + proxy_port, grpc.credentials.createInsecure()); call = proxy_client.bidiStream(); + call.on('data', function() {}); call.on('error', function(err) { done(); }); @@ -1074,6 +1078,7 @@ describe('Call propagation', function() { proxy_impl.bidiStream = function(parent) { var child = client.bidiStream( null, {parent: parent, propagate_flags: deadline_flags}); + child.on('data', function() {}); child.on('error', function(err) { assert(err); assert(err.code === grpc.status.DEADLINE_EXCEEDED || @@ -1089,6 +1094,7 @@ describe('Call propagation', function() { var deadline = new Date(); deadline.setSeconds(deadline.getSeconds() + 1); var call = proxy_client.bidiStream(null, {deadline: deadline}); + call.on('data', function() {}); call.on('error', function(err) { done(); }); @@ -1130,6 +1136,7 @@ describe('Cancelling surface client', function() { }); it('Should correctly cancel a server stream call', function(done) { var call = client.fib({'limit': 5}); + call.on('data', function() {}); call.on('error', function(error) { assert.strictEqual(error.code, surface_client.status.CANCELLED); done(); @@ -1138,6 +1145,7 @@ describe('Cancelling surface client', function() { }); it('Should correctly cancel a bidi stream call', function(done) { var call = client.divMany(); + call.on('data', function() {}); call.on('error', function(error) { assert.strictEqual(error.code, surface_client.status.CANCELLED); done(); From f1b6d61965adf0cf264c826b0a08b5a8b32e57c3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 15:00:02 -0800 Subject: [PATCH 070/160] Fix formatting --- src/core/iomgr/pollset_multipoller_with_poll_posix.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 0a447ba96c1..92d6fb72414 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -137,8 +137,8 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( for (i = 2; i < pfd_count; i++) { grpc_fd *fd = watchers[i].fd; - pfds[i].events = (short)grpc_fd_begin_poll(fd, pollset, worker, - POLLIN, POLLOUT, &watchers[i]); + pfds[i].events = (short)grpc_fd_begin_poll(fd, pollset, worker, POLLIN, + POLLOUT, &watchers[i]); GRPC_FD_UNREF(fd, "multipoller_start"); } From c1edf224797367b62cd88ecff37dae23d333d308 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 15:01:03 -0800 Subject: [PATCH 071/160] Fix sanity --- src/objective-c/GRPCClient/private/GRPCCompletionQueue.h | 2 +- src/objective-c/GRPCClient/private/GRPCCompletionQueue.m | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h index fe3b8f39d12..7b66cd4c329 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index ea2b01ee1d7..ff3031678c5 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 208fd6b339364f9677bbcdfc3bb955ca908fbe2e Mon Sep 17 00:00:00 2001 From: VcamX Date: Mon, 7 Mar 2016 13:08:38 +0800 Subject: [PATCH 072/160] Set grace=0 to server stop in route_guide Fix #5619 --- examples/python/route_guide/route_guide_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python/route_guide/route_guide_server.py b/examples/python/route_guide/route_guide_server.py index f23b98bf367..24f948c42c5 100644 --- a/examples/python/route_guide/route_guide_server.py +++ b/examples/python/route_guide/route_guide_server.py @@ -128,7 +128,7 @@ def serve(): while True: time.sleep(_ONE_DAY_IN_SECONDS) except KeyboardInterrupt: - server.stop() + server.stop(0) if __name__ == '__main__': serve() From 0211098e935165801a63a866d19006bd8c49cccf Mon Sep 17 00:00:00 2001 From: Lisa Carey Date: Mon, 7 Mar 2016 17:41:51 +0000 Subject: [PATCH 073/160] Removes confusing/unnecessary section on generating code. --- examples/cpp/README.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 979ba55dbb5..e4b0eb698bf 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -23,21 +23,6 @@ Change your current directory to examples/cpp/helloworld $ cd examples/cpp/helloworld/ ``` - -### Generating gRPC code - -To generate the client and server side interfaces: - -```sh -$ make helloworld.grpc.pb.cc helloworld.pb.cc -``` -Which internally invokes the proto-compiler as: - -```sh -$ protoc -I ../../protos/ --grpc_out=. --plugin=protoc-gen-grpc=grpc_cpp_plugin ../../protos/helloworld.proto -$ protoc -I ../../protos/ --cpp_out=. ../../protos/helloworld.proto -``` - ### Client and server implementations The client implementation is at [greeter_client.cc](helloworld/greeter_client.cc). From f880e62e04319c92caf361bc6d6a308209a56938 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 10:14:48 -0800 Subject: [PATCH 074/160] Potential fix for race condition exposed by Node --- src/core/channel/subchannel_call_holder.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c index 81297c8d449..0f765c218b4 100644 --- a/src/core/channel/subchannel_call_holder.c +++ b/src/core/channel/subchannel_call_holder.c @@ -168,21 +168,24 @@ retry: static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) { grpc_subchannel_call_holder *holder = arg; - grpc_subchannel_call *call; gpr_mu_lock(&holder->mu); GPR_ASSERT(holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL); - call = GET_CALL(holder); - GPR_ASSERT(call == NULL || call == CANCELLED_CALL); holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; if (holder->connected_subchannel == NULL) { fail_locked(exec_ctx, holder); } else { - gpr_atm_rel_store( + if (!gpr_atm_rel_cas( &holder->subchannel_call, + 0, (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( - exec_ctx, holder->connected_subchannel, holder->pollset)); - retry_waiting_locked(exec_ctx, holder); + exec_ctx, holder->connected_subchannel, holder->pollset))) { + GPR_ASSERT(gpr_atm_acq_load(&holder->subchannel_call) == 1); + /* if this cas fails, the call was cancelled before the pick completed */ + fail_locked(exec_ctx, holder); + } else { + retry_waiting_locked(exec_ctx, holder); + } } gpr_mu_unlock(&holder->mu); GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel"); From 84774b6c4a25ee38a1778778e00cdd81203d874f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 10:26:31 -0800 Subject: [PATCH 075/160] clang-format --- src/core/channel/subchannel_call_holder.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c index 0f765c218b4..8f46885a043 100644 --- a/src/core/channel/subchannel_call_holder.c +++ b/src/core/channel/subchannel_call_holder.c @@ -176,10 +176,9 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) { fail_locked(exec_ctx, holder); } else { if (!gpr_atm_rel_cas( - &holder->subchannel_call, - 0, - (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( - exec_ctx, holder->connected_subchannel, holder->pollset))) { + &holder->subchannel_call, 0, + (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( + exec_ctx, holder->connected_subchannel, holder->pollset))) { GPR_ASSERT(gpr_atm_acq_load(&holder->subchannel_call) == 1); /* if this cas fails, the call was cancelled before the pick completed */ fail_locked(exec_ctx, holder); From b0d1567e8ea9c4ed528e63df9a8649c953b4e349 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 7 Mar 2016 10:51:02 -0800 Subject: [PATCH 076/160] Revert "Revert "Properly integrate async API with server-side cancellations."" --- .../grpc++/impl/codegen/completion_queue.h | 1 + include/grpc++/impl/codegen/server_context.h | 3 + src/cpp/server/server_context.cc | 26 +- test/cpp/end2end/async_end2end_test.cc | 232 +++++++++++++----- 4 files changed, 198 insertions(+), 64 deletions(-) diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h index 102831e1c9b..928ab2db317 100644 --- a/include/grpc++/impl/codegen/completion_queue.h +++ b/include/grpc++/impl/codegen/completion_queue.h @@ -184,6 +184,7 @@ class CompletionQueue : private GrpcLibrary { bool Pluck(CompletionQueueTag* tag); /// Performs a single polling pluck on \a tag. + /// \warning Must not be mixed with calls to \a Next. void TryPluck(CompletionQueueTag* tag); grpc_completion_queue* cq_; // owned diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h index ad08b8210d6..91ebe574b14 100644 --- a/include/grpc++/impl/codegen/server_context.h +++ b/include/grpc++/impl/codegen/server_context.h @@ -103,6 +103,9 @@ class ServerContext { void AddInitialMetadata(const grpc::string& key, const grpc::string& value); void AddTrailingMetadata(const grpc::string& key, const grpc::string& value); + // IsCancelled is always safe to call when using sync API + // When using async API, it is only safe to call IsCancelled after + // the AsyncNotifyWhenDone tag has been delivered bool IsCancelled() const; // Cancel the Call from the server. This is a best-effort API and depending on diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index e205a1969b3..eb49b210379 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -62,7 +62,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void FillOps(grpc_op* ops, size_t* nops) GRPC_OVERRIDE; bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE; - bool CheckCancelled(CompletionQueue* cq); + bool CheckCancelled(CompletionQueue* cq) { + cq->TryPluck(this); + return CheckCancelledNoPluck(); + } + bool CheckCancelledAsync() { return CheckCancelledNoPluck(); } void set_tag(void* tag) { has_tag_ = true; @@ -72,6 +76,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void Unref(); private: + bool CheckCancelledNoPluck() { + grpc::lock_guard g(mu_); + return finalized_ ? (cancelled_ != 0) : false; + } + bool has_tag_; void* tag_; grpc::mutex mu_; @@ -88,12 +97,6 @@ void ServerContext::CompletionOp::Unref() { } } -bool ServerContext::CompletionOp::CheckCancelled(CompletionQueue* cq) { - cq->TryPluck(this); - grpc::lock_guard g(mu_); - return finalized_ ? cancelled_ != 0 : false; -} - void ServerContext::CompletionOp::FillOps(grpc_op* ops, size_t* nops) { ops->op = GRPC_OP_RECV_CLOSE_ON_SERVER; ops->data.recv_close_on_server.cancelled = &cancelled_; @@ -182,7 +185,14 @@ void ServerContext::TryCancel() const { } bool ServerContext::IsCancelled() const { - return completion_op_ && completion_op_->CheckCancelled(cq_); + if (has_notify_when_done_tag_) { + // when using async API, but the result is only valid + // if the tag has already been delivered at the completion queue + return completion_op_ && completion_op_->CheckCancelledAsync(); + } else { + // when using sync API + return completion_op_ && completion_op_->CheckCancelled(cq_); + } } void ServerContext::set_compression_level(grpc_compression_level level) { diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 9ca3bf98f85..dc8c2bb6e5b 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -68,6 +68,7 @@ namespace testing { namespace { void* tag(int i) { return (void*)(intptr_t)i; } +int detag(void* p) { return static_cast(reinterpret_cast(p)); } #ifdef GPR_POSIX_SOCKET static int maybe_assert_non_blocking_poll(struct pollfd* pfds, nfds_t nfds, @@ -106,37 +107,50 @@ class PollingOverrider { class Verifier { public: explicit Verifier(bool spin) : spin_(spin) {} + // Expect sets the expected ok value for a specific tag Verifier& Expect(int i, bool expect_ok) { expectations_[tag(i)] = expect_ok; return *this; } + // Next waits for 1 async tag to complete, checks its + // expectations, and returns the tag + int Next(CompletionQueue* cq, bool ignore_ok) { + bool ok; + void* got_tag; + if (spin_) { + for (;;) { + auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); + if (r == CompletionQueue::TIMEOUT) continue; + if (r == CompletionQueue::GOT_EVENT) break; + gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); + abort(); + } + } else { + EXPECT_TRUE(cq->Next(&got_tag, &ok)); + } + auto it = expectations_.find(got_tag); + EXPECT_TRUE(it != expectations_.end()); + if (!ignore_ok) { + EXPECT_EQ(it->second, ok); + } + expectations_.erase(it); + return detag(got_tag); + } + + // Verify keeps calling Next until all currently set + // expected tags are complete void Verify(CompletionQueue* cq) { Verify(cq, false); } + // This version of Verify allows optionally ignoring the + // outcome of the expectation void Verify(CompletionQueue* cq, bool ignore_ok) { GPR_ASSERT(!expectations_.empty()); while (!expectations_.empty()) { - bool ok; - void* got_tag; - if (spin_) { - for (;;) { - auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); - if (r == CompletionQueue::TIMEOUT) continue; - if (r == CompletionQueue::GOT_EVENT) break; - gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); - abort(); - } - } else { - EXPECT_TRUE(cq->Next(&got_tag, &ok)); - } - auto it = expectations_.find(got_tag); - EXPECT_TRUE(it != expectations_.end()); - if (!ignore_ok) { - EXPECT_EQ(it->second, ok); - } - expectations_.erase(it); + Next(cq, ignore_ok); } } + // This version of Verify stops after a certain deadline void Verify(CompletionQueue* cq, std::chrono::system_clock::time_point deadline) { if (expectations_.empty()) { @@ -793,7 +807,8 @@ TEST_P(AsyncEnd2endTest, UnimplementedRpc) { } // This class is for testing scenarios where RPCs are cancelled on the server -// by calling ServerContext::TryCancel() +// by calling ServerContext::TryCancel(). Server uses AsyncNotifyWhenDone +// API to check for cancellation class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { protected: typedef enum { @@ -803,13 +818,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { CANCEL_AFTER_PROCESSING } ServerTryCancelRequestPhase; - void ServerTryCancel(ServerContext* context) { - EXPECT_FALSE(context->IsCancelled()); - context->TryCancel(); - gpr_log(GPR_INFO, "Server called TryCancel()"); - EXPECT_TRUE(context->IsCancelled()); - } - // Helper for testing client-streaming RPCs which are cancelled on the server. // Depending on the value of server_try_cancel parameter, this will test one // of the following three scenarios: @@ -843,6 +851,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of 'RequestStream' calls // and receive the 'RequestStream' call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestRequestStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -858,9 +867,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_server_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // Since cancellation is done before server reads any results, we know // for sure that all cq results will return false from this point forward @@ -868,22 +880,39 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while reading the // requests from the client. Since the cancellation can happen at anytime, // some of the cq results (i.e those until cancellation) might be true but // its non deterministic. So better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } // Server reads 3 messages (tags 6, 7 and 8) + // But if want_done_tag is true, we might also see tag 11 for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { srv_stream.Read(&recv_request, tag(tag_idx)); - Verifier(GetParam()) - .Expect(tag_idx, expected_server_cq_result) - .Verify(cq_.get(), ignore_cq_result); + // Note that we'll add something to the verifier and verify that + // something was seen, but it might be tag 11 and not what we + // just added + int got_tag = verif.Expect(tag_idx, expected_server_cq_result) + .Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); + } } if (server_try_cancel_thd != NULL) { @@ -892,7 +921,15 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); + } + + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; } // The RPC has been cancelled at this point for sure (i.e irrespective of @@ -945,6 +982,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { Verifier(GetParam()).Expect(1, true).Verify(cq_.get()); // On the server, request to be notified of 'ResponseStream' calls and // receive the call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -952,9 +990,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -962,24 +1003,41 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while writing responses // to the client. Since the cancellation can happen at anytime, some of // the cq results (i.e those until cancellation) might be true but it is // non deterministic. So better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } // Server sends three messages (tags 3, 4 and 5) + // But if want_done tag is true, we might also see tag 11 for (int tag_idx = 3; tag_idx <= 5; tag_idx++) { send_response.set_message("Pong " + std::to_string(tag_idx)); srv_stream.Write(send_response, tag(tag_idx)); - Verifier(GetParam()) - .Expect(tag_idx, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + // Note that we'll add something to the verifier and verify that + // something was seen, but it might be tag 11 and not what we + // just added + int got_tag = verif.Expect(tag_idx, expected_cq_result) + .Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); + } } if (server_try_cancel_thd != NULL) { @@ -988,13 +1046,21 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); // Client reads may fail bacause it is notified that the stream is // cancelled. ignore_cq_result = true; } + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + } + // Client attemts to read the three messages from the server for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { cli_stream->Read(&recv_response, tag(tag_idx)); @@ -1052,6 +1118,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of the 'BidiStream' call and // receive the call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestBidiStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -1063,9 +1130,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -1073,42 +1143,84 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Since server is going to cancel the RPC in a parallel thread, some of // the cq results (i.e those until the cancellation) might be true. Since // that number is non-deterministic, it is better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } + int got_tag; srv_stream.Read(&recv_request, tag(4)); - Verifier(GetParam()) - .Expect(4, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(4, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 4) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 4); + } send_response.set_message("Pong"); srv_stream.Write(send_response, tag(5)); - Verifier(GetParam()) - .Expect(5, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(5, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 5) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 5); + } cli_stream->Read(&recv_response, tag(6)); - Verifier(GetParam()) - .Expect(6, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(6, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 6) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 6); + } // This is expected to succeed in all cases cli_stream->WritesDone(tag(7)); - Verifier(GetParam()).Expect(7, true).Verify(cq_.get()); + verif.Expect(7, true); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 7) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 7); + } // This is expected to fail in all cases i.e for all values of // server_try_cancel. This is because at this point, either there are no // more msgs from the client (because client called WritesDone) or the RPC // is cancelled on the server srv_stream.Read(&recv_request, tag(8)); - Verifier(GetParam()).Expect(8, false).Verify(cq_.get()); + verif.Expect(8, false); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 8) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 8); + } if (server_try_cancel_thd != NULL) { server_try_cancel_thd->join(); @@ -1116,7 +1228,15 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); + } + + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; } // The RPC has been cancelled at this point for sure (i.e irrespective of From de14e50a914c9db99d90812cbaec0110b3e1a52d Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 7 Mar 2016 12:11:47 -0800 Subject: [PATCH 077/160] text update --- INSTALL.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index ee4bc2b73d1..3c694ba5822 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -37,9 +37,10 @@ terminal: By default gRPC uses [protocol buffers](https://github.com/google/protobuf), you will need the `protoc` compiler to generate stub server and client code. -If you compile from source, see below, the Makefile will automatically try -and compile the one present in third_party if you cloned the repository -recursively, and that it detects your system is lacking it. +If you compile gRPC from source, as described below, the Makefile will +automatically try and compile the `protoc` in third_party if you cloned the +repository recursively and it detects that you don't already have it +installed. #Build from Source From 67b05a2513d739c5ab8fc63829ef5fa0340dee59 Mon Sep 17 00:00:00 2001 From: Harsh Vardhan Date: Tue, 8 Mar 2016 02:18:57 +0530 Subject: [PATCH 078/160] updates contributing.md --- CONTRIBUTING.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9423c46547a..518ba2c4809 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,9 +36,25 @@ In order to run most of the available tests, one would need to run: `./tools/run_tests/run_tests.py` -If you want to run all the possible tests for any of the languages {c, c++, node, php, python}, do this: - -`./tools/run_tests/run_tests.py -l -c all` +If you want to run tests for any of the languages {c, c++, node, php, python}, do this: + +`./tools/run_tests/run_tests.py -l ` + +For list of available commands, refer to this: + +``` +run_tests.py [-h] + [-c {asan,asan-noleaks,basicprof,dbg,gcov,helgrind,memcheck,msan,mutrace,opt,stapprof,tsan,ubsan}] + [-n RUNS_PER_TEST] [-r REGEX] [-j JOBS] [-s SLOWDOWN] [-f] + [-t] [--newline_on_success] + [-l {all,c,c++,csharp,node,objc,php,python,ruby,sanity} [{all,c,c++,csharp,node,objc,php,python,ruby,sanity} ...]] + [-S] [--use_docker] [--allow_flakes] + [--arch {default,x86,x64}] + [--compiler {default,gcc4.4,gcc4.9,gcc5.3,vs2010,vs2013,vs2015}] + [--build_only] [--measure_cpu_costs] + [--update_submodules [UPDATE_SUBMODULES [UPDATE_SUBMODULES ...]]] + [-a ANTAGONISTS] [-x XML_REPORT] +``` ## Adding or removing source code From e6cc9c7a253464e17abfd0fbd7dd5f1a52df36da Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 7 Mar 2016 12:36:38 -0800 Subject: [PATCH 079/160] custom test target for parallel test running --- setup.py | 1 + src/python/grpcio/commands.py | 35 +++++++++++++++++++++++++++++++++++ tools/run_tests/run_python.sh | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cfb578e2158..e1abe7fd8ff 100644 --- a/setup.py +++ b/setup.py @@ -165,6 +165,7 @@ COMMAND_CLASS = { 'build_tagged_ext': precompiled.BuildTaggedExt, 'gather': commands.Gather, 'run_interop': commands.RunInterop, + 'test_lite': commands.TestLite } # Ensure that package data is copied over before any commands have been run: diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index aa29c728f25..ec116f58abe 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -264,6 +264,41 @@ class Gather(setuptools.Command): self.distribution.fetch_build_eggs(self.distribution.tests_require) +class TestLite(setuptools.Command): + """Command to run tests without fetching or building anything.""" + + description = 'run tests without fetching or building anything.' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + # distutils requires this override. + pass + + def run(self): + self._add_eggs_to_path() + + import tests + loader = tests.Loader() + loader.loadTestsFromNames(['tests']) + runner = tests.Runner() + result = runner.run(loader.suite) + if not result.wasSuccessful(): + sys.exit(1) + + def _add_eggs_to_path(self): + """Adds all egg files under .eggs to sys.path""" + import pkg_resources + eggs_dir = os.path.join(PYTHON_STEM, '../../../.eggs') + eggs = [os.path.join(eggs_dir, filename) + for filename in os.listdir(eggs_dir) + if filename.endswith('.egg')] + for egg in eggs: + sys.path.insert(0, pkg_resources.normalize_path(egg)) + + class RunInterop(test.test): description = 'run interop test client/server' diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index beb747a6169..ace49e15142 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -46,7 +46,7 @@ if [ "$CONFIG" = "gcov" ] then tox else - $ROOT/.tox/py27/bin/python $ROOT/setup.py test + $ROOT/.tox/py27/bin/python $ROOT/setup.py test_lite fi mkdir -p $ROOT/reports From 8395bfc0b6b443830291b76b961548fc28682f3d Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 7 Mar 2016 14:07:56 -0800 Subject: [PATCH 080/160] Put back the gflags/gtest requirement to doc --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9423c46547a..6fb9a846098 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,13 @@ In order to run all of the tests we provide, you will need valgrind and clang. More specifically, under debian, you will need the package libc++-dev to properly run all the tests. +Compiling and running grpc C++ tests depend on protobuf 3.0.0, gtest and gflags. +Although gflags is provided in third_party, you will need to manually install +that dependency on your system to run these tests. Under a Debian or Ubuntu +system, you can install the gtests and gflags packages using apt-get: + +`apt-get install libgflags-dev libgtest-dev` + If you are planning to work on any of the languages other than C and C++, you will also need their appropriate development environments. From 3a84f46a29a0a4db44a70a0026f726327e074877 Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 7 Mar 2016 14:44:46 -0800 Subject: [PATCH 081/160] resolve comments --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6fb9a846098..d549c5d2084 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,9 @@ Although gflags is provided in third_party, you will need to manually install that dependency on your system to run these tests. Under a Debian or Ubuntu system, you can install the gtests and gflags packages using apt-get: -`apt-get install libgflags-dev libgtest-dev` +```sh + $ [sudo] apt-get install libgflags-dev libgtest-dev +``` If you are planning to work on any of the languages other than C and C++, you will also need their appropriate development environments. From 17908c168def9cba95c548786c39bfe49de00a3f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 7 Mar 2016 15:04:17 -0800 Subject: [PATCH 082/160] Address comments and add a TODO. --- src/python/grpcio/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index ec116f58abe..3207ea3052e 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -286,10 +286,11 @@ class TestLite(setuptools.Command): runner = tests.Runner() result = runner.run(loader.suite) if not result.wasSuccessful(): - sys.exit(1) + sys.exit('Test failure') def _add_eggs_to_path(self): """Adds all egg files under .eggs to sys.path""" + # TODO(jtattemusch): there has to be a cleaner way to do this import pkg_resources eggs_dir = os.path.join(PYTHON_STEM, '../../../.eggs') eggs = [os.path.join(eggs_dir, filename) From ac731e051e755b4392ffa272804b7ada0d0bb8b3 Mon Sep 17 00:00:00 2001 From: vjpai Date: Mon, 7 Mar 2016 15:07:33 -0800 Subject: [PATCH 083/160] @a11r asked me to respond to some BSD folks, so I figured that I should also be a potential mentor if needed. --- summerofcode/ideas.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/summerofcode/ideas.md b/summerofcode/ideas.md index c7f368e7cc2..83f2cecd48c 100644 --- a/summerofcode/ideas.md +++ b/summerofcode/ideas.md @@ -19,7 +19,9 @@ gRPC C Core: 1. Port gRPC to one of the major BSD platforms ([FreeBSD](https://freebsd.org), [NetBSD](https://netbsd.org), and [OpenBSD](https://openbsd.org)) and create packages for them. Add [kqueue](https://www.freebsd.org/cgi/man.cgi?query=kqueue) support in the process. * **Required skills:** C programming language, BSD operating system. - * **Likely mentors:** [Craig Tiller](https://github.com/ctiller), [Nicolas Noble](https://github.com/nicolasnoble). + * **Likely mentors:** [Craig Tiller](https://github.com/ctiller), + [Nicolas Noble](https://github.com/nicolasnoble), + [Vijay Pai](https://github.com/vjpai). 1. Fix gRPC C-core's URI parser. The current parser does not qualify as a standard parser according to [RFC3986]( https://tools.ietf.org/html/rfc3986). Write test suites to verify this and make changes necessary to make the URI parser compliant. * **Required skills:** C programming language, HTTP standard compliance. * **Likely mentors:** [Craig Tiller](https://github.com/ctiller). From a41a6775d4487cce3eea5084928da39ceff923fd Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 7 Mar 2016 17:10:07 -0800 Subject: [PATCH 084/160] php: update readme --- examples/php/README.md | 10 ++-- src/php/README.md | 115 +++++++++++++++++++++++++++++------------ 2 files changed, 87 insertions(+), 38 deletions(-) diff --git a/examples/php/README.md b/examples/php/README.md index 8fb060863a3..ea9ccb67908 100644 --- a/examples/php/README.md +++ b/examples/php/README.md @@ -4,16 +4,15 @@ gRPC in 3 minutes (PHP) PREREQUISITES ------------- -This requires PHP 5.5 or greater. +This requires `php` >=5.5, `phpize`, `pecl`, `phpunit` INSTALL ------- - - On Mac OS X, install [homebrew][]. Run the following command to install gRPC. + - Install the gRPC PHP extension ```sh - $ curl -fsSL https://goo.gl/getgrpc | bash -s php + $ [sudo] pecl install grpc-beta ``` - This will download and run the [gRPC install script][] and compile the gRPC PHP extension. - Clone this repository @@ -37,6 +36,7 @@ TRY IT! Please follow the instruction in [Node][] to run the server ``` $ cd examples/node + $ npm install $ nodejs greeter_server.js ``` @@ -58,7 +58,5 @@ TUTORIAL You can find a more detailed tutorial in [gRPC Basics: PHP][] -[homebrew]:http://brew.sh -[gRPC install script]:https://raw.githubusercontent.com/grpc/homebrew-grpc/master/scripts/install [Node]:https://github.com/grpc/grpc/tree/master/examples/node [gRPC Basics: PHP]:http://www.grpc.io/docs/tutorials/basic/php.html diff --git a/src/php/README.md b/src/php/README.md index b368482f068..c97c3796cda 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -9,14 +9,21 @@ Beta ## Environment -Prerequisite: PHP 5.5 or later, `phpunit`, `pecl` +Prerequisite: `php` >=5.5, `phpize`, `pecl`, `phpunit` -**Linux:** +**Linux (Debian):** ```sh $ sudo apt-get install php5 php5-dev php-pear ``` +**Linux (CentOS):** + +```sh +$ yum install php55w +$ yum --enablerepo=remi,remi-php55 install php-devel php-pear +``` + **Mac OS X:** ```sh @@ -24,11 +31,11 @@ $ curl -O http://pear.php.net/go-pear.phar $ sudo php -d detect_unicode=0 go-pear.phar ``` -**PHPUnit: (Both Linux and Mac OS X)** +**PHPUnit:** ```sh -$ curl https://phar.phpunit.de/phpunit.phar -o phpunit.phar -$ chmod +x phpunit.phar -$ sudo mv phpunit.phar /usr/local/bin/phpunit +$ wget https://phar.phpunit.de/phpunit-old.phar +$ chmod +x phpunit-old.phar +$ sudo mv phpunit-old.phar /usr/bin/phpunit ``` ## Quick Install @@ -39,15 +46,22 @@ Install the gRPC PHP extension sudo pecl install grpc-beta ``` +This will compile and install the gRPC PHP extension into the standard PHP extension directory. You should be able to run the [unit tests](#unit-tests), with the PHP extension installed. + +To run tests with generated stub code from `.proto` files, you will need the `composer`, `protoc` and `protoc-gen-php` binaries additionally. See sections [below](#generated-code-tests). + ## Build from Source + +### gRPC C core library + Clone this repository ```sh $ git clone https://github.com/grpc/grpc.git ``` -Build and install the gRPC C core libraries +Build and install the gRPC C core library ```sh $ cd grpc @@ -56,20 +70,15 @@ $ make $ sudo make install ``` -Note: you may encounter a warning about the Protobuf compiler `protoc` 3.0.0+ not being installed. The following might help, and will be useful later on when we need to compile the `protoc-gen-php` tool. +### gRPC PHP extension -```sh -$ cd grpc/third_party/protobuf -$ sudo make install # 'make' should have been run by core grpc -``` - -Install the gRPC PHP extension +Install the gRPC PHP extension from PECL ```sh $ sudo pecl install grpc-beta ``` -OR +Or, compile from source ```sh $ cd grpc/src/php/ext/grpc @@ -79,58 +88,96 @@ $ make $ sudo make install ``` +### Update php.ini + Add this line to your `php.ini` file, e.g. `/etc/php5/cli/php.ini` ```sh extension=grpc.so ``` -Install Composer +## Unit Tests + +You will need the source code to run tests + +```sh +$ git clone https://github.com/grpc/grpc.git +$ cd grpc +$ git pull --recurse-submodules && git submodule update --init --recursive +``` + +Run unit tests ```sh $ cd grpc/src/php +$ ./bin/run_tests.sh +``` + +## Generated Code Tests + +### Composer + +You need to install `composer`, to pull in some runtime dependencies based on the `composer.json` file. + +```sh $ curl -sS https://getcomposer.org/installer | php $ sudo mv composer.phar /usr/local/bin/composer + +$ cd grpc/src/php $ composer install ``` -## Unit Tests +### Protobuf compiler -Run unit tests +You need the install the protobuf compiler, `protoc`, 3.0.0+. + +If you had compiled the gRPC C core library from source above, the `protoc` binary should have been installed as well. In the case it wasn't, you can run the following commands to install it. ```sh -$ cd grpc/src/php -$ ./bin/run_tests.sh +$ cd grpc/third_party/protobuf +$ sudo make install # 'make' should have been run by core grpc ``` -## Generated Code Tests +Or you can download a `protoc` binaries from [here](https://github.com/google/protobuf/releases). + -Install `protoc-gen-php` +### PHP protobuf compiler + +You need to install `protoc-gen-php`, so that you can generate stub classes `.php` files from service definition `.proto` files. ```sh -$ cd grpc/src/php/vendor/datto/protobuf-php +$ cd grpc/src/php/vendor/datto/protobuf-php # if you had run `composer install` in the previous step + +OR + +$ git clone https://github.com/stanley-cheung/Protobuf-PHP # clone from github repo + $ gem install rake ronn $ rake pear:package version=1.0 $ sudo pear install Protobuf-1.0.tgz ``` -Generate client stub code +### Client Stub + +Generate client stub classes from `.proto` files ```sh $ cd grpc/src/php $ ./bin/generate_proto_php.sh ``` -Run a local server serving the math services +### Run test server - - Please see [Node][] on how to run an example server +Run a local server serving the math services. Please see [Node][] on how to run an example server ```sh -$ cd grpc/src/node +$ cd grpc $ npm install -$ nodejs examples/math_server.js +$ nodejs src/node/test/math/math_server.js ``` +### Run test client + Run the generated code tests ```sh @@ -161,13 +208,15 @@ $ sudo service apache2 restart Make sure the Node math server is still running, as above. ```sh -$ cd grpc/src/node -$ nodejs examples/math_server.js +$ cd grpc +$ npm install +$ nodejs src/node/test/math/math_server.js ``` Make sure you have run `composer install` to generate the `vendor/autoload.php` file ```sh +$ cd grpc/src/php $ composer install ``` @@ -229,13 +278,15 @@ $ sudo service php5-fpm restart Make sure the Node math server is still running, as above. ```sh -$ cd grpc/src/node -$ nodejs examples/math_server.js +$ cd grpc +$ npm install +$ nodejs src/node/test/math/math_server.js ``` Make sure you have run `composer install` to generate the `vendor/autoload.php` file ```sh +$ cd grpc/src/php $ composer install ``` From 0e65fb36f24bceb05b51a9970068d6ba6b1f409d Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 7 Mar 2016 17:49:37 -0800 Subject: [PATCH 085/160] Node: propagate read errors back down to core --- src/node/src/client.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/node/src/client.js b/src/node/src/client.js index 9acf51bd98b..81299b337ca 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -149,6 +149,9 @@ function _readsDone(status) { if (!status) { status = {code: grpc.status.OK, details: 'OK'}; } + if (status.code !== grpc.status.OK) { + this.call.cancelWithStatus(status.code, status.details); + } this.finished = true; this.read_status = status; this._emitStatusIfDone(); From 710d58cfae07c1815cc4791977069beb47aedf78 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 20:00:50 -0800 Subject: [PATCH 086/160] Disable profiling in qps_worker for now --- test/cpp/qps/qps_worker.cc | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index 9442017ddf5..ce4b773d92b 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -101,6 +101,19 @@ static std::unique_ptr CreateServer(const ServerConfig& config) { abort(); } +class ScopedProfile GRPC_FINAL { + public: + ScopedProfile(const char *filename, bool enable) : enable_(enable) { + if (enable_) grpc_profiler_start(filename); + } + ~ScopedProfile() { + if (enable_) grpc_profiler_stop(); + } + + private: + const bool enable_; +}; + class WorkerServiceImpl GRPC_FINAL : public WorkerService::Service { public: WorkerServiceImpl(int server_port, QpsWorker* worker) @@ -114,9 +127,8 @@ class WorkerServiceImpl GRPC_FINAL : public WorkerService::Service { return Status(StatusCode::RESOURCE_EXHAUSTED, ""); } - grpc_profiler_start("qps_client.prof"); + ScopedProfile profile("qps_client.prof", false); Status ret = RunClientBody(ctx, stream); - grpc_profiler_stop(); return ret; } @@ -128,9 +140,8 @@ class WorkerServiceImpl GRPC_FINAL : public WorkerService::Service { return Status(StatusCode::RESOURCE_EXHAUSTED, ""); } - grpc_profiler_start("qps_server.prof"); + ScopedProfile profile("qps_server.prof", false); Status ret = RunServerBody(ctx, stream); - grpc_profiler_stop(); return ret; } From 9d0638c03874b99ad3b0d42c50bad660d5f97c77 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 20:23:37 -0800 Subject: [PATCH 087/160] Fix copyright --- examples/python/route_guide/route_guide_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python/route_guide/route_guide_server.py b/examples/python/route_guide/route_guide_server.py index 24f948c42c5..f95b36b0736 100644 --- a/examples/python/route_guide/route_guide_server.py +++ b/examples/python/route_guide/route_guide_server.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From baaf65546ace6e8ab44f7ac869d868d7864d7578 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 22:39:11 -0800 Subject: [PATCH 088/160] Fix formatting --- test/cpp/qps/qps_worker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index ce4b773d92b..b83e9d1dd7f 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -103,7 +103,7 @@ static std::unique_ptr CreateServer(const ServerConfig& config) { class ScopedProfile GRPC_FINAL { public: - ScopedProfile(const char *filename, bool enable) : enable_(enable) { + ScopedProfile(const char* filename, bool enable) : enable_(enable) { if (enable_) grpc_profiler_start(filename); } ~ScopedProfile() { From 56188658d771ecdb4981e8a175313fecc78a61df Mon Sep 17 00:00:00 2001 From: Harsh Vardhan Date: Tue, 8 Mar 2016 12:55:02 +0530 Subject: [PATCH 089/160] updates contributing.md --- CONTRIBUTING.md | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 518ba2c4809..e147074943c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,21 +40,9 @@ If you want to run tests for any of the languages {c, c++, node, php, python}, d `./tools/run_tests/run_tests.py -l ` -For list of available commands, refer to this: - -``` -run_tests.py [-h] - [-c {asan,asan-noleaks,basicprof,dbg,gcov,helgrind,memcheck,msan,mutrace,opt,stapprof,tsan,ubsan}] - [-n RUNS_PER_TEST] [-r REGEX] [-j JOBS] [-s SLOWDOWN] [-f] - [-t] [--newline_on_success] - [-l {all,c,c++,csharp,node,objc,php,python,ruby,sanity} [{all,c,c++,csharp,node,objc,php,python,ruby,sanity} ...]] - [-S] [--use_docker] [--allow_flakes] - [--arch {default,x86,x64}] - [--compiler {default,gcc4.4,gcc4.9,gcc5.3,vs2010,vs2013,vs2015}] - [--build_only] [--measure_cpu_costs] - [--update_submodules [UPDATE_SUBMODULES [UPDATE_SUBMODULES ...]]] - [-a ANTAGONISTS] [-x XML_REPORT] -``` +To know about the list of available commands, do this: + +`./tools/run_tests/run_tests.py -h` ## Adding or removing source code From a2c8b20ac785d88e4df9bfe1a832e48173458c3e Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 8 Mar 2016 08:21:56 -0800 Subject: [PATCH 090/160] review changes --- src/php/README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/php/README.md b/src/php/README.md index c97c3796cda..cf8f2c11b0f 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -48,7 +48,7 @@ sudo pecl install grpc-beta This will compile and install the gRPC PHP extension into the standard PHP extension directory. You should be able to run the [unit tests](#unit-tests), with the PHP extension installed. -To run tests with generated stub code from `.proto` files, you will need the `composer`, `protoc` and `protoc-gen-php` binaries additionally. See sections [below](#generated-code-tests). +To run tests with generated stub code from `.proto` files, you will also need the `composer`, `protoc` and `protoc-gen-php` binaries. You can find out how to get these [below](#generated-code-tests). ## Build from Source @@ -115,9 +115,11 @@ $ ./bin/run_tests.sh ## Generated Code Tests +This section specifies the prerequisites for running the generated code tests, as well as how to run the tests themselves. + ### Composer -You need to install `composer`, to pull in some runtime dependencies based on the `composer.json` file. +If you don't have it already, install `composer` to pull in some runtime dependencies based on the `composer.json` file. ```sh $ curl -sS https://getcomposer.org/installer | php @@ -129,21 +131,21 @@ $ composer install ### Protobuf compiler -You need the install the protobuf compiler, `protoc`, 3.0.0+. +Again if you don't have it already, you need to install the protobuf compiler `protoc`, version 3.0.0+. -If you had compiled the gRPC C core library from source above, the `protoc` binary should have been installed as well. In the case it wasn't, you can run the following commands to install it. +If you compiled the gRPC C core library from source above, the `protoc` binary should have been installed as well. If it hasn't been installed, you can run the following commands to install it. ```sh $ cd grpc/third_party/protobuf $ sudo make install # 'make' should have been run by core grpc ``` -Or you can download a `protoc` binaries from [here](https://github.com/google/protobuf/releases). +Alternatively, you can download `protoc` binaries from [the protocol buffers Github repository](https://github.com/google/protobuf/releases). ### PHP protobuf compiler -You need to install `protoc-gen-php`, so that you can generate stub classes `.php` files from service definition `.proto` files. +You need to install `protoc-gen-php` to generate stub class `.php` files from service definition `.proto` files. ```sh $ cd grpc/src/php/vendor/datto/protobuf-php # if you had run `composer install` in the previous step @@ -168,7 +170,7 @@ $ ./bin/generate_proto_php.sh ### Run test server -Run a local server serving the math services. Please see [Node][] on how to run an example server +Run a local server serving the math services. Please see [Node][] for how to run an example server. ```sh $ cd grpc From 3be6801b72e91b0e3a1c6b497db02e144e32f7af Mon Sep 17 00:00:00 2001 From: Lisa Carey Date: Tue, 8 Mar 2016 17:47:14 +0000 Subject: [PATCH 091/160] Clarifying how to install for gRPC C++ --- INSTALL.md | 2 +- examples/cpp/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 3c694ba5822..454a8b7b2fd 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -3,7 +3,7 @@ For language-specific installation instructions for gRPC runtime, please refer to these documents - * [C++](examples/cpp) + * [C++](examples/cpp): Currently to install gRPC for C++, you need to build from source as described below. * [C#](src/csharp): NuGet package `Grpc` * [Go](https://github.com/grpc/grpc-go): `go get google.golang.org/grpc` * [Java](https://github.com/grpc/grpc-java) diff --git a/examples/cpp/README.md b/examples/cpp/README.md index e4b0eb698bf..d93cbacf7b2 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -2,7 +2,7 @@ ## Installation -To install gRPC on your system, follow the instructions [here](../../INSTALL.md). +To install gRPC on your system, follow the instructions to build from source [here](../../INSTALL.md). This also installs the protocol buffer compiler `protoc` (if you don't have it already), and the C++ gRPC plugin for `protoc`. ## Hello C++ gRPC! From 9d1476364a002372f2ab375c3aceeb29b7a227cd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 8 Mar 2016 17:00:53 -0800 Subject: [PATCH 092/160] Fix timers --- src/core/iomgr/timer.c | 2 +- src/core/iomgr/timer_heap.c | 14 +++--- test/core/iomgr/timer_heap_test.c | 84 ++----------------------------- 3 files changed, 12 insertions(+), 88 deletions(-) diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 8379fffad02..1bf8387d74b 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -41,7 +41,7 @@ #define INVALID_HEAP_INDEX 0xffffffffu -#define LOG2_NUM_SHARDS 5 +#define LOG2_NUM_SHARDS 2 #define NUM_SHARDS (1 << LOG2_NUM_SHARDS) #define ADD_DEADLINE_SCALE 0.33 #define MIN_QUEUE_WINDOW_DURATION 0.01 diff --git a/src/core/iomgr/timer_heap.c b/src/core/iomgr/timer_heap.c index 9d8be5c1fcb..167bb7c4d11 100644 --- a/src/core/iomgr/timer_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -46,7 +46,7 @@ static void adjust_upwards(grpc_timer **first, uint32_t i, grpc_timer *t) { while (i > 0) { uint32_t parent = (uint32_t)(((int)i - 1) / 2); - if (gpr_time_cmp(first[parent]->deadline, t->deadline) >= 0) break; + if (gpr_time_cmp(first[parent]->deadline, t->deadline) <= 0) break; first[i] = first[parent]; first[i]->heap_index = i; i = parent; @@ -62,16 +62,14 @@ static void adjust_downwards(grpc_timer **first, uint32_t i, uint32_t length, grpc_timer *t) { for (;;) { uint32_t left_child = 1u + 2u * i; - uint32_t right_child; - uint32_t next_i; if (left_child >= length) break; - right_child = left_child + 1; - next_i = right_child < length && + uint32_t right_child = left_child + 1; + uint32_t next_i = right_child < length && gpr_time_cmp(first[left_child]->deadline, - first[right_child]->deadline) < 0 + first[right_child]->deadline) > 0 ? right_child : left_child; - if (gpr_time_cmp(t->deadline, first[next_i]->deadline) >= 0) break; + if (gpr_time_cmp(t->deadline, first[next_i]->deadline) <= 0) break; first[i] = first[next_i]; first[i]->heap_index = i; i = next_i; @@ -95,7 +93,7 @@ static void maybe_shrink(grpc_timer_heap *heap) { static void note_changed_priority(grpc_timer_heap *heap, grpc_timer *timer) { uint32_t i = timer->heap_index; uint32_t parent = (uint32_t)(((int)i - 1) / 2); - if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) < 0) { + if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) > 0) { adjust_upwards(heap->timers, i, timer); } else { adjust_downwards(heap->timers, i, heap->timer_count, timer); diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 077a9fd6bd8..9b51b912003 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -57,79 +57,6 @@ static grpc_timer *create_test_elements(size_t num_elements) { return elems; } -static int cmp_elem(const void *a, const void *b) { - int i = *(const int *)a; - int j = *(const int *)b; - return i - j; -} - -static size_t *all_top(grpc_timer_heap *pq, size_t *n) { - size_t *vec = NULL; - size_t *need_to_check_children; - size_t num_need_to_check_children = 0; - - *n = 0; - if (pq->timer_count == 0) return vec; - need_to_check_children = - gpr_malloc(pq->timer_count * sizeof(*need_to_check_children)); - need_to_check_children[num_need_to_check_children++] = 0; - vec = gpr_malloc(pq->timer_count * sizeof(*vec)); - while (num_need_to_check_children > 0) { - size_t ind = need_to_check_children[0]; - size_t leftchild, rightchild; - num_need_to_check_children--; - memmove(need_to_check_children, need_to_check_children + 1, - num_need_to_check_children * sizeof(*need_to_check_children)); - vec[(*n)++] = ind; - leftchild = 1u + 2u * ind; - if (leftchild < pq->timer_count) { - if (gpr_time_cmp(pq->timers[leftchild]->deadline, - pq->timers[ind]->deadline) >= 0) { - need_to_check_children[num_need_to_check_children++] = leftchild; - } - rightchild = leftchild + 1; - if (rightchild < pq->timer_count && - gpr_time_cmp(pq->timers[rightchild]->deadline, - pq->timers[ind]->deadline) >= 0) { - need_to_check_children[num_need_to_check_children++] = rightchild; - } - } - } - - gpr_free(need_to_check_children); - - return vec; -} - -static void check_pq_top(grpc_timer *elements, grpc_timer_heap *pq, - uint8_t *inpq, size_t num_elements) { - gpr_timespec max_deadline = gpr_inf_past(GPR_CLOCK_REALTIME); - size_t *max_deadline_indices = - gpr_malloc(num_elements * sizeof(*max_deadline_indices)); - size_t *top_elements; - size_t num_max_deadline_indices = 0; - size_t num_top_elements; - size_t i; - for (i = 0; i < num_elements; ++i) { - if (inpq[i] && gpr_time_cmp(elements[i].deadline, max_deadline) >= 0) { - if (gpr_time_cmp(elements[i].deadline, max_deadline) > 0) { - num_max_deadline_indices = 0; - max_deadline = elements[i].deadline; - } - max_deadline_indices[num_max_deadline_indices++] = elements[i].heap_index; - } - } - qsort(max_deadline_indices, num_max_deadline_indices, - sizeof(*max_deadline_indices), cmp_elem); - top_elements = all_top(pq, &num_top_elements); - GPR_ASSERT(num_top_elements == num_max_deadline_indices); - for (i = 0; i < num_top_elements; i++) { - GPR_ASSERT(max_deadline_indices[i] == top_elements[i]); - } - gpr_free(max_deadline_indices); - gpr_free(top_elements); -} - static int contains(grpc_timer_heap *pq, grpc_timer *el) { size_t i; for (i = 0; i < pq->timer_count; i++) { @@ -145,11 +72,11 @@ static void check_valid(grpc_timer_heap *pq) { size_t right_child = left_child + 1u; if (left_child < pq->timer_count) { GPR_ASSERT(gpr_time_cmp(pq->timers[i]->deadline, - pq->timers[left_child]->deadline) >= 0); + pq->timers[left_child]->deadline) <= 0); } if (right_child < pq->timer_count) { GPR_ASSERT(gpr_time_cmp(pq->timers[i]->deadline, - pq->timers[right_child]->deadline) >= 0); + pq->timers[right_child]->deadline) <= 0); } } } @@ -162,6 +89,8 @@ static void test1(void) { grpc_timer *test_elements = create_test_elements(num_test_elements); uint8_t *inpq = gpr_malloc(num_test_elements); + gpr_log(GPR_DEBUG, "******************************************* test1"); + grpc_timer_heap_init(&pq); memset(inpq, 0, num_test_elements); GPR_ASSERT(grpc_timer_heap_is_empty(&pq)); @@ -172,7 +101,6 @@ static void test1(void) { check_valid(&pq); GPR_ASSERT(contains(&pq, &test_elements[i])); inpq[i] = 1; - check_pq_top(test_elements, &pq, inpq, num_test_elements); } for (i = 0; i < num_test_elements; ++i) { /* Test that check still succeeds even for element that wasn't just @@ -182,7 +110,7 @@ static void test1(void) { GPR_ASSERT(pq.timer_count == num_test_elements); - check_pq_top(test_elements, &pq, inpq, num_test_elements); + check_valid(&pq); for (i = 0; i < num_test_operations; ++i) { size_t elem_num = (size_t)rand() % num_test_elements; @@ -193,14 +121,12 @@ static void test1(void) { grpc_timer_heap_add(&pq, el); GPR_ASSERT(contains(&pq, el)); inpq[elem_num] = 1; - check_pq_top(test_elements, &pq, inpq, num_test_elements); check_valid(&pq); } else { GPR_ASSERT(contains(&pq, el)); grpc_timer_heap_remove(&pq, el); GPR_ASSERT(!contains(&pq, el)); inpq[elem_num] = 0; - check_pq_top(test_elements, &pq, inpq, num_test_elements); check_valid(&pq); } } From 08e0c191beaa9fc818ae2363c7684307fde084bf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 8 Mar 2016 17:04:50 -0800 Subject: [PATCH 093/160] Revert mistaken change --- src/core/iomgr/timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 1bf8387d74b..8379fffad02 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -41,7 +41,7 @@ #define INVALID_HEAP_INDEX 0xffffffffu -#define LOG2_NUM_SHARDS 2 +#define LOG2_NUM_SHARDS 5 #define NUM_SHARDS (1 << LOG2_NUM_SHARDS) #define ADD_DEADLINE_SCALE 0.33 #define MIN_QUEUE_WINDOW_DURATION 0.01 From 341b5abef5a039a731c9da260c4e9b76b1849415 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 8 Mar 2016 17:16:29 -0800 Subject: [PATCH 094/160] Doc Fixit: src/cpp/README need more information --- src/cpp/README.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/cpp/README.md b/src/cpp/README.md index baeba083155..a13669b99e1 100644 --- a/src/cpp/README.md +++ b/src/cpp/README.md @@ -6,3 +6,62 @@ This directory contains source code for C++ implementation of gRPC. #Status Beta + +#Pre-requisites + +##Linux + +```sh + $ [sudo] apt-get install build-essential autoconf libtool +``` + +##Mac OSX + +For a Mac system, git is not available by default. You will first need to +install Xcode from the Mac AppStore and then run the following command from a +terminal: + +```sh + $ [sudo] xcode-select --install +``` + +##Protoc + +By default gRPC uses [protocol buffers](https://github.com/google/protobuf), +you will need the `protoc` compiler to generate stub server and client code. + +If you compile gRPC from source, as described below, the Makefile will +automatically try and compile the `protoc` in third party if you cloned the +repository recursively and it detects that you don't already have it +installed. + +If it hasn't been installed, you can run the following commands to install it. + +```sh +$ cd grpc/third_party/protobuf +$ sudo make install # 'make' should have been run by core grpc +``` + +Alternatively, you can download `protoc` binaries from +[the protocol buffers Github repository](https://github.com/google/protobuf/releases). + +#INSTALLATION + +Currently to install gRPC for C++, you need to build from source as described below. + +#Build from Source + +```sh + $ git clone https://github.com/grpc/grpc.git + $ cd grpc + $ git submodule update --init + $ make + $ [sudo] make install +``` + +#DOCUMENTATION + +- The gRPC C++ refenrence documentation is available online at + [grpc.io](http://www.grpc.io/docs/tutorials/basic/c.html) +- [Helloworld example](../../examples/cpp/helloworld) + From 72dbc4aa2d84cdf03dee863af22b4c5870615806 Mon Sep 17 00:00:00 2001 From: Harsh Vardhan Date: Wed, 9 Mar 2016 10:50:58 +0530 Subject: [PATCH 095/160] updates contributing.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e147074943c..011ce43bcdc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ In order to run most of the available tests, one would need to run: `./tools/run_tests/run_tests.py` -If you want to run tests for any of the languages {c, c++, node, php, python}, do this: +If you want to run tests for any of the languages {all, c, c++, csharp, node, objc, php, python, ruby}, do this: `./tools/run_tests/run_tests.py -l ` From 1b393da4332ad97aac62ee8ecb422d6d0fe108f3 Mon Sep 17 00:00:00 2001 From: Lisa Carey Date: Wed, 9 Mar 2016 15:16:56 +0000 Subject: [PATCH 096/160] Updated main examples README to link to main docs and quickstarts --- examples/README.md | 447 ++------------------------------------------- 1 file changed, 12 insertions(+), 435 deletions(-) diff --git a/examples/README.md b/examples/README.md index cd85417ca20..287a80266c9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,450 +1,27 @@ +# Examples -# Getting started +This directory contains code examples for all the C-based gRPC implementations: C++, Node.js, Python, Ruby, Objective-C, PHP, and C#. You can find examples and instructions specific to your +favourite language in the relevant subdirectory. + +Examples for Go and Java gRPC live in their own repositories: -Welcome to the developer documentation for gRPC, a language-neutral, -platform-neutral remote procedure call (RPC) system developed at Google. +* [Java](https://github.com/grpc/grpc-java/tree/master/examples) +* [Android Java](https://github.com/grpc/grpc-java/tree/master/examples/android) +* [Go](https://github.com/grpc/grpc-go/tree/master/examples) -This document introduces you to gRPC with a quick overview and a simple -Hello World example. You'll find more tutorials and reference docs in this repository - more documentation is coming soon! +For more comprehensive documentation, including an [overview](http://www.grpc.io/docs/) and tutorials that use this example code, visit [grpc.io](http://www.grpc.io/docs/). - ## Quick start -You can find quick start guides for each language, including installation instructions, examples, and tutorials here: + +Each example directory has quick start instructions for the appropriate language, including installation instructions and how to run our simplest Hello World example: + * [C++](cpp) -* [Java](https://github.com/grpc/grpc-java/tree/master/examples) -* [Go](https://github.com/grpc/grpc-go/tree/master/examples) * [Ruby](ruby) * [Node.js](node) -* [Android Java](https://github.com/grpc/grpc-java/tree/master/examples/android) * [Python](python/helloworld) * [C#](csharp) * [Objective-C](objective-c/helloworld) * [PHP](php) -## What's in this repository? - -The `examples` directory contains documentation, resources, and examples -for all gRPC users. You can find examples and instructions specific to your -favourite language in the relevant subdirectory. - -You can find out about the gRPC source code repositories in -[`grpc`](https://github.com/grpc/grpc). Each repository provides instructions -for building the appropriate libraries for your language. - - -## What is gRPC? - -In gRPC a *client* application can directly call -methods on a *server* application on a different machine as if it was a -local object, making it easier for you to create distributed applications and -services. As in many RPC systems, gRPC is based around the idea of defining -a *service*, specifying the methods that can be called remotely with their -parameters and return types. On the server side, the server implements this -interface and runs a gRPC server to handle client calls. On the client side, -the client has a *stub* that provides exactly the same methods as the server. - - - -gRPC clients and servers can run and talk to each other in a variety of -environments - from servers inside Google to your own desktop - and can -be written in any of gRPC's [supported languages](#quickstart). So, for -example, you can easily create a gRPC server in Java with clients in Go, -Python, or Ruby. In addition, the latest Google APIs will have gRPC versions -of their interfaces, letting you easily build Google functionality into -your applications. - - -### Working with protocol buffers - -By default gRPC uses *protocol buffers*, Google’s -mature open source mechanism for serializing structured data (although it -can be used with other data formats such as JSON). As you'll -see in our example below, you define gRPC services using *proto files*, -with method parameters and return types specified as protocol buffer message -types. You -can find out lots more about protocol buffers in the [Protocol Buffers -documentation](https://developers.google.com/protocol-buffers/docs/overview). - -#### Protocol buffer versions - -While protocol buffers have been available for open source users for some -time, our examples use a new flavour of protocol buffers called proto3, -which has a slightly simplified syntax, some useful new features, and supports -lots more languages. This is currently available as an alpha release in -Java, C++, Java_nano (Android Java), Python, and Ruby from [the protocol buffers Github -repo](https://github.com/google/protobuf/releases), as well as a Go language -generator from [the golang/protobuf Github repo](https://github.com/golang/protobuf), with more languages in development. You can find out more in the [proto3 language guide](https://developers.google.com/protocol-buffers/docs/proto3), and see -the major differences from the current default version in the [release notes](https://github.com/google/protobuf/releases). More proto3 documentation is coming soon. - -In general, while you *can* use proto2 (the current default protocol buffers version), we recommend that you use proto3 with gRPC as it lets you use the full range of gRPC-supported languages, as well as avoiding compatibility -issues with proto2 clients talking to proto3 servers and vice versa. - - -## Hello gRPC! - -Now that you know a bit more about gRPC, the easiest way to see how it -works is to look at a simple example. Our Hello World walks you through the -construction of a simple gRPC client-server application, showing you how to: - -- Create a protocol buffers schema that defines a simple RPC service with -a single -Hello World method. -- Create a Java server that implements this interface. -- Create a Java client that accesses the Java server. -- Create a Go client that accesses -the same Java server. - -The complete code for the example is available in the `examples` -directory. We use the Git versioning system for source code management: -however, you don't need to know anything about Git to follow along other -than how to install and run a few git commands. - -This is an introductory example rather than a comprehensive tutorial, so -don't worry if you're not a Go or -Java developer - the concepts are similar for all languages, and you can -find more implementations of our Hello World example in other languages (and full tutorials where available) in -the [language-specific folders](#quickstart) in this repository. Complete tutorials and -reference documentation for all gRPC languages are coming soon. - - -### Setup - -This section explains how to set up your local machine to work with -the example code. If you just want to read the example, you can go straight -to the [next step](#servicedef). - -#### Install Git - -You can download and install Git from http://git-scm.com/download. Once -installed you should have access to the git command line tool. The main -commands that you will need to use are: - -- git clone ... : clone a remote repository onto your local machine -- git checkout ... : check out a particular branch or a tagged version of -the code to hack on - -#### Install gRPC - -To build and install gRPC plugins and related tools: -- For Java, see the [Java quick start](https://github.com/grpc/grpc-java). -- For Go, see the [Go quick start](https://github.com/grpc/grpc-go). - -#### Get the source code - -The example code for our Java example lives in the `grpc-java` -GitHub repository. Clone this repository to your local machine by running the -following command: - - -``` -git clone https://github.com/grpc/grpc-java.git -``` - -Change your current directory to grpc-java/examples - -``` -cd grpc-java/examples -``` - - - - -### Defining a service - -The first step in creating our example is to define a *service*: an RPC -service specifies the methods that can be called remotely with their parameters -and return types. As you saw in the -[overview](#protocolbuffers) above, gRPC does this using [protocol -buffers](https://developers.google.com/protocol-buffers/docs/overview). We -use the protocol buffers interface definition language (IDL) to define our -service methods, and define the parameters and return -types as protocol buffer message types. Both the client and the -server use interface code generated from the service definition. - -Here's our example service definition, defined using protocol buffers IDL in -[helloworld.proto](https://github.com/grpc/grpc-java/tree/master/examples/src/main/proto). The `Greeter` -service has one method, `SayHello`, that lets the server receive a single -`HelloRequest` -message from the remote client containing the user's name, then send back -a greeting in a single `HelloReply`. This is the simplest type of RPC you -can specify in gRPC - you can find out about other types in the tutorial for your chosen language. - -```proto -syntax = "proto3"; - -option java_package = "io.grpc.examples"; - -package helloworld; - -// The greeter service definition. -service Greeter { - // Sends a greeting - rpc SayHello (HelloRequest) returns (HelloReply) {} -} - -// The request message containing the user's name. -message HelloRequest { - string name = 1; -} - -// The response message containing the greetings -message HelloReply { - string message = 1; -} - -``` - - -### Generating gRPC code - -Once we've defined our service, we use the protocol buffer compiler -`protoc` to generate the special client and server code we need to create -our application - right now we're going to generate Java code, though you -can generate gRPC code in any gRPC-supported language (as you'll see later -in this example). The generated code contains both stub code for clients to -use and an abstract interface for servers to implement, both with the method -defined in our `Greeter` service. - -(If you didn't install the gRPC plugins and protoc on your system and are just reading along with -the example, you can skip this step and move -onto the next one where we examine the generated code.) - -For simplicity, we've provided a [Gradle build file](https://github.com/grpc/grpc-java/blob/master/examples/build.gradle) with our Java examples that runs `protoc` for you with the appropriate plugin, input, and output: - -```shell -../gradlew build -``` - -This generates the following classes from our .proto, which contain all the generated code -we need to create our example: - -- `Helloworld.java`, which -has all the protocol buffer code to populate, serialize, and retrieve our -`HelloRequest` and `HelloReply` message types -- `GreeterGrpc.java`, which contains (along with some other useful code): - - an interface for `Greeter` servers to implement - - ```java - public static interface Greeter { - public void sayHello(io.grpc.examples.Helloworld.HelloRequest request, - io.grpc.stub.StreamObserver responseObserver); - } - ``` - - - _stub_ classes that clients can use to talk to a `Greeter` server. As you can see, they also implement the `Greeter` interface. - - ```java - public static class GreeterStub extends - io.grpc.stub.AbstractStub - implements Greeter { - ... - } - ``` - - -### Writing a server - -Now let's write some code! First we'll create a server application to implement -our service. Note that we're not going to go into a lot of detail about how -to create a server in this section. More detailed information will be in the -tutorial for your chosen language: check if there's one available yet in the relevant [quick start](#quickstart). - -Our server application has two classes: - -- a main server class that hosts the service implementation and allows access over the -network: [HelloWorldServer.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java). - - -- a simple service implementation class [GreeterImpl.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java#L51). - - -#### Service implementation - -[GreeterImpl.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java#L51) -actually implements our `Greeter` service's required behaviour. - -As you can see, the class `GreeterImpl` implements the interface -`GreeterGrpc.Greeter` that we [generated](#generating) from our proto -[IDL](https://github.com/grpc/grpc-java/tree/master/examples/src/main/proto) by implementing the method `sayHello`: - -```java - @Override - public void sayHello(HelloRequest req, StreamObserver responseObserver) { - HelloReply reply = HelloReply.newBuilder().setMessage("Hello " + req.getName()).build(); - responseObserver.onValue(reply); - responseObserver.onCompleted(); - } -``` -- `sayHello` takes two parameters: - - `HelloRequest`: the request - - `StreamObserver`: a response observer, which is - a special interface for the server to call with its response - -To return our response to the client and complete the call: - -1. We construct and populate a `HelloReply` response object with our exciting -message, as specified in our interface definition. -2. We return the `HelloReply` to the client and then specify that we've finished dealing with the RPC. - - -#### Server implementation - -[HelloWorldServer.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java) -shows the other main feature required to provide a gRPC service; making the service -implementation available from the network. - -```java - /* The port on which the server should run */ - private int port = 50051; - private ServerImpl server; - - private void start() throws Exception { - server = NettyServerBuilder.forPort(port) - .addService(GreeterGrpc.bindService(new GreeterImpl())) - .build().start(); - logger.info("Server started, listening on " + port); - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - // Use stderr here since the logger may have been reset by its JVM shutdown hook. - System.err.println("*** shutting down gRPC server since JVM is shutting down"); - HelloWorldServer.this.stop(); - System.err.println("*** server shut down"); - } - }); - } - -``` - -Here we create an appropriate gRPC server, binding the `Greeter` service -implementation that we created to a port. Then we start the server running: the server is now ready to receive -requests from `Greeter` service clients on our specified port. We'll cover -how all this works in a bit more detail in our language-specific documentation. - - -### Writing a client - -Client-side gRPC is pretty simple. In this step, we'll use the generated code -to write a simple client that can access the `Greeter` server we created -in the [previous section](#server). You can see the complete client code in -[HelloWorldClient.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldClient.java). - -Again, we're not going to go into much detail about how to implement a client; -we'll leave that for the tutorial. - -#### Connecting to the service - -First let's look at how we connect to the `Greeter` server. First we need -to create a gRPC channel, specifying the hostname and port of the server we -want to connect to. Then we use the channel to construct the stub instance. - - -```java - private final ChannelImpl channel; - private final GreeterGrpc.GreeterBlockingStub blockingStub; - - public HelloWorldClient(String host, int port) { - channel = - NettyChannelBuilder.forAddress(host, port).negotiationType(NegotiationType.PLAINTEXT) - .build(); - blockingStub = GreeterGrpc.newBlockingStub(channel); - } - -``` - -In this case, we create a blocking stub. This means that the RPC call waits -for the server to respond, and will either return a response or raise an -exception. gRPC Java has other kinds of stubs that make non-blocking calls -to the server, where the response is returned asynchronously. - -#### Calling an RPC - -Now we can contact the service and obtain a greeting: - -1. We construct and fill in a `HelloRequest` to send to the service. -2. We call the stub's `hello()` RPC with our request and get a `HelloReply` -back, from which we can get our greeting. - - -```java - HelloRequest req = HelloRequest.newBuilder().setName(name).build(); - HelloReply reply = blockingStub.sayHello(req); - -``` - - -### Try it out! - -Our [Gradle build file](https://github.com/grpc/grpc-java/blob/master/examples/build.gradle) simplifies building and running the examples. - -You can build and run the server from the `grpc-java` root folder with: - -```sh -$ ./gradlew :grpc-examples:helloWorldServer -``` - -and in another terminal window confirm that it receives a message. - -```sh -$ ./gradlew :grpc-examples:helloWorldClient -``` - -### Adding another client - -Finally, let's look at one of gRPC's most useful features - interoperability -between code in different languages. So far, we've just looked at Java code -generated from and implementing our `Greeter` service definition. However, -as you'll see if you look at the language-specific subdirectories -in this repository, we've also generated and implemented `Greeter` -in some of gRPC's other supported languages. Each service -and client uses interface code generated from the same proto -that we used for the Java example. - -So, for example, if we visit the [`go` example -directory](https://github.com/grpc/grpc-go/tree/master/examples) and look at the -[`greeter_client`](https://github.com/grpc/grpc-go/blob/master/examples/greeter_client/main.go), -we can see that like the Java client, it connects to a `Greeter` service -at `localhost:50051` and uses a stub to call the `SayHello` method with a -`HelloRequest`: - -```go -const ( - address = "localhost:50051" - defaultName = "world" -) - -func main() { - // Set up a connection to the server. - conn, err := grpc.Dial(address) - if err != nil { - log.Fatalf("did not connect: %v", err) - } - defer conn.Close() - c := pb.NewGreeterClient(conn) - - // Contact the server and print out its response. - name := defaultName - if len(os.Args) > 1 { - name = os.Args[1] - } - r, err := c.SayHello(context.Background(), &pb.HelloRequest{Name: - name}) - if err != nil { - log.Fatalf("could not greet: %v", err) - } - log.Printf("Greeting: %s", r.Message) -} -``` - - -If we run the Java server from earlier in another terminal window, we can -run the Go client and connect to it just like the Java client, even though -it's written in a different language. -``` -$ greeter_client -``` -## Read more! -- You can find links to language-specific tutorials, examples, and other docs in each language's [quick start](#quickstart). -- [gRPC Authentication Support](http://www.grpc.io/docs/guides/auth.html) introduces authentication support in gRPC with supported mechanisms and examples. From 0e6e34e4fae218bbff1d586fe42f2f4151615282 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:21:43 -0800 Subject: [PATCH 097/160] Add an additional test --- test/core/iomgr/timer_heap_test.c | 108 +++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 9b51b912003..3c78a6c1171 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -38,6 +38,8 @@ #include #include +#include + #include "test/core/util/test_config.h" static gpr_timespec random_deadline(void) { @@ -81,6 +83,10 @@ static void check_valid(grpc_timer_heap *pq) { } } +/******************************************************************************* + * test1 + */ + static void test1(void) { grpc_timer_heap pq; const size_t num_test_elements = 200; @@ -89,7 +95,7 @@ static void test1(void) { grpc_timer *test_elements = create_test_elements(num_test_elements); uint8_t *inpq = gpr_malloc(num_test_elements); - gpr_log(GPR_DEBUG, "******************************************* test1"); + gpr_log(GPR_INFO, "test1"); grpc_timer_heap_init(&pq); memset(inpq, 0, num_test_elements); @@ -136,7 +142,106 @@ static void test1(void) { gpr_free(inpq); } +/******************************************************************************* + * test2 + */ + +typedef struct { + grpc_timer elem; + bool inserted; +} elem_struct; + +static elem_struct *search_elems(elem_struct *elems, size_t count, bool inserted) { + size_t *search_order = gpr_malloc(count * sizeof(*search_order)); + for (size_t i = 0; i < count; i++) { + search_order[i] = i; + } + for (size_t i = 0; i < count * 2; i++) { + size_t a = (size_t)rand() % count; + size_t b = (size_t)rand() % count; + GPR_SWAP(size_t, search_order[a], search_order[b]); + } + elem_struct *out = NULL; + for (size_t i = 0; out == NULL && i < count; i++) { + if (elems[search_order[i]].inserted == inserted) { + out = &elems[search_order[i]]; + } + } + gpr_free(search_order); + return out; +} + +static void test2(void) { +gpr_log(GPR_INFO, "test2"); + + grpc_timer_heap pq; + + elem_struct elems[1000]; + size_t num_inserted = 0; + + grpc_timer_heap_init(&pq); + memset(elems, 0, sizeof(elems)); + + for (size_t round = 0; round < 10000; round++) { + int r = (size_t)rand() % 1000; + if (r <= 550) { + /* 55% of the time we try to add something */ + elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), false); + if (el != NULL) { + el->elem.deadline = random_deadline(); + grpc_timer_heap_add(&pq, &el->elem); + el->inserted = true; + num_inserted++; + check_valid(&pq); + } + } else if (r <= 650) { + /* 10% of the time we try to remove something */ + elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), true); + if (el != NULL) { + grpc_timer_heap_remove(&pq, &el->elem); + el->inserted = false; + num_inserted--; + check_valid(&pq); + } + } else { + /* the remaining times we pop */ + if (num_inserted > 0) { + grpc_timer *top = grpc_timer_heap_top(&pq); + grpc_timer_heap_pop(&pq); + for (size_t i = 0; i < GPR_ARRAY_SIZE(elems); i++) { + if (top == &elems[i].elem) { + GPR_ASSERT(elems[i].inserted); + elems[i].inserted = false; + } + } + num_inserted--; + check_valid(&pq); + } + } + + if (num_inserted) { + gpr_timespec *min_deadline = NULL; + for (size_t i = 0; i < GPR_ARRAY_SIZE(elems); i++) { + if (elems[i].inserted) { + if (min_deadline == NULL) { + min_deadline = &elems[i].elem.deadline; + } else { + if (gpr_time_cmp(elems[i].elem.deadline, *min_deadline) < 0) { + min_deadline = &elems[i].elem.deadline; + } + } + } + } + GPR_ASSERT(0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline)); + } + } + + grpc_timer_heap_destroy(&pq); +} + static void shrink_test(void) { + gpr_log(GPR_INFO, "shrink_test"); + grpc_timer_heap pq; size_t i; size_t expected_size; @@ -200,6 +305,7 @@ int main(int argc, char **argv) { for (i = 0; i < 5; i++) { test1(); + test2(); shrink_test(); } From 3a951bd975cb14ba7fe2b59e7c6b598932998a73 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:23:57 -0800 Subject: [PATCH 098/160] clang-format --- src/core/iomgr/timer_heap.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/iomgr/timer_heap.c b/src/core/iomgr/timer_heap.c index 167bb7c4d11..9d4aaf3eee7 100644 --- a/src/core/iomgr/timer_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -65,10 +65,10 @@ static void adjust_downwards(grpc_timer **first, uint32_t i, uint32_t length, if (left_child >= length) break; uint32_t right_child = left_child + 1; uint32_t next_i = right_child < length && - gpr_time_cmp(first[left_child]->deadline, - first[right_child]->deadline) > 0 - ? right_child - : left_child; + gpr_time_cmp(first[left_child]->deadline, + first[right_child]->deadline) > 0 + ? right_child + : left_child; if (gpr_time_cmp(t->deadline, first[next_i]->deadline) <= 0) break; first[i] = first[next_i]; first[i]->heap_index = i; From a39c1995623ed5ef48c49fc5212f2140b3390462 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:26:52 -0800 Subject: [PATCH 099/160] clang-format --- test/core/iomgr/timer_heap_test.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 3c78a6c1171..4510d51bd70 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -151,7 +151,8 @@ typedef struct { bool inserted; } elem_struct; -static elem_struct *search_elems(elem_struct *elems, size_t count, bool inserted) { +static elem_struct *search_elems(elem_struct *elems, size_t count, + bool inserted) { size_t *search_order = gpr_malloc(count * sizeof(*search_order)); for (size_t i = 0; i < count; i++) { search_order[i] = i; @@ -172,7 +173,7 @@ static elem_struct *search_elems(elem_struct *elems, size_t count, bool inserted } static void test2(void) { -gpr_log(GPR_INFO, "test2"); + gpr_log(GPR_INFO, "test2"); grpc_timer_heap pq; @@ -232,7 +233,8 @@ gpr_log(GPR_INFO, "test2"); } } } - GPR_ASSERT(0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline)); + GPR_ASSERT( + 0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline)); } } From 5f3ba8c834b829254f85fedee6441fc82f14bcfc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:44:51 -0800 Subject: [PATCH 100/160] Copyright, casting fixes --- src/core/iomgr/timer_heap.c | 2 +- test/core/iomgr/timer_heap_test.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/timer_heap.c b/src/core/iomgr/timer_heap.c index 9d4aaf3eee7..b5df566c453 100644 --- a/src/core/iomgr/timer_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 4510d51bd70..cd34696f7dd 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -184,7 +184,7 @@ static void test2(void) { memset(elems, 0, sizeof(elems)); for (size_t round = 0; round < 10000; round++) { - int r = (size_t)rand() % 1000; + int r = rand() % 1000; if (r <= 550) { /* 55% of the time we try to add something */ elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), false); From 76a5c0e433eb0a4a941e0bf9172e55c6f8058ed4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 09:05:30 -0800 Subject: [PATCH 101/160] Fix memory leak: keep all resolver calls under the same lock --- src/core/channel/client_channel.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index d4ba9508185..f021a8ae329 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -165,7 +165,6 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, channel_data *chand = arg; grpc_lb_policy *lb_policy = NULL; grpc_lb_policy *old_lb_policy; - grpc_resolver *old_resolver; grpc_connectivity_state state = GRPC_CHANNEL_TRANSIENT_FAILURE; int exit_idle = 0; @@ -201,28 +200,25 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, } if (iomgr_success && chand->resolver) { - grpc_resolver *resolver = chand->resolver; - GRPC_RESOLVER_REF(resolver, "channel-next"); grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, state, "new_lb+resolver"); if (lb_policy != NULL) { watch_lb_policy(exec_ctx, chand, lb_policy, state); } - gpr_mu_unlock(&chand->mu_config); GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver"); - grpc_resolver_next(exec_ctx, resolver, &chand->incoming_configuration, + grpc_resolver_next(exec_ctx, chand->resolver, + &chand->incoming_configuration, &chand->on_config_changed); - GRPC_RESOLVER_UNREF(exec_ctx, resolver, "channel-next"); + gpr_mu_unlock(&chand->mu_config); } else { - old_resolver = chand->resolver; - chand->resolver = NULL; + if (chand->resolver != NULL) { + grpc_resolver_shutdown(exec_ctx, chand->resolver); + GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel"); + chand->resolver = NULL; + } grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, GRPC_CHANNEL_FATAL_FAILURE, "resolver_gone"); gpr_mu_unlock(&chand->mu_config); - if (old_resolver != NULL) { - grpc_resolver_shutdown(exec_ctx, old_resolver); - GRPC_RESOLVER_UNREF(exec_ctx, old_resolver, "channel"); - } } if (exit_idle) { @@ -247,7 +243,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_transport_op *op) { channel_data *chand = elem->channel_data; - grpc_resolver *destroy_resolver = NULL; grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL); @@ -279,7 +274,8 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, if (op->disconnect && chand->resolver != NULL) { grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, GRPC_CHANNEL_FATAL_FAILURE, "disconnect"); - destroy_resolver = chand->resolver; + grpc_resolver_shutdown(exec_ctx, chand->resolver); + GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel"); chand->resolver = NULL; if (chand->lb_policy != NULL) { grpc_pollset_set_del_pollset_set(exec_ctx, @@ -290,11 +286,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, } } gpr_mu_unlock(&chand->mu_config); - - if (destroy_resolver) { - grpc_resolver_shutdown(exec_ctx, destroy_resolver); - GRPC_RESOLVER_UNREF(exec_ctx, destroy_resolver, "channel"); - } } typedef struct { From d426d9dfdbd1712b1b241a17452ed15441b502a1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 09:30:18 -0800 Subject: [PATCH 102/160] Fix memory leak if call is already cancelled Obvious in hindsight, if the cas failed, we still created the subchannel call object and then left it dangling. Fixes a leak observed on master in the past 48 hours. --- src/core/channel/subchannel_call_holder.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c index 8f46885a043..9c087dc2a1d 100644 --- a/src/core/channel/subchannel_call_holder.c +++ b/src/core/channel/subchannel_call_holder.c @@ -174,17 +174,15 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) { holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; if (holder->connected_subchannel == NULL) { fail_locked(exec_ctx, holder); + } else if (1 == gpr_atm_acq_load(&holder->subchannel_call)) { + /* already cancelled before subchannel became ready */ + fail_locked(exec_ctx, holder); } else { - if (!gpr_atm_rel_cas( - &holder->subchannel_call, 0, - (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( - exec_ctx, holder->connected_subchannel, holder->pollset))) { - GPR_ASSERT(gpr_atm_acq_load(&holder->subchannel_call) == 1); - /* if this cas fails, the call was cancelled before the pick completed */ - fail_locked(exec_ctx, holder); - } else { - retry_waiting_locked(exec_ctx, holder); - } + gpr_atm_rel_store( + &holder->subchannel_call, + (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( + exec_ctx, holder->connected_subchannel, holder->pollset)); + retry_waiting_locked(exec_ctx, holder); } gpr_mu_unlock(&holder->mu); GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel"); From d48d84da195642c89d1108567398d576f2d3382d Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 9 Mar 2016 11:10:20 -0800 Subject: [PATCH 103/160] Ruby: fix some synchronization code in server implementation --- src/ruby/.rubocop.yml | 3 + src/ruby/lib/grpc/generic/rpc_server.rb | 96 ++++++++++++++---------- src/ruby/spec/generic/rpc_server_spec.rb | 23 +----- 3 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/ruby/.rubocop.yml b/src/ruby/.rubocop.yml index ff5cf8db831..d13ce426553 100644 --- a/src/ruby/.rubocop.yml +++ b/src/ruby/.rubocop.yml @@ -15,3 +15,6 @@ Metrics/CyclomaticComplexity: Metrics/PerceivedComplexity: Max: 8 + +Metrics/ClassLength: + Max: 250 diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index ef2997c9918..eefa9577c47 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -107,7 +107,9 @@ module GRPC # Starts running the jobs in the thread pool. def start - fail 'already stopped' if @stopped + @stop_mutex.synchronize do + fail 'already stopped' if @stopped + end until @workers.size == @size.to_i next_thread = Thread.new do catch(:exit) do # allows { throw :exit } to kill a thread @@ -264,10 +266,13 @@ module GRPC @pool = Pool.new(@pool_size) @run_cond = ConditionVariable.new @run_mutex = Mutex.new - @running = false + # running_state can take 4 values: :not_started, :running, :stopping, and + # :stopped. State transitions can only proceed in that order. + @running_state = :not_started @server = RpcServer.setup_srv(server_override, @cq, **kw) - @stopped = false - @stop_mutex = Mutex.new + # Mutex to synchronize registration of services and registered service + # count. @run_mutex should not be acquired while holding @handle_mutex + @handle_mutex = Mutex.new end # stops a running server @@ -275,27 +280,28 @@ module GRPC # the call has no impact if the server is already stopped, otherwise # server's current call loop is it's last. def stop - return unless @running - @stop_mutex.synchronize do - @stopped = true + @run_mutex.synchronize do + fail 'Cannot stop before starting' if @running_state == :not_started + return if @running_state != :running + @running_state = :stopping end deadline = from_relative_time(@poll_period) - return if @server.close(@cq, deadline) - deadline = from_relative_time(@poll_period) @server.close(@cq, deadline) @pool.stop end - # determines if the server has been stopped - def stopped? - @stop_mutex.synchronize do - return @stopped + def running_state + @run_mutex.synchronize do + return @running_state end end - # determines if the server is currently running def running? - @running + running_state == :running + end + + def stopped? + running_state == :stopped end # Is called from other threads to wait for #run to start up the server. @@ -304,13 +310,11 @@ module GRPC # # @param timeout [Numeric] number of seconds to wait # @result [true, false] true if the server is running, false otherwise - def wait_till_running(timeout = 0.1) - end_time, sleep_period = Time.now + timeout, (1.0 * timeout) / 100 - while Time.now < end_time - @run_mutex.synchronize { @run_cond.wait(@run_mutex) } unless running? - sleep(sleep_period) + def wait_till_running(timeout = nil) + @run_mutex.synchronize do + @run_cond.wait(@run_mutex, timeout) if @running_state == :not_started + return @running_state == :running end - running? end # Runs the server in its own thread, then waits for signal INT or TERM on @@ -360,11 +364,16 @@ module GRPC # @param service [Object|Class] a service class or object as described # above def handle(service) - fail 'cannot add services if the server is running' if running? - fail 'cannot add services if the server is stopped' if stopped? - cls = service.is_a?(Class) ? service : service.class - assert_valid_service_class(cls) - add_rpc_descs_for(service) + @run_mutex.synchronize do + unless @running_state == :not_started + fail 'cannot add services if the server has been started' + end + cls = service.is_a?(Class) ? service : service.class + assert_valid_service_class(cls) + @handle_mutex.synchronize do + add_rpc_descs_for(service) + end + end end # runs the server @@ -375,16 +384,13 @@ module GRPC # - #running? returns true after this is called, until #stop cause the # the server to stop. def run - if rpc_descs.size.zero? - GRPC.logger.warn('did not run as no services were present') - return - end @run_mutex.synchronize do - @running = true - @run_cond.signal + fail 'cannot run without registering services' if rpc_descs.size.zero? + @pool.start + @server.start + @running_state = :running + @run_cond.broadcast end - @pool.start - @server.start loop_handle_server_calls end @@ -413,9 +419,9 @@ module GRPC # handles calls to the server def loop_handle_server_calls - fail 'not running' unless @running + fail 'not started' if running_state == :not_started loop_tag = Object.new - until stopped? + while running_state == :running begin an_rpc = @server.request_call(@cq, loop_tag, INFINITE_FUTURE) break if (!an_rpc.nil?) && an_rpc.call.nil? @@ -430,11 +436,14 @@ module GRPC rescue Core::CallError, RuntimeError => e # these might happen for various reasonse. The correct behaviour of # the server is to log them and continue, if it's not shutting down. - GRPC.logger.warn("server call failed: #{e}") unless stopped? + if running_state == :running + GRPC.logger.warn("server call failed: #{e}") + end next end end - @running = false + # @running_state should be :stopping here + @run_mutex.synchronize { @running_state = :stopped } GRPC.logger.info("stopped: #{self}") end @@ -467,11 +476,15 @@ module GRPC protected def rpc_descs - @rpc_descs ||= {} + @handle_mutex.synchronize do + return @rpc_descs ||= {} + end end def rpc_handlers - @rpc_handlers ||= {} + @handle_mutex.synchronize do + @rpc_handlers ||= {} + end end def assert_valid_service_class(cls) @@ -484,9 +497,10 @@ module GRPC cls.assert_rpc_descs_have_methods end + # This should be called while holding @handle_mutex def add_rpc_descs_for(service) cls = service.is_a?(Class) ? service : service.class - specs, handlers = rpc_descs, rpc_handlers + specs, handlers = (@rpc_descs ||= {}), (@rpc_handlers ||= {}) cls.rpc_descs.each_pair do |name, spec| route = "/#{cls.service_name}/#{name}".to_sym fail "already registered: rpc #{route} from #{spec}" if specs.key? route diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index be6331d68bb..e16ba60387e 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -220,19 +220,10 @@ describe GRPC::RpcServer do @srv = RpcServer.new(**opts) end - after(:each) do - @srv.stop - end - it 'starts out false' do expect(@srv.stopped?).to be(false) end - it 'stays false after a #stop is called before #run' do - @srv.stop - expect(@srv.stopped?).to be(false) - end - it 'stays false after the server starts running', server: true do @srv.handle(EchoService) t = Thread.new { @srv.run } @@ -247,8 +238,8 @@ describe GRPC::RpcServer do t = Thread.new { @srv.run } @srv.wait_till_running @srv.stop - expect(@srv.stopped?).to be(true) t.join + expect(@srv.stopped?).to be(true) end end @@ -266,9 +257,7 @@ describe GRPC::RpcServer do server_override: @server } r = RpcServer.new(**opts) - r.run - expect(r.running?).to be(false) - r.stop + expect { r.run }.to raise_error(RuntimeError) end it 'is true after run is called with a registered service' do @@ -293,10 +282,6 @@ describe GRPC::RpcServer do @srv = RpcServer.new(**@opts) end - after(:each) do - @srv.stop - end - it 'raises if #run has already been called' do @srv.handle(EchoService) t = Thread.new { @srv.run } @@ -528,10 +513,6 @@ describe GRPC::RpcServer do @srv = RpcServer.new(**server_opts) end - after(:each) do - @srv.stop - end - it 'should be added to BadStatus when requests fail', server: true do service = FailingService.new @srv.handle(service) From 3c77ff452fce5265c3454d26613bc6df529475ed Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 9 Mar 2016 11:44:56 -0800 Subject: [PATCH 104/160] Fix copyright --- src/ruby/spec/generic/rpc_server_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index e16ba60387e..dfaec6d6edc 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From ec49e156fe62c4c451cd65686efaa07999178166 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 11:51:23 -0800 Subject: [PATCH 105/160] Fix typo --- src/python/grpcio/tests/_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py index 38a5432e791..b0dbd92a495 100644 --- a/src/python/grpcio/tests/_runner.py +++ b/src/python/grpcio/tests/_runner.py @@ -144,7 +144,7 @@ class Runner(object): def run(self, suite): """See setuptools' test_runner setup argument for information.""" # only run test cases with id starting with given prefix - testcase_filter = os.getenv('GPRC_PYTHON_TESTRUNNER_FILTER') + testcase_filter = os.getenv('GRPC_PYTHON_TESTRUNNER_FILTER') filtered_cases = [] for case in _loader.iterate_suite_cases(suite): if not testcase_filter or case.id().startswith(testcase_filter): From 798b0dc4426f4485ad108a04c2dd44285ba2e700 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 9 Mar 2016 12:00:09 -0800 Subject: [PATCH 106/160] Doc Fixit: src/cpp/README need more information, revised 03092016 - Revised 'protoc' section and 'Documentation' section as LisaFC suggested. - Added 'Examples' section as LisaFC suggested. Resolves: #5661 --- src/cpp/README.md | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/cpp/README.md b/src/cpp/README.md index a13669b99e1..83d37aa2ed8 100644 --- a/src/cpp/README.md +++ b/src/cpp/README.md @@ -30,10 +30,8 @@ terminal: By default gRPC uses [protocol buffers](https://github.com/google/protobuf), you will need the `protoc` compiler to generate stub server and client code. -If you compile gRPC from source, as described below, the Makefile will -automatically try and compile the `protoc` in third party if you cloned the -repository recursively and it detects that you don't already have it -installed. +If you compile gRPC from source, as described below, this also installs the +`protoc` compiler. If it hasn't been installed, you can run the following commands to install it. @@ -45,9 +43,10 @@ $ sudo make install # 'make' should have been run by core grpc Alternatively, you can download `protoc` binaries from [the protocol buffers Github repository](https://github.com/google/protobuf/releases). -#INSTALLATION +#Installation -Currently to install gRPC for C++, you need to build from source as described below. +Currently to install gRPC for C++, you need to build from source as described +below. #Build from Source @@ -59,9 +58,25 @@ Currently to install gRPC for C++, you need to build from source as described be $ [sudo] make install ``` -#DOCUMENTATION +#Documentation -- The gRPC C++ refenrence documentation is available online at - [grpc.io](http://www.grpc.io/docs/tutorials/basic/c.html) -- [Helloworld example](../../examples/cpp/helloworld) +You can find out how to build and run our simplest gRPC C++ example in our +[C++ quick start](https://github.com/grpc/grpc/tree/{{ site.data.config.grpc_release_branch }}/examples/cpp). +For more detailed documentation on using gRPC in C++ , see our main +documentation site at [grpc.io](http://grpc.io), specifically: + +* [Overview](http://www.grpc.io/docs/): An introduction to gRPC with a simple + Hello World example in all our supported languages, including C++. +* [gRPC Basics - C++](http://www.grpc.io/docs/tutorials/basic/c.html): + A tutorial that steps you through creating a simple gRPC C++ example + application. +* [Asynchronous Basics - C++](http://www.grpc.io/docs/tutorials/async/helloasync-cpp.html): + A tutorial that shows you how to use gRPC C++'s asynchronous/non-blocking + APIs. + + +# Examples + +Code examples for gRPC C++ live in this repository's +[examples/cpp](https://github.com/grpc/grpc/tree/{{ site.data.config.grpc_release_branch }}/examples/cpp) directory. From 1f646dc8873db8ce9bc1aac189cc2cc471e25bb1 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 14:46:45 -0800 Subject: [PATCH 107/160] Release GIL in queue __dealloc__ --- .../grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index c139147114b..e11138b1cd8 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -140,7 +140,8 @@ cdef class CompletionQueue: grpc_completion_queue_shutdown(self.c_completion_queue) # Pump the queue while not self.is_shutdown: - event = grpc_completion_queue_next( - self.c_completion_queue, c_deadline, NULL) + with nogil: + event = grpc_completion_queue_next( + self.c_completion_queue, c_deadline, NULL) self._interpret_event(event) grpc_completion_queue_destroy(self.c_completion_queue) From f7ff8be43a721511b5b4cc0e277fe32e894f794d Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 15:27:28 -0800 Subject: [PATCH 108/160] Fix other typo to match previous typo fix --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index cc004f38d7f..2df82adf0a7 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -360,7 +360,7 @@ class PythonLanguage(object): ['tools/run_tests/run_python.sh'], None, environ=dict(environment.items() + - [('GPRC_PYTHON_TESTRUNNER_FILTER', suite_name)]), + [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]), shortname='py.test.%s' % suite_name, timeout_seconds=5*60) for suite_name in tests_json] From 68a94c8f072f14764130215fa0da45d5965471b2 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 11:45:07 -0800 Subject: [PATCH 109/160] Don't set linetrace unless testing coverage --- setup.py | 7 ++++++- tools/run_tests/build_python.sh | 6 +++++- tools/run_tests/run_python.sh | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e1abe7fd8ff..b21265c5756 100644 --- a/setup.py +++ b/setup.py @@ -108,8 +108,13 @@ if "linux" in sys.platform or "darwin" in sys.platform: def cython_extensions(package_names, module_names, extra_sources, include_dirs, libraries, define_macros, build_with_cython=False): + # Set compiler directives linetrace argument only if we care about tracing; + # this is due to Cython having different behavior between linetrace being + # False and linetrace being unset. See issue #5689. + cython_compiler_directives = {} if ENABLE_CYTHON_TRACING: define_macros = define_macros + [('CYTHON_TRACE_NOGIL', 1)] + cython_compiler_directives['linetrace'] = True file_extension = 'pyx' if build_with_cython else 'c' module_files = [os.path.join(PYTHON_STEM, name.replace('.', '/') + '.' + file_extension) @@ -129,7 +134,7 @@ def cython_extensions(package_names, module_names, extra_sources, include_dirs, return Cython.Build.cythonize( extensions, include_path=include_dirs, - compiler_directives={'linetrace': bool(ENABLE_CYTHON_TRACING)}) + compiler_directives=cython_compiler_directives) else: return extensions diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index f120fc7ed69..79a148faf1b 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -40,7 +40,11 @@ export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH export CFLAGS="-I$ROOT/include -std=gnu99" export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 -export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 + +if [ "$CONFIG" = "gcov" ] +then + export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 +fi tox --notest diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index ace49e15142..d4b7250cbb7 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -40,10 +40,10 @@ export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH export CFLAGS="-I$ROOT/include -std=c89" export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 -export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 if [ "$CONFIG" = "gcov" ] then + export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 tox else $ROOT/.tox/py27/bin/python $ROOT/setup.py test_lite From 1d68520c22fc708cdd6043962a0f59964a8aaf60 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 9 Mar 2016 17:30:37 -0800 Subject: [PATCH 110/160] Remove extraneous mutex, restrict state transitions --- src/ruby/lib/grpc/generic/rpc_server.rb | 37 ++++++++++++++----------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index eefa9577c47..b30d19dd2bb 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -270,9 +270,6 @@ module GRPC # :stopped. State transitions can only proceed in that order. @running_state = :not_started @server = RpcServer.setup_srv(server_override, @cq, **kw) - # Mutex to synchronize registration of services and registered service - # count. @run_mutex should not be acquired while holding @handle_mutex - @handle_mutex = Mutex.new end # stops a running server @@ -283,7 +280,7 @@ module GRPC @run_mutex.synchronize do fail 'Cannot stop before starting' if @running_state == :not_started return if @running_state != :running - @running_state = :stopping + transition_running_state(:stopping) end deadline = from_relative_time(@poll_period) @server.close(@cq, deadline) @@ -296,6 +293,20 @@ module GRPC end end + # Can only be called while holding @run_mutex + def transition_running_state(target_state) + state_transitions = { + not_started: :running, + running: :stopping, + stopping: :stopped + } + if state_transitions[@running_state] == target_state + @running_state = target_state + else + fail "Bad server state transition: #{@running_state}->#{target_state}" + end + end + def running? running_state == :running end @@ -370,9 +381,7 @@ module GRPC end cls = service.is_a?(Class) ? service : service.class assert_valid_service_class(cls) - @handle_mutex.synchronize do - add_rpc_descs_for(service) - end + add_rpc_descs_for(service) end end @@ -388,7 +397,7 @@ module GRPC fail 'cannot run without registering services' if rpc_descs.size.zero? @pool.start @server.start - @running_state = :running + transition_running_state(:running) @run_cond.broadcast end loop_handle_server_calls @@ -443,7 +452,7 @@ module GRPC end end # @running_state should be :stopping here - @run_mutex.synchronize { @running_state = :stopped } + @run_mutex.synchronize { transition_running_state(:stopped) } GRPC.logger.info("stopped: #{self}") end @@ -476,15 +485,11 @@ module GRPC protected def rpc_descs - @handle_mutex.synchronize do - return @rpc_descs ||= {} - end + @rpc_descs ||= {} end def rpc_handlers - @handle_mutex.synchronize do - @rpc_handlers ||= {} - end + @rpc_handlers ||= {} end def assert_valid_service_class(cls) @@ -497,7 +502,7 @@ module GRPC cls.assert_rpc_descs_have_methods end - # This should be called while holding @handle_mutex + # This should be called while holding @run_mutex def add_rpc_descs_for(service) cls = service.is_a?(Class) ? service : service.class specs, handlers = (@rpc_descs ||= {}), (@rpc_handlers ||= {}) From d4726c1ae63315726faa88e924c5749f13816ead Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 23 Feb 2016 16:57:36 -0800 Subject: [PATCH 111/160] add support for clang3.4 and 3.6 --- .../cxx_ubuntu1404_x64/Dockerfile.template | 39 +++++++++ .../test/cxx_ubuntu1404_x64/Dockerfile | 86 +++++++++++++++++++ tools/run_tests/run_tests.py | 24 ++++-- 3 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template create mode 100644 tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile diff --git a/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template new file mode 100644 index 00000000000..d824220afea --- /dev/null +++ b/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template @@ -0,0 +1,39 @@ +%YAML 1.2 +--- | + # Copyright 2015-2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM ubuntu:14.04 + + <%include file="../../apt_get_basic.include"/> + <%include file="../../cxx_deps.include"/> + <%include file="../../run_tests_addons.include"/> + # Define the default command. + CMD ["bash"] + \ No newline at end of file diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile new file mode 100644 index 00000000000..e8dab1b4712 --- /dev/null +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -0,0 +1,86 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM ubuntu:14.04 + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#====================== +# Zookeeper dependencies +# TODO(jtattermusch): is zookeeper still needed? +RUN apt-get install -y libzookeeper-mt-dev + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index cc004f38d7f..e687d463427 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -142,9 +142,8 @@ class CLanguage(object): self._make_options = [_windows_toolset_option(self.args.compiler), _windows_arch_option(self.args.arch)] else: - self._make_options = [] - self._docker_distro = self._get_docker_distro(self.args.use_docker, - self.args.compiler) + self._docker_distro, self._make_options = self._compiler_options(self.args.use_docker, + self.args.compiler) def test_specs(self): out = [] @@ -229,18 +228,26 @@ class CLanguage(object): def makefile_name(self): return 'Makefile' - def _get_docker_distro(self, use_docker, compiler): + def _clang_make_options(self): + return ['CC=clang', 'CXX=clang++', 'LD=clang', 'LDXX=clang++'] + + def _compiler_options(self, use_docker, compiler): + """Returns docker distro and make options to use for given compiler.""" if _is_use_docker_child(): - return "already_under_docker" + return ("already_under_docker", []) if not use_docker: _check_compiler(compiler, ['default']) if compiler == 'gcc4.9' or compiler == 'default': - return 'jessie' + return ('jessie', []) elif compiler == 'gcc4.4': - return 'squeeze' + return ('squeeze', []) elif compiler == 'gcc5.3': - return 'ubuntu1604' + return ('ubuntu1604', []) + elif compiler == 'clang3.4': + return ('ubuntu1404', self._clang_make_options()) + elif compiler == 'clang3.6': + return ('ubuntu1604', self._clang_make_options()) else: raise Exception('Compiler %s not supported.' % compiler) @@ -774,6 +781,7 @@ argp.add_argument('--arch', argp.add_argument('--compiler', choices=['default', 'gcc4.4', 'gcc4.9', 'gcc5.3', + 'clang3.4', 'clang3.6', 'vs2010', 'vs2013', 'vs2015'], default='default', help='Selects compiler to use. Allowed values depend on the platform and language.') From 60cbec7c19516065e49800b31d55d7bb260ebf40 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 26 Feb 2016 13:47:16 -0800 Subject: [PATCH 112/160] fix C# artifacts compatibility --- tools/run_tests/artifact_targets.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 288a3f01542..e61c46d8b5a 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -69,16 +69,6 @@ def create_jobspec(name, cmdline, environ=None, shell=False, return jobspec -def macos_arch_env(arch): - """Returns environ specifying -arch arguments for make.""" - if arch == 'x86': - arch_arg = '-arch i386' - elif arch == 'x64': - arch_arg = '-arch x86_64' - else: - raise Exception('Unsupported arch') - return {'CFLAGS': arch_arg, 'LDFLAGS': arch_arg} - _MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' _ARCH_FLAG_MAP = { @@ -191,13 +181,17 @@ class CSharpExtArtifact: environ = {'CONFIG': 'opt', 'EMBED_OPENSSL': 'true', 'EMBED_ZLIB': 'true', - 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE'} + 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', + 'LDFLAGS': ''} if self.platform == 'linux': return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, - 'tools/run_tests/build_artifact_csharp.sh') + 'tools/run_tests/build_artifact_csharp.sh', + environ=environ) else: - environ.update(macos_arch_env(self.arch)) + archflag = _ARCH_FLAG_MAP[self.arch] + environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) + environ['LDFLAGS'] += ' %s' % archflag return create_jobspec(self.name, ['tools/run_tests/build_artifact_csharp.sh'], environ=environ) From cbdd9844f9cd42123ee6334e8f2aa1b49c500c27 Mon Sep 17 00:00:00 2001 From: Harsh Vardhan Date: Thu, 10 Mar 2016 20:51:20 +0530 Subject: [PATCH 113/160] updates contributing.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 011ce43bcdc..850ea14399c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,7 +36,7 @@ In order to run most of the available tests, one would need to run: `./tools/run_tests/run_tests.py` -If you want to run tests for any of the languages {all, c, c++, csharp, node, objc, php, python, ruby}, do this: +If you want to run tests for any of the languages {c, c++, csharp, node, objc, php, python, ruby}, do this: `./tools/run_tests/run_tests.py -l ` From fa45a5e96c2f31682047bb96508c85bf7746642b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 10 Mar 2016 14:38:27 -0800 Subject: [PATCH 114/160] fix race between send message and send initial metadata --- src/core/transport/chttp2_transport.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 19265252ca3..03444fd4c24 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -851,9 +851,11 @@ static void perform_stream_op_locked( if (stream_global->write_closed) { grpc_chttp2_complete_closure_step( exec_ctx, &stream_global->send_message_finished, 0); - } else if (stream_global->id != 0) { + } else { stream_global->send_message = op->send_message; - grpc_chttp2_become_writable(transport_global, stream_global); + if (stream_global->id != 0) { + grpc_chttp2_become_writable(transport_global, stream_global); + } } } From 7212f16700f765f9acd9681d945401a7ba81081a Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 19:29:11 -0800 Subject: [PATCH 115/160] php: fix examples ClientStub constructor --- examples/php/greeter_client.php | 4 +++- examples/php/route_guide/route_guide_client.php | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/php/greeter_client.php b/examples/php/greeter_client.php index e5e4c2651e9..718ef88c644 100644 --- a/examples/php/greeter_client.php +++ b/examples/php/greeter_client.php @@ -36,7 +36,9 @@ require dirname(__FILE__) . '/vendor/autoload.php'; require dirname(__FILE__) . '/helloworld.php'; function greet($name) { - $client = new helloworld\GreeterClient('localhost:50051', []); + $client = new helloworld\GreeterClient('localhost:50051', [ + 'credentials' => Grpc\ChannelCredentials::createInsecure() + ]); $request = new helloworld\HelloRequest(); $request->setName($name); list($reply, $status) = $client->SayHello($request)->wait(); diff --git a/examples/php/route_guide/route_guide_client.php b/examples/php/route_guide/route_guide_client.php index 3cd1df72541..2f9533be4b7 100644 --- a/examples/php/route_guide/route_guide_client.php +++ b/examples/php/route_guide/route_guide_client.php @@ -37,7 +37,9 @@ require dirname(__FILE__) . '/route_guide.php'; define('COORD_FACTOR', 1e7); -$client = new routeguide\RouteGuideClient('localhost:50051', []); +$client = new routeguide\RouteGuideClient('localhost:50051', [ + 'credentials' => Grpc\ChannelCredentials::createInsecure() +]); function printFeature($feature) { $name = $feature->getName(); From 40202158fddf89d16b0e638d9cf415489e948926 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 21:16:39 -0800 Subject: [PATCH 116/160] fix copyright --- examples/php/greeter_client.php | 2 +- examples/php/route_guide/route_guide_client.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/php/greeter_client.php b/examples/php/greeter_client.php index 718ef88c644..3fab14f33e1 100644 --- a/examples/php/greeter_client.php +++ b/examples/php/greeter_client.php @@ -1,7 +1,7 @@ Date: Mon, 7 Mar 2016 16:09:21 -0800 Subject: [PATCH 117/160] one more --- src/php/tests/generated_code/math_client.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/php/tests/generated_code/math_client.php b/src/php/tests/generated_code/math_client.php index 76ccabc0684..b8652ceb088 100644 --- a/src/php/tests/generated_code/math_client.php +++ b/src/php/tests/generated_code/math_client.php @@ -43,7 +43,9 @@ function p($line) $host = 'localhost:50051'; p("Connecting to host: $host"); -$client = new math\MathClient($host, []); +$client = new math\MathClient($host, [ + 'credentials' => Grpc\ChannelCredentials::createInsecure() +]); p('Client class: '.get_class($client)); p(''); From cfad24393931706e667554a867aa98a6c88236ea Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 8 Mar 2016 08:41:08 -0800 Subject: [PATCH 118/160] fixed copyright --- src/php/tests/generated_code/math_client.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/php/tests/generated_code/math_client.php b/src/php/tests/generated_code/math_client.php index b8652ceb088..2085560d19e 100644 --- a/src/php/tests/generated_code/math_client.php +++ b/src/php/tests/generated_code/math_client.php @@ -1,7 +1,7 @@ Date: Thu, 3 Mar 2016 20:39:01 -0800 Subject: [PATCH 119/160] update PHP composer.json files --- composer.json | 11 +++++++++-- examples/php/composer.json | 11 +---------- src/php/composer.json | 6 ++++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/composer.json b/composer.json index 61f81e02bf4..97b1a5cb49b 100644 --- a/composer.json +++ b/composer.json @@ -2,13 +2,20 @@ "name": "grpc/grpc", "type": "library", "description": "gRPC library for PHP", - "version": "0.6.0", + "version": "0.14.0", "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/stanley-cheung/Protobuf-PHP" + } + ], "require": { "php": ">=5.5.0", - "google/auth": "dev-master" + "datto/protobuf-php": "dev-master", + "google/auth": "v0.7" }, "autoload": { "psr-4": { diff --git a/examples/php/composer.json b/examples/php/composer.json index 9f44f0b186a..314066b22df 100644 --- a/examples/php/composer.json +++ b/examples/php/composer.json @@ -1,17 +1,8 @@ { - "repositories": [ - { - "type": "vcs", - "url": "https://github.com/stanley-cheung/Protobuf-PHP" - } - ], "name": "grpc/grpc-demo", "description": "gRPC example for PHP", "minimum-stability": "dev", "require": { - "php": ">=5.5.0", - "datto/protobuf-php": "dev-master", - "google/auth": "dev-master", - "grpc/grpc": "dev-release-0_11" + "grpc/grpc": "dev-release-0_13" } } diff --git a/src/php/composer.json b/src/php/composer.json index 1d41f847ac0..01674a25db8 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -1,7 +1,9 @@ { "name": "grpc/grpc", + "type": "library", "description": "gRPC library for PHP", - "version": "0.6.0", + "version": "0.14.0", + "keywords": ["rpc"], "homepage": "http://grpc.io", "license": "BSD-3-Clause", "repositories": [ @@ -13,7 +15,7 @@ "require": { "php": ">=5.5.0", "datto/protobuf-php": "dev-master", - "google/auth": "dev-master" + "google/auth": "v0.7" }, "autoload": { "psr-4": { From 7fc1208a5cee894cada499a33d6a7e36d4209edc Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 22:14:54 -0800 Subject: [PATCH 120/160] still need this block for the example composer.json file --- examples/php/composer.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/php/composer.json b/examples/php/composer.json index 314066b22df..c837bf7ac0f 100644 --- a/examples/php/composer.json +++ b/examples/php/composer.json @@ -2,6 +2,12 @@ "name": "grpc/grpc-demo", "description": "gRPC example for PHP", "minimum-stability": "dev", + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/stanley-cheung/Protobuf-PHP" + } + ], "require": { "grpc/grpc": "dev-release-0_13" } From 5709e930faec4e88e63cec7edb396264031246a7 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 7 Mar 2016 17:34:47 -0800 Subject: [PATCH 121/160] remove stale src/php/ext/grpc/README.md --- package.xml | 2 +- src/php/ext/grpc/README.md | 67 ---------------------------------- templates/package.xml.template | 2 +- 3 files changed, 2 insertions(+), 69 deletions(-) delete mode 100644 src/php/ext/grpc/README.md diff --git a/package.xml b/package.xml index e65ab73b54a..bb4dcc2037a 100644 --- a/package.xml +++ b/package.xml @@ -27,9 +27,9 @@ + - diff --git a/src/php/ext/grpc/README.md b/src/php/ext/grpc/README.md deleted file mode 100644 index 6e1cb2002f5..00000000000 --- a/src/php/ext/grpc/README.md +++ /dev/null @@ -1,67 +0,0 @@ -gRPC PHP Extension -================== - -# Requirements - - * PHP 5.5+ - * [gRPC core library](https://github.com/grpc/grpc) 0.11.0 - -# Installation - -## Install PHP 5 - -``` -$ sudo apt-get install git php5 php5-dev php-pear unzip -``` - -## Compile gRPC Core Library - -Clone the gRPC source code repository - -``` -$ git clone https://github.com/grpc/grpc.git -``` - -Build and install the gRPC C core libraries - -```sh -$ cd grpc -$ git checkout --track origin/release-0_11 -$ git pull --recurse-submodules && git submodule update --init --recursive -$ make -$ sudo make install -``` - -Note: you may encounter a warning about the Protobuf compiler `protoc` 3.0.0+ not being installed. The following might help, and will be useful later on when we need to compile the `protoc-gen-php` tool. - -```sh -$ cd grpc/third_party/protobuf -$ sudo make install # 'make' should have been run by core grpc -``` - -## Install the gRPC PHP extension - -Quick install - -```sh -$ sudo pecl install grpc -``` - -Note: before a stable release, you may need to do - -```sh -$ sudo pecl install grpc-beta -``` - -OR - -Compile from source - -```sh -$ # from grpc -$ cd src/php/ext/grpc -$ phpize -$ ./configure -$ make -$ sudo make install -``` diff --git a/templates/package.xml.template b/templates/package.xml.template index d309bfddbc3..8f757401e4c 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -29,9 +29,9 @@ + - % for source in php_config_m4.src + php_config_m4.headers: % endfor From e1e4559690eb5ecea7b8c1a376936f1095dd0c11 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 08:01:58 -0800 Subject: [PATCH 122/160] Make blocking_resolve_address overridable --- src/core/iomgr/resolve_address.h | 4 ++-- src/core/iomgr/resolve_address_posix.c | 19 +++++++++++-------- src/core/iomgr/resolve_address_windows.c | 19 +++++++++++-------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/core/iomgr/resolve_address.h b/src/core/iomgr/resolve_address.h index 01eedffa889..508e161a7e0 100644 --- a/src/core/iomgr/resolve_address.h +++ b/src/core/iomgr/resolve_address.h @@ -66,7 +66,7 @@ void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addresses); /* Resolve addr in a blocking fashion. Returns NULL on failure. On success, result must be freed with grpc_resolved_addresses_destroy. */ -grpc_resolved_addresses *grpc_blocking_resolve_address( - const char *addr, const char *default_port); +extern grpc_resolved_addresses *(*grpc_blocking_resolve_address)( + const char *name, const char *default_port); #endif /* GRPC_INTERNAL_CORE_IOMGR_RESOLVE_ADDRESS_H */ diff --git a/src/core/iomgr/resolve_address_posix.c b/src/core/iomgr/resolve_address_posix.c index c51745b9187..a6c9893f231 100644 --- a/src/core/iomgr/resolve_address_posix.c +++ b/src/core/iomgr/resolve_address_posix.c @@ -34,18 +34,13 @@ #include #ifdef GPR_POSIX_SOCKET -#include "src/core/iomgr/sockaddr.h" #include "src/core/iomgr/resolve_address.h" +#include "src/core/iomgr/sockaddr.h" +#include #include #include -#include -#include "src/core/iomgr/executor.h" -#include "src/core/iomgr/iomgr_internal.h" -#include "src/core/iomgr/sockaddr_utils.h" -#include "src/core/support/block_annotate.h" -#include "src/core/support/string.h" #include #include #include @@ -53,6 +48,11 @@ #include #include #include +#include "src/core/iomgr/executor.h" +#include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/sockaddr_utils.h" +#include "src/core/support/block_annotate.h" +#include "src/core/support/string.h" typedef struct { char *name; @@ -62,7 +62,7 @@ typedef struct { void *arg; } request; -grpc_resolved_addresses *grpc_blocking_resolve_address( +static grpc_resolved_addresses *blocking_resolve_address_impl( const char *name, const char *default_port) { struct addrinfo hints; struct addrinfo *result = NULL, *resp; @@ -150,6 +150,9 @@ done: return addrs; } +grpc_resolved_addresses *(*grpc_blocking_resolve_address)( + const char *name, const char *default_port) = blocking_resolve_address_impl; + /* Callback to be passed to grpc_executor to asynch-ify * grpc_blocking_resolve_address */ static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) { diff --git a/src/core/iomgr/resolve_address_windows.c b/src/core/iomgr/resolve_address_windows.c index 28c8661e73b..472e7971634 100644 --- a/src/core/iomgr/resolve_address_windows.c +++ b/src/core/iomgr/resolve_address_windows.c @@ -34,17 +34,12 @@ #include #ifdef GPR_WINSOCK_SOCKET -#include "src/core/iomgr/sockaddr.h" #include "src/core/iomgr/resolve_address.h" +#include "src/core/iomgr/sockaddr.h" -#include #include +#include -#include "src/core/iomgr/executor.h" -#include "src/core/iomgr/iomgr_internal.h" -#include "src/core/iomgr/sockaddr_utils.h" -#include "src/core/support/block_annotate.h" -#include "src/core/support/string.h" #include #include #include @@ -52,6 +47,11 @@ #include #include #include +#include "src/core/iomgr/executor.h" +#include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/sockaddr_utils.h" +#include "src/core/support/block_annotate.h" +#include "src/core/support/string.h" typedef struct { char *name; @@ -61,7 +61,7 @@ typedef struct { void *arg; } request; -grpc_resolved_addresses *grpc_blocking_resolve_address( +static grpc_resolved_addresses *blocking_resolve_address_impl( const char *name, const char *default_port) { struct addrinfo hints; struct addrinfo *result = NULL, *resp; @@ -133,6 +133,9 @@ done: return addrs; } +grpc_resolved_addresses *(*grpc_blocking_resolve_address)( + const char *name, const char *default_port) = blocking_resolve_address_impl; + /* Callback to be passed to grpc_executor to asynch-ify * grpc_blocking_resolve_address */ static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, bool success) { From e91ef68d2a9077641d6ece47b388e523942164fe Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 08:59:17 -0800 Subject: [PATCH 123/160] Add failing test --- Makefile | 36 ++++ build.yaml | 10 + .../dns_resolver_connectivity_test.c | 133 ++++++++++++ tools/run_tests/sources_and_headers.json | 16 ++ tools/run_tests/tests.json | 21 ++ vsprojects/buildtests_c.sln | 27 +++ .../dns_resolver_connectivity_test.vcxproj | 199 ++++++++++++++++++ ...resolver_connectivity_test.vcxproj.filters | 24 +++ 8 files changed, 466 insertions(+) create mode 100644 test/core/client_config/resolvers/dns_resolver_connectivity_test.c create mode 100644 vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj create mode 100644 vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj.filters diff --git a/Makefile b/Makefile index 5be0d146afc..171a83c2a3a 100644 --- a/Makefile +++ b/Makefile @@ -849,6 +849,7 @@ chttp2_status_conversion_test: $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test chttp2_stream_map_test: $(BINDIR)/$(CONFIG)/chttp2_stream_map_test chttp2_varint_test: $(BINDIR)/$(CONFIG)/chttp2_varint_test compression_test: $(BINDIR)/$(CONFIG)/compression_test +dns_resolver_connectivity_test: $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test dns_resolver_test: $(BINDIR)/$(CONFIG)/dns_resolver_test dualstack_socket_test: $(BINDIR)/$(CONFIG)/dualstack_socket_test endpoint_pair_test: $(BINDIR)/$(CONFIG)/endpoint_pair_test @@ -1159,6 +1160,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/chttp2_stream_map_test \ $(BINDIR)/$(CONFIG)/chttp2_varint_test \ $(BINDIR)/$(CONFIG)/compression_test \ + $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test \ $(BINDIR)/$(CONFIG)/dns_resolver_test \ $(BINDIR)/$(CONFIG)/dualstack_socket_test \ $(BINDIR)/$(CONFIG)/endpoint_pair_test \ @@ -1402,6 +1404,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/chttp2_varint_test || ( echo test chttp2_varint_test failed ; exit 1 ) $(E) "[RUN] Testing compression_test" $(Q) $(BINDIR)/$(CONFIG)/compression_test || ( echo test compression_test failed ; exit 1 ) + $(E) "[RUN] Testing dns_resolver_connectivity_test" + $(Q) $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test || ( echo test dns_resolver_connectivity_test failed ; exit 1 ) $(E) "[RUN] Testing dns_resolver_test" $(Q) $(BINDIR)/$(CONFIG)/dns_resolver_test || ( echo test dns_resolver_test failed ; exit 1 ) $(E) "[RUN] Testing dualstack_socket_test" @@ -6075,6 +6079,38 @@ endif endif +DNS_RESOLVER_CONNECTIVITY_TEST_SRC = \ + test/core/client_config/resolvers/dns_resolver_connectivity_test.c \ + +DNS_RESOLVER_CONNECTIVITY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(DNS_RESOLVER_CONNECTIVITY_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test: $(DNS_RESOLVER_CONNECTIVITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(DNS_RESOLVER_CONNECTIVITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/client_config/resolvers/dns_resolver_connectivity_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_dns_resolver_connectivity_test: $(DNS_RESOLVER_CONNECTIVITY_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(DNS_RESOLVER_CONNECTIVITY_TEST_OBJS:.o=.dep) +endif +endif + + DNS_RESOLVER_TEST_SRC = \ test/core/client_config/resolvers/dns_resolver_test.c \ diff --git a/build.yaml b/build.yaml index a277539a054..7cafb0ac6fe 100644 --- a/build.yaml +++ b/build.yaml @@ -1057,6 +1057,16 @@ targets: - grpc - gpr_test_util - gpr +- name: dns_resolver_connectivity_test + build: test + language: c + src: + - test/core/client_config/resolvers/dns_resolver_connectivity_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: dns_resolver_test build: test language: c diff --git a/test/core/client_config/resolvers/dns_resolver_connectivity_test.c b/test/core/client_config/resolvers/dns_resolver_connectivity_test.c new file mode 100644 index 00000000000..e4d83539fb9 --- /dev/null +++ b/test/core/client_config/resolvers/dns_resolver_connectivity_test.c @@ -0,0 +1,133 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/client_config/resolvers/dns_resolver.h" + +#include + +#include +#include + +#include "src/core/iomgr/resolve_address.h" +#include "test/core/util/test_config.h" + +static void subchannel_factory_ref(grpc_subchannel_factory *scv) {} +static void subchannel_factory_unref(grpc_exec_ctx *exec_ctx, + grpc_subchannel_factory *scv) {} +static grpc_subchannel *subchannel_factory_create_subchannel( + grpc_exec_ctx *exec_ctx, grpc_subchannel_factory *factory, + grpc_subchannel_args *args) { + GPR_UNREACHABLE_CODE(return NULL); +} + +static const grpc_subchannel_factory_vtable sc_vtable = { + subchannel_factory_ref, subchannel_factory_unref, + subchannel_factory_create_subchannel}; + +static grpc_subchannel_factory sc_factory = {&sc_vtable}; + +static gpr_mu g_mu; +static bool g_fail_resolution = true; + +static grpc_resolved_addresses *my_resolve_address(const char *name, + const char *addr) { + gpr_mu_lock(&g_mu); + GPR_ASSERT(0 == strcmp("test", name)); + if (g_fail_resolution) { + g_fail_resolution = false; + gpr_mu_unlock(&g_mu); + return NULL; + } else { + gpr_mu_unlock(&g_mu); + grpc_resolved_addresses *addrs = gpr_malloc(sizeof(*addrs)); + addrs->naddrs = 1; + addrs->addrs = gpr_malloc(sizeof(*addrs->addrs)); + addrs->addrs[0].len = 123; + return addrs; + } +} + +static grpc_resolver *create_resolver(const char *name) { + grpc_resolver_factory *factory = grpc_dns_resolver_factory_create(); + grpc_uri *uri = grpc_uri_parse(name, 0); + GPR_ASSERT(uri); + grpc_resolver_args args; + memset(&args, 0, sizeof(args)); + args.uri = uri; + args.subchannel_factory = &sc_factory; + grpc_resolver *resolver = + grpc_resolver_factory_create_resolver(factory, &args); + grpc_resolver_factory_unref(factory); + grpc_uri_destroy(uri); + return resolver; +} + +static void on_done(grpc_exec_ctx *exec_ctx, void *ev, bool success) { + gpr_event_set(ev, (void *)1); +} + +int main(int argc, char **argv) { + grpc_test_init(argc, argv); + + grpc_init(); + gpr_mu_init(&g_mu); + grpc_blocking_resolve_address = my_resolve_address; + + grpc_resolver *resolver = create_resolver("dns:test"); + + grpc_client_config *config = (grpc_client_config *)1; + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_event ev1; + gpr_event_init(&ev1); + grpc_resolver_next(&exec_ctx, resolver, &config, + grpc_closure_create(on_done, &ev1)); + grpc_exec_ctx_flush(&exec_ctx); + GPR_ASSERT(gpr_event_wait(&ev1, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + GPR_ASSERT(config == NULL); + + gpr_event ev2; + gpr_event_init(&ev2); + grpc_resolver_next(&exec_ctx, resolver, &config, + grpc_closure_create(on_done, &ev2)); + grpc_exec_ctx_flush(&exec_ctx); + GPR_ASSERT(gpr_event_wait(&ev2, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + GPR_ASSERT(config != NULL); + + grpc_client_config_unref(&exec_ctx, config); + GRPC_RESOLVER_UNREF(&exec_ctx, resolver, "test"); + grpc_exec_ctx_finish(&exec_ctx); + + grpc_shutdown(); + gpr_mu_destroy(&g_mu); +} diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 0c7a6c7b5f3..0039b99a55e 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -189,6 +189,22 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "dns_resolver_connectivity_test", + "src": [ + "test/core/client_config/resolvers/dns_resolver_connectivity_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index d91245cd06a..c2d6cadd530 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -253,6 +253,27 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "dns_resolver_connectivity_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 7cdc3b7ee81..be2c4a3bb14 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -251,6 +251,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "compression_test", "vcxproj {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dns_resolver_connectivity_test", "vcxproj\test\dns_resolver_connectivity_test\dns_resolver_connectivity_test.vcxproj", "{F7B6FE68-E847-D7CA-4062-E737E542BCC3}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dns_resolver_test", "vcxproj\test\dns_resolver_test\dns_resolver_test.vcxproj", "{D06E10DC-272A-5203-7066-2698A247DF26}" ProjectSection(myProperties) = preProject lib = "False" @@ -1736,6 +1747,22 @@ Global {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|Win32.Build.0 = Release|Win32 {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|x64.ActiveCfg = Release|x64 {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|x64.Build.0 = Release|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|Win32.ActiveCfg = Debug|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|x64.ActiveCfg = Debug|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|Win32.ActiveCfg = Release|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|x64.ActiveCfg = Release|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|Win32.Build.0 = Debug|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|x64.Build.0 = Debug|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|Win32.Build.0 = Release|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|x64.Build.0 = Release|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug-DLL|x64.Build.0 = Debug|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release-DLL|Win32.Build.0 = Release|Win32 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release-DLL|x64.ActiveCfg = Release|x64 + {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release-DLL|x64.Build.0 = Release|x64 {D06E10DC-272A-5203-7066-2698A247DF26}.Debug|Win32.ActiveCfg = Debug|Win32 {D06E10DC-272A-5203-7066-2698A247DF26}.Debug|x64.ActiveCfg = Debug|x64 {D06E10DC-272A-5203-7066-2698A247DF26}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj b/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj new file mode 100644 index 00000000000..76ac196dda6 --- /dev/null +++ b/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj @@ -0,0 +1,199 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {F7B6FE68-E847-D7CA-4062-E737E542BCC3} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + dns_resolver_connectivity_test + static + Debug + static + Debug + + + dns_resolver_connectivity_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj.filters b/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj.filters new file mode 100644 index 00000000000..e49318df110 --- /dev/null +++ b/vsprojects/vcxproj/test/dns_resolver_connectivity_test/dns_resolver_connectivity_test.vcxproj.filters @@ -0,0 +1,24 @@ + + + + + test\core\client_config\resolvers + + + + + + {7743e287-3311-1206-2766-32381628ff07} + + + {cc06b800-cd26-f7a8-7ecf-ec789e78881b} + + + {4fbd5c0a-d1e3-6658-5788-2fc6f2cc9071} + + + {eab637a2-7ac1-f6be-4fc2-c8989c66070b} + + + + From 1e55bd455d5fa901e19c1ccf64996c0adc7a4584 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 09:47:43 -0800 Subject: [PATCH 124/160] Add retry for dns resolution --- src/core/client_config/client_config.c | 4 +- .../client_config/lb_policies/pick_first.c | 2 +- .../client_config/resolvers/dns_resolver.c | 53 ++++++++++++++++--- .../dns_resolver_connectivity_test.c | 21 ++++++-- 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/src/core/client_config/client_config.c b/src/core/client_config/client_config.c index 6ecffb38541..25f4ac25177 100644 --- a/src/core/client_config/client_config.c +++ b/src/core/client_config/client_config.c @@ -53,7 +53,9 @@ void grpc_client_config_ref(grpc_client_config *c) { gpr_ref(&c->refs); } void grpc_client_config_unref(grpc_exec_ctx *exec_ctx, grpc_client_config *c) { if (gpr_unref(&c->refs)) { - GRPC_LB_POLICY_UNREF(exec_ctx, c->lb_policy, "client_config"); + if (c->lb_policy != NULL) { + GRPC_LB_POLICY_UNREF(exec_ctx, c->lb_policy, "client_config"); + } gpr_free(c); } } diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c index 81167b31c8c..8ed1223d399 100644 --- a/src/core/client_config/lb_policies/pick_first.c +++ b/src/core/client_config/lb_policies/pick_first.c @@ -387,8 +387,8 @@ static void pick_first_factory_unref(grpc_lb_policy_factory *factory) {} static grpc_lb_policy *create_pick_first(grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { + if (args->num_subchannels == 0) return NULL; pick_first_lb_policy *p = gpr_malloc(sizeof(*p)); - GPR_ASSERT(args->num_subchannels > 0); memset(p, 0, sizeof(*p)); grpc_lb_policy_init(&p->base, &pick_first_lb_policy_vtable); p->subchannels = diff --git a/src/core/client_config/resolvers/dns_resolver.c b/src/core/client_config/resolvers/dns_resolver.c index 376b6b3d768..55457647b3c 100644 --- a/src/core/client_config/resolvers/dns_resolver.c +++ b/src/core/client_config/resolvers/dns_resolver.c @@ -41,6 +41,7 @@ #include "src/core/client_config/lb_policy_registry.h" #include "src/core/iomgr/resolve_address.h" +#include "src/core/iomgr/timer.h" #include "src/core/support/string.h" typedef struct { @@ -71,6 +72,9 @@ typedef struct { grpc_client_config **target_config; /** current (fully resolved) config */ grpc_client_config *resolved_config; + /** retry timer */ + bool have_retry_timer; + grpc_timer retry_timer; } dns_resolver; static void dns_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *r); @@ -125,6 +129,21 @@ static void dns_next(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver, gpr_mu_unlock(&r->mu); } +static void dns_on_retry_timer(grpc_exec_ctx *exec_ctx, void *arg, + bool success) { + dns_resolver *r = arg; + + if (success) { + gpr_mu_lock(&r->mu); + if (!r->resolving) { + dns_start_resolving_locked(r); + } + gpr_mu_unlock(&r->mu); + } + + GRPC_RESOLVER_UNREF(exec_ctx, &r->base, "retry-timer"); +} + static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_resolved_addresses *addresses) { dns_resolver *r = arg; @@ -133,29 +152,47 @@ static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_subchannel_args args; grpc_lb_policy *lb_policy; size_t i; - if (addresses) { + gpr_mu_lock(&r->mu); + GPR_ASSERT(r->resolving); + r->resolving = 0; + if (addresses != NULL) { grpc_lb_policy_args lb_policy_args; config = grpc_client_config_create(); subchannels = gpr_malloc(sizeof(grpc_subchannel *) * addresses->naddrs); + size_t naddrs = 0; for (i = 0; i < addresses->naddrs; i++) { memset(&args, 0, sizeof(args)); args.addr = (struct sockaddr *)(addresses->addrs[i].addr); args.addr_len = (size_t)addresses->addrs[i].len; - subchannels[i] = grpc_subchannel_factory_create_subchannel( + grpc_subchannel *subchannel = grpc_subchannel_factory_create_subchannel( exec_ctx, r->subchannel_factory, &args); + if (subchannel != NULL) { + subchannels[naddrs++] = subchannel; + } } memset(&lb_policy_args, 0, sizeof(lb_policy_args)); lb_policy_args.subchannels = subchannels; - lb_policy_args.num_subchannels = addresses->naddrs; + lb_policy_args.num_subchannels = naddrs; lb_policy = grpc_lb_policy_create(r->lb_policy_name, &lb_policy_args); - grpc_client_config_set_lb_policy(config, lb_policy); - GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction"); + if (lb_policy != NULL) { + grpc_client_config_set_lb_policy(config, lb_policy); + GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction"); + } grpc_resolved_addresses_destroy(addresses); gpr_free(subchannels); + } else { + int retry_seconds = 15; + gpr_log(GPR_DEBUG, "dns resolution failed: retrying in %d seconds", + retry_seconds); + GPR_ASSERT(!r->have_retry_timer); + r->have_retry_timer = true; + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + GRPC_RESOLVER_REF(&r->base, "retry-timer"); + grpc_timer_init( + exec_ctx, &r->retry_timer, + gpr_time_add(now, gpr_time_from_seconds(retry_seconds, GPR_TIMESPAN)), + dns_on_retry_timer, r, now); } - gpr_mu_lock(&r->mu); - GPR_ASSERT(r->resolving); - r->resolving = 0; if (r->resolved_config) { grpc_client_config_unref(exec_ctx, r->resolved_config); } diff --git a/test/core/client_config/resolvers/dns_resolver_connectivity_test.c b/test/core/client_config/resolvers/dns_resolver_connectivity_test.c index e4d83539fb9..75d1eb674f6 100644 --- a/test/core/client_config/resolvers/dns_resolver_connectivity_test.c +++ b/test/core/client_config/resolvers/dns_resolver_connectivity_test.c @@ -39,6 +39,7 @@ #include #include "src/core/iomgr/resolve_address.h" +#include "src/core/iomgr/timer.h" #include "test/core/util/test_config.h" static void subchannel_factory_ref(grpc_subchannel_factory *scv) {} @@ -47,7 +48,7 @@ static void subchannel_factory_unref(grpc_exec_ctx *exec_ctx, static grpc_subchannel *subchannel_factory_create_subchannel( grpc_exec_ctx *exec_ctx, grpc_subchannel_factory *factory, grpc_subchannel_args *args) { - GPR_UNREACHABLE_CODE(return NULL); + return NULL; } static const grpc_subchannel_factory_vtable sc_vtable = { @@ -96,6 +97,20 @@ static void on_done(grpc_exec_ctx *exec_ctx, void *ev, bool success) { gpr_event_set(ev, (void *)1); } +// interleave waiting for an event with a timer check +static bool wait_loop(int deadline_seconds, gpr_event *ev) { + while (deadline_seconds) { + gpr_log(GPR_DEBUG, "Test: waiting for %d more seconds", deadline_seconds); + if (gpr_event_wait(ev, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1))) return true; + deadline_seconds--; + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_timer_check(&exec_ctx, gpr_now(GPR_CLOCK_MONOTONIC), NULL); + grpc_exec_ctx_finish(&exec_ctx); + } + return false; +} + int main(int argc, char **argv) { grpc_test_init(argc, argv); @@ -113,7 +128,7 @@ int main(int argc, char **argv) { grpc_resolver_next(&exec_ctx, resolver, &config, grpc_closure_create(on_done, &ev1)); grpc_exec_ctx_flush(&exec_ctx); - GPR_ASSERT(gpr_event_wait(&ev1, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + GPR_ASSERT(wait_loop(5, &ev1)); GPR_ASSERT(config == NULL); gpr_event ev2; @@ -121,7 +136,7 @@ int main(int argc, char **argv) { grpc_resolver_next(&exec_ctx, resolver, &config, grpc_closure_create(on_done, &ev2)); grpc_exec_ctx_flush(&exec_ctx); - GPR_ASSERT(gpr_event_wait(&ev2, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))); + GPR_ASSERT(wait_loop(30, &ev2)); GPR_ASSERT(config != NULL); grpc_client_config_unref(&exec_ctx, config); From e2327dbb8e45936c81802119142a7cda28230f24 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 09:52:42 -0800 Subject: [PATCH 125/160] Fit and finish for dns retry loop --- build.yaml | 1 + src/core/client_config/resolvers/dns_resolver.c | 8 ++++++-- tools/run_tests/tests.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/build.yaml b/build.yaml index 7cafb0ac6fe..e003dd83396 100644 --- a/build.yaml +++ b/build.yaml @@ -1058,6 +1058,7 @@ targets: - gpr_test_util - gpr - name: dns_resolver_connectivity_test + cpu_cost: 0.1 build: test language: c src: diff --git a/src/core/client_config/resolvers/dns_resolver.c b/src/core/client_config/resolvers/dns_resolver.c index 55457647b3c..e28e4757a10 100644 --- a/src/core/client_config/resolvers/dns_resolver.c +++ b/src/core/client_config/resolvers/dns_resolver.c @@ -95,6 +95,9 @@ static const grpc_resolver_vtable dns_resolver_vtable = { static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver) { dns_resolver *r = (dns_resolver *)resolver; gpr_mu_lock(&r->mu); + if (r->have_retry_timer) { + grpc_timer_cancel(exec_ctx, &r->retry_timer); + } if (r->next_completion != NULL) { *r->target_config = NULL; grpc_exec_ctx_enqueue(exec_ctx, r->next_completion, true, NULL); @@ -133,13 +136,14 @@ static void dns_on_retry_timer(grpc_exec_ctx *exec_ctx, void *arg, bool success) { dns_resolver *r = arg; + gpr_mu_lock(&r->mu); + r->have_retry_timer = false; if (success) { - gpr_mu_lock(&r->mu); if (!r->resolving) { dns_start_resolving_locked(r); } - gpr_mu_unlock(&r->mu); } + gpr_mu_unlock(&r->mu); GRPC_RESOLVER_UNREF(exec_ctx, &r->base, "retry-timer"); } diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index c2d6cadd530..724cd64d19c 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -261,7 +261,7 @@ "posix", "windows" ], - "cpu_cost": 1.0, + "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, "gtest": false, From b18fe559629cea0c2a6d345449071cfb6c31bdc1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 10:23:16 -0800 Subject: [PATCH 126/160] Update copyrights --- src/core/client_config/client_config.c | 2 +- src/core/iomgr/resolve_address.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/client_config/client_config.c b/src/core/client_config/client_config.c index 25f4ac25177..c500af25eec 100644 --- a/src/core/client_config/client_config.c +++ b/src/core/client_config/client_config.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/resolve_address.h b/src/core/iomgr/resolve_address.h index 508e161a7e0..b0596304576 100644 --- a/src/core/iomgr/resolve_address.h +++ b/src/core/iomgr/resolve_address.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 9aa4f1b062686e01c2e1dfb792e5ec9c3a91e1c9 Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 2 Mar 2016 16:39:13 -0800 Subject: [PATCH 127/160] fix windows leaks --- src/core/iomgr/iocp_windows.c | 14 +++++++++----- src/core/iomgr/iocp_windows.h | 11 +++++++++-- src/core/iomgr/tcp_server_windows.c | 3 +-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/core/iomgr/iocp_windows.c b/src/core/iomgr/iocp_windows.c index 807729708ea..fa87e5246b0 100644 --- a/src/core/iomgr/iocp_windows.c +++ b/src/core/iomgr/iocp_windows.c @@ -71,7 +71,8 @@ static DWORD deadline_to_millis_timeout(gpr_timespec deadline, timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); } -void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { +grpc_iocp_work_status grpc_iocp_work(grpc_exec_ctx *exec_ctx, + gpr_timespec deadline) { BOOL success; DWORD bytes = 0; DWORD flags = 0; @@ -84,14 +85,14 @@ void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { g_iocp, &bytes, &completion_key, &overlapped, deadline_to_millis_timeout(deadline, gpr_now(deadline.clock_type))); if (success == 0 && overlapped == NULL) { - return; + return GRPC_IOCP_WORK_TIMEOUT; } GPR_ASSERT(completion_key && overlapped); if (overlapped == &g_iocp_custom_overlap) { gpr_atm_full_fetch_add(&g_custom_events, -1); if (completion_key == (ULONG_PTR)&g_iocp_kick_token) { /* We were awoken from a kick. */ - return; + return GRPC_IOCP_WORK_KICK; } gpr_log(GPR_ERROR, "Unknown custom completion key."); abort(); @@ -121,6 +122,7 @@ void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { } gpr_mu_unlock(&socket->state_mu); grpc_exec_ctx_enqueue(exec_ctx, closure, true, NULL); + return GRPC_IOCP_WORK_WORK; } void grpc_iocp_init(void) { @@ -140,10 +142,12 @@ void grpc_iocp_kick(void) { void grpc_iocp_flush(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_iocp_work_status work_status; do { - grpc_iocp_work(&exec_ctx, gpr_inf_past(GPR_CLOCK_MONOTONIC)); - } while (grpc_exec_ctx_flush(&exec_ctx)); + work_status = grpc_iocp_work(&exec_ctx, gpr_inf_past(GPR_CLOCK_MONOTONIC)); + } while (work_status == GRPC_IOCP_WORK_KICK || + grpc_exec_ctx_flush(&exec_ctx)); } void grpc_iocp_shutdown(void) { diff --git a/src/core/iomgr/iocp_windows.h b/src/core/iomgr/iocp_windows.h index 75f3ba84770..8b2b1aeb5c2 100644 --- a/src/core/iomgr/iocp_windows.h +++ b/src/core/iomgr/iocp_windows.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,7 +38,14 @@ #include "src/core/iomgr/socket_windows.h" -void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline); +typedef enum { + GRPC_IOCP_WORK_WORK, + GRPC_IOCP_WORK_TIMEOUT, + GRPC_IOCP_WORK_KICK +} grpc_iocp_work_status; + +grpc_iocp_work_status grpc_iocp_work(grpc_exec_ctx *exec_ctx, + gpr_timespec deadline); void grpc_iocp_init(void); void grpc_iocp_kick(void); void grpc_iocp_flush(void); diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index ce930b8f41d..a4abc5b9743 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -240,8 +240,7 @@ static void decrement_active_ports_and_notify(grpc_exec_ctx *exec_ctx, sp->shutting_down = 0; gpr_mu_lock(&sp->server->mu); GPR_ASSERT(sp->server->active_ports > 0); - if (0 == --sp->server->active_ports && - sp->server->shutdown_complete != NULL) { + if (0 == --sp->server->active_ports) { notify = 1; } gpr_mu_unlock(&sp->server->mu); From 9ee163557bda7a86e33c12626cf160342796014a Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Fri, 11 Mar 2016 11:13:59 -0800 Subject: [PATCH 128/160] Remove unnecessary (potentially bad) safety-refs --- src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi | 1 - src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi | 1 - 2 files changed, 2 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index 1f1833d5ecc..bc03c4dcf19 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -95,7 +95,6 @@ cdef class Channel: self, grpc_connectivity_state last_observed_state, Timespec deadline not None, CompletionQueue queue not None, tag): cdef OperationTag operation_tag = OperationTag(tag) - operation_tag.references = [self, queue] cpython.Py_INCREF(operation_tag) grpc_channel_watch_connectivity_state( self.c_channel, last_observed_state, deadline.c_time, diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index fe93da6c124..5f859235246 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -106,7 +106,6 @@ cdef class Server: self.is_shutting_down = True operation_tag = OperationTag(tag) operation_tag.shutting_down_server = self - operation_tag.references.extend([self, queue]) cpython.Py_INCREF(operation_tag) grpc_server_shutdown_and_notify( self.c_server, queue.c_completion_queue, From a41940d0e0dc384318e6ea116e4ff8e10f7747be Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 11 Mar 2016 20:31:32 +0100 Subject: [PATCH 129/160] Disabling check_version. --- tools/run_tests/sanity/sanity_tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index cffc180fb07..3840f4d8df6 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -2,7 +2,6 @@ - script: tools/run_tests/sanity/check_cache_mk.sh - script: tools/run_tests/sanity/check_sources_and_headers.py - script: tools/run_tests/sanity/check_submodules.sh -- script: tools/run_tests/sanity/check_version.py - script: tools/buildgen/generate_projects.sh -j 3 cpu_cost: 3 - script: tools/distrib/check_copyright.py From f1d084a4d159eb07536e4b24f2bf65a3771a747c Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 30 Oct 2015 11:14:09 -0700 Subject: [PATCH 130/160] Add monitoring of connectivity Move C-layer call creation into GRPCChannel, so that it always acts on a non-destroyed channel. Listen for connectivity in the GRPCCall, and destroy the channel. --- src/objective-c/GRPCClient/GRPCCall.m | 37 +++- .../GRPCClient/private/GRPCChannel.h | 3 + .../GRPCClient/private/GRPCChannel.m | 14 ++ .../private/GRPCConnectivityMonitor.h | 77 +++++++ .../private/GRPCConnectivityMonitor.m | 188 ++++++++++++++++++ src/objective-c/GRPCClient/private/GRPCHost.h | 26 ++- src/objective-c/GRPCClient/private/GRPCHost.m | 93 ++++----- .../GRPCReachabilityFlagNames.xmacro.h | 65 ++++++ .../GRPCClient/private/GRPCWrappedCall.h | 2 +- 9 files changed, 445 insertions(+), 60 deletions(-) create mode 100644 src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h create mode 100644 src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m create mode 100644 src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index f79b7d0bc0c..2d45818b6e9 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -37,6 +37,8 @@ #include #import +#import "private/GRPCConnectivityMonitor.h" +#import "private/GRPCHost.h" #import "private/GRPCRequestHeaders.h" #import "private/GRPCWrappedCall.h" #import "private/NSData+GRPC.h" @@ -71,8 +73,11 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; @implementation GRPCCall { dispatch_queue_t _callQueue; + NSString *_host; + NSString *_path; GRPCWrappedCall *_wrappedCall; dispatch_once_t _callAlreadyInvoked; + GRPCConnectivityMonitor *_connectivityMonitor; // The C gRPC library has less guarantees on the ordering of events than we // do. Particularly, in the face of errors, there's no ordering guarantee at @@ -115,13 +120,11 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; format:@"The requests writer can't be already started."]; } if ((self = [super init])) { - _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:host path:path]; - if (!_wrappedCall) { - return nil; - } + _host = [host copy]; + _path = [path copy]; // Serial queue to invoke the non-reentrant methods of the grpc_call object. - _callQueue = dispatch_queue_create("org.grpc.call", NULL); + _callQueue = dispatch_queue_create("io.grpc.call", NULL); _requestWriter = requestWriter; @@ -156,7 +159,7 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; - (void)cancel { [self finishWithError:[NSError errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeCancelled - userInfo:nil]]; + userInfo:@{NSLocalizedDescriptionKey: @"Canceled by app"}]]; [self cancelCall]; } @@ -354,8 +357,29 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; _retainSelf = self; _responseWriteable = [[GRXConcurrentWriteable alloc] initWithWriteable:writeable]; + + _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:_host path:_path]; + NSAssert(_wrappedCall, @"Error allocating RPC objects. Low memory?"); + [self sendHeaders:_requestHeaders]; [self invokeCall]; + // TODO(jcanizales): Extract this logic somewhere common. + NSString *host = [NSURL URLWithString:[@"https://" stringByAppendingString:_host]].host; + if (!host) { + // TODO(jcanizales): Check this on init. + [NSException raise:NSInvalidArgumentException format:@"host of %@ is nil", _host]; + } + __weak typeof(self) weakSelf = self; + _connectivityMonitor = [GRPCConnectivityMonitor monitorWithHost:host]; + [_connectivityMonitor handleLossWithHandler:^{ + typeof(self) strongSelf = weakSelf; + if (strongSelf) { + [strongSelf finishWithError:[NSError errorWithDomain:kGRPCErrorDomain + code:GRPCErrorCodeUnavailable + userInfo:@{NSLocalizedDescriptionKey: @"Connectivity lost."}]]; + [[GRPCHost hostWithAddress:strongSelf->_host] disconnect]; + } + }]; } - (void)setState:(GRXWriterState)newState { @@ -385,4 +409,5 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey"; return; } } + @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h index 8661ae6f972..e49a6aca297 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.h +++ b/src/objective-c/GRPCClient/private/GRPCChannel.h @@ -35,6 +35,7 @@ #include +@class GRPCCompletionQueue; struct grpc_channel_credentials; @@ -80,4 +81,6 @@ struct grpc_channel_credentials; + (nonnull GRPCChannel *)insecureChannelWithHost:(nonnull NSString *)host channelArgs:(nullable NSDictionary *)channelArgs; +- (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path + completionQueue:(nonnull GRPCCompletionQueue *)queue; @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 7e55a473d74..d7de025e21d 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -38,6 +38,8 @@ #include #include +#import "GRPCCompletionQueue.h" + /** * Returns @c grpc_channel_credentials from the specified @c path. If the file at the path could not * be read then NULL is returned. If NULL is returned, @c errorPtr may not be NULL if there are @@ -205,4 +207,16 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) { channelArgs:channelArgs]; } +- (grpc_call *)unmanagedCallWithPath:(NSString *)path + completionQueue:(GRPCCompletionQueue *)queue { + return grpc_channel_create_call(_unmanagedChannel, + NULL, GRPC_PROPAGATE_DEFAULTS, + queue.unmanagedQueue, + path.UTF8String, + // Get "host" from "host:port" + // TODO(jcanizales): Use NSURLs throughout, to clarify these. + [_host componentsSeparatedByString:@":"][0].UTF8String, + gpr_inf_future(GPR_CLOCK_REALTIME), NULL); +} + @end diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h new file mode 100644 index 00000000000..d9d6c22ae9d --- /dev/null +++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h @@ -0,0 +1,77 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#import +#import + +@interface GRPCReachabilityFlags : NSObject + ++ (nonnull instancetype)flagsWithFlags:(SCNetworkReachabilityFlags)flags; + +/** + * One accessor method to query each of the different flags. Example: + +@property(nonatomic, readonly) BOOL isCell; + + */ +#define GRPC_XMACRO_ITEM(methodName, FlagName) \ +@property(nonatomic, readonly) BOOL methodName; + +#include "GRPCReachabilityFlagNames.xmacro.h" +#undef GRPC_XMACRO_ITEM + +@property(nonatomic, readonly) BOOL isHostReachable; +@end + + +@interface GRPCConnectivityMonitor : NSObject + ++ (nullable instancetype)monitorWithHost:(nonnull NSString *)hostName; + +- (nonnull instancetype)init NS_UNAVAILABLE; + +/** + * Queue on which callbacks will be dispatched. Default is the main queue. Set it before calling + * handleLossWithHandler:. + */ +// TODO(jcanizales): Default to a serial background queue instead. +@property(nonatomic, strong, null_resettable) dispatch_queue_t queue; + +/** + * Calls handler the next time the connectivity to this instance's host is lost. If this instance is + * released before that happens, the handler won't be called. + * Only one handler is active at a time, so if this method is called again before the previous + * handler has been called, it might never be called at all (or yes, if it has already been queued). + */ +- (void)handleLossWithHandler:(nonnull void (^)())handler; +@end diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m new file mode 100644 index 00000000000..f66a77c9b20 --- /dev/null +++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m @@ -0,0 +1,188 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#import "GRPCConnectivityMonitor.h" + +#pragma mark Flags + +@implementation GRPCReachabilityFlags { + SCNetworkReachabilityFlags _flags; +} + ++ (instancetype)flagsWithFlags:(SCNetworkReachabilityFlags)flags { + return [[self alloc] initWithFlags:flags]; +} + +- (instancetype)initWithFlags:(SCNetworkReachabilityFlags)flags { + if ((self = [super init])) { + _flags = flags; + } + return self; +} + +/* + * One accessor method implementation per flag. Example: + +- (BOOL)isCell { \ + return !!(_flags & kSCNetworkReachabilityFlagsIsWWAN); \ +} + + */ +#define GRPC_XMACRO_ITEM(methodName, FlagName) \ +- (BOOL)methodName { \ + return !!(_flags & kSCNetworkReachabilityFlags ## FlagName); \ +} +#include "GRPCReachabilityFlagNames.xmacro.h" +#undef GRPC_XMACRO_ITEM + +- (BOOL)isHostReachable { + // Note: connectionOnDemand means it'll be reachable only if using the CFSocketStream API or APIs + // on top of it. + // connectionRequired means we can't tell until a connection is attempted (e.g. for VPN on + // demand). + return self.reachable && !self.interventionRequired && !self.connectionOnDemand; +} + +- (NSString *)description { + NSMutableArray *activeOptions = [NSMutableArray arrayWithCapacity:9]; + + /* + * For each flag, add its name to the array if it's ON. Example: + + if (self.isCell) { + [activeOptions addObject:@"isCell"]; + } + + */ +#define GRPC_XMACRO_ITEM(methodName, FlagName) \ + if (self.methodName) { \ + [activeOptions addObject:@#methodName]; \ + } +#include "GRPCReachabilityFlagNames.xmacro.h" +#undef GRPC_XMACRO_ITEM + + return activeOptions.count == 0 ? @"(none)" : [activeOptions componentsJoinedByString:@", "]; +} + +- (BOOL)isEqual:(id)object { + return [object isKindOfClass:[GRPCReachabilityFlags class]] && + _flags == ((GRPCReachabilityFlags *)object)->_flags; +} + +- (NSUInteger)hash { + return _flags; +} +@end + +#pragma mark Connectivity Monitor + +// Assumes the third argument is a block that accepts SCNetworkReachabilityFlags, and passes the +// received ones to it. +static void PassFlagsToContextInfoBlock(SCNetworkReachabilityRef target, + SCNetworkReachabilityFlags flags, + void *info) { + #pragma unused (target) + ((__bridge void (^)(SCNetworkReachabilityFlags))info)(flags); +} + +@implementation GRPCConnectivityMonitor { + SCNetworkReachabilityRef _reachabilityRef; +} + +- (nullable instancetype)initWithReachability:(nullable SCNetworkReachabilityRef)reachability { + if (!reachability) { + return nil; + } + if ((self = [super init])) { + _reachabilityRef = CFRetain(reachability); + _queue = dispatch_get_main_queue(); + } + return self; +} + ++ (nullable instancetype)monitorWithHost:(nonnull NSString *)host { + const char *hostName = host.UTF8String; + if (!hostName) { + [NSException raise:NSInvalidArgumentException + format:@"host.UTF8String returns NULL for %@", host]; + } + SCNetworkReachabilityRef reachability = + SCNetworkReachabilityCreateWithName(NULL, hostName); + + GRPCConnectivityMonitor *returnValue = [[self alloc] initWithReachability:reachability]; + if (reachability) { + CFRelease(reachability); + } + return returnValue; +} + +- (void)handleLossWithHandler:(void (^)())handler { + __weak typeof(self) weakSelf = self; + [self startListeningWithHandler:^(GRPCReachabilityFlags *flags) { + if (!flags.isHostReachable) { + [weakSelf stopListening]; + handler(); + } + }]; +} + +- (void)startListeningWithHandler:(void (^)(GRPCReachabilityFlags *))handler { + SCNetworkReachabilityContext context = { + .version = 0, + .info = (__bridge_retained void *)^(SCNetworkReachabilityFlags flags){ + // Pass the flags as as Objective-C wrapper. + handler([[GRPCReachabilityFlags alloc] initWithFlags:flags]); + }, + .release = CFRelease + }; + SCNetworkReachabilitySetCallback(_reachabilityRef, PassFlagsToContextInfoBlock, &context); + SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, _queue); +} + +- (void)stopListening { + SCNetworkReachabilitySetCallback(_reachabilityRef, NULL, NULL); + SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, NULL); +} + +- (void)setQueue:(dispatch_queue_t)queue { + _queue = queue ?: dispatch_get_main_queue(); +} + +- (void)dealloc { + if (_reachabilityRef) { + [self stopListening]; + CFRelease(_reachabilityRef); + } +} + +@end diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index 82c0ad6cf63..987d3e9f597 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -33,27 +33,39 @@ #import +NS_ASSUME_NONNULL_BEGIN + @class GRPCCompletionQueue; struct grpc_call; @interface GRPCHost : NSObject @property(nonatomic, readonly) NSString *address; -@property(nonatomic, copy) NSString *userAgentPrefix; +@property(nonatomic, copy, nullable) NSString *userAgentPrefix; /** The following properties should only be modified for testing: */ @property(nonatomic, getter=isSecure) BOOL secure; -@property(nonatomic, copy) NSString *pathToCertificates; -@property(nonatomic, copy) NSString *hostNameOverride; +@property(nonatomic, copy, nullable) NSString *pathToCertificates; +@property(nonatomic, copy, nullable) NSString *hostNameOverride; +- (nullable instancetype)init NS_UNAVAILABLE; /** Host objects initialized with the same address are the same. */ -+ (instancetype)hostWithAddress:(NSString *)address; -- (instancetype)initWithAddress:(NSString *)address NS_DESIGNATED_INITIALIZER; ++ (nullable instancetype)hostWithAddress:(NSString *)address; +- (nullable instancetype)initWithAddress:(NSString *)address NS_DESIGNATED_INITIALIZER; /** Create a grpc_call object to the provided path on this host. */ -- (struct grpc_call *)unmanagedCallWithPath:(NSString *)path - completionQueue:(GRPCCompletionQueue *)queue; +- (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path + completionQueue:(GRPCCompletionQueue *)queue; +// TODO: There's a race when a new RPC is coming through just as an existing one is getting +// notified that there's no connectivity. If connectivity comes back at that moment, the new RPC +// will have its channel destroyed by the other RPC, and will never get notified of a problem, so +// it'll hang (the C layer logs a timeout, with exponential back off). One solution could be to pass +// the GRPCChannel to the GRPCCall, renaming this as "disconnectChannel:channel", which would only +// act on that specific channel. +- (void)disconnect; @end + +NS_ASSUME_NONNULL_END diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index eb1db899b70..b6a6d48d964 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -34,33 +34,30 @@ #import "GRPCHost.h" #include +#import #import #import "GRPCChannel.h" #import "GRPCCompletionQueue.h" #import "NSDictionary+GRPC.h" +NS_ASSUME_NONNULL_BEGIN + // TODO(jcanizales): Generate the version in a standalone header, from templates. Like // templates/src/core/surface/version.c.template . #define GRPC_OBJC_VERSION_STRING @"0.13.0" -@interface GRPCHost () -// TODO(mlumish): Investigate whether caching channels with strong links is a good idea. -@property(nonatomic, strong) GRPCChannel *channel; -@end - -@implementation GRPCHost - -+ (instancetype)hostWithAddress:(NSString *)address { - return [[self alloc] initWithAddress:address]; +@implementation GRPCHost { + // TODO(mlumish): Investigate whether caching channels with strong links is a good idea. + GRPCChannel *_channel; } -- (instancetype)init { - return [self initWithAddress:nil]; ++ (nullable instancetype)hostWithAddress:(NSString *)address { + return [[self alloc] initWithAddress:address]; } // Default initializer. -- (instancetype)initWithAddress:(NSString *)address { +- (nullable instancetype)initWithAddress:(NSString *)address { if (!address) { return nil; } @@ -95,46 +92,42 @@ return self; } -- (grpc_call *)unmanagedCallWithPath:(NSString *)path completionQueue:(GRPCCompletionQueue *)queue { - if (!queue || !path || !self.channel) { - return NULL; +- (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path + completionQueue:(GRPCCompletionQueue *)queue { + @synchronized(self) { + if (!_channel) { + _channel = [self newChannel]; + } + return [_channel unmanagedCallWithPath:path completionQueue:queue]; } - return grpc_channel_create_call(self.channel.unmanagedChannel, - NULL, GRPC_PROPAGATE_DEFAULTS, - queue.unmanagedQueue, - path.UTF8String, - self.hostName.UTF8String, - gpr_inf_future(GPR_CLOCK_REALTIME), NULL); } -- (GRPCChannel *)channel { - // Create it lazily, because we don't want to open a connection just because someone is - // configuring a host. +- (NSDictionary *)channelArgs { + NSMutableDictionary *args = [NSMutableDictionary dictionary]; - if (!_channel) { - NSMutableDictionary *args = [NSMutableDictionary dictionary]; + // TODO(jcanizales): Add OS and device information (see + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#user-agents ). + NSString *userAgent = @"grpc-objc/" GRPC_OBJC_VERSION_STRING; + if (_userAgentPrefix) { + userAgent = [_userAgentPrefix stringByAppendingFormat:@" %@", userAgent]; + } + args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] = userAgent; - // TODO(jcanizales): Add OS and device information (see - // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#user-agents ). - NSString *userAgent = @"grpc-objc/" GRPC_OBJC_VERSION_STRING; - if (_userAgentPrefix) { - userAgent = [@[_userAgentPrefix, userAgent] componentsJoinedByString:@" "]; - } - args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] = userAgent; - - if (_secure) { - if (_hostNameOverride) { - args[@GRPC_SSL_TARGET_NAME_OVERRIDE_ARG] = _hostNameOverride; - } - - _channel = [GRPCChannel secureChannelWithHost:_address - pathToCertificates:_pathToCertificates - channelArgs:args]; - } else { - _channel = [GRPCChannel insecureChannelWithHost:_address channelArgs:args]; - } + if (_secure && _hostNameOverride) { + args[@GRPC_SSL_TARGET_NAME_OVERRIDE_ARG] = _hostNameOverride; + } + return args; +} + +- (GRPCChannel *)newChannel { + NSDictionary *args = [self channelArgs]; + if (_secure) { + return [GRPCChannel secureChannelWithHost:_address + pathToCertificates:_pathToCertificates + channelArgs:args]; + } else { + return [GRPCChannel insecureChannelWithHost:_address channelArgs:args]; } - return _channel; } - (NSString *)hostName { @@ -142,7 +135,15 @@ return _hostNameOverride ?: _address; } +- (void)disconnect { + @synchronized(self) { + _channel = nil; + } +} + // TODO(jcanizales): Don't let set |secure| to |NO| if |pathToCertificates| or |hostNameOverride| // have been set. Don't let set either of the latter if |secure| has been set to |NO|. @end + +NS_ASSUME_NONNULL_END diff --git a/src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h b/src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h new file mode 100644 index 00000000000..02871d5d028 --- /dev/null +++ b/src/objective-c/GRPCClient/private/GRPCReachabilityFlagNames.xmacro.h @@ -0,0 +1,65 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/** + * "X-macro" file that lists the flags names of Apple's Network Reachability API, along with a nice + * Objective-C method name used to query each of them. + * + * Example usage: To generate a dictionary from flag value to name, one can do: + + NSDictionary *flagNames = @{ +#define GRPC_XMACRO_ITEM(methodName, FlagName) \ + @(kSCNetworkReachabilityFlags ## FlagName): @#methodName, +#include "GRXReachabilityFlagNames.xmacro.h" +#undef GRPC_XMACRO_ITEM + }; + + XCTAssertEqualObjects(flagNames[@(kSCNetworkReachabilityFlagsIsWWAN)], @"isCell"); + + */ + +#ifndef GRPC_XMACRO_ITEM +#error This file is to be used with the "X-macro" pattern: Please #define \ + GRPC_XMACRO_ITEM(methodName, FlagName), then #include this file, and then #undef \ + GRPC_XMACRO_ITEM. +#endif + +GRPC_XMACRO_ITEM(isCell, IsWWAN) +GRPC_XMACRO_ITEM(reachable, Reachable) +GRPC_XMACRO_ITEM(transientConnection, TransientConnection) +GRPC_XMACRO_ITEM(connectionRequired, ConnectionRequired) +GRPC_XMACRO_ITEM(connectionOnTraffic, ConnectionOnTraffic) +GRPC_XMACRO_ITEM(interventionRequired, InterventionRequired) +GRPC_XMACRO_ITEM(connectionOnDemand, ConnectionOnDemand) +GRPC_XMACRO_ITEM(isLocalAddress, IsLocalAddress) +GRPC_XMACRO_ITEM(isDirect, IsDirect) diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index 71e7e0e54ee..e37ed1b59fb 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -34,7 +34,6 @@ #import #include -#import "GRPCChannel.h" #import "GRPCRequestHeaders.h" @interface GRPCOperation : NSObject @@ -94,4 +93,5 @@ - (void)startBatchWithOperations:(NSArray *)ops; - (void)cancel; + @end From 07a2a70bca9a2ffa3c69b43a49f29c4d52808c98 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 10:49:56 -0800 Subject: [PATCH 131/160] Not call external methods from within a critical section --- src/objective-c/GRPCClient/private/GRPCHost.m | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index b6a6d48d964..508cb206444 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -94,12 +94,15 @@ NS_ASSUME_NONNULL_BEGIN - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path completionQueue:(GRPCCompletionQueue *)queue { + GRPCChannel *channel; + // This is racing -[GRPCHost disconnect]. @synchronized(self) { if (!_channel) { _channel = [self newChannel]; } - return [_channel unmanagedCallWithPath:path completionQueue:queue]; + channel = _channel; } + return [channel unmanagedCallWithPath:path completionQueue:queue]; } - (NSDictionary *)channelArgs { @@ -136,6 +139,7 @@ NS_ASSUME_NONNULL_BEGIN } - (void)disconnect { + // This is racing -[GRPCHost unmanagedCallWithPath:completionQueue:]. @synchronized(self) { _channel = nil; } From 8b414f184db9295e30164140d54357c6ad310368 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 10:28:40 -0800 Subject: [PATCH 132/160] Clean up ownership of the connection loss handler --- .../private/GRPCConnectivityMonitor.h | 2 +- .../private/GRPCConnectivityMonitor.m | 22 +++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h index d9d6c22ae9d..2fae4103315 100644 --- a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h +++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.h @@ -68,7 +68,7 @@ @property(nonatomic, strong, null_resettable) dispatch_queue_t queue; /** - * Calls handler the next time the connectivity to this instance's host is lost. If this instance is + * Calls handler every time the connectivity to this instance's host is lost. If this instance is * released before that happens, the handler won't be called. * Only one handler is active at a time, so if this method is called again before the previous * handler has been called, it might never be called at all (or yes, if it has already been queued). diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m index f66a77c9b20..b4061bd5ef5 100644 --- a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m +++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m @@ -106,13 +106,16 @@ #pragma mark Connectivity Monitor -// Assumes the third argument is a block that accepts SCNetworkReachabilityFlags, and passes the +// Assumes the third argument is a block that accepts a GRPCReachabilityFlags object, and passes the // received ones to it. static void PassFlagsToContextInfoBlock(SCNetworkReachabilityRef target, SCNetworkReachabilityFlags flags, void *info) { #pragma unused (target) - ((__bridge void (^)(SCNetworkReachabilityFlags))info)(flags); + // This can be called many times with the same info. The info is retained by SCNetworkReachability + // while this function is being executed. + void (^handler)(GRPCReachabilityFlags *) = (__bridge void (^)(GRPCReachabilityFlags *))info; + handler([[GRPCReachabilityFlags alloc] initWithFlags:flags]); } @implementation GRPCConnectivityMonitor { @@ -147,29 +150,30 @@ static void PassFlagsToContextInfoBlock(SCNetworkReachabilityRef target, } - (void)handleLossWithHandler:(void (^)())handler { - __weak typeof(self) weakSelf = self; [self startListeningWithHandler:^(GRPCReachabilityFlags *flags) { if (!flags.isHostReachable) { - [weakSelf stopListening]; handler(); } }]; } - (void)startListeningWithHandler:(void (^)(GRPCReachabilityFlags *))handler { + // Copy to ensure the handler block is in the heap (and so can't be deallocated when this method + // returns). + void (^copiedHandler)(GRPCReachabilityFlags *) = [handler copy]; SCNetworkReachabilityContext context = { .version = 0, - .info = (__bridge_retained void *)^(SCNetworkReachabilityFlags flags){ - // Pass the flags as as Objective-C wrapper. - handler([[GRPCReachabilityFlags alloc] initWithFlags:flags]); - }, - .release = CFRelease + .info = (__bridge void *)copiedHandler, + .retain = CFRetain, + .release = CFRelease, }; + // The following will retain context.info, and release it when the callback is set to NULL. SCNetworkReachabilitySetCallback(_reachabilityRef, PassFlagsToContextInfoBlock, &context); SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, _queue); } - (void)stopListening { + // This releases the block on context.info. SCNetworkReachabilitySetCallback(_reachabilityRef, NULL, NULL); SCNetworkReachabilitySetDispatchQueue(_reachabilityRef, NULL); } From 232b6a8f191d9cbb905680dd2a95f3b1815d1eb7 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 12:26:27 -0800 Subject: [PATCH 133/160] Test robustness of WriteableSingleHandler against non-single Writers --- src/objective-c/tests/RxLibraryUnitTests.m | 51 ++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/objective-c/tests/RxLibraryUnitTests.m b/src/objective-c/tests/RxLibraryUnitTests.m index d3426628141..ae9465f58cc 100644 --- a/src/objective-c/tests/RxLibraryUnitTests.m +++ b/src/objective-c/tests/RxLibraryUnitTests.m @@ -64,6 +64,8 @@ } @end +// TODO(jcanizales): Split into one file per tested class. + @interface RxLibraryUnitTests : XCTestCase @end @@ -79,6 +81,7 @@ // If: id writeable = [GRXWriteable writeableWithSingleHandler:handler.block]; [writeable writeValue:anyValue]; + [writeable writesFinishedWithError:nil]; // Then: XCTAssertEqual(handler.timesCalled, 1); @@ -101,6 +104,54 @@ XCTAssertEqualObjects(handler.errorOrNil, anyError); } +- (void)testWriteableSingleHandlerIsCalledOnlyOnce_ValueThenError { + // Given: + CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler]; + id anyValue = @7; + NSError *anyError = [NSError errorWithDomain:@"domain" code:7 userInfo:nil]; + + // If: + id writeable = [GRXWriteable writeableWithSingleHandler:handler.block]; + [writeable writeValue:anyValue]; + [writeable writesFinishedWithError:anyError]; + + // Then: + XCTAssertEqual(handler.timesCalled, 1); + XCTAssertEqualObjects(handler.value, anyValue); + XCTAssertEqualObjects(handler.errorOrNil, nil); +} + +- (void)testWriteableSingleHandlerIsCalledOnlyOnce_ValueThenValue { + // Given: + CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler]; + id anyValue = @7; + + // If: + id writeable = [GRXWriteable writeableWithSingleHandler:handler.block]; + [writeable writeValue:anyValue]; + [writeable writeValue:anyValue]; + [writeable writesFinishedWithError:nil]; + + // Then: + XCTAssertEqual(handler.timesCalled, 1); + XCTAssertEqualObjects(handler.value, anyValue); + XCTAssertEqualObjects(handler.errorOrNil, nil); +} + +- (void)testWriteableSingleHandlerFailsOnEmptyWriter { + // Given: + CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler]; + + // If: + id writeable = [GRXWriteable writeableWithSingleHandler:handler.block]; + [writeable writesFinishedWithError:nil]; + + // Then: + XCTAssertEqual(handler.timesCalled, 1); + XCTAssertEqualObjects(handler.value, nil); + XCTAssertNotNil(handler.errorOrNil); +} + #pragma mark BufferedPipe - (void)testBufferedPipePropagatesValue { From 5e6723203013b365c2a60158578f73abf480b8bf Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Wed, 9 Mar 2016 13:34:28 -0800 Subject: [PATCH 134/160] Make Writeable with single handler robust against stream Writers --- src/objective-c/RxLibrary/GRXWriteable.m | 37 ++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/objective-c/RxLibrary/GRXWriteable.m b/src/objective-c/RxLibrary/GRXWriteable.m index 2729d62b72f..b122c04ce0c 100644 --- a/src/objective-c/RxLibrary/GRXWriteable.m +++ b/src/objective-c/RxLibrary/GRXWriteable.m @@ -42,11 +42,38 @@ if (!handler) { return [[self alloc] init]; } - return [[self alloc] initWithValueHandler:^(id value) { - handler(value, nil); - } completionHandler:^(NSError *errorOrNil) { - if (errorOrNil) { - handler(nil, errorOrNil); + // We nilify this variable when the block is invoked, so that handler is only invoked once even if + // the writer tries to write multiple values. + __block GRXEventHandler eventHandler = ^(BOOL done, id value, NSError *error) { + // Nillify eventHandler before invoking handler, in case the latter causes the former to be + // executed recursively. Because blocks can be deallocated even during execution, we have to + // first retain handler locally to guarantee it's valid. + // TODO(jcanizales): Just turn this craziness into a simple subclass of GRXWriteable. + GRXSingleHandler singleHandler = handler; + eventHandler = nil; + + if (value) { + singleHandler(value, nil); + } else if (error) { + singleHandler(nil, error); + } else { + NSDictionary *userInfo = @{ + NSLocalizedDescriptionKey: @"The writer finished without producing any value." + }; + // Even though RxLibrary is independent of gRPC, the domain and code here are, for the moment, + // set to the values of kGRPCErrorDomain and GRPCErrorCodeInternal. This way, the error formed + // is the one user of gRPC would expect if the server failed to produce a response. + // + // TODO(jcanizales): Figure out a way to keep errors of RxLibrary generic without making users + // of gRPC take care of two different error domains and error code enums. A possibility is to + // add error handling to GRXWriters or GRXWriteables, and use them to translate errors between + // the two domains. + singleHandler(nil, [NSError errorWithDomain:@"io.grpc" code:13 userInfo:userInfo]); + } + }; + return [self writeableWithEventHandler:^(BOOL done, id value, NSError *error) { + if (eventHandler) { + eventHandler(done, value, error); } }]; } From cfcaccc6d20c67a2595b02b18fc31a3a2da83daf Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 12:36:18 -0800 Subject: [PATCH 135/160] Revert "Revert "Pass a non-infinite deadline to grpc_completion_queue_next() to prevent queues from blocking indefinitely in poll()"" --- .../GRPCClient/private/GRPCCompletionQueue.h | 7 ++++++ .../GRPCClient/private/GRPCCompletionQueue.m | 24 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h index 7b66cd4c329..a52095dd011 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h @@ -36,6 +36,8 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); +extern const int64_t kGRPCCompletionQueueDefaultTimeoutSecs; + /** * This class lets one more easily use |grpc_completion_queue|. To use it, pass the value of the * |unmanagedQueue| property of an instance of this class to |grpc_channel_create_call|. Then for @@ -49,6 +51,11 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); */ @interface GRPCCompletionQueue : NSObject @property(nonatomic, readonly) grpc_completion_queue *unmanagedQueue; +@property(nonatomic, readonly) int64_t timeoutSecs; + (instancetype)completionQueue; + +- (instancetype)init; +- (instancetype)initWithTimeout:(int64_t)timeoutSecs NS_DESIGNATED_INITIALIZER; + @end diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index ff3031678c5..250bbf03a66 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -35,6 +35,9 @@ #import + +const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; + @implementation GRPCCompletionQueue + (instancetype)completionQueue { @@ -42,8 +45,13 @@ } - (instancetype)init { + return [self initWithTimeout:kGRPCCompletionQueueDefaultTimeoutSecs]; +} + +- (instancetype)initWithTimeout:(int64_t)timeoutSecs { if ((self = [super init])) { _unmanagedQueue = grpc_completion_queue_create(NULL); + _timeoutSecs = timeoutSecs; // This is for the following block to capture the pointer by value (instead // of retaining self and doing self->_unmanagedQueue). This is essential @@ -61,22 +69,28 @@ gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ + // Using a non-infinite deadline to re-enter grpc_completion_queue_next() + // alleviates https://github.com/grpc/grpc/issues/5593 + gpr_timespec deadline = (timeoutSecs < 0) + ? gpr_inf_future(GPR_CLOCK_REALTIME) + : gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); while (YES) { - // The following call blocks until an event is available. - grpc_event event = grpc_completion_queue_next(unmanagedQueue, - gpr_inf_future(GPR_CLOCK_REALTIME), - NULL); + // The following call blocks until an event is available or the deadline elapses. + grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); GRPCQueueCompletionHandler handler; switch (event.type) { case GRPC_OP_COMPLETE: handler = (__bridge_transfer GRPCQueueCompletionHandler)event.tag; handler(event.success); break; + case GRPC_QUEUE_TIMEOUT: + // Nothing to do here + break; case GRPC_QUEUE_SHUTDOWN: grpc_completion_queue_destroy(unmanagedQueue); return; default: - [NSException raise:@"Unrecognized completion type" format:@""]; + [NSException raise:@"Unrecognized completion type" format:@"type=%d", event.type]; } }; }); From aad4156a0ba79d963bfa3cd37e48a1074f956809 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 11 Mar 2016 22:21:11 +0100 Subject: [PATCH 136/160] Disabling iOS user-agent test. --- src/objective-c/tests/GRPCClientTests.m | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 624958f4b97..7dd6873c802 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -273,10 +273,12 @@ static ProtoMethod *kUnaryCallMethod; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { XCTAssertNotNil(value, @"nil value received as response."); XCTAssertEqual([value length], 0, @"Non-empty response received: %@", value); + /* This test needs to be more clever in regards to changing the version of the core. XCTAssertEqualObjects(call.responseHeaders[@"x-grpc-test-echo-useragent"], @"Foo grpc-objc/0.13.0 grpc-c/0.14.0-dev (ios)", @"Did not receive expected user agent %@", call.responseHeaders[@"x-grpc-test-echo-useragent"]); + */ [response fulfill]; } completionHandler:^(NSError *errorOrNil) { XCTAssertNil(errorOrNil, @"Finished with unexpected error: %@", errorOrNil); From afc59cbc886424e9c3ca9b5da1ffd2a817ee903c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 11 Mar 2016 15:06:15 -0800 Subject: [PATCH 137/160] remove tests for EventInvocation API that is going to be removed --- src/python/grpcio/tests/tests.json | 9 - ...nt_invocation_synchronous_event_service.py | 381 ------------------ .../framework/interfaces/face/test_cases.py | 2 - 3 files changed, 392 deletions(-) delete mode 100644 src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py diff --git a/src/python/grpcio/tests/tests.json b/src/python/grpcio/tests/tests.json index 388d040d5ca..84870aaa5cb 100644 --- a/src/python/grpcio/tests/tests.json +++ b/src/python/grpcio/tests/tests.json @@ -12,31 +12,22 @@ "_core_over_links_base_interface_test.SyncEasyTest", "_core_over_links_base_interface_test.SyncPeasyTest", "_crust_over_core_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_over_links_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_over_links_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", - "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", - "_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", - "_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", - "_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", "_implementations_test.ChannelCredentialsTest", "_insecure_interop_test.InsecureInteropTest", diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py b/src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py deleted file mode 100644 index 34db6c3e55f..00000000000 --- a/src/python/grpcio/tests/unit/framework/interfaces/face/_event_invocation_synchronous_event_service.py +++ /dev/null @@ -1,381 +0,0 @@ -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Test code for the Face layer of RPC Framework.""" - -import abc -import unittest - -# test_interfaces is referenced from specification in this module. -from grpc.framework.interfaces.face import face -from tests.unit.framework.common import test_constants -from tests.unit.framework.common import test_control -from tests.unit.framework.common import test_coverage -from tests.unit.framework.interfaces.face import _3069_test_constant -from tests.unit.framework.interfaces.face import _digest -from tests.unit.framework.interfaces.face import _receiver -from tests.unit.framework.interfaces.face import _stock_service -from tests.unit.framework.interfaces.face import test_interfaces # pylint: disable=unused-import - - -class TestCase(test_coverage.Coverage, unittest.TestCase): - """A test of the Face layer of RPC Framework. - - Concrete subclasses must have an "implementation" attribute of type - test_interfaces.Implementation and an "invoker_constructor" attribute of type - _invocation.InvokerConstructor. - """ - __metaclass__ = abc.ABCMeta - - NAME = 'EventInvocationSynchronousEventServiceTest' - - def setUp(self): - """See unittest.TestCase.setUp for full specification. - - Overriding implementations must call this implementation. - """ - self._control = test_control.PauseFailControl() - self._digest = _digest.digest( - _stock_service.STOCK_TEST_SERVICE, self._control, None) - - generic_stub, dynamic_stubs, self._memo = self.implementation.instantiate( - self._digest.methods, self._digest.event_method_implementations, None) - self._invoker = self.invoker_constructor.construct_invoker( - generic_stub, dynamic_stubs, self._digest.methods) - - def tearDown(self): - """See unittest.TestCase.tearDown for full specification. - - Overriding implementations must call this implementation. - """ - self._invoker = None - self.implementation.destantiate(self._memo) - - def testSuccessfulUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - receiver.block_until_terminated() - response = receiver.unary_response() - - test_messages.verify(request, response, self) - - def testSuccessfulUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - receiver.block_until_terminated() - responses = receiver.stream_responses() - - test_messages.verify(request, responses, self) - - def testSuccessfulStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.terminate() - receiver.block_until_terminated() - response = receiver.unary_response() - - test_messages.verify(requests, response, self) - - def testSuccessfulStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.terminate() - receiver.block_until_terminated() - responses = receiver.stream_responses() - - test_messages.verify(requests, responses, self) - - def testSequentialInvocations(self): - # pylint: disable=cell-var-from-loop - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - first_request = test_messages.request() - second_request = test_messages.request() - second_receiver = _receiver.Receiver() - - def make_second_invocation(): - self._invoker.event(group, method)( - second_request, second_receiver, second_receiver.abort, - test_constants.LONG_TIMEOUT) - - class FirstReceiver(_receiver.Receiver): - - def complete(self, terminal_metadata, code, details): - super(FirstReceiver, self).complete( - terminal_metadata, code, details) - make_second_invocation() - - first_receiver = FirstReceiver() - - self._invoker.event(group, method)( - first_request, first_receiver, first_receiver.abort, - test_constants.LONG_TIMEOUT) - second_receiver.block_until_terminated() - - first_response = first_receiver.unary_response() - second_response = second_receiver.unary_response() - test_messages.verify(first_request, first_response, self) - test_messages.verify(second_request, second_response, self) - - def testParallelInvocations(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - first_request = test_messages.request() - first_receiver = _receiver.Receiver() - second_request = test_messages.request() - second_receiver = _receiver.Receiver() - - self._invoker.event(group, method)( - first_request, first_receiver, first_receiver.abort, - test_constants.LONG_TIMEOUT) - self._invoker.event(group, method)( - second_request, second_receiver, second_receiver.abort, - test_constants.LONG_TIMEOUT) - first_receiver.block_until_terminated() - second_receiver.block_until_terminated() - - first_response = first_receiver.unary_response() - second_response = second_receiver.unary_response() - test_messages.verify(first_request, first_response, self) - test_messages.verify(second_request, second_response, self) - - @unittest.skip('TODO(nathaniel): implement.') - def testWaitingForSomeButNotAllParallelInvocations(self): - raise NotImplementedError() - - def testCancelledUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.pause(): - call = self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - call.cancel() - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind) - - def testCancelledUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - call = self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - call.cancel() - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind) - - def testCancelledStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.cancel() - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind) - - def testCancelledStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_stream_messages_sequences.iteritems()): - for unused_test_messages in test_messages_sequence: - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - call_consumer.cancel() - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.CANCELLED, receiver.abortion().kind) - - def testExpiredUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.pause(): - self._invoker.event(group, method)( - request, receiver, receiver.abort, - _3069_test_constant.REALLY_SHORT_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind) - - def testExpiredUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.pause(): - self._invoker.event(group, method)( - request, receiver, receiver.abort, - _3069_test_constant.REALLY_SHORT_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind) - - def testExpiredStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_unary_messages_sequences.iteritems()): - for unused_test_messages in test_messages_sequence: - receiver = _receiver.Receiver() - - self._invoker.event(group, method)( - receiver, receiver.abort, _3069_test_constant.REALLY_SHORT_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind) - - def testExpiredStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, _3069_test_constant.REALLY_SHORT_TIMEOUT) - for request in requests: - call_consumer.consume(request) - receiver.block_until_terminated() - - self.assertIs(face.Abortion.Kind.EXPIRED, receiver.abortion().kind) - - def testFailedUnaryRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.fail(): - self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs( - face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind) - - def testFailedUnaryRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.unary_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - request = test_messages.request() - receiver = _receiver.Receiver() - - with self._control.fail(): - self._invoker.event(group, method)( - request, receiver, receiver.abort, test_constants.LONG_TIMEOUT) - receiver.block_until_terminated() - - self.assertIs( - face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind) - - def testFailedStreamRequestUnaryResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_unary_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - with self._control.fail(): - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.terminate() - receiver.block_until_terminated() - - self.assertIs( - face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind) - - def testFailedStreamRequestStreamResponse(self): - for (group, method), test_messages_sequence in ( - self._digest.stream_stream_messages_sequences.iteritems()): - for test_messages in test_messages_sequence: - requests = test_messages.requests() - receiver = _receiver.Receiver() - - with self._control.fail(): - call_consumer = self._invoker.event(group, method)( - receiver, receiver.abort, test_constants.LONG_TIMEOUT) - for request in requests: - call_consumer.consume(request) - call_consumer.terminate() - receiver.block_until_terminated() - - self.assertIs( - face.Abortion.Kind.REMOTE_FAILURE, receiver.abortion().kind) diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py index 462829b6606..71de9d835e9 100644 --- a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py +++ b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py @@ -34,14 +34,12 @@ import unittest # pylint: disable=unused-import # test_interfaces is referenced from specification in this module. from tests.unit.framework.interfaces.face import _blocking_invocation_inline_service -from tests.unit.framework.interfaces.face import _event_invocation_synchronous_event_service from tests.unit.framework.interfaces.face import _future_invocation_asynchronous_event_service from tests.unit.framework.interfaces.face import _invocation from tests.unit.framework.interfaces.face import test_interfaces # pylint: disable=unused-import _TEST_CASE_SUPERCLASSES = ( _blocking_invocation_inline_service.TestCase, - _event_invocation_synchronous_event_service.TestCase, _future_invocation_asynchronous_event_service.TestCase, ) From 8734e02f52025219de2a3908f1eb0002568d1ecd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 11 Mar 2016 15:52:53 -0800 Subject: [PATCH 138/160] fix copyright --- .../grpcio/tests/unit/framework/interfaces/face/test_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py index 71de9d835e9..06b9d77e521 100644 --- a/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py +++ b/src/python/grpcio/tests/unit/framework/interfaces/face/test_cases.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From 57e2743b6580ae19326ceccb5ac9ed88e0691a94 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 16:53:58 -0800 Subject: [PATCH 139/160] Add contributed test --- Makefile | 36 ++++ build.yaml | 10 + .../surface/concurrent_connectivity_test.c | 48 +++++ tools/run_tests/sources_and_headers.json | 16 ++ tools/run_tests/tests.json | 21 ++ vsprojects/buildtests_c.sln | 27 +++ .../concurrent_connectivity_test.vcxproj | 199 ++++++++++++++++++ ...ncurrent_connectivity_test.vcxproj.filters | 21 ++ 8 files changed, 378 insertions(+) create mode 100644 test/core/surface/concurrent_connectivity_test.c create mode 100644 vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj create mode 100644 vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj.filters diff --git a/Makefile b/Makefile index 171a83c2a3a..f118d542538 100644 --- a/Makefile +++ b/Makefile @@ -849,6 +849,7 @@ chttp2_status_conversion_test: $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test chttp2_stream_map_test: $(BINDIR)/$(CONFIG)/chttp2_stream_map_test chttp2_varint_test: $(BINDIR)/$(CONFIG)/chttp2_varint_test compression_test: $(BINDIR)/$(CONFIG)/compression_test +concurrent_connectivity_test: $(BINDIR)/$(CONFIG)/concurrent_connectivity_test dns_resolver_connectivity_test: $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test dns_resolver_test: $(BINDIR)/$(CONFIG)/dns_resolver_test dualstack_socket_test: $(BINDIR)/$(CONFIG)/dualstack_socket_test @@ -1160,6 +1161,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/chttp2_stream_map_test \ $(BINDIR)/$(CONFIG)/chttp2_varint_test \ $(BINDIR)/$(CONFIG)/compression_test \ + $(BINDIR)/$(CONFIG)/concurrent_connectivity_test \ $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test \ $(BINDIR)/$(CONFIG)/dns_resolver_test \ $(BINDIR)/$(CONFIG)/dualstack_socket_test \ @@ -1404,6 +1406,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/chttp2_varint_test || ( echo test chttp2_varint_test failed ; exit 1 ) $(E) "[RUN] Testing compression_test" $(Q) $(BINDIR)/$(CONFIG)/compression_test || ( echo test compression_test failed ; exit 1 ) + $(E) "[RUN] Testing concurrent_connectivity_test" + $(Q) $(BINDIR)/$(CONFIG)/concurrent_connectivity_test || ( echo test concurrent_connectivity_test failed ; exit 1 ) $(E) "[RUN] Testing dns_resolver_connectivity_test" $(Q) $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test || ( echo test dns_resolver_connectivity_test failed ; exit 1 ) $(E) "[RUN] Testing dns_resolver_test" @@ -6079,6 +6083,38 @@ endif endif +CONCURRENT_CONNECTIVITY_TEST_SRC = \ + test/core/surface/concurrent_connectivity_test.c \ + +CONCURRENT_CONNECTIVITY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(CONCURRENT_CONNECTIVITY_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/concurrent_connectivity_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/concurrent_connectivity_test: $(CONCURRENT_CONNECTIVITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(CONCURRENT_CONNECTIVITY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/concurrent_connectivity_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/surface/concurrent_connectivity_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_concurrent_connectivity_test: $(CONCURRENT_CONNECTIVITY_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(CONCURRENT_CONNECTIVITY_TEST_OBJS:.o=.dep) +endif +endif + + DNS_RESOLVER_CONNECTIVITY_TEST_SRC = \ test/core/client_config/resolvers/dns_resolver_connectivity_test.c \ diff --git a/build.yaml b/build.yaml index e003dd83396..83b7714e655 100644 --- a/build.yaml +++ b/build.yaml @@ -1057,6 +1057,16 @@ targets: - grpc - gpr_test_util - gpr +- name: concurrent_connectivity_test + build: test + language: c + src: + - test/core/surface/concurrent_connectivity_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: dns_resolver_connectivity_test cpu_cost: 0.1 build: test diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c new file mode 100644 index 00000000000..d8639f5e510 --- /dev/null +++ b/test/core/surface/concurrent_connectivity_test.c @@ -0,0 +1,48 @@ +#include + +#include +#include +#include + +#define NUM_THREADS 100 +static grpc_channel* channels[NUM_THREADS]; +static grpc_completion_queue* queues[NUM_THREADS]; + +void create_loop_destroy(void* actually_an_int) { + int thread_index = (int)(actually_an_int); + for (int i = 0; i < 10; ++i) { + grpc_completion_queue* cq = grpc_completion_queue_create(NULL); + grpc_channel* chan = grpc_insecure_channel_create("localhost", NULL, NULL); + + channels[thread_index] = chan; + queues[thread_index] = cq; + + gpr_timespec inf_future = gpr_inf_future(GPR_CLOCK_REALTIME); + gpr_timespec delta = gpr_time_from_millis(10, GPR_TIMESPAN); + for (int j = 0; j < 10; ++j) { + gpr_timespec later_time = + gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), delta); + grpc_connectivity_state state = + grpc_channel_check_connectivity_state(chan, 1); + grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); + grpc_completion_queue_next(cq, inf_future, NULL); + } + grpc_channel_destroy(channels[thread_index]); + grpc_completion_queue_destroy(queues[thread_index]); + } +} + +int main() { + grpc_init(); + gpr_thd_id threads[NUM_THREADS]; + for (intptr_t i = 0; i < NUM_THREADS; ++i) { + gpr_thd_options options = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&options); + gpr_thd_new(&threads[i], create_loop_destroy, (void*)i, &options); + } + for (int i = 0; i < NUM_THREADS; ++i) { + gpr_thd_join(threads[i]); + } + grpc_shutdown(); + return 0; +} diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 0039b99a55e..092ed35ad97 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -189,6 +189,22 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "concurrent_connectivity_test", + "src": [ + "test/core/surface/concurrent_connectivity_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 724cd64d19c..000315f2f4a 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -253,6 +253,27 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "concurrent_connectivity_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index be2c4a3bb14..413ed3e3f38 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -251,6 +251,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "compression_test", "vcxproj {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "concurrent_connectivity_test", "vcxproj\test\concurrent_connectivity_test\concurrent_connectivity_test.vcxproj", "{391B366C-D916-45AA-3FE5-67363A46193B}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dns_resolver_connectivity_test", "vcxproj\test\dns_resolver_connectivity_test\dns_resolver_connectivity_test.vcxproj", "{F7B6FE68-E847-D7CA-4062-E737E542BCC3}" ProjectSection(myProperties) = preProject lib = "False" @@ -1747,6 +1758,22 @@ Global {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|Win32.Build.0 = Release|Win32 {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|x64.ActiveCfg = Release|x64 {5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release-DLL|x64.Build.0 = Release|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug|Win32.ActiveCfg = Debug|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug|x64.ActiveCfg = Debug|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release|Win32.ActiveCfg = Release|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release|x64.ActiveCfg = Release|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug|Win32.Build.0 = Debug|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug|x64.Build.0 = Debug|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release|Win32.Build.0 = Release|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release|x64.Build.0 = Release|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Debug-DLL|x64.Build.0 = Debug|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release-DLL|Win32.Build.0 = Release|Win32 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release-DLL|x64.ActiveCfg = Release|x64 + {391B366C-D916-45AA-3FE5-67363A46193B}.Release-DLL|x64.Build.0 = Release|x64 {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|Win32.ActiveCfg = Debug|Win32 {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Debug|x64.ActiveCfg = Debug|x64 {F7B6FE68-E847-D7CA-4062-E737E542BCC3}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj b/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj new file mode 100644 index 00000000000..6d69eae302f --- /dev/null +++ b/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj @@ -0,0 +1,199 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {391B366C-D916-45AA-3FE5-67363A46193B} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + concurrent_connectivity_test + static + Debug + static + Debug + + + concurrent_connectivity_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj.filters b/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj.filters new file mode 100644 index 00000000000..a6492c775a2 --- /dev/null +++ b/vsprojects/vcxproj/test/concurrent_connectivity_test/concurrent_connectivity_test.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\core\surface + + + + + + {1320b717-a107-004b-c517-074e6a7baf97} + + + {f205219c-cee2-c0e8-a922-5486f2d31026} + + + {6804b6ae-88f4-acf9-b3fc-6b12f8e28d4d} + + + + From 0ce7bec923867aae3b115a9a7e4fc2e56f021353 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 17:02:07 -0800 Subject: [PATCH 140/160] Fail faster --- test/core/surface/concurrent_connectivity_test.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index d8639f5e510..1046d422774 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -2,7 +2,9 @@ #include #include +#include #include +#include "test/core/util/test_config.h" #define NUM_THREADS 100 static grpc_channel* channels[NUM_THREADS]; @@ -17,22 +19,22 @@ void create_loop_destroy(void* actually_an_int) { channels[thread_index] = chan; queues[thread_index] = cq; - gpr_timespec inf_future = gpr_inf_future(GPR_CLOCK_REALTIME); - gpr_timespec delta = gpr_time_from_millis(10, GPR_TIMESPAN); for (int j = 0; j < 10; ++j) { - gpr_timespec later_time = - gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), delta); + gpr_timespec later_time = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10); grpc_connectivity_state state = grpc_channel_check_connectivity_state(chan, 1); grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); - grpc_completion_queue_next(cq, inf_future, NULL); + GPR_ASSERT(grpc_completion_queue_next( + cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL) + .type == GRPC_OP_COMPLETE); } grpc_channel_destroy(channels[thread_index]); grpc_completion_queue_destroy(queues[thread_index]); } } -int main() { +int main(int argc, char** argv) { + grpc_test_init(argc, argv); grpc_init(); gpr_thd_id threads[NUM_THREADS]; for (intptr_t i = 0; i < NUM_THREADS; ++i) { From 2ba4133d3a260f3fe218160ef6fe68e0edc1e4a7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 17:09:12 -0800 Subject: [PATCH 141/160] Fix race in subchannel.c --- src/core/client_config/subchannel.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index d91dd116b8c..ddd129c5397 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -184,8 +184,8 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(c); } -void grpc_connected_subchannel_ref(grpc_connected_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_connected_subchannel_ref( + grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON); } @@ -226,8 +226,8 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta, return old_val; } -grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_ref( + grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS), 0 REF_MUTATE_PURPOSE("STRONG_REF")); @@ -235,8 +235,8 @@ grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c return c; } -grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_weak_ref( + grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF")); GPR_ASSERT(old_refs != 0); @@ -646,19 +646,23 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg, if (c->connecting_result.transport != NULL) { publish_transport(exec_ctx, c); - } else if (c->disconnected) { + } + + GRPC_SUBCHANNEL_WEAK_REF(c, "connected"); + gpr_mu_lock(&c->mu); + if (c->disconnected) { GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); } else { gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_mu_lock(&c->mu); GPR_ASSERT(!c->have_alarm); c->have_alarm = 1; grpc_connectivity_state_set(exec_ctx, &c->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE, "connect_failed"); grpc_timer_init(exec_ctx, &c->alarm, c->next_attempt, on_alarm, c, now); - gpr_mu_unlock(&c->mu); } + gpr_mu_unlock(&c->mu); + GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); } static gpr_timespec compute_connect_deadline(grpc_subchannel *c) { @@ -686,8 +690,8 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); } -void grpc_subchannel_call_ref(grpc_subchannel_call *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_subchannel_call_ref( + grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } From 64baf2cbab3bc3caa98907758ee0a6c7db72e68a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 17:16:33 -0800 Subject: [PATCH 142/160] Extend timeout --- test/core/surface/concurrent_connectivity_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 1046d422774..7306a394ebd 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -25,7 +25,7 @@ void create_loop_destroy(void* actually_an_int) { grpc_channel_check_connectivity_state(chan, 1); grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); GPR_ASSERT(grpc_completion_queue_next( - cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL) + cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(30), NULL) .type == GRPC_OP_COMPLETE); } grpc_channel_destroy(channels[thread_index]); From d24fc85b59b4224d7932c5a7a9f17e73402f388b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 17:18:19 -0800 Subject: [PATCH 143/160] Fix cast --- test/core/surface/concurrent_connectivity_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 7306a394ebd..cd1aead5e51 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -11,7 +11,7 @@ static grpc_channel* channels[NUM_THREADS]; static grpc_completion_queue* queues[NUM_THREADS]; void create_loop_destroy(void* actually_an_int) { - int thread_index = (int)(actually_an_int); + int thread_index = (int)(intptr_t)(actually_an_int); for (int i = 0; i < 10; ++i) { grpc_completion_queue* cq = grpc_completion_queue_create(NULL); grpc_channel* chan = grpc_insecure_channel_create("localhost", NULL, NULL); From 5a8ec75012862815bc28a2145c6d294b1993cbd4 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 18:41:54 -0800 Subject: [PATCH 144/160] Use a named constant for error domain and code --- src/objective-c/RxLibrary/GRXWriteable.m | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/objective-c/RxLibrary/GRXWriteable.m b/src/objective-c/RxLibrary/GRXWriteable.m index b122c04ce0c..c15ccb17fb9 100644 --- a/src/objective-c/RxLibrary/GRXWriteable.m +++ b/src/objective-c/RxLibrary/GRXWriteable.m @@ -68,7 +68,11 @@ // of gRPC take care of two different error domains and error code enums. A possibility is to // add error handling to GRXWriters or GRXWriteables, and use them to translate errors between // the two domains. - singleHandler(nil, [NSError errorWithDomain:@"io.grpc" code:13 userInfo:userInfo]); + static NSString *kGRPCErrorDomain = @"io.grpc"; + static NSUInteger kGRPCErrorCodeInternal = 13; + singleHandler(nil, [NSError errorWithDomain:kGRPCErrorDomain + code:kGRPCErrorCodeInternal + userInfo:userInfo]); } }; return [self writeableWithEventHandler:^(BOOL done, id value, NSError *error) { From 3591482a372dee047a7c0867ada8cc8d360d247d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 19:29:23 -0800 Subject: [PATCH 145/160] Expand lock --- src/core/client_config/subchannel.c | 14 +++++--------- test/core/surface/concurrent_connectivity_test.c | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index ddd129c5397..e8f308b6076 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -505,7 +505,8 @@ void grpc_connected_subchannel_ping(grpc_exec_ctx *exec_ctx, elem->filter->start_transport_op(exec_ctx, elem, &op); } -static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { +static void publish_transport_locked(grpc_exec_ctx *exec_ctx, + grpc_subchannel *c) { size_t channel_stack_size; grpc_connected_subchannel *con; grpc_channel_stack *stk; @@ -541,8 +542,6 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { grpc_closure_init(&sw_subchannel->closure, subchannel_on_child_state_changed, sw_subchannel); - gpr_mu_lock(&c->mu); - if (c->disconnected) { gpr_mu_unlock(&c->mu); gpr_free(sw_subchannel); @@ -575,7 +574,6 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { grpc_connectivity_state_set(exec_ctx, &c->state_tracker, GRPC_CHANNEL_READY, "connected"); - gpr_mu_unlock(&c->mu); gpr_free((void *)filters); } @@ -644,13 +642,11 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) { grpc_subchannel *c = arg; - if (c->connecting_result.transport != NULL) { - publish_transport(exec_ctx, c); - } - GRPC_SUBCHANNEL_WEAK_REF(c, "connected"); gpr_mu_lock(&c->mu); - if (c->disconnected) { + if (c->connecting_result.transport != NULL) { + publish_transport_locked(exec_ctx, c); + } else if (c->disconnected) { GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); } else { gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 7306a394ebd..1046d422774 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -25,7 +25,7 @@ void create_loop_destroy(void* actually_an_int) { grpc_channel_check_connectivity_state(chan, 1); grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); GPR_ASSERT(grpc_completion_queue_next( - cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(30), NULL) + cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL) .type == GRPC_OP_COMPLETE); } grpc_channel_destroy(channels[thread_index]); From 9fb89ea9e31dbe376fc1ce9c173d4e52f4b31290 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 20:12:45 -0800 Subject: [PATCH 146/160] If we cant check timers due to contention, at least make sure we do a follow up check soon --- src/core/client_config/subchannel.c | 16 ++++---- src/core/iomgr/timer.c | 3 ++ src/core/iomgr/timer.h | 1 - src/core/surface/completion_queue.c | 2 +- .../surface/concurrent_connectivity_test.c | 39 +++++++++++++++++-- test/core/surface/lame_client_test.c | 2 +- 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index e8f308b6076..ec9e47a6ddb 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -184,8 +184,8 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(c); } -void grpc_connected_subchannel_ref( - grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_connected_subchannel_ref(grpc_connected_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON); } @@ -226,8 +226,8 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta, return old_val; } -grpc_subchannel *grpc_subchannel_ref( - grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS), 0 REF_MUTATE_PURPOSE("STRONG_REF")); @@ -235,8 +235,8 @@ grpc_subchannel *grpc_subchannel_ref( return c; } -grpc_subchannel *grpc_subchannel_weak_ref( - grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF")); GPR_ASSERT(old_refs != 0); @@ -686,8 +686,8 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); } -void grpc_subchannel_call_ref( - grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_subchannel_call_ref(grpc_subchannel_call *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 8379fffad02..8badd83bbb1 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -330,6 +330,9 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_mu_unlock(&g_mu); gpr_mu_unlock(&g_checker_mu); + } else if (next != NULL) { + *next = gpr_time_min( + *next, gpr_time_add(now, gpr_time_from_millis(100, GPR_TIMESPAN))); } return (int)n; diff --git a/src/core/iomgr/timer.h b/src/core/iomgr/timer.h index 9ad1e92f42e..e239e884e7b 100644 --- a/src/core/iomgr/timer.h +++ b/src/core/iomgr/timer.h @@ -96,7 +96,6 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer); *next is never guaranteed to be updated on any given execution; however, with high probability at least one thread in the system will see an update at any time slice. */ - bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_timespec *next); void grpc_timer_list_init(gpr_timespec now); diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index f6a95ebbd3f..b22818ea87d 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -86,7 +86,7 @@ struct grpc_completion_queue { #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1)) static gpr_mu g_freelist_mu; -grpc_completion_queue *g_freelist; +static grpc_completion_queue *g_freelist; static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc, bool success); diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index b380d4471fc..5361201db72 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -1,3 +1,36 @@ +/* +* +* Copyright 2015-2016, Google Inc. +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following disclaimer +* in the documentation and/or other materials provided with the +* distribution. +* * Neither the name of Google Inc. nor the names of its +* contributors may be used to endorse or promote products derived from +* this software without specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +*/ + #include #include @@ -24,9 +57,9 @@ void create_loop_destroy(void* actually_an_int) { grpc_connectivity_state state = grpc_channel_check_connectivity_state(chan, 1); grpc_channel_watch_connectivity_state(chan, state, later_time, cq, NULL); - GPR_ASSERT(grpc_completion_queue_next( - cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), NULL) - .type == GRPC_OP_COMPLETE); + GPR_ASSERT(grpc_completion_queue_next(cq, + GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3), + NULL).type == GRPC_OP_COMPLETE); } grpc_channel_destroy(channels[thread_index]); grpc_completion_queue_destroy(queues[thread_index]); diff --git a/test/core/surface/lame_client_test.c b/test/core/surface/lame_client_test.c index 79e53cb4222..c154d99d4cc 100644 --- a/test/core/surface/lame_client_test.c +++ b/test/core/surface/lame_client_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From a45782f090dfd997d2e80bb7761ddc8db4481b0b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 11 Mar 2016 20:15:51 -0800 Subject: [PATCH 147/160] Fix copyright --- test/core/surface/concurrent_connectivity_test.c | 2 +- test/core/surface/lame_client_test.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 5361201db72..4d3b7bf22a3 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -1,6 +1,6 @@ /* * -* Copyright 2015-2016, Google Inc. +* Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/surface/lame_client_test.c b/test/core/surface/lame_client_test.c index c154d99d4cc..79e53cb4222 100644 --- a/test/core/surface/lame_client_test.c +++ b/test/core/surface/lame_client_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 822f1d7509fe7426a291de1054ea5e9f4892ee15 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 12 Mar 2016 06:54:47 -0800 Subject: [PATCH 148/160] Add comment, tighten loop --- src/core/iomgr/timer.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 8badd83bbb1..f444643428d 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -33,11 +33,11 @@ #include "src/core/iomgr/timer.h" -#include "src/core/iomgr/timer_heap.h" -#include "src/core/iomgr/time_averaged_stats.h" #include #include #include +#include "src/core/iomgr/time_averaged_stats.h" +#include "src/core/iomgr/timer_heap.h" #define INVALID_HEAP_INDEX 0xffffffffu @@ -331,8 +331,17 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_mu_unlock(&g_mu); gpr_mu_unlock(&g_checker_mu); } else if (next != NULL) { + /* TODO(ctiller): this forces calling code to do an short poll, and + then retry the timer check (because this time through the timer list was + contended). + + We could reduce the cost here dramatically by keeping a count of how many + currently active pollers got through the uncontended case above + successfully, and waking up other pollers IFF that count drops to zero. + + Once that count is in place, this entire else branch could disappear. */ *next = gpr_time_min( - *next, gpr_time_add(now, gpr_time_from_millis(100, GPR_TIMESPAN))); + *next, gpr_time_add(now, gpr_time_from_millis(1, GPR_TIMESPAN))); } return (int)n; From 2480f98e192f9e9a00c7cabcd6a77321f1d9525e Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Fri, 26 Feb 2016 14:18:46 -0800 Subject: [PATCH 149/160] Add a script to run interop tests against all versions of prod servers. --- tools/jenkins/run_full_interop.sh | 37 ++++++++++++++++++++++++++++ tools/run_tests/run_interop_tests.py | 6 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tools/jenkins/run_full_interop.sh diff --git a/tools/jenkins/run_full_interop.sh b/tools/jenkins/run_full_interop.sh new file mode 100644 index 00000000000..a82da1cb68c --- /dev/null +++ b/tools/jenkins/run_full_interop.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and runs interop test suite. +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../.. + +tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --prod_servers default cloud_gateway gateway_v2 cloud_gateway_v2 gateway_v4 cloud_gateway_v4 --use_docker --http2_interop -t -j 12 $@ || true diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 1dc772a8565..86eb52bbde1 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -589,7 +589,11 @@ prod_servers = { 'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com', False), 'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com', - True) + True), + 'gateway_v4': ('grpc-test4.sandbox.googleapis.com', + 'grpc-test4.sandbox.googleapis.com', True), + 'cloud_gateway_v4': ('216.239.32.255', 'grpc-test4.sandbox.googleapis.com', + True), } argp = argparse.ArgumentParser(description='Run interop tests.') From c5dd4d47aa0828cb7ba180cef35bf565a57231e5 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 11 Mar 2016 12:57:43 -0800 Subject: [PATCH 150/160] Use a singleton completion queue --- src/objective-c/GRPCClient/private/GRPCCompletionQueue.m | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index 250bbf03a66..be214d4d36b 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -41,7 +41,12 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; @implementation GRPCCompletionQueue + (instancetype)completionQueue { - return [[self alloc] init]; + static GRPCCompletionQueue *singleton = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + singleton = [[self alloc] init]; + }); + return singleton; } - (instancetype)init { From cc43d693e5c715fcbdab1717efc4c84ef836a1fb Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 14 Mar 2016 15:53:30 -0700 Subject: [PATCH 151/160] Reduce the number of threads so that this test works on x86 systems --- test/core/support/thd_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/support/thd_test.c b/test/core/support/thd_test.c index f7807d280a2..2671dd3d3d5 100644 --- a/test/core/support/thd_test.c +++ b/test/core/support/thd_test.c @@ -79,9 +79,9 @@ static void test_options(void) { static void test(void) { int i; gpr_thd_id thd; - gpr_thd_id thds[1000]; + gpr_thd_id thds[300]; struct test t; - int n = 1000; + int n = 300; gpr_thd_options options = gpr_thd_options_default(); gpr_mu_init(&t.mu); gpr_cv_init(&t.done_cv); From 9506ef292259625f675d58875937053e2623a400 Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Wed, 2 Mar 2016 13:53:34 -0800 Subject: [PATCH 152/160] Fix code coverage. --- .../multilang_jessie_x64/Dockerfile.template | 43 +++++ .../test/multilang_jessie_x64/Dockerfile | 155 ++++++++++++++++++ tools/run_tests/run_tests.py | 15 +- 3 files changed, 209 insertions(+), 4 deletions(-) create mode 100644 templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template create mode 100644 tools/dockerfile/test/multilang_jessie_x64/Dockerfile diff --git a/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template new file mode 100644 index 00000000000..fdf44312ec3 --- /dev/null +++ b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template @@ -0,0 +1,43 @@ +%YAML 1.2 +--- | + # Copyright 2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM debian:jessie + + <%include file="../../apt_get_basic.include"/> + <%include file="../../csharp_deps.include"/> + <%include file="../../cxx_deps.include"/> + <%include file="../../node_deps.include"/> + <%include file="../../php_deps.include"/> + <%include file="../../ruby_deps.include"/> + <%include file="../../python_deps.include"/> + <%include file="../../run_tests_addons.include"/> + # Define the default command. + CMD ["bash"] diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile new file mode 100644 index 00000000000..71ebf2bf715 --- /dev/null +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -0,0 +1,155 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +#================ +# C# dependencies + +# Update to a newer version of mono +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list + +# Install dependencies +RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ + mono-devel \ + ca-certificates-mono \ + nuget \ + && apt-get clean + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +#================== +# Node dependencies + +# Install nvm +RUN touch .profile +RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +RUN /bin/bash -l -c "nvm install 0.12 && npm config set cache /tmp/npm-cache" + +#================= +# PHP dependencies + +# Install dependencies + +RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \ + >> /etc/apt/sources.list.d/dotdeb.list" +RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \ + >> /etc/apt/sources.list.d/dotdeb.list" +RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add - + +RUN apt-get update && apt-get install -y \ + git php5 php5-dev phpunit unzip + +#================== +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm use --default ruby-2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +#==================== +# Python dependencies + +# Install dependencies + +RUN apt-get update && apt-get install -y \ + python-all-dev \ + python3-all-dev \ + python-pip + +# Install Python packages from PyPI +RUN pip install pip --upgrade +RUN pip install virtualenv +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 tox + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#====================== +# Zookeeper dependencies +# TODO(jtattermusch): is zookeeper still needed? +RUN apt-get install -y libzookeeper-mt-dev + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index aa5073fbbd2..99bf5df1f92 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -869,10 +869,17 @@ if args.use_docker: dockerfile_dirs = set([l.dockerfile_dir() for l in languages]) if len(dockerfile_dirs) > 1: - print 'Languages to be tested require running under different docker images.' - sys.exit(1) - dockerfile_dir = next(iter(dockerfile_dirs)) - + if 'gcov' in args.config: + dockerfile_dir = 'tools/dockerfile/test/multilang_jessie_x64' + print ('Using multilang_jessie_x64 docker image for code coverage for ' + 'all languages.') + else: + print ('Languages to be tested require running under different docker ' + 'images.') + sys.exit(1) + else: + dockerfile_dir = next(iter(dockerfile_dirs)) + child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) From b19489c6c8ae7d3da0bb3830eef0ad91ddcfe506 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 14 Mar 2016 16:12:14 -0700 Subject: [PATCH 153/160] Minor code cleanup --- test/core/support/thd_test.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/core/support/thd_test.c b/test/core/support/thd_test.c index 2671dd3d3d5..4b5e41eac78 100644 --- a/test/core/support/thd_test.c +++ b/test/core/support/thd_test.c @@ -41,6 +41,8 @@ #include #include "test/core/util/test_config.h" +#define NUM_THREADS 300 + struct test { gpr_mu mu; int n; @@ -79,15 +81,14 @@ static void test_options(void) { static void test(void) { int i; gpr_thd_id thd; - gpr_thd_id thds[300]; + gpr_thd_id thds[NUM_THREADS]; struct test t; - int n = 300; gpr_thd_options options = gpr_thd_options_default(); gpr_mu_init(&t.mu); gpr_cv_init(&t.done_cv); - t.n = n; + t.n = NUM_THREADS; t.is_done = 0; - for (i = 0; i != n; i++) { + for (i = 0; i < NUM_THREADS; i++) { GPR_ASSERT(gpr_thd_new(&thd, &thd_body, &t, NULL)); } gpr_mu_lock(&t.mu); @@ -97,10 +98,10 @@ static void test(void) { gpr_mu_unlock(&t.mu); GPR_ASSERT(t.n == 0); gpr_thd_options_set_joinable(&options); - for (i = 0; i < n; i++) { + for (i = 0; i < NUM_THREADS; i++) { GPR_ASSERT(gpr_thd_new(&thds[i], &thd_body_joinable, NULL, &options)); } - for (i = 0; i < n; i++) { + for (i = 0; i < NUM_THREADS; i++) { gpr_thd_join(thds[i]); } } From 47d841de4d56314153775349df74feeaa814c6c7 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Thu, 10 Mar 2016 11:19:17 -0800 Subject: [PATCH 154/160] Add support for IP Addresses in Subject Alt Names. --- src/core/tsi/ssl_transport_security.c | 67 +++++++-- src/core/tsi/ssl_transport_security.h | 3 +- test/core/tsi/transport_security_test.c | 186 ++++++++++++++---------- test/cpp/end2end/end2end_test.cc | 3 +- 4 files changed, 160 insertions(+), 99 deletions(-) diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 6adcaac9ede..7d0c74bb878 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -33,8 +33,15 @@ #include "src/core/tsi/ssl_transport_security.h" +#include + #include #include +#ifdef GPR_WINSOCK_SOCKET +#include +#else +#include +#endif #include #include @@ -296,21 +303,44 @@ static tsi_result add_subject_alt_names_properties_to_peer( sk_GENERAL_NAME_value(subject_alt_names, TSI_SIZE_AS_SIZE(i)); /* Filter out the non-dns entries names. */ if (subject_alt_name->type == GEN_DNS) { - unsigned char *dns_name = NULL; - int dns_name_size = - ASN1_STRING_to_UTF8(&dns_name, subject_alt_name->d.dNSName); - if (dns_name_size < 0) { + unsigned char *name = NULL; + int name_size; + name_size = ASN1_STRING_to_UTF8(&name, subject_alt_name->d.dNSName); + if (name_size < 0) { gpr_log(GPR_ERROR, "Could not get utf8 from asn1 string."); result = TSI_INTERNAL_ERROR; break; } result = tsi_construct_string_peer_property( - TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, - (const char *)dns_name, (size_t)dns_name_size, + TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, (const char *)name, + (size_t)name_size, &peer->properties[peer->property_count++]); + OPENSSL_free(name); + } else if (subject_alt_name->type == GEN_IPADD) { + char ntop_buf[INET6_ADDRSTRLEN]; + int af; + + if (subject_alt_name->d.iPAddress->length == 4) { + af = AF_INET; + } else if (subject_alt_name->d.iPAddress->length == 16) { + af = AF_INET6; + } else { + gpr_log(GPR_ERROR, "SAN IP Address contained invalid IP"); + result = TSI_INTERNAL_ERROR; + break; + } + const char *name = inet_ntop(af, subject_alt_name->d.iPAddress->data, + ntop_buf, INET6_ADDRSTRLEN); + if (name == NULL) { + gpr_log(GPR_ERROR, "Could not get IP string from asn1 octet."); + result = TSI_INTERNAL_ERROR; + break; + } + + result = tsi_construct_string_peer_property_from_cstring( + TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, name, &peer->properties[peer->property_count++]); - OPENSSL_free(dns_name); - if (result != TSI_OK) break; } + if (result != TSI_OK) break; } return result; } @@ -1436,9 +1466,7 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) { size_t i = 0; size_t san_count = 0; const tsi_peer_property *cn_property = NULL; - - /* For now reject what looks like an IP address. */ - if (looks_like_ip_address(name)) return 0; + int like_ip = looks_like_ip_address(name); /* Check the SAN first. */ for (i = 0; i < peer->property_count; i++) { @@ -1447,8 +1475,15 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) { if (strcmp(property->name, TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY) == 0) { san_count++; - if (does_entry_match_name(property->value.data, property->value.length, - name)) { + + if (!like_ip && does_entry_match_name(property->value.data, + property->value.length, name)) { + return 1; + } else if (like_ip && + strncmp(name, property->value.data, property->value.length) == + 0 && + strlen(name) == property->value.length) { + /* IP Addresses are exact matches only. */ return 1; } } else if (strcmp(property->name, @@ -1457,8 +1492,8 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) { } } - /* If there's no SAN, try the CN. */ - if (san_count == 0 && cn_property != NULL) { + /* If there's no SAN, try the CN, but only if its not like an IP Address */ + if (san_count == 0 && cn_property != NULL && !like_ip) { if (does_entry_match_name(cn_property->value.data, cn_property->value.length, name)) { return 1; diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index 51c0003a854..b587d7ce31a 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -162,8 +162,7 @@ void tsi_ssl_handshaker_factory_destroy(tsi_ssl_handshaker_factory *self); Still TODO(jboeuf): - handle mixed case. - handle %encoded chars. - - handle public suffix wildchar more strictly (e.g. *.co.uk) - - handle IP addresses in SAN. */ + - handle public suffix wildchar more strictly (e.g. *.co.uk) */ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name); #ifdef __cplusplus diff --git a/test/core/tsi/transport_security_test.c b/test/core/tsi/transport_security_test.c index 7ce343987bb..f1e7f7f8a87 100644 --- a/test/core/tsi/transport_security_test.c +++ b/test/core/tsi/transport_security_test.c @@ -61,35 +61,37 @@ typedef struct { of '#' will be replaced with a null character before processing. */ const char *dns_names; + /* Comma separated list of IP SANs to match aggainst */ + const char *ip_names; } cert_name_test_entry; /* Largely inspired from: chromium/src/net/cert/x509_certificate_unittest.cc. TODO(jboeuf) uncomment test cases as we fix tsi_ssl_peer_matches_name. */ const cert_name_test_entry cert_name_test_entries[] = { - {1, "foo.com", "foo.com", NULL}, - {1, "f", "f", NULL}, - {0, "h", "i", NULL}, - {1, "bar.foo.com", "*.foo.com", NULL}, + {1, "foo.com", "foo.com", NULL, NULL}, + {1, "f", "f", NULL, NULL}, + {0, "h", "i", NULL, NULL}, + {1, "bar.foo.com", "*.foo.com", NULL, NULL}, {1, "www.test.fr", "common.name", - "*.test.com,*.test.co.uk,*.test.de,*.test.fr"}, + "*.test.com,*.test.co.uk,*.test.de,*.test.fr", NULL}, /* {1, "wwW.tESt.fr", "common.name", ",*.*,*.test.de,*.test.FR,www"}, */ - {0, "f.uk", ".uk", NULL}, - {0, "w.bar.foo.com", "?.bar.foo.com", NULL}, - {0, "www.foo.com", "(www|ftp).foo.com", NULL}, - {0, "www.foo.com", "www.foo.com#", NULL}, /* # = null char. */ - {0, "www.foo.com", "", "www.foo.com#*.foo.com,#,#"}, - {0, "www.house.example", "ww.house.example", NULL}, - {0, "test.org", "", "www.test.org,*.test.org,*.org"}, - {0, "w.bar.foo.com", "w*.bar.foo.com", NULL}, - {0, "www.bar.foo.com", "ww*ww.bar.foo.com", NULL}, - {0, "wwww.bar.foo.com", "ww*ww.bar.foo.com", NULL}, - {0, "wwww.bar.foo.com", "w*w.bar.foo.com", NULL}, - {0, "wwww.bar.foo.com", "w*w.bar.foo.c0m", NULL}, - {0, "WALLY.bar.foo.com", "wa*.bar.foo.com", NULL}, - {0, "wally.bar.foo.com", "*Ly.bar.foo.com", NULL}, + {0, "f.uk", ".uk", NULL, NULL}, + {0, "w.bar.foo.com", "?.bar.foo.com", NULL, NULL}, + {0, "www.foo.com", "(www|ftp).foo.com", NULL, NULL}, + {0, "www.foo.com", "www.foo.com#", NULL, NULL}, /* # = null char. */ + {0, "www.foo.com", "", "www.foo.com#*.foo.com,#,#", NULL}, + {0, "www.house.example", "ww.house.example", NULL, NULL}, + {0, "test.org", "", "www.test.org,*.test.org,*.org", NULL}, + {0, "w.bar.foo.com", "w*.bar.foo.com", NULL, NULL}, + {0, "www.bar.foo.com", "ww*ww.bar.foo.com", NULL, NULL}, + {0, "wwww.bar.foo.com", "ww*ww.bar.foo.com", NULL, NULL}, + {0, "wwww.bar.foo.com", "w*w.bar.foo.com", NULL, NULL}, + {0, "wwww.bar.foo.com", "w*w.bar.foo.c0m", NULL, NULL}, + {0, "WALLY.bar.foo.com", "wa*.bar.foo.com", NULL, NULL}, + {0, "wally.bar.foo.com", "*Ly.bar.foo.com", NULL, NULL}, /* {1, "ww%57.foo.com", "", "www.foo.com"}, {1, "www&.foo.com", "www%26.foo.com", NULL}, @@ -97,94 +99,101 @@ const cert_name_test_entry cert_name_test_entries[] = { /* Common name must not be used if subject alternative name was provided. */ {0, "www.test.co.jp", "www.test.co.jp", - "*.test.de,*.jp,www.test.co.uk,www.*.co.jp"}, + "*.test.de,*.jp,www.test.co.uk,www.*.co.jp", NULL}, {0, "www.bar.foo.com", "www.bar.foo.com", - "*.foo.com,*.*.foo.com,*.*.bar.foo.com,*..bar.foo.com,"}, + "*.foo.com,*.*.foo.com,*.*.bar.foo.com,*..bar.foo.com,", NULL}, /* IDN tests */ - {1, "xn--poema-9qae5a.com.br", "xn--poema-9qae5a.com.br", NULL}, - {1, "www.xn--poema-9qae5a.com.br", "*.xn--poema-9qae5a.com.br", NULL}, + {1, "xn--poema-9qae5a.com.br", "xn--poema-9qae5a.com.br", NULL, NULL}, + {1, "www.xn--poema-9qae5a.com.br", "*.xn--poema-9qae5a.com.br", NULL, NULL}, {0, "xn--poema-9qae5a.com.br", "", "*.xn--poema-9qae5a.com.br," "xn--poema-*.com.br," "xn--*-9qae5a.com.br," - "*--poema-9qae5a.com.br"}, + "*--poema-9qae5a.com.br", + NULL}, /* The following are adapted from the examples quoted from http://tools.ietf.org/html/rfc6125#section-6.4.3 (e.g., *.example.com would match foo.example.com but not bar.foo.example.com or example.com). */ - {1, "foo.example.com", "*.example.com", NULL}, - {0, "bar.foo.example.com", "*.example.com", NULL}, - {0, "example.com", "*.example.com", NULL}, + {1, "foo.example.com", "*.example.com", NULL, NULL}, + {0, "bar.foo.example.com", "*.example.com", NULL, NULL}, + {0, "example.com", "*.example.com", NULL, NULL}, /* Partial wildcards are disallowed, though RFC 2818 rules allow them. That is, forms such as baz*.example.net, *baz.example.net, and b*z.example.net should NOT match domains. Instead, the wildcard must always be the left-most label, and only a single label. */ - {0, "baz1.example.net", "baz*.example.net", NULL}, - {0, "foobaz.example.net", "*baz.example.net", NULL}, - {0, "buzz.example.net", "b*z.example.net", NULL}, - {0, "www.test.example.net", "www.*.example.net", NULL}, + {0, "baz1.example.net", "baz*.example.net", NULL, NULL}, + {0, "foobaz.example.net", "*baz.example.net", NULL, NULL}, + {0, "buzz.example.net", "b*z.example.net", NULL, NULL}, + {0, "www.test.example.net", "www.*.example.net", NULL, NULL}, /* Wildcards should not be valid for public registry controlled domains, and unknown/unrecognized domains, at least three domain components must be present. */ - {1, "www.test.example", "*.test.example", NULL}, - {1, "test.example.co.uk", "*.example.co.uk", NULL}, - {0, "test.example", "*.example", NULL}, + {1, "www.test.example", "*.test.example", NULL, NULL}, + {1, "test.example.co.uk", "*.example.co.uk", NULL, NULL}, + {0, "test.example", "*.example", NULL, NULL}, /* {0, "example.co.uk", "*.co.uk", NULL}, */ - {0, "foo.com", "*.com", NULL}, - {0, "foo.us", "*.us", NULL}, - {0, "foo", "*", NULL}, + {0, "foo.com", "*.com", NULL, NULL}, + {0, "foo.us", "*.us", NULL, NULL}, + {0, "foo", "*", NULL, NULL}, /* IDN variants of wildcards and registry controlled domains. */ - {1, "www.xn--poema-9qae5a.com.br", "*.xn--poema-9qae5a.com.br", NULL}, - {1, "test.example.xn--mgbaam7a8h", "*.example.xn--mgbaam7a8h", NULL}, + {1, "www.xn--poema-9qae5a.com.br", "*.xn--poema-9qae5a.com.br", NULL, NULL}, + {1, "test.example.xn--mgbaam7a8h", "*.example.xn--mgbaam7a8h", NULL, NULL}, /* {0, "xn--poema-9qae5a.com.br", "*.com.br", NULL}, */ - {0, "example.xn--mgbaam7a8h", "*.xn--mgbaam7a8h", NULL}, + {0, "example.xn--mgbaam7a8h", "*.xn--mgbaam7a8h", NULL, NULL}, /* Wildcards should be permissible for 'private' registry controlled domains. */ - {1, "www.appspot.com", "*.appspot.com", NULL}, - {1, "foo.s3.amazonaws.com", "*.s3.amazonaws.com", NULL}, + {1, "www.appspot.com", "*.appspot.com", NULL, NULL}, + {1, "foo.s3.amazonaws.com", "*.s3.amazonaws.com", NULL, NULL}, /* Multiple wildcards are not valid. */ - {0, "foo.example.com", "*.*.com", NULL}, - {0, "foo.bar.example.com", "*.bar.*.com", NULL}, + {0, "foo.example.com", "*.*.com", NULL, NULL}, + {0, "foo.bar.example.com", "*.bar.*.com", NULL, NULL}, /* Absolute vs relative DNS name tests. Although not explicitly specified in RFC 6125, absolute reference names (those ending in a .) should match either absolute or relative presented names. */ - {1, "foo.com", "foo.com.", NULL}, - {1, "foo.com.", "foo.com", NULL}, - {1, "foo.com.", "foo.com.", NULL}, - {1, "f", "f.", NULL}, - {1, "f.", "f", NULL}, - {1, "f.", "f.", NULL}, - {1, "www-3.bar.foo.com", "*.bar.foo.com.", NULL}, - {1, "www-3.bar.foo.com.", "*.bar.foo.com", NULL}, - {1, "www-3.bar.foo.com.", "*.bar.foo.com.", NULL}, - {0, ".", ".", NULL}, - {0, "example.com", "*.com.", NULL}, - {0, "example.com.", "*.com", NULL}, - {0, "example.com.", "*.com.", NULL}, - {0, "foo.", "*.", NULL}, - {0, "foo", "*.", NULL}, + {1, "foo.com", "foo.com.", NULL, NULL}, + {1, "foo.com.", "foo.com", NULL, NULL}, + {1, "foo.com.", "foo.com.", NULL, NULL}, + {1, "f", "f.", NULL, NULL}, + {1, "f.", "f", NULL, NULL}, + {1, "f.", "f.", NULL, NULL}, + {1, "www-3.bar.foo.com", "*.bar.foo.com.", NULL, NULL}, + {1, "www-3.bar.foo.com.", "*.bar.foo.com", NULL, NULL}, + {1, "www-3.bar.foo.com.", "*.bar.foo.com.", NULL, NULL}, + {0, ".", ".", NULL, NULL}, + {0, "example.com", "*.com.", NULL, NULL}, + {0, "example.com.", "*.com", NULL, NULL}, + {0, "example.com.", "*.com.", NULL, NULL}, + {0, "foo.", "*.", NULL, NULL}, + {0, "foo", "*.", NULL, NULL}, /* {0, "foo.co.uk", "*.co.uk.", NULL}, {0, "foo.co.uk.", "*.co.uk.", NULL}, */ /* An empty CN is OK. */ - {1, "test.foo.com", "", "test.foo.com"}, + {1, "test.foo.com", "", "test.foo.com", NULL}, /* An IP should not be used for the CN. */ - {0, "173.194.195.139", "173.194.195.139", NULL}, + {0, "173.194.195.139", "173.194.195.139", NULL, NULL}, + /* An IP can be used if the SAN IP is present */ + {1, "173.194.195.139", "foo.example.com", NULL, "173.194.195.139"}, + {0, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8"}, + {0, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8,8.8.4.4"}, + {1, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8,173.194.195.139"}, + {0, "173.194.195.139", "foo.example.com", NULL, "173.194.195.13"}, }; typedef struct name_list { @@ -196,7 +205,7 @@ typedef struct { size_t name_count; char *buffer; name_list *names; -} parsed_dns_names; +} parsed_names; name_list *name_list_add(const char *n) { name_list *result = gpr_malloc(sizeof(name_list)); @@ -205,18 +214,18 @@ name_list *name_list_add(const char *n) { return result; } -static parsed_dns_names parse_dns_names(const char *dns_names_str) { - parsed_dns_names result; +static parsed_names parse_names(const char *names_str) { + parsed_names result; name_list *current_nl; size_t i; - memset(&result, 0, sizeof(parsed_dns_names)); - if (dns_names_str == 0) return result; + memset(&result, 0, sizeof(parsed_names)); + if (names_str == 0) return result; result.name_count = 1; - result.buffer = gpr_strdup(dns_names_str); + result.buffer = gpr_strdup(names_str); result.names = name_list_add(result.buffer); current_nl = result.names; - for (i = 0; i < strlen(dns_names_str); i++) { - if (dns_names_str[i] == ',') { + for (i = 0; i < strlen(names_str); i++) { + if (names_str[i] == ',') { result.buffer[i] = '\0'; result.name_count++; i++; @@ -227,7 +236,7 @@ static parsed_dns_names parse_dns_names(const char *dns_names_str) { return result; } -static void destruct_parsed_dns_names(parsed_dns_names *pdn) { +static void destruct_parsed_names(parsed_names *pdn) { name_list *nl = pdn->names; if (pdn->buffer != NULL) gpr_free(pdn->buffer); while (nl != NULL) { @@ -237,8 +246,8 @@ static void destruct_parsed_dns_names(parsed_dns_names *pdn) { } } -static char *processed_dns_name(const char *dns_name) { - char *result = gpr_strdup(dns_name); +static char *processed_name(const char *name) { + char *result = gpr_strdup(name); size_t i; for (i = 0; i < strlen(result); i++) { if (result[i] == '#') { @@ -253,31 +262,48 @@ static tsi_peer peer_from_cert_name_test_entry( size_t i; tsi_peer peer; name_list *nl; - parsed_dns_names dns_entries = parse_dns_names(entry->dns_names); + parsed_names dns_entries = parse_names(entry->dns_names); + parsed_names ip_entries = parse_names(entry->ip_names); nl = dns_entries.names; - GPR_ASSERT(tsi_construct_peer(1 + dns_entries.name_count, &peer) == TSI_OK); + GPR_ASSERT(tsi_construct_peer( + 1 + dns_entries.name_count + ip_entries.name_count, &peer) == + TSI_OK); GPR_ASSERT(tsi_construct_string_peer_property_from_cstring( TSI_X509_SUBJECT_COMMON_NAME_PEER_PROPERTY, entry->common_name, &peer.properties[0]) == TSI_OK); i = 1; while (nl != NULL) { - char *processed = processed_dns_name(nl->name); + char *processed = processed_name(nl->name); GPR_ASSERT(tsi_construct_string_peer_property( TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, processed, strlen(nl->name), &peer.properties[i++]) == TSI_OK); nl = nl->next; gpr_free(processed); } - destruct_parsed_dns_names(&dns_entries); + + nl = ip_entries.names; + while (nl != NULL) { + char *processed = processed_name(nl->name); + GPR_ASSERT(tsi_construct_string_peer_property( + TSI_X509_SUBJECT_ALTERNATIVE_NAME_PEER_PROPERTY, processed, + strlen(nl->name), &peer.properties[i++]) == TSI_OK); + nl = nl->next; + gpr_free(processed); + } + destruct_parsed_names(&dns_entries); + destruct_parsed_names(&ip_entries); return peer; } char *cert_name_test_entry_to_string(const cert_name_test_entry *entry) { char *s; - gpr_asprintf( - &s, "{ success = %s, host_name = %s, common_name = %s, dns_names = %s}", - entry->expected ? "true" : "false", entry->host_name, entry->common_name, - entry->dns_names != NULL ? entry->dns_names : ""); + gpr_asprintf(&s, + "{ success = %s, host_name = %s, common_name = %s, dns_names = " + "%s, ip_names = %s}", + entry->expected ? "true" : "false", entry->host_name, + entry->common_name, + entry->dns_names != NULL ? entry->dns_names : "", + entry->ip_names != NULL ? entry->ip_names : ""); return s; } diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index 8131a14ff76..47598183220 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -1371,11 +1371,12 @@ TEST_P(SecureEnd2endTest, ClientAuthContext) { if (GetParam().credentials_type == kTlsCredentialsType) { EXPECT_EQ("x509_subject_alternative_name", auth_ctx->GetPeerIdentityPropertyName()); - EXPECT_EQ(3u, auth_ctx->GetPeerIdentity().size()); + EXPECT_EQ(4u, auth_ctx->GetPeerIdentity().size()); EXPECT_EQ("*.test.google.fr", ToString(auth_ctx->GetPeerIdentity()[0])); EXPECT_EQ("waterzooi.test.google.be", ToString(auth_ctx->GetPeerIdentity()[1])); EXPECT_EQ("*.test.youtube.com", ToString(auth_ctx->GetPeerIdentity()[2])); + EXPECT_EQ("192.168.1.3", ToString(auth_ctx->GetPeerIdentity()[3])); } } From 27df68940577764f4a98a1972d9958005e0ef858 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sun, 13 Mar 2016 13:34:27 -0700 Subject: [PATCH 155/160] Add note about refactor portability header --- src/core/tsi/ssl_transport_security.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 7d0c74bb878..d8f7b5b0e0f 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -37,6 +37,8 @@ #include #include + +/* TODO(jboeuf): refactor inet_ntop into a portability header. */ #ifdef GPR_WINSOCK_SOCKET #include #else From 4a9e7c4cb155ac3284bdca098227b1a78495a26f Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sun, 13 Mar 2016 14:00:11 -0700 Subject: [PATCH 156/160] Improve looks_like_ip_address for IPv6 addresses, and add tests --- src/core/tsi/ssl_transport_security.c | 7 +++++-- src/core/tsi/ssl_transport_security.h | 2 +- test/core/tsi/transport_security_test.c | 9 ++++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index d8f7b5b0e0f..fcbd910f07f 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -206,13 +206,16 @@ static void ssl_info_callback(const SSL *ssl, int where, int ret) { } /* Returns 1 if name looks like an IP address, 0 otherwise. - This is a very rough heuristic as it does not handle IPV6 or things like: - 0300.0250.00.01, 0xC0.0Xa8.0x0.0x1, 000030052000001, 0xc0.052000001 */ + This is a very rough heuristic, and only handles IPv6 in hexadecimal form. */ static int looks_like_ip_address(const char *name) { size_t i; size_t dot_count = 0; size_t num_size = 0; for (i = 0; i < strlen(name); i++) { + if (name[i] == ':') { + /* IPv6 Address in hexadecimal form, : is not allowed in DNS names. */ + return 1; + } if (name[i] >= '0' && name[i] <= '9') { if (num_size > 3) return 0; num_size++; diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index b587d7ce31a..4909af4c470 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/tsi/transport_security_test.c b/test/core/tsi/transport_security_test.c index f1e7f7f8a87..667d3f03497 100644 --- a/test/core/tsi/transport_security_test.c +++ b/test/core/tsi/transport_security_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -194,6 +194,13 @@ const cert_name_test_entry cert_name_test_entries[] = { {0, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8,8.8.4.4"}, {1, "173.194.195.139", "foo.example.com", NULL, "8.8.8.8,173.194.195.139"}, {0, "173.194.195.139", "foo.example.com", NULL, "173.194.195.13"}, + {0, "2001:db8:a0b:12f0::1", "foo.example.com", NULL, "173.194.195.13"}, + {1, "2001:db8:a0b:12f0::1", "foo.example.com", NULL, + "2001:db8:a0b:12f0::1"}, + {0, "2001:db8:a0b:12f0::1", "foo.example.com", NULL, + "2001:db8:a0b:12f0::2"}, + {1, "2001:db8:a0b:12f0::1", "foo.example.com", NULL, + "2001:db8:a0b:12f0::2,2001:db8:a0b:12f0::1,8.8.8.8"}, }; typedef struct name_list { From f7cea976581a19a06eca5d54089920ead4f788ae Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 14 Mar 2016 17:01:15 -0700 Subject: [PATCH 157/160] Fix copyright --- test/core/support/thd_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/support/thd_test.c b/test/core/support/thd_test.c index 4b5e41eac78..0c176da2d3c 100644 --- a/test/core/support/thd_test.c +++ b/test/core/support/thd_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 7cfb396bf6c2194ca9c1bb6514cc57ce1403f3d8 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Mon, 14 Mar 2016 19:03:54 -0700 Subject: [PATCH 158/160] Make copyright not expire one year too soon. --- src/objective-c/RxLibrary/GRXWriteable.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/RxLibrary/GRXWriteable.m b/src/objective-c/RxLibrary/GRXWriteable.m index c15ccb17fb9..028ba9b551a 100644 --- a/src/objective-c/RxLibrary/GRXWriteable.m +++ b/src/objective-c/RxLibrary/GRXWriteable.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 9db726812eceef6705c71fd1817b0658a00aa38f Mon Sep 17 00:00:00 2001 From: Paddy Byers Date: Thu, 10 Mar 2016 22:56:15 +0000 Subject: [PATCH 159/160] Use gpr_free() in gpr_free_aligned() --- src/core/support/alloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/support/alloc.c b/src/core/support/alloc.c index 0a064b2c188..b99584bd200 100644 --- a/src/core/support/alloc.c +++ b/src/core/support/alloc.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -87,4 +87,4 @@ void *gpr_malloc_aligned(size_t size, size_t alignment_log) { return (void *)ret; } -void gpr_free_aligned(void *ptr) { free(((void **)ptr)[-1]); } +void gpr_free_aligned(void *ptr) { gpr_free(((void **)ptr)[-1]); } From cbd1bce45a605df97aa00a42b7568d08419253f9 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Tue, 15 Mar 2016 13:24:10 -0700 Subject: [PATCH 160/160] Fix two ways tests can hang Both have to do with the test runner's handling of the tests. With one it's the read thread somehow outliving the other threads (e.g. with ctrl-C). The other is due to a filled OS-level pipe's buffer causing a block while code is still holding the GIL in some gRPC core function. We can't empty the buffer from Python because the GIL is held, and the OS can't unblock because it's waiting for the buffer to get cleared: deadlock. --- src/python/grpcio/tests/_runner.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py index b0dbd92a495..32a31ce00e1 100644 --- a/src/python/grpcio/tests/_runner.py +++ b/src/python/grpcio/tests/_runner.py @@ -43,6 +43,13 @@ import uuid from tests import _loader from tests import _result +# This number needs to be large enough to outpace output on stdout and stderr +# from the gRPC core, otherwise we could end up in a potential deadlock. This +# stems from the OS waiting on someone to clear a filled pipe buffer while the +# GIL is held from a write to stderr from gRPC core, but said someone is in +# Python code thus necessitating GIL acquisition. +_READ_BYTES = 2**20 + class CapturePipe(object): """A context-manager pipe to redirect output to a byte array. @@ -76,6 +83,10 @@ class CapturePipe(object): flags = fcntl.fcntl(self._read_fd, fcntl.F_GETFL) fcntl.fcntl(self._read_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) self._read_thread = threading.Thread(target=self._read) + # If the user wants to exit from the Python program and hits ctrl-C and the + # read thread is somehow deadlocked with something else, the Python code may + # refuse to exit. This prevents that by making the read thread second-class. + self._read_thread.daemon = True self._read_thread.start() def stop(self): @@ -93,7 +104,7 @@ class CapturePipe(object): self.output = bytearray() while True: select.select([self._read_fd], [], []) - read_bytes = os.read(self._read_fd, 1024) + read_bytes = os.read(self._read_fd, _READ_BYTES) if read_bytes: self.output.extend(read_bytes) else: