From 7225408dc7fd7af82348f4e123b0906ef411caa6 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 21 Aug 2017 17:05:44 -0700 Subject: [PATCH 001/227] Add Z_FINISH flush at end of stream --- src/core/ext/transport/chttp2/transport/writing.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 80eb51ff0d5..304f5f199eb 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -325,6 +325,19 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( is_last_frame = is_last_data_frame && s->send_trailing_metadata != NULL && grpc_metadata_batch_is_empty(s->send_trailing_metadata); + if (is_last_frame && s->stream_compression_ctx) { + GPR_ASSERT(grpc_stream_compress( + s->stream_compression_ctx, &s->flow_controlled_buffer, + s->compressed_data_buffer, NULL, MAX_SIZE_T, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + grpc_stream_compression_context_destroy(s->stream_compression_ctx); + s->stream_compression_ctx = NULL; + /* After finish, bytes in s->compressed_data_buffer may be + * more than max_outgoing. Start another round of the current + * while loop so that send_bytes, is_last_data_frame and + * is_last_frame are recalculated. */ + continue; + } grpc_chttp2_encode_data(s->id, s->compressed_data_buffer, send_bytes, is_last_frame, &s->stats.outgoing, &t->outbuf); @@ -345,6 +358,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, GRPC_STREAM_COMPRESSION_FLUSH_SYNC)); + GPR_ASSERT(s->compressed_data_buffer->length > 0); } } } else { From 1e95aa1e3b79d3094197eb3af8e5613442a18241 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 21 Aug 2017 18:06:02 -0700 Subject: [PATCH 002/227] clang-format --- src/core/ext/transport/chttp2/transport/writing.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 304f5f199eb..50f0d8588c8 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -330,7 +330,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); - grpc_stream_compression_context_destroy(s->stream_compression_ctx); + grpc_stream_compression_context_destroy( + s->stream_compression_ctx); s->stream_compression_ctx = NULL; /* After finish, bytes in s->compressed_data_buffer may be * more than max_outgoing. Start another round of the current From b02f1bf6fc3dd1943ab2e4f9e7ed7d9278a0a113 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 10:03:16 -0700 Subject: [PATCH 003/227] Minor bug fix --- src/core/ext/transport/chttp2/transport/writing.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 50f0d8588c8..e338caf39ff 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -322,10 +322,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( (send_bytes == s->compressed_data_buffer->length && s->flow_controlled_buffer.length == 0 && s->fetching_send_message == NULL); - is_last_frame = - is_last_data_frame && s->send_trailing_metadata != NULL && - grpc_metadata_batch_is_empty(s->send_trailing_metadata); - if (is_last_frame && s->stream_compression_ctx) { + if (is_last_data_frame && s->stream_compression_ctx) { GPR_ASSERT(grpc_stream_compress( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, @@ -335,10 +332,13 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( s->stream_compression_ctx = NULL; /* After finish, bytes in s->compressed_data_buffer may be * more than max_outgoing. Start another round of the current - * while loop so that send_bytes, is_last_data_frame and - * is_last_frame are recalculated. */ + * while loop so that send_bytes and is_last_data_frame are + * recalculated. */ continue; } + is_last_frame = + is_last_data_frame && s->send_trailing_metadata != NULL && + grpc_metadata_batch_is_empty(s->send_trailing_metadata); grpc_chttp2_encode_data(s->id, s->compressed_data_buffer, send_bytes, is_last_frame, &s->stats.outgoing, &t->outbuf); From 2e0e4e91ed46c5fe39a327ab6484f4531d29788f Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 22 Aug 2017 11:33:15 -0700 Subject: [PATCH 004/227] Increase Python mac target build time 30m -> 60m --- tools/run_tests/artifacts/artifact_targets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 841bbdf9120..41ab8b9339e 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -157,6 +157,7 @@ class PythonArtifact: return create_jobspec(self.name, ['tools/run_tests/artifacts/build_artifact_python.sh'], environ=environ, + timeout_seconds=60*60, use_workspace=True) def __str__(self): From f85caa63829211614a3a252bc993ea10c53c8d2e Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 12:31:54 -0700 Subject: [PATCH 005/227] Advance dependency version of Protobuf on ObjC --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler.podspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index b7387c897a7..7fc065c60c1 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.3.0' + s.dependency '!ProtoCompiler', '3.4.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index c3f95f9f425..25c437911f6 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.3.0' + v = '3.4.0' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC From 9d3f3a2a39db7907a29e2e0d86b7ea242dc7d7cc Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 17:15:58 -0700 Subject: [PATCH 006/227] fix the template --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 7fc065c60c1..b7387c897a7 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.4.0' + s.dependency '!ProtoCompiler', '3.3.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' From c457b47fcf4c5e63a545a60096b9783650586773 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 17:19:21 -0700 Subject: [PATCH 007/227] fix the template --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- .../src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index b7387c897a7..7fc065c60c1 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.3.0' + s.dependency '!ProtoCompiler', '3.4.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template index b8223417b71..196c4054686 100644 --- a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template +++ b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template @@ -103,7 +103,7 @@ s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.3.0' + s.dependency '!ProtoCompiler', '3.4.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' From 2746d84282136cea8e101b392ba53a25ade863cc Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 18:19:14 -0700 Subject: [PATCH 008/227] Add missing files to podspec --- gRPC-Core.podspec | 1 + templates/gRPC-Core.podspec.template | 1 + 2 files changed, 2 insertions(+) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 77ddc6a33e2..083305762a1 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -987,6 +987,7 @@ Pod::Spec.new do |s| 'test/core/end2end/end2end_tests.{c,h}', 'test/core/end2end/end2end_test_utils.c', 'test/core/end2end/tests/*.{c,h}', + 'test/core/end2end/fixtures/*.h', 'test/core/end2end/data/*.{c,h}', 'test/core/util/debugger_macros.{c,h}', 'test/core/util/test_config.{c,h}', diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 616f2cbbff9..6077f8098dd 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -171,6 +171,7 @@ 'test/core/end2end/end2end_tests.{c,h}', 'test/core/end2end/end2end_test_utils.c', 'test/core/end2end/tests/*.{c,h}', + 'test/core/end2end/fixtures/*.h', 'test/core/end2end/data/*.{c,h}', 'test/core/util/debugger_macros.{c,h}', 'test/core/util/test_config.{c,h}', From 103fc3e3f45941ab471de19ff17c7a51a373b80b Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 23 Aug 2017 11:24:47 -0700 Subject: [PATCH 009/227] nit fix --- src/core/ext/transport/chttp2/transport/writing.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index e338caf39ff..0ae10ee883d 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -322,7 +322,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( (send_bytes == s->compressed_data_buffer->length && s->flow_controlled_buffer.length == 0 && s->fetching_send_message == NULL); - if (is_last_data_frame && s->stream_compression_ctx) { + if (is_last_data_frame && s->stream_compression_ctx != NULL) { GPR_ASSERT(grpc_stream_compress( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, From 66999d219274b278c02ffcd3e40c0902e904c6c7 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 23 Aug 2017 13:51:42 -0700 Subject: [PATCH 010/227] fix build.yaml to expand nanopb headers file group --- build.yaml | 2 +- grpc.gemspec | 4 ++++ package.xml | 4 ++++ tools/doxygen/Doxyfile.core.internal | 6 +++++- vsprojects/vcxproj/grpc/grpc.vcxproj | 4 ++++ vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 12 ++++++++++++ .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 4 ++++ .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 12 ++++++++++++ 8 files changed, 46 insertions(+), 2 deletions(-) diff --git a/build.yaml b/build.yaml index dba7da12450..905d0e4cd9d 100644 --- a/build.yaml +++ b/build.yaml @@ -911,7 +911,7 @@ filegroups: - third_party/nanopb/pb_common.c - third_party/nanopb/pb_decode.c - third_party/nanopb/pb_encode.c - filegroups: + uses: - nanopb_headers - name: nanopb_headers headers: diff --git a/grpc.gemspec b/grpc.gemspec index f04a14167b1..9957f050d0c 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -369,6 +369,10 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h ) + s.files += %w( third_party/nanopb/pb.h ) + s.files += %w( third_party/nanopb/pb_common.h ) + s.files += %w( third_party/nanopb/pb_decode.h ) + s.files += %w( third_party/nanopb/pb_encode.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h ) diff --git a/package.xml b/package.xml index e21fa6212ca..c970a918d2f 100644 --- a/package.xml +++ b/package.xml @@ -383,6 +383,10 @@ + + + + diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ea0ebf1704f..771e7fc66bb 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1424,9 +1424,13 @@ src/core/tsi/transport_security_adapter.h \ src/core/tsi/transport_security_grpc.c \ src/core/tsi/transport_security_grpc.h \ src/core/tsi/transport_security_interface.h \ +third_party/nanopb/pb.h \ third_party/nanopb/pb_common.c \ +third_party/nanopb/pb_common.h \ third_party/nanopb/pb_decode.c \ -third_party/nanopb/pb_encode.c +third_party/nanopb/pb_decode.h \ +third_party/nanopb/pb_encode.c \ +third_party/nanopb/pb_encode.h # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 8a659280a46..33e3901e382 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -494,6 +494,10 @@ + + + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 6879047f55b..d5ef8221727 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -1454,6 +1454,18 @@ src\core\ext\filters\client_channel\lb_policy\grpclb\proto\grpc\lb\v1 + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + src\core\ext\filters\client_channel\resolver\fake diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 945ca2f3d72..38e8a3a52a4 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -462,6 +462,10 @@ + + + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index deb3433168d..cceb79936c6 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -1292,6 +1292,18 @@ src\core\ext\filters\client_channel\lb_policy\grpclb\proto\grpc\lb\v1 + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + src\core\ext\census From c353643c378cecf96a580eee901a8be3308e18b9 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 23 Aug 2017 14:03:30 -0700 Subject: [PATCH 011/227] Log error when stream compression fails --- src/core/ext/transport/chttp2/transport/writing.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 0ae10ee883d..b6bf0450eea 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -323,10 +323,12 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( s->flow_controlled_buffer.length == 0 && s->fetching_send_message == NULL); if (is_last_data_frame && s->stream_compression_ctx != NULL) { - GPR_ASSERT(grpc_stream_compress( - s->stream_compression_ctx, &s->flow_controlled_buffer, - s->compressed_data_buffer, NULL, MAX_SIZE_T, - GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + if (!grpc_stream_compress( + s->stream_compression_ctx, &s->flow_controlled_buffer, + s->compressed_data_buffer, NULL, MAX_SIZE_T, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)) { + gpr_log(GPR_ERROR, "Stream compression failed."); + } grpc_stream_compression_context_destroy( s->stream_compression_ctx); s->stream_compression_ctx = NULL; From f5f5ed0305d1d4d59b85f9a00786a876be098f1a Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 24 Aug 2017 11:46:03 -0700 Subject: [PATCH 012/227] Fix syntax error on classifiers --- setup.py | 2 +- src/python/grpcio_health_checking/setup.py | 2 +- src/python/grpcio_reflection/setup.py | 2 +- tools/distrib/python/grpcio_tools/setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 403c13562cc..c90ca8652a1 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] # Environment variable to determine whether or not the Cython extension should # *use* Cython or use the generated C files. Note that this requires the C files diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 0299b4cca95..1f5e9c5130b 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -34,7 +34,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] PACKAGE_DIRECTORIES = { '': '.', diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index bed2311b598..9360550afbc 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -35,7 +35,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] PACKAGE_DIRECTORIES = { '': '.', diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 5c0329bff0c..e0e9226211d 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -47,7 +47,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] PY3 = sys.version_info.major == 3 From 5920abc5358e854a0f4a2217bf8be230977d1853 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 24 Aug 2017 12:26:05 -0700 Subject: [PATCH 013/227] Revert "Don't clear alarm in jobset when running performance tests" This reverts commit c15d32bbe89a2bf950992ded06d1b3da7f1f39a6. --- tools/run_tests/python_utils/jobset.py | 13 ++++--------- tools/run_tests/run_performance_tests.py | 10 +++++----- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 08d652ae3f3..044c6f3aa4f 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -367,10 +367,9 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, add_env, quiet_success, max_time, clear_alarms): + stop_on_failure, add_env, quiet_success, max_time): self._running = set() self._check_cancelled = check_cancelled - self._clear_alarms = clear_alarms self._cancelled = False self._failures = 0 self._completed = 0 @@ -474,10 +473,7 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - # Clear the alarms when finished to avoid a race condition causing job - # failures. Don't do this when running multi-VM tests because clearing - # the alarms causes the test to stall - if platform_string() != 'windows' and self._clear_alarms: + if platform_string() != 'windows': signal.alarm(0) return not self.cancelled() and self._failures == 0 @@ -507,8 +503,7 @@ def run(cmdlines, add_env={}, skip_jobs=False, quiet_success=False, - max_time=-1, - clear_alarms=True): + max_time=-1): if skip_jobs: resultset = {} skipped_job_result = JobResult() @@ -520,7 +515,7 @@ def run(cmdlines, js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, - quiet_success, max_time, clear_alarms) + quiet_success, max_time) for cmdline, remaining in tag_remaining(cmdlines): if not js.start(cmdline): break diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 0db5e4ef830..b9277c919b5 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -183,7 +183,7 @@ def archive_repo(languages): jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( - [archive_job], newline_on_success=True, maxjobs=1, clear_alarms=False) + [archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository created successfully.', @@ -215,7 +215,7 @@ def prepare_remote_hosts(hosts, prepare_local=False): timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( - prepare_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', @@ -248,7 +248,7 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), bui timeout_seconds=build_timeout)) jobset.message('START', 'Building.', do_newline=True) num_failures, _ = jobset.run( - build_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + build_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Built successfully.', @@ -414,7 +414,7 @@ def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name): perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) - failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures @@ -556,7 +556,7 @@ for scenario in scenarios: jobs = [scenario.jobspec] if scenario.workers: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) - scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset), six.iteritems(resultset))) From abbeefa8a462530b29f2341bd798c6f3f47f1350 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 24 Aug 2017 12:28:14 -0700 Subject: [PATCH 014/227] PHP: backport master features into v1.6.x branch --- package.xml | 12 +++---- src/php/ext/grpc/call_credentials.c | 2 +- src/php/ext/grpc/channel.c | 40 +++++++++++----------- src/php/ext/grpc/channel_credentials.c | 8 +++-- src/php/ext/grpc/php_grpc.c | 42 +++++++++++++++++++----- src/php/ext/grpc/php_grpc.h | 6 ++-- src/php/lib/Grpc/ServerStreamingCall.php | 28 ++++++++++++---- src/php/lib/Grpc/UnaryCall.php | 28 ++++++++++++---- src/php/tests/unit_tests/ChannelTest.php | 15 +++++---- templates/package.xml.template | 12 +++---- 10 files changed, 124 insertions(+), 69 deletions(-) diff --git a/package.xml b/package.xml index c970a918d2f..ce678492735 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ grpc-packages@google.com yes - 2017-05-22 + 2017-08-24 1.6.0RC1 @@ -22,13 +22,9 @@ Apache 2.0 -- Fixed some memory leaks #9559, #10996 -- Disabled cares dependency from gRPC C Core #10940 -- De-coupled protobuf dependency #11112 -- Fixed extension reported version #10842 -- Added config.w32 for Windows support #8161 -- Fixed PHP distrib test after cc files were added #11193 -- Fixed protoc plugin comment escape bug #11025 +- License changed to Apache 2.0 +- Added support for php_namespace option in codegen plugin #11886 +- Updated gRPC C Core library version 1.6 diff --git a/src/php/ext/grpc/call_credentials.c b/src/php/ext/grpc/call_credentials.c index f46091d7098..1eee8645df9 100644 --- a/src/php/ext/grpc/call_credentials.c +++ b/src/php/ext/grpc/call_credentials.c @@ -179,7 +179,7 @@ void plugin_get_metadata(void *ptr, grpc_auth_metadata_context context, grpc_metadata_array metadata; bool cleanup = true; - if (Z_TYPE_P(retval) != IS_ARRAY) { + if (retval == NULL || Z_TYPE_P(retval) != IS_ARRAY) { cleanup = false; code = GRPC_STATUS_INVALID_ARGUMENT; } else if (!create_metadata_array(retval, &metadata)) { diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c index f1187e8722f..dc3acc89bb7 100644 --- a/src/php/ext/grpc/channel.c +++ b/src/php/ext/grpc/channel.c @@ -66,6 +66,8 @@ PHP_GRPC_FREE_WRAPPED_FUNC_START(wrapped_grpc_channel) if (!(PHP_GRPC_PERSISTENT_LIST_FIND(&EG(persistent_list), p->wrapper->key, key_len, rsrc))) { grpc_channel_destroy(p->wrapper->wrapped); + free(p->wrapper->target); + free(p->wrapper->args_hashstr); } gpr_mu_unlock(&global_persistent_list_mu); } @@ -158,7 +160,7 @@ void create_and_add_channel_to_persistent_list( grpc_channel_args args, wrapped_grpc_channel_credentials *creds, char *key, - php_grpc_int key_len) { + php_grpc_int key_len TSRMLS_DC) { php_grpc_zend_resource new_rsrc; channel_persistent_le_t *le; // this links each persistent list entry to a destructor @@ -187,10 +189,8 @@ void create_and_add_channel_to_persistent_list( * credentials. * * If the $args array contains a "force_new" key mapping to a boolean value - * of "true", a new underlying grpc_channel will be created regardless. If - * there are any opened channels on the same hostname, user must manually - * call close() on those dangling channels before the end of the PHP - * script. + * of "true", a new and separate underlying grpc_channel will be created + * and returned. This will not affect existing channels. * * @param string $target The hostname to associate with this channel * @param array $args_array The arguments to pass to the Channel @@ -273,19 +273,15 @@ PHP_METHOD(Channel, __construct) { } channel->wrapper = malloc(sizeof(grpc_channel_wrapper)); channel->wrapper->key = key; - channel->wrapper->target = target; - channel->wrapper->args_hashstr = sha1str; + channel->wrapper->target = strdup(target); + channel->wrapper->args_hashstr = strdup(sha1str); if (creds != NULL && creds->hashstr != NULL) { channel->wrapper->creds_hashstr = creds->hashstr; } gpr_mu_init(&channel->wrapper->mu); smart_str_free(&buf); - if (force_new) { - php_grpc_delete_persistent_list_entry(key, key_len TSRMLS_CC); - } - - if (creds != NULL && creds->has_call_creds) { + if (force_new || (creds != NULL && creds->has_call_creds)) { // If the ChannelCredentials object was composed with a CallCredentials // object, there is no way we can tell them apart. Do NOT persist // them. They should be individually destroyed. @@ -293,7 +289,7 @@ PHP_METHOD(Channel, __construct) { } else if (!(PHP_GRPC_PERSISTENT_LIST_FIND(&EG(persistent_list), key, key_len, rsrc))) { create_and_add_channel_to_persistent_list( - channel, target, args, creds, key, key_len); + channel, target, args, creds, key, key_len TSRMLS_CC); } else { // Found a previously stored channel in the persistent list channel_persistent_le_t *le = (channel_persistent_le_t *)rsrc->ptr; @@ -303,7 +299,7 @@ PHP_METHOD(Channel, __construct) { strcmp(creds->hashstr, le->channel->creds_hashstr) != 0)) { // somehow hash collision create_and_add_channel_to_persistent_list( - channel, target, args, creds, key, key_len); + channel, target, args, creds, key, key_len TSRMLS_CC); } else { channel->wrapper = le->channel; } @@ -416,12 +412,14 @@ PHP_METHOD(Channel, close) { gpr_mu_lock(&channel->wrapper->mu); if (channel->wrapper->wrapped != NULL) { grpc_channel_destroy(channel->wrapper->wrapped); + free(channel->wrapper->target); + free(channel->wrapper->args_hashstr); channel->wrapper->wrapped = NULL; - } - php_grpc_delete_persistent_list_entry(channel->wrapper->key, - strlen(channel->wrapper->key) - TSRMLS_CC); + php_grpc_delete_persistent_list_entry(channel->wrapper->key, + strlen(channel->wrapper->key) + TSRMLS_CC); + } gpr_mu_unlock(&channel->wrapper->mu); } @@ -449,12 +447,11 @@ static void php_grpc_channel_plink_dtor(php_grpc_zend_resource *rsrc gpr_mu_lock(&le->channel->mu); if (le->channel->wrapped != NULL) { grpc_channel_destroy(le->channel->wrapped); - free(le->channel->key); - free(le->channel); + free(le->channel->target); + free(le->channel->args_hashstr); } gpr_mu_unlock(&le->channel->mu); } - free(le); } ZEND_BEGIN_ARG_INFO_EX(arginfo_construct, 0, 0, 2) @@ -496,6 +493,7 @@ GRPC_STARTUP_FUNCTION(channel) { INIT_CLASS_ENTRY(ce, "Grpc\\Channel", channel_methods); ce.create_object = create_wrapped_grpc_channel; grpc_ce_channel = zend_register_internal_class(&ce TSRMLS_CC); + gpr_mu_init(&global_persistent_list_mu); le_plink = zend_register_list_destructors_ex( NULL, php_grpc_channel_plink_dtor, "Persistent Channel", module_number); PHP_GRPC_INIT_HANDLER(wrapped_grpc_channel, channel_ce_handlers); diff --git a/src/php/ext/grpc/channel_credentials.c b/src/php/ext/grpc/channel_credentials.c index 19e1cefb6f5..86e4f46c67a 100644 --- a/src/php/ext/grpc/channel_credentials.c +++ b/src/php/ext/grpc/channel_credentials.c @@ -135,6 +135,8 @@ PHP_METHOD(ChannelCredentials, createSsl) { pem_key_cert_pair.private_key = pem_key_cert_pair.cert_chain = NULL; + grpc_set_ssl_roots_override_callback(get_ssl_roots_override); + /* "|s!s!s!" == 3 optional nullable strings */ if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "|s!s!s!", &pem_root_certs, &root_certs_length, @@ -148,7 +150,7 @@ PHP_METHOD(ChannelCredentials, createSsl) { } php_grpc_int hashkey_len = root_certs_length + cert_chain_length; - char hashkey[hashkey_len]; + char *hashkey = emalloc(hashkey_len); if (root_certs_length > 0) { strcpy(hashkey, pem_root_certs); } @@ -164,6 +166,7 @@ PHP_METHOD(ChannelCredentials, createSsl) { pem_key_cert_pair.private_key == NULL ? NULL : &pem_key_cert_pair, NULL); zval *creds_object = grpc_php_wrap_channel_credentials(creds, hashstr, false TSRMLS_CC); + efree(hashkey); RETURN_DESTROY_ZVAL(creds_object); } @@ -177,6 +180,8 @@ PHP_METHOD(ChannelCredentials, createComposite) { zval *cred1_obj; zval *cred2_obj; + grpc_set_ssl_roots_override_callback(get_ssl_roots_override); + /* "OO" == 2 Objects */ if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "OO", &cred1_obj, grpc_ce_channel_credentials, &cred2_obj, @@ -245,7 +250,6 @@ void grpc_init_channel_credentials(TSRMLS_D) { zend_class_entry ce; INIT_CLASS_ENTRY(ce, "Grpc\\ChannelCredentials", channel_credentials_methods); - grpc_set_ssl_roots_override_callback(get_ssl_roots_override); ce.create_object = create_wrapped_grpc_channel_credentials; grpc_ce_channel_credentials = zend_register_internal_class(&ce TSRMLS_CC); PHP_GRPC_INIT_HANDLER(wrapped_grpc_channel_credentials, diff --git a/src/php/ext/grpc/php_grpc.c b/src/php/ext/grpc/php_grpc.c index a96daf7d9b8..4ed56de9930 100644 --- a/src/php/ext/grpc/php_grpc.c +++ b/src/php/ext/grpc/php_grpc.c @@ -34,7 +34,8 @@ #include #include "php_grpc.h" -// ZEND_DECLARE_MODULE_GLOBALS(grpc) +ZEND_DECLARE_MODULE_GLOBALS(grpc) +static PHP_GINIT_FUNCTION(grpc); /* {{{ grpc_functions[] * @@ -55,13 +56,17 @@ zend_module_entry grpc_module_entry = { grpc_functions, PHP_MINIT(grpc), PHP_MSHUTDOWN(grpc), - NULL, + PHP_RINIT(grpc), NULL, PHP_MINFO(grpc), #if ZEND_MODULE_API_NO >= 20010901 PHP_GRPC_VERSION, #endif - STANDARD_MODULE_PROPERTIES}; + PHP_MODULE_GLOBALS(grpc), + PHP_GINIT(grpc), + NULL, + NULL, + STANDARD_MODULE_PROPERTIES_EX}; /* }}} */ #ifdef COMPILE_DL_GRPC @@ -99,7 +104,6 @@ PHP_MINIT_FUNCTION(grpc) { REGISTER_INI_ENTRIES(); */ /* Register call error constants */ - grpc_init(); REGISTER_LONG_CONSTANT("Grpc\\CALL_OK", GRPC_CALL_OK, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("Grpc\\CALL_ERROR", GRPC_CALL_ERROR, @@ -227,7 +231,6 @@ PHP_MINIT_FUNCTION(grpc) { grpc_init_channel_credentials(TSRMLS_C); grpc_init_call_credentials(TSRMLS_C); grpc_init_server_credentials(TSRMLS_C); - grpc_php_init_completion_queue(TSRMLS_C); return SUCCESS; } /* }}} */ @@ -240,9 +243,12 @@ PHP_MSHUTDOWN_FUNCTION(grpc) { */ // WARNING: This function IS being called by PHP when the extension // is unloaded but the logs were somehow suppressed. - grpc_shutdown_timeval(TSRMLS_C); - grpc_php_shutdown_completion_queue(TSRMLS_C); - grpc_shutdown(); + if (GRPC_G(initialized)) { + grpc_shutdown_timeval(TSRMLS_C); + grpc_php_shutdown_completion_queue(TSRMLS_C); + grpc_shutdown(); + GRPC_G(initialized) = 0; + } return SUCCESS; } /* }}} */ @@ -259,6 +265,26 @@ PHP_MINFO_FUNCTION(grpc) { */ } /* }}} */ + +/* {{{ PHP_RINIT_FUNCTION + */ +PHP_RINIT_FUNCTION(grpc) { + if (!GRPC_G(initialized)) { + grpc_init(); + grpc_php_init_completion_queue(TSRMLS_C); + GRPC_G(initialized) = 1; + } + return SUCCESS; +} +/* }}} */ + +/* {{{ PHP_GINIT_FUNCTION + */ +static PHP_GINIT_FUNCTION(grpc) { + grpc_globals->initialized = 0; +} +/* }}} */ + /* The previous line is meant for vim and emacs, so it can correctly fold and unfold functions in source code. See the corresponding marks just before function definition, where the functions purpose is also documented. Please diff --git a/src/php/ext/grpc/php_grpc.h b/src/php/ext/grpc/php_grpc.h index 32329178dc4..e30f011c395 100644 --- a/src/php/ext/grpc/php_grpc.h +++ b/src/php/ext/grpc/php_grpc.h @@ -49,14 +49,16 @@ PHP_MINIT_FUNCTION(grpc); PHP_MSHUTDOWN_FUNCTION(grpc); /* Displays information about the module */ PHP_MINFO_FUNCTION(grpc); +/* Code that runs at request start */ +PHP_RINIT_FUNCTION(grpc); /* Declare any global variables you may need between the BEGIN and END macros here: - +*/ ZEND_BEGIN_MODULE_GLOBALS(grpc) + zend_bool initialized; ZEND_END_MODULE_GLOBALS(grpc) -*/ /* In every utility function you add that needs to use variables in php_grpc_globals, call TSRMLS_FETCH(); after declaring other diff --git a/src/php/lib/Grpc/ServerStreamingCall.php b/src/php/lib/Grpc/ServerStreamingCall.php index 269a99ab593..f8fddfea01c 100644 --- a/src/php/lib/Grpc/ServerStreamingCall.php +++ b/src/php/lib/Grpc/ServerStreamingCall.php @@ -40,13 +40,11 @@ class ServerStreamingCall extends AbstractCall if (array_key_exists('flags', $options)) { $message_array['flags'] = $options['flags']; } - $event = $this->call->startBatch([ + $this->call->startBatch([ OP_SEND_INITIAL_METADATA => $metadata, - OP_RECV_INITIAL_METADATA => true, OP_SEND_MESSAGE => $message_array, OP_SEND_CLOSE_FROM_CLIENT => true, ]); - $this->metadata = $event->metadata; } /** @@ -54,9 +52,15 @@ class ServerStreamingCall extends AbstractCall */ public function responses() { - $response = $this->call->startBatch([ - OP_RECV_MESSAGE => true, - ])->message; + $batch = [OP_RECV_MESSAGE => true]; + if ($this->metadata === null) { + $batch[OP_RECV_INITIAL_METADATA] = true; + } + $read_event = $this->call->startBatch($batch); + if ($this->metadata === null) { + $this->metadata = $read_event->metadata; + } + $response = $read_event->message; while ($response !== null) { yield $this->_deserializeResponse($response); $response = $this->call->startBatch([ @@ -81,4 +85,16 @@ class ServerStreamingCall extends AbstractCall return $status_event->status; } + + /** + * @return mixed The metadata sent by the server + */ + public function getMetadata() + { + if ($this->metadata === null) { + $event = $this->call->startBatch([OP_RECV_INITIAL_METADATA => true]); + $this->metadata = $event->metadata; + } + return $this->metadata; + } } diff --git a/src/php/lib/Grpc/UnaryCall.php b/src/php/lib/Grpc/UnaryCall.php index dd55d02165e..e8496492b86 100644 --- a/src/php/lib/Grpc/UnaryCall.php +++ b/src/php/lib/Grpc/UnaryCall.php @@ -40,13 +40,11 @@ class UnaryCall extends AbstractCall if (isset($options['flags'])) { $message_array['flags'] = $options['flags']; } - $event = $this->call->startBatch([ + $this->call->startBatch([ OP_SEND_INITIAL_METADATA => $metadata, - OP_RECV_INITIAL_METADATA => true, OP_SEND_MESSAGE => $message_array, OP_SEND_CLOSE_FROM_CLIENT => true, ]); - $this->metadata = $event->metadata; } /** @@ -56,14 +54,32 @@ class UnaryCall extends AbstractCall */ public function wait() { - $event = $this->call->startBatch([ + $batch = [ OP_RECV_MESSAGE => true, OP_RECV_STATUS_ON_CLIENT => true, - ]); - + ]; + if ($this->metadata === null) { + $batch[OP_RECV_INITIAL_METADATA] = true; + } + $event = $this->call->startBatch($batch); + if ($this->metadata === null) { + $this->metadata = $event->metadata; + } $status = $event->status; $this->trailing_metadata = $status->metadata; return [$this->_deserializeResponse($event->message), $status]; } + + /** + * @return mixed The metadata sent by the server + */ + public function getMetadata() + { + if ($this->metadata === null) { + $event = $this->call->startBatch([OP_RECV_INITIAL_METADATA => true]); + $this->metadata = $event->metadata; + } + return $this->metadata; + } } diff --git a/src/php/tests/unit_tests/ChannelTest.php b/src/php/tests/unit_tests/ChannelTest.php index 400df0fb66d..c375a16269d 100644 --- a/src/php/tests/unit_tests/ChannelTest.php +++ b/src/php/tests/unit_tests/ChannelTest.php @@ -517,8 +517,6 @@ class ChannelTest extends PHPUnit_Framework_TestCase $state = $this->channel2->getConnectivityState(); $this->assertEquals(GRPC\CHANNEL_IDLE, $state); - // any dangling old connection to the same host must be - // manually closed $this->channel1->close(); $this->channel2->close(); } @@ -529,6 +527,7 @@ class ChannelTest extends PHPUnit_Framework_TestCase $this->channel1 = new Grpc\Channel('localhost:1', []); $this->channel2 = new Grpc\Channel('localhost:1', ["force_new" => true]); + // channel3 shares with channel1 $this->channel3 = new Grpc\Channel('localhost:1', []); // try to connect on channel2 @@ -540,29 +539,31 @@ class ChannelTest extends PHPUnit_Framework_TestCase $state = $this->channel2->getConnectivityState(); $this->assertConnecting($state); $state = $this->channel3->getConnectivityState(); - $this->assertConnecting($state); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); $this->channel1->close(); $this->channel2->close(); } + /** + * @expectedException RuntimeException + */ public function testPersistentChannelForceNewOldChannelClose() { $this->channel1 = new Grpc\Channel('localhost:1', []); $this->channel2 = new Grpc\Channel('localhost:1', ["force_new" => true]); + // channel3 shares with channel1 $this->channel3 = new Grpc\Channel('localhost:1', []); $this->channel1->close(); $state = $this->channel2->getConnectivityState(); $this->assertEquals(GRPC\CHANNEL_IDLE, $state); - $state = $this->channel3->getConnectivityState(); - $this->assertEquals(GRPC\CHANNEL_IDLE, $state); - $this->channel2->close(); - $this->channel3->close(); + // channel3 already closed + $state = $this->channel3->getConnectivityState(); } public function testPersistentChannelForceNewNewChannelClose() diff --git a/templates/package.xml.template b/templates/package.xml.template index 27cc4d01a99..a378b58e0cb 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -12,7 +12,7 @@ grpc-packages@google.com yes - 2017-05-22 + 2017-08-24 ${settings.php_version.php()} @@ -24,13 +24,9 @@ Apache 2.0 - - Fixed some memory leaks #9559, #10996 - - Disabled cares dependency from gRPC C Core #10940 - - De-coupled protobuf dependency #11112 - - Fixed extension reported version #10842 - - Added config.w32 for Windows support #8161 - - Fixed PHP distrib test after cc files were added #11193 - - Fixed protoc plugin comment escape bug #11025 + - License changed to Apache 2.0 + - Added support for php_namespace option in codegen plugin #11886 + - Updated gRPC C Core library version 1.6 From 3da8c5defbc9a83d3db0bf84fb0dce012802d9b3 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 24 Aug 2017 12:35:04 -0700 Subject: [PATCH 015/227] Let alarms at end of jobset.py trigger isntead of clearing --- tools/run_tests/python_utils/jobset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 044c6f3aa4f..50fe7d7df86 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -473,8 +473,10 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - if platform_string() != 'windows': - signal.alarm(0) + global have_alarm + if platform_string() != 'windows' and have_alarm: + signal.alarm(1) + signal.pause() return not self.cancelled() and self._failures == 0 From dfde2881487f05ec4e528c40510029833ed5c8c9 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 29 Aug 2017 11:31:50 -0700 Subject: [PATCH 016/227] 1.6.0-pre1 -> 1.6.0 --- BUILD | 2 +- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 2 +- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 30 files changed, 34 insertions(+), 34 deletions(-) diff --git a/BUILD b/BUILD index 26a8af9fc26..4d1121a79d5 100644 --- a/BUILD +++ b/BUILD @@ -38,7 +38,7 @@ g_stands_for = "garcia" core_version = "4.0.0" -version = "1.6.0-pre1" +version = "1.6.0" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 8fea5079870..ebbaa6a9ec7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.6.0-pre1") +set(PACKAGE_VERSION "1.6.0") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index b47fd222d57..801f8a7d6ed 100644 --- a/Makefile +++ b/Makefile @@ -411,8 +411,8 @@ Q = @ endif CORE_VERSION = 4.0.0 -CPP_VERSION = 1.6.0-pre1 -CSHARP_VERSION = 1.6.0-pre1 +CPP_VERSION = 1.6.0 +CSHARP_VERSION = 1.6.0 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 905d0e4cd9d..723c8fe4460 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 4.0.0 g_stands_for: garcia - version: 1.6.0-pre1 + version: 1.6.0 filegroups: - name: census public_headers: diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 083305762a1..222d3e4f0f5 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.6.0-pre1' + version = '1.6.0' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index e15d4d9bd7f..2d2f31d3239 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.6.0-pre1' + version = '1.6.0' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 50c3f36d62b..38a159404fd 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.6.0-pre1' + version = '1.6.0' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index b8a4d92f054..12591cda336 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.6.0-pre1' + version = '1.6.0' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.json b/package.json index 4dd51237ce1..94e6391a8d8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.6.0-pre1", + "version": "1.6.0", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "https://grpc.io/", diff --git a/package.xml b/package.xml index ce678492735..30018620dd7 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2017-08-24 - 1.6.0RC1 - 1.6.0RC1 + 1.6.0 + 1.6.0 beta diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index de303036654..c65185c77ee 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.6.0-pre1"; } +grpc::string Version() { return "1.6.0"; } } diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 178b0369535..23f6e409091 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.6.0-pre1 + 1.6.0 3.3.0 diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 619955423ba..d402c323c1f 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -38,6 +38,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.6.0-pre1"; + public const string CurrentVersion = "1.6.0"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index f47f8daa4b8..b7aa22b2928 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.6.0-pre1 +set VERSION=1.6.0 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 125b1dfdd2e..455b930525b 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -39,7 +39,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.6.0-pre1" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.6.0-pre1" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.6.0" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.6.0" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 30e6df6b381..66ab2a292fe 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.6.0-pre1", + "version": "1.6.0", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.6.0-pre1", + "grpc": "^1.6.0", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index 2a5c1723539..b0b1e8679a5 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.6.0-pre1", + "version": "1.6.0", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "https://grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 7fc065c60c1..0e180a47113 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.6.0-pre1' + v = '1.6.0' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index bd029903913..35215375787 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -23,4 +23,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.6.0-pre1" +#define GRPC_OBJC_VERSION_STRING @"1.6.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 7a20adf8c31..1a46570117f 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.6.0RC1" +#define PHP_GRPC_VERSION "1.6.0" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 4be88e1a4fc..28f5a1a8a26 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.6.0rc1""" +__version__ = """1.6.0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 76bf737cfe0..ce35bda91c4 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 75b0e47da1f..fb5c5f5f651 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 4194086f405..73e7d2e4977 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index ca09db0bf0f..99510bbe092 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index fedb824c541..66834d20e16 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.6.0.pre1' + VERSION = '1.6.0' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 4de06c9db6f..73fa15f85f8 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.6.0.pre1' + VERSION = '1.6.0' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index f1ad8537002..db3b39ae683 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 478a8da685b..f9225688ad6 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.6.0-pre1 +PROJECT_NUMBER = 1.6.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 3756dc92acf..1b41266c715 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.6.0-pre1 +PROJECT_NUMBER = 1.6.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 738b1bb424292b4138e82a3c2085ab4a3a95507e Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 29 Aug 2017 12:45:53 -0700 Subject: [PATCH 017/227] Get rid of have_alarm var in jobset.py --- tools/run_tests/python_utils/jobset.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 50fe7d7df86..6151a7276a9 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -71,10 +71,8 @@ def platform_string(): if platform_string() == 'windows': pass else: - have_alarm = False def alarm_handler(unused_signum, unused_frame): - global have_alarm - have_alarm = False + pass signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None) signal.signal(signal.SIGALRM, alarm_handler) @@ -454,10 +452,7 @@ class Jobset(object): if platform_string() == 'windows': time.sleep(0.1) else: - global have_alarm - if not have_alarm: - have_alarm = True - signal.alarm(10) + signal.alarm(10) signal.pause() def cancelled(self): @@ -473,10 +468,8 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - global have_alarm - if platform_string() != 'windows' and have_alarm: - signal.alarm(1) - signal.pause() + if platform_string() != 'windows': + signal.alarm(0) return not self.cancelled() and self._failures == 0 From f282c8f525788e6f15331bb6c94471012a7e5346 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 30 Aug 2017 14:40:15 -0700 Subject: [PATCH 018/227] Surface call deadline to Objective C API --- src/objective-c/GRPCClient/GRPCCall.h | 5 +++++ src/objective-c/GRPCClient/GRPCCall.m | 3 ++- .../GRPCClient/private/GRPCChannel.h | 1 + .../GRPCClient/private/GRPCChannel.m | 8 ++++++- src/objective-c/GRPCClient/private/GRPCHost.h | 1 + src/objective-c/GRPCClient/private/GRPCHost.m | 6 +++++- .../GRPCClient/private/GRPCWrappedCall.h | 3 ++- .../GRPCClient/private/GRPCWrappedCall.m | 10 ++++++--- src/objective-c/tests/GRPCClientTests.m | 21 +++++++++++++++++++ 9 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 4d90cfd3847..bcf4ae9cc7f 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -169,6 +169,11 @@ extern id const kGRPCTrailersKey; */ @property (atomic, copy, readwrite) NSString *serverName; +/** + * The deadline for the RPC call in milliseconds. If set to 0, the deadline will be infinity. + */ +@property UInt64 deadline; + /** * The container of the request headers of an RPC conforms to this protocol, which is a subset of * NSMutableDictionary's interface. It will become a NSMutableDictionary later on. diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 436c19e354a..0f8f93cdae8 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -423,7 +423,8 @@ static NSString * const kBearerPrefix = @"Bearer "; _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:_host serverName:_serverName - path:_path]; + path:_path + deadline:_deadline]; NSAssert(_wrappedCall, @"Error allocating RPC objects. Low memory?"); [self sendHeaders:_requestHeaders]; diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h index e2aa5bd036f..67d78c3a44c 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.h +++ b/src/objective-c/GRPCClient/private/GRPCChannel.h @@ -63,5 +63,6 @@ struct grpc_channel_credentials; - (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path serverName:(nonnull NSString *)serverName + deadline:(UInt64)deadline completionQueue:(nonnull GRPCCompletionQueue *)queue; @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 52dbc70b997..72dc961d099 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -182,18 +182,24 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { - (grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName + deadline:(UInt64)deadline completionQueue:(GRPCCompletionQueue *)queue { grpc_slice host_slice; if (serverName) { host_slice = grpc_slice_from_copied_string(serverName.UTF8String); } grpc_slice path_slice = grpc_slice_from_copied_string(path.UTF8String); + gpr_timespec deadline_ms = deadline == 0 ? + gpr_inf_future(GPR_CLOCK_REALTIME) : + gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_millis(deadline, GPR_TIMESPAN)); grpc_call *call = grpc_channel_create_call(_unmanagedChannel, NULL, GRPC_PROPAGATE_DEFAULTS, queue.unmanagedQueue, path_slice, serverName ? &host_slice : NULL, - gpr_inf_future(GPR_CLOCK_REALTIME), NULL); + deadline_ms, NULL); if (serverName) { grpc_slice_unref(host_slice); } diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index 0c1d7152405..7513cd0b8d8 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -55,6 +55,7 @@ struct grpc_channel_credentials; /** Create a grpc_call object to the provided path on this host. */ - (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName + deadline:(UInt64)deadline completionQueue:(GRPCCompletionQueue *)queue; // TODO: There's a race when a new RPC is coming through just as an existing one is getting diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index 23794c1fed2..b847637ac59 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -121,6 +121,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName + deadline:(UInt64)deadline completionQueue:(GRPCCompletionQueue *)queue { GRPCChannel *channel; // This is racing -[GRPCHost disconnect]. @@ -130,7 +131,10 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; } channel = _channel; } - return [channel unmanagedCallWithPath:path serverName:serverName completionQueue:queue]; + return [channel unmanagedCallWithPath:path + serverName:serverName + deadline:deadline + completionQueue:queue]; } - (BOOL)setTLSPEMRootCerts:(nullable NSString *)pemRootCerts diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index 64075591a37..1995f1d0b59 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -76,7 +76,8 @@ - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName - path:(NSString *)path NS_DESIGNATED_INITIALIZER; + path:(NSString *)path + deadline:(UInt64)deadline NS_DESIGNATED_INITIALIZER; - (void)startBatchWithOperations:(NSArray *)ops errorHandler:(void(^)())errorHandler; diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index 87dc33af88b..b855a96bee5 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -238,12 +238,13 @@ } - (instancetype)init { - return [self initWithHost:nil serverName:nil path:nil]; + return [self initWithHost:nil serverName:nil path:nil deadline:0]; } - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName - path:(NSString *)path { + path:(NSString *)path + deadline:(UInt64)deadline { if (!path || !host) { [NSException raise:NSInvalidArgumentException format:@"path and host cannot be nil."]; @@ -255,7 +256,10 @@ // queue. Currently we use a singleton queue. _queue = [GRPCCompletionQueue completionQueue]; - _call = [[GRPCHost hostWithAddress:host] unmanagedCallWithPath:path serverName:serverName completionQueue:_queue]; + _call = [[GRPCHost hostWithAddress:host] unmanagedCallWithPath:path + serverName:serverName + deadline:deadline + completionQueue:_queue]; if (_call == NULL) { return nil; } diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 9afe5071217..d9c6932ea8b 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -422,4 +422,25 @@ static GRPCProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +- (void)testDeadline { + __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; + + GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress + path:kEmptyCallMethod.HTTPPath + requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; + + id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + XCTAssert(0, @"Failure: response received; Expect: no response received."); + } completionHandler:^(NSError *errorOrNil) { + XCTAssertNotNil(errorOrNil, @"Failure: no error received; Expect: receive deadline exceeded."); + XCTAssertEqual(errorOrNil.code, GRPCErrorCodeDeadlineExceeded); + [completion fulfill]; + }]; + + call.deadline = 1; + [call startWithWriteable:responsesWriteable]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; +} + @end From 1ebe1e89155159a933ffadc58a07d8aa5e49b844 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Wed, 30 Aug 2017 14:52:57 -0700 Subject: [PATCH 019/227] PHP: fix c++ error after c core 1.6 uptake --- config.m4 | 2 +- templates/config.m4.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.m4 b/config.m4 index f6f8531b2fd..393250fff11 100644 --- a/config.m4 +++ b/config.m4 @@ -12,7 +12,7 @@ if test "$PHP_GRPC" != "no"; then LIBS="-lpthread $LIBS" CFLAGS="-Wall -Werror -Wno-parentheses-equality -Wno-unused-value -std=c11" - CXXFLAGS="-std=c++11" + CXXFLAGS="-std=c++11 -fno-exceptions -fno-rtti" GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" PHP_REQUIRE_CXX() PHP_ADD_LIBRARY(pthread) diff --git a/templates/config.m4.template b/templates/config.m4.template index f91893c2bd3..cd93fbd0fbe 100644 --- a/templates/config.m4.template +++ b/templates/config.m4.template @@ -14,7 +14,7 @@ LIBS="-lpthread $LIBS" CFLAGS="-Wall -Werror -Wno-parentheses-equality -Wno-unused-value -std=c11" - CXXFLAGS="-std=c++11" + CXXFLAGS="-std=c++11 -fno-exceptions -fno-rtti" GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" PHP_REQUIRE_CXX() PHP_ADD_LIBRARY(pthread) From c7c8e3cb1712438c64b1212911b6c22165982426 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 30 Aug 2017 15:36:47 -0700 Subject: [PATCH 020/227] Rename deadline to timeout --- src/objective-c/GRPCClient/GRPCCall.h | 4 ++-- src/objective-c/GRPCClient/GRPCCall.m | 2 +- src/objective-c/GRPCClient/private/GRPCChannel.h | 2 +- src/objective-c/GRPCClient/private/GRPCChannel.m | 12 ++++++------ src/objective-c/GRPCClient/private/GRPCHost.h | 2 +- src/objective-c/GRPCClient/private/GRPCHost.m | 4 ++-- src/objective-c/GRPCClient/private/GRPCWrappedCall.h | 2 +- src/objective-c/GRPCClient/private/GRPCWrappedCall.m | 6 +++--- src/objective-c/tests/GRPCClientTests.m | 4 ++-- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index bcf4ae9cc7f..26c920b973a 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -170,9 +170,9 @@ extern id const kGRPCTrailersKey; @property (atomic, copy, readwrite) NSString *serverName; /** - * The deadline for the RPC call in milliseconds. If set to 0, the deadline will be infinity. + * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. */ -@property UInt64 deadline; +@property UInt64 timeout; /** * The container of the request headers of an RPC conforms to this protocol, which is a subset of diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 0f8f93cdae8..d6c3a3c165a 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -424,7 +424,7 @@ static NSString * const kBearerPrefix = @"Bearer "; _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:_host serverName:_serverName path:_path - deadline:_deadline]; + timeout:_timeout]; NSAssert(_wrappedCall, @"Error allocating RPC objects. Low memory?"); [self sendHeaders:_requestHeaders]; diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h index 67d78c3a44c..9dff86c271f 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.h +++ b/src/objective-c/GRPCClient/private/GRPCChannel.h @@ -63,6 +63,6 @@ struct grpc_channel_credentials; - (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path serverName:(nonnull NSString *)serverName - deadline:(UInt64)deadline + timeout:(UInt64)timeout completionQueue:(nonnull GRPCCompletionQueue *)queue; @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 72dc961d099..39b4e6c1a7e 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -182,18 +182,18 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { - (grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - deadline:(UInt64)deadline + timeout:(UInt64)timeout completionQueue:(GRPCCompletionQueue *)queue { grpc_slice host_slice; if (serverName) { host_slice = grpc_slice_from_copied_string(serverName.UTF8String); } grpc_slice path_slice = grpc_slice_from_copied_string(path.UTF8String); - gpr_timespec deadline_ms = deadline == 0 ? - gpr_inf_future(GPR_CLOCK_REALTIME) : - gpr_time_add( - gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_millis(deadline, GPR_TIMESPAN)); + gpr_timespec deadline_ms = timeout == 0 ? + gpr_inf_future(GPR_CLOCK_REALTIME) : + gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_millis(timeout, GPR_TIMESPAN)); grpc_call *call = grpc_channel_create_call(_unmanagedChannel, NULL, GRPC_PROPAGATE_DEFAULTS, queue.unmanagedQueue, diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index 7513cd0b8d8..a9b3032c9c6 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -55,7 +55,7 @@ struct grpc_channel_credentials; /** Create a grpc_call object to the provided path on this host. */ - (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - deadline:(UInt64)deadline + timeout:(UInt64)timeout completionQueue:(GRPCCompletionQueue *)queue; // TODO: There's a race when a new RPC is coming through just as an existing one is getting diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index b847637ac59..a0791827db3 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -121,7 +121,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - deadline:(UInt64)deadline + timeout:(UInt64)timeout completionQueue:(GRPCCompletionQueue *)queue { GRPCChannel *channel; // This is racing -[GRPCHost disconnect]. @@ -133,7 +133,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; } return [channel unmanagedCallWithPath:path serverName:serverName - deadline:deadline + timeout:timeout completionQueue:queue]; } diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index 1995f1d0b59..c0a6c71c72f 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -77,7 +77,7 @@ - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName path:(NSString *)path - deadline:(UInt64)deadline NS_DESIGNATED_INITIALIZER; + timeout:(UInt64)timeout NS_DESIGNATED_INITIALIZER; - (void)startBatchWithOperations:(NSArray *)ops errorHandler:(void(^)())errorHandler; diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index b855a96bee5..7366f319312 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -238,13 +238,13 @@ } - (instancetype)init { - return [self initWithHost:nil serverName:nil path:nil deadline:0]; + return [self initWithHost:nil serverName:nil path:nil timeout:0]; } - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName path:(NSString *)path - deadline:(UInt64)deadline { + timeout:(UInt64)timeout { if (!path || !host) { [NSException raise:NSInvalidArgumentException format:@"path and host cannot be nil."]; @@ -258,7 +258,7 @@ _call = [[GRPCHost hostWithAddress:host] unmanagedCallWithPath:path serverName:serverName - deadline:deadline + timeout:timeout completionQueue:_queue]; if (_call == NULL) { return nil; diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index d9c6932ea8b..d050319533a 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -422,7 +422,7 @@ static GRPCProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -- (void)testDeadline { +- (void)testTimeout { __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress @@ -437,7 +437,7 @@ static GRPCProtoMethod *kUnaryCallMethod; [completion fulfill]; }]; - call.deadline = 1; + call.timeout = 1; [call startWithWriteable:responsesWriteable]; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; From 9c437c2067e9e0142ad5799696388c2acb25f3d5 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 31 Aug 2017 11:23:03 +0200 Subject: [PATCH 021/227] Use docfx to generate C# API reference --- src/csharp/doc/.gitignore | 2 + src/csharp/doc/README.md | 9 ++- src/csharp/doc/docfx.json | 37 ++++++++++ src/csharp/doc/grpc_csharp_public.shfbproj | 83 ---------------------- src/csharp/doc/toc.yml | 3 + 5 files changed, 50 insertions(+), 84 deletions(-) create mode 100644 src/csharp/doc/.gitignore create mode 100644 src/csharp/doc/docfx.json delete mode 100644 src/csharp/doc/grpc_csharp_public.shfbproj create mode 100644 src/csharp/doc/toc.yml diff --git a/src/csharp/doc/.gitignore b/src/csharp/doc/.gitignore new file mode 100644 index 00000000000..09ee235efcd --- /dev/null +++ b/src/csharp/doc/.gitignore @@ -0,0 +1,2 @@ +html +obj diff --git a/src/csharp/doc/README.md b/src/csharp/doc/README.md index 585500b5cab..46cce013a18 100644 --- a/src/csharp/doc/README.md +++ b/src/csharp/doc/README.md @@ -1,2 +1,9 @@ +DocFX-generated C# API Reference +-------------------------------- -SandCastle project files to generate HTML reference documentation. \ No newline at end of file +Install docfx based on instructions here: https://github.com/dotnet/docfx + +``` +# generate docfx documentation into ./html directory +$ docfx +``` diff --git a/src/csharp/doc/docfx.json b/src/csharp/doc/docfx.json new file mode 100644 index 00000000000..7219d0e7a69 --- /dev/null +++ b/src/csharp/doc/docfx.json @@ -0,0 +1,37 @@ +{ + "metadata": [ + { + "src": [ + { + "files": ["Grpc.Core/Grpc.Core.csproj", + "Grpc.Auth/Grpc.Auth.csproj", + "Grpc.Core.Testing/Grpc.Core.Testing.csproj", + "Grpc.HealthCheck/Grpc.HealthCheck.csproj", + "Grpc.Reflection/Grpc.HealthCheck.csproj"], + "exclude": [ "**/bin/**", "**/obj/**" ], + "cwd": ".." + } + ], + "properties": { "TargetFramework": "net45" }, + "dest": "obj/api" + } + ], + "build": { + "content": [ + { + "files": [ "**/*.yml" ], + "cwd": "obj/api", + "dest": "api" + }, + { + "files": [ "toc.yml"], + } + ], + "globalMetadata": { + "_appTitle": "gRPC C#", + "_enableSearch": true, + "_disableContribution": true + }, + "dest": "html" + } +} diff --git a/src/csharp/doc/grpc_csharp_public.shfbproj b/src/csharp/doc/grpc_csharp_public.shfbproj deleted file mode 100644 index fab953da353..00000000000 --- a/src/csharp/doc/grpc_csharp_public.shfbproj +++ /dev/null @@ -1,83 +0,0 @@ - - - - - Debug - AnyCPU - 2.0 - {77e3da09-fc92-486f-a90a-99ca788e8b59} - 2015.6.5.0 - - Documentation - Documentation - Documentation - - .NET Framework 4.5 - ..\..\..\doc\ref\csharp\html - en-US - - - - - - - OnlyWarningsAndErrors - Website - False - True - False - True - 1.0.0.0 - 2 - False - Standard - Blank - True - VS2013 - False - MemberName - gRPC C# - AboveNamespaces - Documentation - - Provides OAuth2 based authentication for gRPC. <c>Grpc.Auth</c> currently consists of a set of very lightweight wrappers and uses C# <a href="https://www.nuget.org/packages/Google.Apis.Auth/">Google.Apis.Auth</a> library. - Main namespace for gRPC C# functionality. Contains concepts representing both client side and server side gRPC logic. - -<seealso cref="Grpc.Core.Channel"/> -<seealso cref="Grpc.Core.Server"/> - Provides functionality to redirect gRPC logs to application-specified destination. - Various utilities for gRPC C#. - - Summary, Parameter, AutoDocumentCtors, Namespace, TypeParameter, AutoDocumentDispose - - - - - - - - - - - - - - - - - - - - - - - - - - - OnBuildSuccess - - \ No newline at end of file diff --git a/src/csharp/doc/toc.yml b/src/csharp/doc/toc.yml new file mode 100644 index 00000000000..c3a1e415ab1 --- /dev/null +++ b/src/csharp/doc/toc.yml @@ -0,0 +1,3 @@ +- name: API Documentation + href: obj/api/ + homepage: obj/api/Grpc.Core.yml From 76e0c1ddd536342bdc5059209da38d5406c7e717 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 31 Aug 2017 14:59:17 -0700 Subject: [PATCH 022/227] Guarantee that Z_FINISH is only applied at the end of stream --- src/core/ext/transport/chttp2/transport/writing.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index b6bf0450eea..410c154206c 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -322,7 +322,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( (send_bytes == s->compressed_data_buffer->length && s->flow_controlled_buffer.length == 0 && s->fetching_send_message == NULL); - if (is_last_data_frame && s->stream_compression_ctx != NULL) { + if (is_last_data_frame && s->send_trailing_metadata != NULL && + s->stream_compression_ctx != NULL) { if (!grpc_stream_compress( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, From 0ff222a23b94a34d03d1cfa4b5c148406e441f4c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 1 Sep 2017 09:41:43 -0700 Subject: [PATCH 023/227] Add pollset_kick stats --- src/core/lib/debug/stats_data.c | 243 +++++++++++++++++- src/core/lib/debug/stats_data.h | 128 +++++++-- src/core/lib/debug/stats_data.yaml | 12 + src/core/lib/iomgr/ev_epoll1_linux.c | 25 +- .../iomgr/ev_epoll_limited_pollers_linux.c | 5 +- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 5 +- src/core/lib/iomgr/ev_epollex_linux.c | 9 +- src/core/lib/iomgr/ev_epollsig_linux.c | 5 +- src/core/lib/iomgr/ev_poll_posix.c | 46 ++-- src/core/lib/iomgr/ev_posix.c | 4 +- src/core/lib/iomgr/ev_posix.h | 2 +- src/core/lib/iomgr/pollset.h | 2 +- .../google_default_credentials.c | 3 +- src/core/lib/surface/call.c | 5 +- src/core/lib/surface/completion_queue.c | 11 +- test/core/http/httpcli_test.c | 2 +- test/core/http/httpscli_test.c | 2 +- test/core/iomgr/endpoint_tests.c | 9 +- test/core/iomgr/fd_posix_test.c | 16 +- test/core/iomgr/resolve_address_test.c | 6 +- test/core/iomgr/tcp_client_posix_test.c | 6 +- test/core/iomgr/tcp_posix_test.c | 11 +- test/core/iomgr/tcp_server_posix_test.c | 4 +- test/core/iomgr/udp_server_test.c | 8 +- test/core/security/oauth2_utils.c | 3 +- .../print_google_default_creds_token.c | 3 +- test/core/security/verify_jwt.c | 3 +- .../surface/concurrent_connectivity_test.c | 3 +- test/core/util/port_server_client.c | 9 +- test/cpp/qps/report.cc | 4 +- 30 files changed, 477 insertions(+), 117 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 9277ee57b23..445dcb67a32 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -24,6 +24,12 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "server_calls_created", "syscall_poll", "syscall_wait", + "pollset_kick", + "pollset_kicked_without_poller", + "pollset_kicked_again", + "pollset_kick_wakeup_fd", + "pollset_kick_wakeup_cv", + "pollset_kick_own_thread", "histogram_slow_lookups", "syscall_write", "syscall_read", @@ -47,10 +53,226 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "executor_queue_drained", }; const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { - "tcp_write_size", "tcp_write_iov_size", "tcp_read_size", - "tcp_read_offer", "tcp_read_iov_size", "http2_send_message_size", + "call_initial_size", "poll_events_returned", "tcp_write_size", + "tcp_write_iov_size", "tcp_read_size", "tcp_read_offer", + "tcp_read_iov_size", "http2_send_message_size", }; const double grpc_stats_table_0[64] = {0, + 1, + 2, + 3, + 4, + 5, + 6.03034139457, + 7.27300346702, + 8.77173877401, + 10.5793158863, + 12.759377303, + 15.3886802239, + 18.5597990724, + 22.3843849243, + 26.9970966002, + 32.5603418323, + 39.2699954346, + 47.3622958068, + 57.1221625891, + 68.8932283217, + 83.0899373109, + 100.212137688, + 120.862680427, + 145.768644968, + 175.806938756, + 212.035172047, + 255.72889502, + 308.426508286, + 371.98342802, + 448.637412817, + 541.087352333, + 652.588291771, + 787.066037897, + 949.255381718, + 1144.86680448, + 1380.78753647, + 1665.32404765, + 2008.49450799, + 2422.38151446, + 2921.55750402, + 3523.59783062, + 4249.69957117, + 5125.4278477, + 6181.61594298, + 7455.45090126, + 8991.78283702, + 10844.7040506, + 13079.4535497, + 15774.7140318, + 19025.3822027, + 22945.9099689, + 27674.3341444, + 33377.1365516, + 40255.1056359, + 48550.4059718, + 58555.104571, + 70621.4541917, + 85174.2957114, + 102726.016236, + 123894.589602, + 149425.334448, + 180217.155944, + 217354.195101, + 262144.0}; +const uint8_t grpc_stats_table_1[124] = { + 0, 2, 2, 4, 4, 6, 6, 8, 8, 11, 11, 11, 13, 13, + 15, 15, 17, 17, 20, 20, 20, 21, 23, 23, 26, 26, 26, 28, + 28, 30, 30, 32, 32, 35, 35, 35, 37, 37, 38, 41, 41, 41, + 43, 43, 45, 45, 47, 47, 50, 50, 50, 52, 52, 54, 54, 56, + 56, 58, 58, 60, 60, 62, 62, 65, 65, 65, 67, 67, 69, 69, + 71, 71, 73, 73, 76, 76, 76, 78, 78, 80, 80, 82, 82, 84, + 84, 86, 86, 88, 88, 91, 91, 91, 93, 93, 95, 95, 97, 97, + 100, 100, 100, 101, 103, 103, 106, 106, 106, 108, 108, 110, 110, 112, + 112, 115, 115, 115, 117, 117, 118, 121, 121, 121, 124, 124}; +const double grpc_stats_table_2[128] = {0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29.0367057593, + 30.1117957626, + 31.2266911945, + 32.382865859, + 33.5818481283, + 34.8252229627, + 36.1146340061, + 37.4517857586, + 38.8384458298, + 40.2764472753, + 41.7676910202, + 43.3141483714, + 44.9178636242, + 46.580956764, + 48.3056262695, + 50.0941520181, + 51.9488983005, + 53.8723169454, + 55.8669505614, + 57.9354358972, + 60.0805073281, + 62.3050004699, + 64.6118559278, + 67.0041231836, + 69.4849646266, + 72.0576597342, + 74.7256094075, + 77.4923404661, + 80.3615103113, + 83.3369117602, + 86.4224780597, + 89.6222880862, + 92.940571737, + 96.3817155226, + 99.9502683646, + 103.65094761, + 107.488645265, + 111.468434465, + 115.595576179, + 119.875526164, + 124.313942178, + 128.916691457, + 133.689858475, + 138.639752984, + 143.772918355, + 149.09614023, + 154.616455489, + 160.341161557, + 166.277826044, + 172.434296755, + 178.818712061, + 185.439511656, + 192.305447719, + 199.425596476, + 206.809370205, + 214.466529676, + 222.407197051, + 230.641869269, + 239.181431919, + 248.037173633, + 257.220801006, + 266.744454071, + 276.62072235, + 286.862661493, + 297.48381054, + 308.498209814, + 319.920419488, + 331.765538824, + 344.04922614, + 356.787719506, + 369.997858208, + 383.697105013, + 397.903569249, + 412.636030746, + 427.913964659, + 443.757567219, + 460.187782422, + 477.226329722, + 494.895732741, + 513.219349041, + 532.221401003, + 551.927007848, + 572.36221884, + 593.554047722, + 615.530508428, + 638.320652111, + 661.954605552, + 686.46361098, + 711.880067376, + 738.237573297, + 765.570971297, + 793.91639398, + 823.311311768, + 853.794582433, + 885.406502465, + 918.188860339, + 952.184991756, + 987.439836931, + 1024.0}; +const uint8_t grpc_stats_table_3[166] = { + 0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 11, 12, 14, + 14, 15, 17, 17, 18, 20, 20, 22, 22, 24, 24, 25, 27, 27, + 29, 29, 31, 31, 34, 34, 34, 36, 36, 38, 38, 39, 40, 42, + 42, 43, 44, 46, 46, 47, 49, 49, 50, 52, 52, 54, 54, 55, + 57, 57, 59, 59, 61, 61, 63, 63, 65, 65, 68, 68, 68, 70, + 70, 71, 72, 73, 75, 75, 76, 78, 78, 79, 81, 81, 82, 84, + 84, 86, 86, 87, 89, 89, 91, 91, 93, 93, 95, 95, 97, 97, + 100, 100, 100, 102, 102, 103, 104, 105, 107, 107, 108, 109, 111, 111, + 113, 113, 114, 116, 116, 117, 119, 119, 121, 121, 123, 123, 125, 125, + 127, 127, 129, 129, 131, 131, 134, 134, 134, 135, 136, 137, 139, 139, + 140, 141, 143, 143, 144, 146, 146, 148, 148, 149, 151, 151, 153, 153, + 155, 155, 157, 157, 159, 159, 161, 161, 163, 163, 166, 166}; +const double grpc_stats_table_4[64] = {0, 1, 2, 3, @@ -114,13 +336,13 @@ const double grpc_stats_table_0[64] = {0, 10005134.9318, 12956014.428, 16777216.0}; -const uint8_t grpc_stats_table_1[87] = { +const uint8_t grpc_stats_table_5[87] = { 0, 1, 3, 3, 4, 6, 6, 7, 9, 9, 10, 12, 12, 13, 15, 15, 16, 18, 18, 19, 21, 21, 22, 24, 24, 25, 27, 27, 28, 30, 30, 31, 32, 34, 34, 36, 36, 37, 39, 39, 40, 42, 42, 43, 44, 46, 46, 47, 49, 49, 51, 51, 52, 53, 55, 55, 56, 58, 58, 59, 61, 61, 63, 63, 64, 65, 67, 67, 68, 70, 70, 71, 73, 73, 75, 75, 76, 77, 79, 79, 80, 82, 82, 83, 85, 85, 87}; -const double grpc_stats_table_2[64] = {0, +const double grpc_stats_table_6[64] = {0, 1, 2, 3, @@ -184,12 +406,13 @@ const double grpc_stats_table_2[64] = {0, 860.147148411, 938.504491184, 1024.0}; -const uint8_t grpc_stats_table_3[52] = { +const uint8_t grpc_stats_table_7[52] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 52}; -const int grpc_stats_histo_buckets[6] = {64, 64, 64, 64, 64, 64}; -const int grpc_stats_histo_start[6] = {0, 64, 128, 192, 256, 320}; -const double *const grpc_stats_histo_bucket_boundaries[6] = { - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0}; +const int grpc_stats_histo_buckets[8] = {64, 128, 64, 64, 64, 64, 64, 64}; +const int grpc_stats_histo_start[8] = {0, 64, 192, 256, 320, 384, 448, 512}; +const double *const grpc_stats_histo_bucket_boundaries[8] = { + grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_4, + grpc_stats_table_6, grpc_stats_table_4, grpc_stats_table_4, + grpc_stats_table_6, grpc_stats_table_4}; diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index 4d1078dfdb8..17e5f22e6c2 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -28,6 +28,12 @@ typedef enum { GRPC_STATS_COUNTER_SERVER_CALLS_CREATED, GRPC_STATS_COUNTER_SYSCALL_POLL, GRPC_STATS_COUNTER_SYSCALL_WAIT, + GRPC_STATS_COUNTER_POLLSET_KICK, + GRPC_STATS_COUNTER_POLLSET_KICKED_WITHOUT_POLLER, + GRPC_STATS_COUNTER_POLLSET_KICKED_AGAIN, + GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD, + GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV, + GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD, GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS, GRPC_STATS_COUNTER_SYSCALL_WRITE, GRPC_STATS_COUNTER_SYSCALL_READ, @@ -53,6 +59,8 @@ typedef enum { } grpc_stats_counters; extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT]; typedef enum { + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, @@ -63,19 +71,23 @@ typedef enum { } grpc_stats_histograms; extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT]; typedef enum { - GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 0, + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE_FIRST_SLOT = 0, + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE_BUCKETS = 64, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED_FIRST_SLOT = 64, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED_BUCKETS = 128, + GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 192, GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 64, + GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 256, GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 128, + GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 320, GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 192, + GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 384, GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_FIRST_SLOT = 256, + GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_FIRST_SLOT = 448, GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 320, + GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 512, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_BUCKETS = 384 + GRPC_STATS_HISTOGRAM_BUCKETS = 576 } grpc_stats_histogram_constants; #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) @@ -85,6 +97,19 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL) #define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WAIT) +#define GRPC_STATS_INC_POLLSET_KICK(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK) +#define GRPC_STATS_INC_POLLSET_KICKED_WITHOUT_POLLER(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_POLLSET_KICKED_WITHOUT_POLLER) +#define GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICKED_AGAIN) +#define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD) +#define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV) +#define GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD) #define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS) #define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \ @@ -138,6 +163,55 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED) #define GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED) +#define GRPC_STATS_INC_CALL_INITIAL_SIZE(exec_ctx, value) \ + do { \ + union { \ + double dbl; \ + uint64_t uint; \ + } _val; \ + _val.dbl = (double)(value); \ + if (_val.dbl < 0) _val.dbl = 0; \ + if (_val.dbl < 6.000000) { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, (int)_val.dbl); \ + } else { \ + if (_val.uint < 4688247212092686336ull) { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, \ + grpc_stats_table_1[((_val.uint - 4618441417868443648ull) >> 49)]); \ + } else { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, \ + grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ + grpc_stats_table_0, 64)); \ + } \ + } \ + } while (false) +#define GRPC_STATS_INC_POLL_EVENTS_RETURNED(exec_ctx, value) \ + do { \ + union { \ + double dbl; \ + uint64_t uint; \ + } _val; \ + _val.dbl = (double)(value); \ + if (_val.dbl < 0) _val.dbl = 0; \ + if (_val.dbl < 29.000000) { \ + GRPC_STATS_INC_HISTOGRAM((exec_ctx), \ + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, \ + (int)_val.dbl); \ + } else { \ + if (_val.uint < 4652218415073722368ull) { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, \ + grpc_stats_table_3[((_val.uint - 4628855992006737920ull) >> 47)]); \ + } else { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, \ + grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ + grpc_stats_table_2, 128)); \ + } \ + } \ + } while (false) #define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \ do { \ union { \ @@ -153,12 +227,12 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) @@ -177,12 +251,12 @@ typedef enum { if (_val.uint < 4652218415073722368ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \ - grpc_stats_table_3[((_val.uint - 4622945017495814144ull) >> 49)]); \ + grpc_stats_table_7[((_val.uint - 4622945017495814144ull) >> 49)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_2, 64)); \ + grpc_stats_table_6, 64)); \ } \ } \ } while (false) @@ -201,12 +275,12 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) @@ -225,12 +299,12 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) @@ -249,12 +323,12 @@ typedef enum { if (_val.uint < 4652218415073722368ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE, \ - grpc_stats_table_3[((_val.uint - 4622945017495814144ull) >> 49)]); \ + grpc_stats_table_7[((_val.uint - 4622945017495814144ull) >> 49)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_2, 64)); \ + grpc_stats_table_6, 64)); \ } \ } \ } while (false) @@ -274,21 +348,25 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) extern const double grpc_stats_table_0[64]; -extern const uint8_t grpc_stats_table_1[87]; -extern const double grpc_stats_table_2[64]; -extern const uint8_t grpc_stats_table_3[52]; -extern const int grpc_stats_histo_buckets[6]; -extern const int grpc_stats_histo_start[6]; -extern const double *const grpc_stats_histo_bucket_boundaries[6]; +extern const uint8_t grpc_stats_table_1[124]; +extern const double grpc_stats_table_2[128]; +extern const uint8_t grpc_stats_table_3[166]; +extern const double grpc_stats_table_4[64]; +extern const uint8_t grpc_stats_table_5[87]; +extern const double grpc_stats_table_6[64]; +extern const uint8_t grpc_stats_table_7[52]; +extern const int grpc_stats_histo_buckets[8]; +extern const int grpc_stats_histo_start[8]; +extern const double *const grpc_stats_histo_bucket_boundaries[8]; #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index a0d042a688b..4e0bc5de588 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -18,9 +18,21 @@ # overall - counter: client_calls_created - counter: server_calls_created +- histogram: call_initial_size + max: 262144 + buckets: 64 # polling - counter: syscall_poll - counter: syscall_wait +- histogram: poll_events_returned + max: 1024 + buckets: 128 +- counter: pollset_kick +- counter: pollset_kicked_without_poller +- counter: pollset_kicked_again +- counter: pollset_kick_wakeup_fd +- counter: pollset_kick_wakeup_cv +- counter: pollset_kick_own_thread # stats system - counter: histogram_slow_lookups # tcp diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 7f053fa728b..6a3c2654a8c 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -502,12 +502,14 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { gpr_mu_destroy(&pollset->mu); } -static grpc_error *pollset_kick_all(grpc_pollset *pollset) { +static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { GPR_TIMER_BEGIN("pollset_kick_all", 0); grpc_error *error = GRPC_ERROR_NONE; if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); switch (worker->kick_state) { case KICKED: break; @@ -550,7 +552,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutdown_closure = closure; pollset->shutting_down = true; - GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(exec_ctx, pollset)); pollset_maybe_finish_shutdown(exec_ctx, pollset); GPR_TIMER_END("pollset_shutdown", 0); } @@ -646,6 +648,8 @@ static grpc_error *do_epoll_wait(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + GRPC_STATS_INC_POLL_EVENTS_RETURNED(exec_ctx, r); + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "ps: %p poll got %d events", ps, r); } @@ -971,9 +975,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, return error; } -static grpc_error *pollset_kick(grpc_pollset *pollset, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); grpc_error *ret_err = GRPC_ERROR_NONE; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_strvec log; @@ -1005,6 +1010,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { grpc_pollset_worker *root_worker = pollset->root_worker; if (root_worker == NULL) { + GRPC_STATS_INC_POLLSET_KICKED_WITHOUT_POLLER(exec_ctx); pollset->kicked_without_poller = true; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked_without_poller"); @@ -1013,12 +1019,14 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } grpc_pollset_worker *next_worker = root_worker->next; if (root_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", root_worker); } SET_KICK_STATE(root_worker, KICKED); goto done; } else if (next_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", next_worker); } @@ -1029,6 +1037,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, // there is no next worker root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load( &g_active_poller)) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", root_worker); } @@ -1036,6 +1045,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; } else if (next_worker->kick_state == UNKICKED) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", next_worker); } @@ -1053,10 +1063,12 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } SET_KICK_STATE(root_worker, KICKED); if (root_worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&root_worker->cv); } goto done; } else { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. non-root poller %p (root=%p)", next_worker, root_worker); @@ -1066,23 +1078,27 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); GPR_ASSERT(next_worker->kick_state == KICKED); SET_KICK_STATE(next_worker, KICKED); goto done; } } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked while waking up"); } goto done; } } else if (specific_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. specific worker already kicked"); } goto done; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { + GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. mark %p kicked", specific_worker); } @@ -1090,6 +1106,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } else if (specific_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick active poller"); } @@ -1097,6 +1114,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; } else if (specific_worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick waiting worker"); } @@ -1104,6 +1122,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, gpr_cv_signal(&specific_worker->cv); goto done; } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick non-waiting worker"); } diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c index e2e3cd9003f..ee3250297fe 100644 --- a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c +++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c @@ -1112,12 +1112,13 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); grpc_error *error = GRPC_ERROR_NONE; const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (worker != NULL) { if (worker == GRPC_POLLSET_KICK_BROADCAST) { if (pollset_has_workers(p)) { @@ -1265,7 +1266,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying polling island. In such a case, we diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index ddb8c74d2de..ace491550b0 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -632,9 +632,10 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); grpc_error *error = GRPC_ERROR_NONE; const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; @@ -731,7 +732,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying epoll set. In such a case, we diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0e42f76af37..165a562bd74 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -561,6 +561,7 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (worker->pollable != &pollset->pollable) { gpr_mu_lock(&worker->pollable->po.mu); } @@ -599,7 +600,8 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, +static grpc_error *pollset_kick_inner(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -662,13 +664,14 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, } /* p->po.mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *pollset, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { pollable *p = pollset->current_pollable; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } - grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); + grpc_error *error = pollset_kick_inner(exec_ctx, pollset, p, specific_worker); if (p != &pollset->pollable) { gpr_mu_unlock(&p->po.mu); } diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 59c7cdc285f..6cc96455266 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1020,10 +1020,11 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); grpc_error *error = GRPC_ERROR_NONE; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; if (worker != NULL) { @@ -1157,7 +1158,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying polling island. In such a case, we diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index fbd265f3ce7..78817144af7 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -209,7 +209,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, #define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2 /* As per pollset_kick, with an extended set of flags (defined above) -- mostly for fd_posix's use. */ -static grpc_error *pollset_kick_ext(grpc_pollset *p, +static grpc_error *pollset_kick_ext(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) GRPC_MUST_USE_RESULT; @@ -365,36 +365,39 @@ static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, return notifier; } -static grpc_error *pollset_kick_locked(grpc_fd_watcher *watcher) { +static grpc_error *pollset_kick_locked(grpc_exec_ctx *exec_ctx, + grpc_fd_watcher *watcher) { gpr_mu_lock(&watcher->pollset->mu); GPR_ASSERT(watcher->worker); - grpc_error *err = pollset_kick_ext(watcher->pollset, watcher->worker, - GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); + grpc_error *err = + pollset_kick_ext(exec_ctx, watcher->pollset, watcher->worker, + GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); gpr_mu_unlock(&watcher->pollset->mu); return err; } -static void maybe_wake_one_watcher_locked(grpc_fd *fd) { +static void maybe_wake_one_watcher_locked(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) { - pollset_kick_locked(fd->inactive_watcher_root.next); + pollset_kick_locked(exec_ctx, fd->inactive_watcher_root.next); } else if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); + pollset_kick_locked(exec_ctx, fd->read_watcher); } else if (fd->write_watcher) { - pollset_kick_locked(fd->write_watcher); + pollset_kick_locked(exec_ctx, fd->write_watcher); } } -static void wake_all_watchers_locked(grpc_fd *fd) { +static void wake_all_watchers_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_fd_watcher *watcher; for (watcher = fd->inactive_watcher_root.next; watcher != &fd->inactive_watcher_root; watcher = watcher->next) { - pollset_kick_locked(watcher); + pollset_kick_locked(exec_ctx, watcher); } if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); + pollset_kick_locked(exec_ctx, fd->read_watcher); } if (fd->write_watcher && fd->write_watcher != fd->read_watcher) { - pollset_kick_locked(fd->write_watcher); + pollset_kick_locked(exec_ctx, fd->write_watcher); } } @@ -435,7 +438,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, if (!has_watchers(fd)) { close_fd_locked(exec_ctx, fd); } else { - wake_all_watchers_locked(fd); + wake_all_watchers_locked(exec_ctx, fd); } gpr_mu_unlock(&fd->mu); UNREF_BY(fd, 2, reason); /* drop the reference */ @@ -479,7 +482,7 @@ static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, /* already ready ==> queue the closure to run immediately */ *st = CLOSURE_NOT_READY; GRPC_CLOSURE_SCHED(exec_ctx, closure, fd_shutdown_error(fd)); - maybe_wake_one_watcher_locked(fd); + maybe_wake_one_watcher_locked(exec_ctx, fd); } else { /* upcallptr was set to a different closure. This is an error! */ gpr_log(GPR_ERROR, @@ -648,7 +651,7 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher, } } if (kick) { - maybe_wake_one_watcher_locked(fd); + maybe_wake_one_watcher_locked(exec_ctx, fd); } if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) { close_fd_locked(exec_ctx, fd); @@ -712,11 +715,12 @@ static void kick_append_error(grpc_error **composite, grpc_error *error) { *composite = grpc_error_add_child(*composite, error); } -static grpc_error *pollset_kick_ext(grpc_pollset *p, +static grpc_error *pollset_kick_ext(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) { GPR_TIMER_BEGIN("pollset_kick_ext", 0); grpc_error *error = GRPC_ERROR_NONE; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); /* pollset->mu already held */ if (specific_worker != NULL) { @@ -782,9 +786,9 @@ static grpc_error *pollset_kick_ext(grpc_pollset *p, return error; } -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { - return pollset_kick_ext(p, specific_worker, 0); + return pollset_kick_ext(exec_ctx, p, specific_worker, 0); } /* global state management */ @@ -847,7 +851,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } pollset->fds[pollset->fd_count++] = fd; GRPC_FD_REF(fd, "multipoller"); - pollset_kick(pollset, NULL); + pollset_kick(exec_ctx, pollset, NULL); exit: gpr_mu_unlock(&pollset->mu); } @@ -1070,7 +1074,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* check shutdown conditions */ if (pollset->shutting_down) { if (pollset_has_workers(pollset)) { - pollset_kick(pollset, NULL); + pollset_kick(exec_ctx, pollset, NULL); } else if (!pollset->called_shutdown && !pollset_has_observers(pollset)) { pollset->called_shutdown = 1; gpr_mu_unlock(&pollset->mu); @@ -1099,7 +1103,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = 1; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset_has_workers(pollset)) { GRPC_CLOSURE_LIST_SCHED(exec_ctx, &pollset->idle_jobs); } diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 424b40e425b..32363765e39 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -214,9 +214,9 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return g_event_engine->pollset_work(exec_ctx, pollset, worker, now, deadline); } -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - return g_event_engine->pollset_kick(pollset, specific_worker); + return g_event_engine->pollset_kick(exec_ctx, pollset, specific_worker); } void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 1108e46ef81..1ff2ff14133 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -54,7 +54,7 @@ typedef struct grpc_event_engine_vtable { grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, gpr_timespec deadline); - grpc_error *(*pollset_kick)(grpc_pollset *pollset, + grpc_error *(*pollset_kick)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker); void (*pollset_add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, struct grpc_fd *fd); diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index a609a3877a7..a0f6b3a9d3a 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -76,7 +76,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* Break one polling thread out of polling work for this pollset. If specific_worker is non-NULL, then kick that worker. */ -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) GRPC_MUST_USE_RESULT; diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index a2a8e289ee3..691d66df698 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -79,7 +79,8 @@ static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx, detector->is_done = 1; GRPC_LOG_IF_ERROR( "Pollset kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&detector->pollent), NULL)); + grpc_pollset_kick(exec_ctx, + grpc_polling_entity_pollset(&detector->pollent), NULL)); gpr_mu_unlock(g_polling_mu); } diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index e68c201134d..2599765510f 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -322,8 +322,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, grpc_channel_get_channel_stack(args->channel); grpc_call *call; GPR_TIMER_BEGIN("grpc_call_create", 0); - gpr_arena *arena = - gpr_arena_create(grpc_channel_get_call_size_estimate(args->channel)); + size_t initial_size = grpc_channel_get_call_size_estimate(args->channel); + GRPC_STATS_INC_CALL_INITIAL_SIZE(exec_ctx, initial_size); + gpr_arena *arena = gpr_arena_create(initial_size); call = gpr_arena_alloc(arena, sizeof(grpc_call) + channel_stack->call_stack_size); gpr_ref_init(&call->ext_ref, 1); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 10e4e5ab0cb..a15b09e55d1 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -54,7 +54,7 @@ typedef struct { bool can_listen; size_t (*size)(void); void (*init)(grpc_pollset *pollset, gpr_mu **mu); - grpc_error *(*kick)(grpc_pollset *pollset, + grpc_error *(*kick)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker); grpc_error *(*work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, @@ -130,7 +130,8 @@ static grpc_error *non_polling_poller_work(grpc_exec_ctx *exec_ctx, } static grpc_error *non_polling_poller_kick( - grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { + grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { non_polling_poller *p = (non_polling_poller *)pollset; if (specific_worker == NULL) specific_worker = (grpc_pollset_worker *)p->root; if (specific_worker != NULL) { @@ -645,7 +646,7 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, if (is_first) { gpr_mu_lock(cq->mu); grpc_error *kick_error = - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); + cq->poller_vtable->kick(exec_ctx, POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); if (kick_error != GRPC_ERROR_NONE) { @@ -731,7 +732,7 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx, } grpc_error *kick_error = - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), pluck_worker); + cq->poller_vtable->kick(exec_ctx, POLLSET_FROM_CQ(cq), pluck_worker); gpr_mu_unlock(cq->mu); @@ -930,7 +931,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, if (cq_event_queue_num_items(&cqd->queue) > 0 && gpr_atm_no_barrier_load(&cqd->pending_events) > 0) { gpr_mu_lock(cq->mu); - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); + cq->poller_vtable->kick(&exec_ctx, POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); } diff --git a/test/core/http/httpcli_test.c b/test/core/http/httpcli_test.c index b8b96d673ce..8a53903763d 100644 --- a/test/core/http/httpcli_test.c +++ b/test/core/http/httpcli_test.c @@ -52,7 +52,7 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { g_done = 1; GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&g_pops), NULL))); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&g_pops), NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/http/httpscli_test.c b/test/core/http/httpscli_test.c index a9d7abdcffd..c7455bd8df3 100644 --- a/test/core/http/httpscli_test.c +++ b/test/core/http/httpscli_test.c @@ -52,7 +52,7 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { g_done = 1; GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&g_pops), NULL))); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&g_pops), NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c index 11b45e8e084..353d79b11d1 100644 --- a/test/core/iomgr/endpoint_tests.c +++ b/test/core/iomgr/endpoint_tests.c @@ -125,7 +125,8 @@ static void read_and_write_test_read_handler(grpc_exec_ctx *exec_ctx, gpr_log(GPR_INFO, "Read handler done"); gpr_mu_lock(g_mu); state->read_done = 1 + (error == GRPC_ERROR_NONE); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL)); gpr_mu_unlock(g_mu); } else if (error == GRPC_ERROR_NONE) { grpc_endpoint_read(exec_ctx, state->read_ep, &state->incoming, @@ -160,7 +161,8 @@ static void read_and_write_test_write_handler(grpc_exec_ctx *exec_ctx, gpr_log(GPR_INFO, "Write handler done"); gpr_mu_lock(g_mu); state->write_done = 1 + (error == GRPC_ERROR_NONE); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL)); gpr_mu_unlock(g_mu); } @@ -252,7 +254,8 @@ static void inc_on_failure(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_mu_lock(g_mu); *(int *)arg += (error != GRPC_ERROR_NONE); - GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT( + GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 5791d17af62..881277a8d6a 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -178,8 +178,8 @@ static void listen_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg /*server */, gpr_mu_lock(g_mu); sv->done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -297,8 +297,8 @@ static void client_session_shutdown_cb(grpc_exec_ctx *exec_ctx, grpc_fd_orphan(exec_ctx, cl->em_fd, NULL, NULL, false /* already_closed */, "c"); cl->done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); } /* Write as much as possible, then register notify_on_write. */ @@ -417,8 +417,8 @@ static void first_read_callback(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); fdc->cb_that_ran = first_read_callback; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -429,8 +429,8 @@ static void second_read_callback(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); fdc->cb_that_ran = second_read_callback; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/resolve_address_test.c b/test/core/iomgr/resolve_address_test.c index 7f5c4073d81..1110c04b6e2 100644 --- a/test/core/iomgr/resolve_address_test.c +++ b/test/core/iomgr/resolve_address_test.c @@ -106,7 +106,8 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *argsp, GPR_ASSERT(args->addrs->naddrs > 0); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } @@ -115,7 +116,8 @@ static void must_fail(grpc_exec_ctx *exec_ctx, void *argsp, grpc_error *err) { GPR_ASSERT(err != GRPC_ERROR_NONE); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 00ea495bbe3..1032da942b3 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -53,8 +53,10 @@ static gpr_timespec test_deadline(void) { static void finish_connection() { gpr_mu_lock(g_mu); g_connections_complete++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(&exec_ctx, g_pollset, NULL))); + grpc_exec_ctx_finish(&exec_ctx); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index c45068e7ec5..8cfc1bcab1a 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -147,7 +147,8 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, gpr_log(GPR_INFO, "Read %" PRIuPTR " bytes of %" PRIuPTR, read_bytes, state->target_read_bytes); if (state->read_bytes >= state->target_read_bytes) { - GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } else { grpc_endpoint_read(exec_ctx, state->ep, &state->incoming, &state->read_cb); @@ -295,8 +296,8 @@ static void write_done(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); gpr_log(GPR_INFO, "Signalling write done"); state->write_done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -406,8 +407,8 @@ static void write_test(size_t num_bytes, size_t slice_size) { void on_fd_released(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *errors) { int *done = arg; *done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); } /* Do a read_test, then release fd and try to read/write again. Verify that diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 2371721a608..4d84608376a 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -159,8 +159,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_mu_lock(g_mu); g_result = temp_result; g_nconnects++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index aa34857dbd1..1d051bea62d 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -61,8 +61,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { g_number_of_reads++; g_number_of_bytes_read += (int)byte_count; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -70,8 +70,8 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { gpr_mu_lock(g_mu); g_number_of_writes++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index fdbc6ea7417..d240403a293 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -60,7 +60,8 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *arg, request->token = token; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&request->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&request->pops), + NULL)); gpr_mu_unlock(request->mu); } diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index e1385a80caa..3144717a853 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -57,7 +57,8 @@ static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *arg, sync->is_done = true; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&sync->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&sync->pops), + NULL)); gpr_mu_unlock(sync->mu); } diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index 259038f4d6c..5faa6352a87 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -66,7 +66,8 @@ static void on_jwt_verification_done(grpc_exec_ctx *exec_ctx, void *user_data, gpr_mu_lock(sync->mu); sync->is_done = 1; - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(sync->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, sync->pollset, NULL)); gpr_mu_unlock(sync->mu); } diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 08079b6091a..ec2cd8610bc 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -108,7 +108,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *vargs, grpc_endpoint *tcp, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Connected")); grpc_endpoint_destroy(exec_ctx, tcp); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index d5739effb32..1b015c95e98 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -54,7 +54,8 @@ static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->done = 1; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); } @@ -153,7 +154,8 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->port = 0; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); return; } @@ -189,7 +191,8 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->port = port; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); } diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 3c99bda1447..42ebeff41d9 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -107,8 +107,8 @@ void GprLogReporter::ReportCoreStats(const char* name, int idx, grpc_stats_counter_name[i], data.counters[i]); } for (int i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) { - gpr_log(GPR_DEBUG, "%s[%d].%s = %lf/%lf/%lf (50/95/99%%-ile)", name, idx, - grpc_stats_histogram_name[i], + gpr_log(GPR_DEBUG, "%s[%d].%s = %.1lf/%.1lf/%.1lf (50/95/99%%-ile)", name, + idx, grpc_stats_histogram_name[i], grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 50), grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 95), grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 99)); From f7acbc40e7e806a8ca3e4793e6ed6831435ab10d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 30 Aug 2017 10:06:28 +0200 Subject: [PATCH 024/227] fix C# distribtest dockerfiles --- .../distribtest/csharp_jessie_x64/Dockerfile | 13 +++++-------- .../distribtest/csharp_jessie_x86/Dockerfile | 13 +++++-------- .../distribtest/csharp_ubuntu1604_x64/Dockerfile | 11 ++++++++--- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile index d13eecaa551..02ec4c278a7 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile @@ -14,18 +14,15 @@ FROM debian:jessie -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ - nuget + nuget \ + && apt-get clean -# make sure we have nuget 2.12+ (in case there's an older cached docker image) -RUN apt-get update && apt-get install -y nuget - -RUN apt-get update && apt-get install -y unzip +RUN apt-get update && apt-get install -y unzip && apt-get clean diff --git a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile index 71845b590be..758f3145727 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile @@ -14,18 +14,15 @@ FROM 32bit/debian:jessie -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ - nuget + nuget \ + && apt-get clean -# make sure we have nuget 2.12+ (in case there's an older cached docker image) -RUN apt-get update && apt-get install -y nuget - -RUN apt-get update && apt-get install -y unzip +RUN apt-get update && apt-get install -y unzip && apt-get clean diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile index 6604caa42c2..0f40f18e385 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile @@ -17,11 +17,16 @@ FROM ubuntu:16.04 RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ - nuget + nuget \ + && apt-get clean # make sure we have nuget 2.12+ (in case there's an older cached docker image) RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list -RUN apt-get update && apt-get install -y nuget +RUN apt-get update && apt-get install -y nuget && apt-get clean -RUN apt-get update && apt-get install -y unzip +# Prevent "Error: SendFailure (Error writing headers)" when fetching nuget packages +# See https://github.com/tianon/docker-brew-ubuntu-core/issues/86 +RUN apt-get update && apt-get install -y tzdata && apt-get clean + +RUN apt-get update && apt-get install -y unzip && apt-get clean From 980744c436793ce63cd539c862959fe653df9c35 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 30 Aug 2017 16:36:12 +0200 Subject: [PATCH 025/227] fix C# windows x64 distrib test --- tools/run_tests/artifacts/distribtest_targets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index fa461efa85e..76407810c05 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -105,7 +105,9 @@ class CSharpDistribTest(object): use_workspace=True) elif self.platform == 'windows': if self.arch == 'x64': - environ={'MSBUILD_EXTRA_ARGS': '/p:Platform=x64', + # Use double leading / as the first occurence gets removed by msys bash + # when invoking the .bat file (side-effect of posix path conversion) + environ={'MSBUILD_EXTRA_ARGS': '//p:Platform=x64', 'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'} else: environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\Debug'} From 60c03d9b2a1ecb5d2c494b08e09a606830e2088e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 30 Aug 2017 16:42:16 +0200 Subject: [PATCH 026/227] fix nonexistent version warning for System.Threading.ThreadPool --- src/csharp/Grpc.Core/Grpc.Core.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index dde800aaddb..08f0e096f10 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -65,7 +65,7 @@ - + From 9eb36afb1124c9bf1753e78f0efa47bf41d72d71 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 13:40:06 +0200 Subject: [PATCH 027/227] dont set NetStandardImplicitPackageVersion --- src/csharp/Grpc.Auth/Grpc.Auth.csproj | 1 - src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj | 1 - src/csharp/Grpc.Core/Grpc.Core.csproj | 1 - src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj | 1 - src/csharp/Grpc.Reflection/Grpc.Reflection.csproj | 1 - 5 files changed, 5 deletions(-) diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj index abf326459c1..bbcbd95be5f 100755 --- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj +++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj @@ -15,7 +15,6 @@ gRPC RPC Protocol HTTP/2 Auth OAuth2 https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true diff --git a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj index 9ad6fd0c616..4d6767fa985 100755 --- a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj +++ b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj @@ -15,7 +15,6 @@ gRPC test testing https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 08f0e096f10..d9950b2f201 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -14,7 +14,6 @@ gRPC RPC Protocol HTTP/2 https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj index 3eb90434f3f..681719d124a 100755 --- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj +++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj @@ -14,7 +14,6 @@ gRPC health check https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true diff --git a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj index b77fd69aeeb..704eea5c17a 100755 --- a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj +++ b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj @@ -14,7 +14,6 @@ gRPC reflection https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true From ec6f7280849aa95f30de32fb85d1e27f7c5688fd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 13:46:48 +0200 Subject: [PATCH 028/227] NUnit.ConsoleRunner dependency is useless --- src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 6df68fda589..be3e312b2a0 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -21,7 +21,6 @@ - From a37b72e9fdee4130af45073714938a39bc83b8e1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 13:55:46 +0200 Subject: [PATCH 029/227] dont set RuntimeFrameworkVersion unneccessarily --- src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj | 1 - .../Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj | 1 - .../Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj | 1 - src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj | 1 - src/csharp/Grpc.Examples/Grpc.Examples.csproj | 1 - src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj | 1 - .../Grpc.IntegrationTesting.Client.csproj | 1 - .../Grpc.IntegrationTesting.QpsWorker.csproj | 1 - .../Grpc.IntegrationTesting.Server.csproj | 1 - .../Grpc.IntegrationTesting.StressClient.csproj | 1 - .../Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj | 1 - src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj | 1 - src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj | 1 - 13 files changed, 13 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index be3e312b2a0..82fb080e6ef 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -9,7 +9,6 @@ Exe Grpc.Core.Tests $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj index 74deed65840..db4e3ef4e3f 100755 --- a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj +++ b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj @@ -8,7 +8,6 @@ Grpc.Examples.MathClient Exe Grpc.Examples.MathClient - 1.0.4 true diff --git a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj index 1abf2614985..b12b418d015 100755 --- a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj +++ b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj @@ -8,7 +8,6 @@ Grpc.Examples.MathServer Exe Grpc.Examples.MathServer - 1.0.4 true diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj index d2a13ed6e14..32362aad87b 100755 --- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj +++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj @@ -9,7 +9,6 @@ Exe Grpc.Examples.Tests $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.Examples/Grpc.Examples.csproj b/src/csharp/Grpc.Examples/Grpc.Examples.csproj index 491d313f178..baa3b4ce6c1 100755 --- a/src/csharp/Grpc.Examples/Grpc.Examples.csproj +++ b/src/csharp/Grpc.Examples/Grpc.Examples.csproj @@ -7,7 +7,6 @@ net45;netcoreapp1.0 Grpc.Examples Grpc.Examples - 1.0.4 true diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj index 2ccf46b9b9c..a9920ff68ec 100755 --- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj +++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj @@ -9,7 +9,6 @@ Exe Grpc.HealthCheck.Tests $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj index c67beea7cd3..0f3971e8d65 100755 --- a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting.Client $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj index e452257b1b1..92ed20b0395 100755 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj +++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj @@ -10,7 +10,6 @@ Grpc.IntegrationTesting.QpsWorker true $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj index a1fb316fdb1..4480ca1bbaa 100755 --- a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting.Server $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj index f64bea3d2be..fef23633ccb 100755 --- a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj +++ b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting.StressClient $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index f5077fe0f74..719f1eebba4 100755 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj index 17797e1e1e8..0da24c6179d 100644 --- a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj +++ b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj @@ -9,7 +9,6 @@ Exe Grpc.Microbenchmarks $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj index cf756c68ad3..33a9106caed 100755 --- a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj +++ b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj @@ -9,7 +9,6 @@ Exe Grpc.Reflection.Tests $(PackageTargetFallback);portable-net45 - 1.0.4 true From f32d3a79b561fc887b91e2bdd5051cf06df502e2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 13:59:23 +0200 Subject: [PATCH 030/227] System.Linq.Expressions dependency in redundant --- .../Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index 719f1eebba4..b0de1dc560f 100755 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -30,10 +30,6 @@ - - - - From 398435122a74f3f6a9f282531bfa91739ac39808 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 14:22:48 +0200 Subject: [PATCH 031/227] PackageTargetFallback not necessary --- src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj | 1 - src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj | 1 - src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj | 1 - .../Grpc.IntegrationTesting.Client.csproj | 1 - .../Grpc.IntegrationTesting.QpsWorker.csproj | 1 - .../Grpc.IntegrationTesting.Server.csproj | 1 - .../Grpc.IntegrationTesting.StressClient.csproj | 1 - .../Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj | 1 - src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj | 1 - src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj | 1 - 10 files changed, 10 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 82fb080e6ef..18993a93e00 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -8,7 +8,6 @@ Grpc.Core.Tests Exe Grpc.Core.Tests - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj index 32362aad87b..3ccc9adfaf7 100755 --- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj +++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj @@ -8,7 +8,6 @@ Grpc.Examples.Tests Exe Grpc.Examples.Tests - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj index a9920ff68ec..9da0539dcb9 100755 --- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj +++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj @@ -8,7 +8,6 @@ Grpc.HealthCheck.Tests Exe Grpc.HealthCheck.Tests - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj index 0f3971e8d65..35713156ea2 100755 --- a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj @@ -8,7 +8,6 @@ Grpc.IntegrationTesting.Client Exe Grpc.IntegrationTesting.Client - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj index 92ed20b0395..3ecefe3bc4f 100755 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj +++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting.QpsWorker true - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj index 4480ca1bbaa..1092b2c21ef 100755 --- a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj @@ -8,7 +8,6 @@ Grpc.IntegrationTesting.Server Exe Grpc.IntegrationTesting.Server - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj index fef23633ccb..22272547f61 100755 --- a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj +++ b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj @@ -8,7 +8,6 @@ Grpc.IntegrationTesting.StressClient Exe Grpc.IntegrationTesting.StressClient - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index b0de1dc560f..c02c9844e3d 100755 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -8,7 +8,6 @@ Grpc.IntegrationTesting Exe Grpc.IntegrationTesting - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj index 0da24c6179d..108357e4eb4 100644 --- a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj +++ b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj @@ -8,7 +8,6 @@ Grpc.Microbenchmarks Exe Grpc.Microbenchmarks - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj index 33a9106caed..d3686971245 100755 --- a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj +++ b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj @@ -8,7 +8,6 @@ Grpc.Reflection.Tests Exe Grpc.Reflection.Tests - $(PackageTargetFallback);portable-net45 true From c3e2510bd492f6bd30a7f5fa538c0a549058b59d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 15:02:38 +0200 Subject: [PATCH 032/227] csharp_coreclr_x64 dockerfile not useful --- .../test/csharp_coreclr_x64/Dockerfile | 114 ------------------ tools/run_tests/artifacts/package_targets.py | 2 +- 2 files changed, 1 insertion(+), 115 deletions(-) delete mode 100644 tools/dockerfile/test/csharp_coreclr_x64/Dockerfile diff --git a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile deleted file mode 100644 index 4ccfbc43c30..00000000000 --- a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM debian:jessie - -# Install Git and basic packages. -RUN apt-get update && apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - gcc-multilib \ - git \ - golang \ - gyp \ - lcov \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libgtest-dev \ - libtool \ - make \ - perl \ - strace \ - python-dev \ - python-setuptools \ - python-yaml \ - telnet \ - unzip \ - wget \ - zip && apt-get clean - -#================ -# Build profiling -RUN apt-get update && apt-get install -y time && apt-get clean - -#==================== -# Python dependencies - -# Install dependencies - -RUN apt-get update && apt-get install -y \ - python-all-dev \ - python3-all-dev \ - python-pip - -# Install Python packages from PyPI -RUN pip install pip --upgrade -RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 - -#================ -# C# dependencies - -# Update to a newer version of mono -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list - -# Install dependencies -RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ - mono-devel \ - ca-certificates-mono \ - nuget \ - && apt-get clean - -RUN nuget update -self - -# Install dotnet SDK based on https://www.microsoft.com/net/core#debian -RUN apt-get update && apt-get install -y curl libunwind8 gettext -# dotnet-dev-1.0.0-preview2-003131 -RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=827530 -RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet -# dotnet-dev-1.0.1 -RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 -RUN mkdir -p /opt/dotnet && tar zxf dotnet101.tar.gz -C /opt/dotnet -RUN ln -s /opt/dotnet/dotnet /usr/local/bin - -# Trigger the population of the local package cache -ENV NUGET_XMLDOC_MODE skip -RUN mkdir warmup \ - && cd warmup \ - && dotnet new \ - && cd .. \ - && rm -rf warmup - -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - - -RUN mkdir /var/local/jenkins - -# Define the default command. -CMD ["bash"] diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index 0da13864f0c..671d0f7b45e 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -78,7 +78,7 @@ class CSharpPackage: if self.linux: return create_docker_jobspec( self.name, - 'tools/dockerfile/test/csharp_coreclr_x64', + 'tools/dockerfile/test/csharp_jessie_x64', 'src/csharp/build_packages_dotnetcli.sh') else: return create_jobspec(self.name, From 561dc32247534d9204d9f68e92259759875c6d93 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 5 Sep 2017 13:00:46 -0700 Subject: [PATCH 033/227] Add stats for server request serving path --- src/core/lib/debug/stats_data.c | 48 +++++++++++++++++++++++++----- src/core/lib/debug/stats_data.h | 23 ++++++++++---- src/core/lib/debug/stats_data.yaml | 6 ++++ src/core/lib/surface/server.c | 5 ++++ 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 57cbafc8176..73703208da1 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -48,10 +48,13 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "executor_scheduled_to_self", "executor_wakeup_initiated", "executor_queue_drained", + "server_requested_calls", + "server_slowpath_requests_queued", }; const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { - "tcp_write_size", "tcp_write_iov_size", "tcp_read_size", - "tcp_read_offer", "tcp_read_iov_size", "http2_send_message_size", + "tcp_write_size", "tcp_write_iov_size", "tcp_read_size", + "tcp_read_offer", "tcp_read_iov_size", "http2_send_message_size", + "server_cqs_checked", }; const int grpc_stats_table_0[64] = { 0, 1, 2, 3, 4, 6, 8, 11, @@ -81,6 +84,8 @@ const uint8_t grpc_stats_table_3[104] = { 24, 24, 25, 25, 26, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32, 32, 33, 33, 34, 34, 35, 36, 36, 37, 38, 38, 39, 39, 40, 40, 41, 41, 42, 42, 42, 43, 44, 45, 45, 46, 47, 47, 48, 48, 49, 49, 50, 50, 51, 51}; +const int grpc_stats_table_4[8] = {0, 1, 2, 4, 8, 16, 32, 64}; +const uint8_t grpc_stats_table_5[4] = {0, 1, 2, 3}; void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { @@ -233,12 +238,39 @@ void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, grpc_stats_histo_find_bucket_slow( (exec_ctx), value, grpc_stats_table_0, 64)); } -const int grpc_stats_histo_buckets[6] = {64, 64, 64, 64, 64, 64}; -const int grpc_stats_histo_start[6] = {0, 64, 128, 192, 256, 320}; -const int *const grpc_stats_histo_bucket_boundaries[6] = { +void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) { + value = GPR_CLAMP(value, 0, 64); + if (value < 3) { + GRPC_STATS_INC_HISTOGRAM((exec_ctx), + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, value); + return; + } + union { + double dbl; + uint64_t uint; + } _val, _bkt; + _val.dbl = value; + if (_val.uint < 4627448617123184640ull) { + int bucket = + grpc_stats_table_5[((_val.uint - 4613937818241073152ull) >> 52)] + 3; + _bkt.dbl = grpc_stats_table_4[bucket]; + bucket -= (_val.uint < _bkt.uint); + GRPC_STATS_INC_HISTOGRAM((exec_ctx), + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, bucket); + return; + } + GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, + grpc_stats_histo_find_bucket_slow( + (exec_ctx), value, grpc_stats_table_4, 8)); +} +const int grpc_stats_histo_buckets[7] = {64, 64, 64, 64, 64, 64, 8}; +const int grpc_stats_histo_start[7] = {0, 64, 128, 192, 256, 320, 384}; +const int *const grpc_stats_histo_bucket_boundaries[7] = { + grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0}; -void (*const grpc_stats_inc_histogram[6])(grpc_exec_ctx *exec_ctx, int x) = { + grpc_stats_table_4}; +void (*const grpc_stats_inc_histogram[7])(grpc_exec_ctx *exec_ctx, int x) = { grpc_stats_inc_tcp_write_size, grpc_stats_inc_tcp_write_iov_size, grpc_stats_inc_tcp_read_size, grpc_stats_inc_tcp_read_offer, - grpc_stats_inc_tcp_read_iov_size, grpc_stats_inc_http2_send_message_size}; + grpc_stats_inc_tcp_read_iov_size, grpc_stats_inc_http2_send_message_size, + grpc_stats_inc_server_cqs_checked}; diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index 9b2d43a03c9..f5dd4868738 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -50,6 +50,8 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_TO_SELF, GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED, GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED, + GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS, + GRPC_STATS_COUNTER_SERVER_SLOWPATH_REQUESTS_QUEUED, GRPC_STATS_COUNTER_COUNT } grpc_stats_counters; extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT]; @@ -60,6 +62,7 @@ typedef enum { GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, GRPC_STATS_HISTOGRAM_COUNT } grpc_stats_histograms; extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT]; @@ -76,7 +79,9 @@ typedef enum { GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_BUCKETS = 64, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 320, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_BUCKETS = 384 + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_FIRST_SLOT = 384, + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_BUCKETS = 8, + GRPC_STATS_HISTOGRAM_BUCKETS = 392 } grpc_stats_histogram_constants; #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) @@ -139,6 +144,11 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED) #define GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED) +#define GRPC_STATS_INC_SERVER_REQUESTED_CALLS(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS) +#define GRPC_STATS_INC_SERVER_SLOWPATH_REQUESTS_QUEUED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_SERVER_SLOWPATH_REQUESTS_QUEUED) #define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \ grpc_stats_inc_tcp_write_size((exec_ctx), (int)(value)) void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int x); @@ -157,10 +167,13 @@ void grpc_stats_inc_tcp_read_iov_size(grpc_exec_ctx *exec_ctx, int x); #define GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE(exec_ctx, value) \ grpc_stats_inc_http2_send_message_size((exec_ctx), (int)(value)) void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, int x); -extern const int grpc_stats_histo_buckets[6]; -extern const int grpc_stats_histo_start[6]; -extern const int *const grpc_stats_histo_bucket_boundaries[6]; -extern void (*const grpc_stats_inc_histogram[6])(grpc_exec_ctx *exec_ctx, +#define GRPC_STATS_INC_SERVER_CQS_CHECKED(exec_ctx, value) \ + grpc_stats_inc_server_cqs_checked((exec_ctx), (int)(value)) +void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int x); +extern const int grpc_stats_histo_buckets[7]; +extern const int grpc_stats_histo_start[7]; +extern const int *const grpc_stats_histo_bucket_boundaries[7]; +extern void (*const grpc_stats_inc_histogram[7])(grpc_exec_ctx *exec_ctx, int x); #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index a0d042a688b..71dc32a8ea5 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -65,3 +65,9 @@ - counter: executor_scheduled_to_self - counter: executor_wakeup_initiated - counter: executor_queue_drained +# server +- counter: server_requested_calls +- histogram: server_cqs_checked + buckets: 8 + max: 64 +- counter: server_slowpath_requests_queued diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 66dcc299aab..786d754b496 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -29,6 +29,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/connected_channel.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/slice/slice_internal.h" @@ -538,6 +539,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, if (request_id == -1) { continue; } else { + GRPC_STATS_INC_SERVER_CQS_CHECKED(exec_ctx, i); gpr_mu_lock(&calld->mu_state); calld->state = ACTIVATED; gpr_mu_unlock(&calld->mu_state); @@ -548,6 +550,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, } /* no cq to take the request found: queue it on the slow list */ + GRPC_STATS_INC_SERVER_SLOWPATH_REQUESTS_QUEUED(exec_ctx); gpr_mu_lock(&server->mu_call); gpr_mu_lock(&calld->mu_state); calld->state = PENDING; @@ -1429,6 +1432,7 @@ grpc_call_error grpc_server_request_call( grpc_call_error error; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; requested_call *rc = gpr_malloc(sizeof(*rc)); + GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx); GRPC_API_TRACE( "grpc_server_request_call(" "server=%p, call=%p, details=%p, initial_metadata=%p, " @@ -1475,6 +1479,7 @@ grpc_call_error grpc_server_request_registered_call( grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; requested_call *rc = gpr_malloc(sizeof(*rc)); registered_method *rm = rmp; + GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx); GRPC_API_TRACE( "grpc_server_request_registered_call(" "server=%p, rmp=%p, call=%p, deadline=%p, initial_metadata=%p, " From 130e07061f6eed9bbb04f58eeab091fa515b0572 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 1 Sep 2017 18:02:51 -0700 Subject: [PATCH 034/227] Add interface abstraction and identity module --- build.yaml | 2 + .../chttp2/transport/chttp2_transport.c | 6 +- .../ext/transport/chttp2/transport/writing.c | 2 +- src/core/lib/compression/stream_compression.c | 173 +++----------- src/core/lib/compression/stream_compression.h | 34 ++- .../lib/compression/stream_compression_gzip.c | 218 ++++++++++++++++++ .../lib/compression/stream_compression_gzip.h | 26 +++ .../compression/stream_compression_identity.c | 77 +++++++ .../compression/stream_compression_identity.h | 26 +++ 9 files changed, 410 insertions(+), 154 deletions(-) create mode 100644 src/core/lib/compression/stream_compression_gzip.c create mode 100644 src/core/lib/compression/stream_compression_gzip.h create mode 100644 src/core/lib/compression/stream_compression_identity.c create mode 100644 src/core/lib/compression/stream_compression_identity.h diff --git a/build.yaml b/build.yaml index 7d8440ce446..0939796bcb3 100644 --- a/build.yaml +++ b/build.yaml @@ -197,6 +197,7 @@ filegroups: - src/core/lib/compression/stream_compression.c - src/core/lib/debug/stats.c - src/core/lib/debug/stats_data.c + - src/core/lib/compression/stream_compression_gzip.c - src/core/lib/http/format_request.c - src/core/lib/http/httpcli.c - src/core/lib/http/parser.c @@ -349,6 +350,7 @@ filegroups: - src/core/lib/compression/stream_compression.h - src/core/lib/debug/stats.h - src/core/lib/debug/stats_data.h + - src/core/lib/compression/stream_compression_gzip.h - src/core/lib/http/format_request.h - src/core/lib/http/httpcli.h - src/core/lib/http/parser.h diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 7541bd5c92c..0e00995d11d 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1731,7 +1731,7 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, @@ -1804,7 +1804,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->frame_storage, @@ -2694,7 +2694,7 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 711938b2782..b877e9f1264 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -329,7 +329,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (s->stream_compression_ctx == NULL) { s->stream_compression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_COMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); } s->uncompressed_data_size = s->flow_controlled_buffer.length; GPR_ASSERT(grpc_stream_compress( diff --git a/src/core/lib/compression/stream_compression.c b/src/core/lib/compression/stream_compression.c index df13d53e06f..274ee100ba5 100644 --- a/src/core/lib/compression/stream_compression.c +++ b/src/core/lib/compression/stream_compression.c @@ -16,176 +16,57 @@ * */ -#include #include #include "src/core/lib/compression/stream_compression.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/compression/stream_compression_gzip.h" -#define OUTPUT_BLOCK_SIZE (1024) - -static bool gzip_flate(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, int flush, - bool *end_of_context) { - GPR_ASSERT(flush == 0 || flush == Z_SYNC_FLUSH || flush == Z_FINISH); - /* Full flush is not allowed when inflating. */ - GPR_ASSERT(!(ctx->flate == inflate && (flush == Z_FINISH))); - - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - int r; - bool eoc = false; - size_t original_max_output_size = max_output_size; - while (max_output_size > 0 && (in->length > 0 || flush) && !eoc) { - size_t slice_size = max_output_size < OUTPUT_BLOCK_SIZE ? max_output_size - : OUTPUT_BLOCK_SIZE; - grpc_slice slice_out = GRPC_SLICE_MALLOC(slice_size); - ctx->zs.avail_out = (uInt)slice_size; - ctx->zs.next_out = GRPC_SLICE_START_PTR(slice_out); - while (ctx->zs.avail_out > 0 && in->length > 0 && !eoc) { - grpc_slice slice = grpc_slice_buffer_take_first(in); - ctx->zs.avail_in = (uInt)GRPC_SLICE_LENGTH(slice); - ctx->zs.next_in = GRPC_SLICE_START_PTR(slice); - r = ctx->flate(&ctx->zs, Z_NO_FLUSH); - if (r < 0 && r != Z_BUF_ERROR) { - gpr_log(GPR_ERROR, "zlib error (%d)", r); - grpc_slice_unref_internal(&exec_ctx, slice_out); - grpc_exec_ctx_finish(&exec_ctx); - return false; - } else if (r == Z_STREAM_END && ctx->flate == inflate) { - eoc = true; - } - if (ctx->zs.avail_in > 0) { - grpc_slice_buffer_undo_take_first( - in, - grpc_slice_sub(slice, GRPC_SLICE_LENGTH(slice) - ctx->zs.avail_in, - GRPC_SLICE_LENGTH(slice))); - } - grpc_slice_unref_internal(&exec_ctx, slice); - } - if (flush != 0 && ctx->zs.avail_out > 0 && !eoc) { - GPR_ASSERT(in->length == 0); - r = ctx->flate(&ctx->zs, flush); - if (flush == Z_SYNC_FLUSH) { - switch (r) { - case Z_OK: - /* Maybe flush is not complete; just made some partial progress. */ - if (ctx->zs.avail_out > 0) { - flush = 0; - } - break; - case Z_BUF_ERROR: - case Z_STREAM_END: - flush = 0; - break; - default: - gpr_log(GPR_ERROR, "zlib error (%d)", r); - grpc_slice_unref_internal(&exec_ctx, slice_out); - grpc_exec_ctx_finish(&exec_ctx); - return false; - } - } else if (flush == Z_FINISH) { - switch (r) { - case Z_OK: - case Z_BUF_ERROR: - /* Wait for the next loop to assign additional output space. */ - GPR_ASSERT(ctx->zs.avail_out == 0); - break; - case Z_STREAM_END: - flush = 0; - break; - default: - gpr_log(GPR_ERROR, "zlib error (%d)", r); - grpc_slice_unref_internal(&exec_ctx, slice_out); - grpc_exec_ctx_finish(&exec_ctx); - return false; - } - } - } - - if (ctx->zs.avail_out == 0) { - grpc_slice_buffer_add(out, slice_out); - } else if (ctx->zs.avail_out < slice_size) { - slice_out.data.refcounted.length -= ctx->zs.avail_out; - grpc_slice_buffer_add(out, slice_out); - } else { - grpc_slice_unref_internal(&exec_ctx, slice_out); - } - max_output_size -= (slice_size - ctx->zs.avail_out); - } - grpc_exec_ctx_finish(&exec_ctx); - if (end_of_context) { - *end_of_context = eoc; - } - if (output_size) { - *output_size = original_max_output_size - max_output_size; - } - return true; -} +extern const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable; bool grpc_stream_compress(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, grpc_stream_compression_flush flush) { - GPR_ASSERT(ctx->flate == deflate); - int gzip_flush; - switch (flush) { - case GRPC_STREAM_COMPRESSION_FLUSH_NONE: - gzip_flush = 0; - break; - case GRPC_STREAM_COMPRESSION_FLUSH_SYNC: - gzip_flush = Z_SYNC_FLUSH; - break; - case GRPC_STREAM_COMPRESSION_FLUSH_FINISH: - gzip_flush = Z_FINISH; - break; - default: - gzip_flush = 0; - } - return gzip_flate(ctx, in, out, output_size, max_output_size, gzip_flush, - NULL); + return ctx->vtable->compress(ctx, in, out, output_size, max_output_size, flush); } bool grpc_stream_decompress(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, bool *end_of_context) { - GPR_ASSERT(ctx->flate == inflate); - return gzip_flate(ctx, in, out, output_size, max_output_size, Z_SYNC_FLUSH, - end_of_context); + return ctx->vtable->decompress(ctx, in, out, output_size, max_output_size, end_of_context); } grpc_stream_compression_context *grpc_stream_compression_context_create( grpc_stream_compression_method method) { - grpc_stream_compression_context *ctx = - gpr_zalloc(sizeof(grpc_stream_compression_context)); - int r; - if (ctx == NULL) { - return NULL; - } - if (method == GRPC_STREAM_COMPRESSION_DECOMPRESS) { - r = inflateInit2(&ctx->zs, 0x1F); - ctx->flate = inflate; - } else { - r = deflateInit2(&ctx->zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, - Z_DEFAULT_STRATEGY); - ctx->flate = deflate; - } - if (r != Z_OK) { - gpr_free(ctx); - return NULL; + switch (method) { + case GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS: + case GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS: + return grpc_stream_compression_identity_vtable.context_create(method); + case GRPC_STREAM_COMPRESSION_GZIP_COMPRESS: + case GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS: + return grpc_stream_compression_gzip_vtable.context_create(method); + default: + gpr_log(GPR_ERROR, "Unknown stream compression method: %d", method); + return NULL; } - - return ctx; } void grpc_stream_compression_context_destroy( grpc_stream_compression_context *ctx) { - if (ctx->flate == inflate) { - inflateEnd(&ctx->zs); + ctx->vtable->context_destroy(ctx); +} + +int grpc_stream_compression_method_parse( + grpc_slice value, bool is_compress, grpc_stream_compression_method *method) { + if (grpc_slice_eq(value, GRPC_MDSTR_IDENTITY)) { + *method = is_compress ? GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS : GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; + return 1; + } else if (grpc_slice_eq(value, GRPC_MDSTR_GZIP)) { + *method = is_compress ? GRPC_STREAM_COMPRESSION_GZIP_COMPRESS : GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS; + return 1; } else { - deflateEnd(&ctx->zs); + return 0; } - gpr_free(ctx); } + diff --git a/src/core/lib/compression/stream_compression.h b/src/core/lib/compression/stream_compression.h index 844dff81a3b..fa07247c310 100644 --- a/src/core/lib/compression/stream_compression.h +++ b/src/core/lib/compression/stream_compression.h @@ -24,15 +24,20 @@ #include #include +#include "src/core/lib/transport/static_metadata.h" + +typedef struct grpc_stream_compression_vtable grpc_stream_compression_vtable; + /* Stream compression/decompression context */ typedef struct grpc_stream_compression_context { - z_stream zs; - int (*flate)(z_stream *zs, int flush); + const grpc_stream_compression_vtable *vtable; } grpc_stream_compression_context; typedef enum grpc_stream_compression_method { - GRPC_STREAM_COMPRESSION_COMPRESS = 0, - GRPC_STREAM_COMPRESSION_DECOMPRESS, + GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS = 0, + GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS, + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS, + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS, GRPC_STREAM_COMPRESSION_METHOD_COUNT } grpc_stream_compression_method; @@ -43,6 +48,21 @@ typedef enum grpc_stream_compression_flush { GRPC_STREAM_COMPRESSION_FLUSH_COUNT } grpc_stream_compression_flush; +struct grpc_stream_compression_vtable { +bool (*compress)(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + grpc_stream_compression_flush flush); +bool (*decompress)(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context); +grpc_stream_compression_context *(*context_create)( + grpc_stream_compression_method method); +void (*context_destroy)( + grpc_stream_compression_context *ctx); +}; + /** * Compress bytes provided in \a in with a given context, with an optional flush * at the end of compression. Emits at most \a max_output_size compressed bytes @@ -87,4 +107,10 @@ grpc_stream_compression_context *grpc_stream_compression_context_create( void grpc_stream_compression_context_destroy( grpc_stream_compression_context *ctx); +/** + * Parse stream compression method based on algorithm name + */ +int grpc_stream_compression_method_parse( + grpc_slice value, bool is_compress, grpc_stream_compression_method *method); + #endif diff --git a/src/core/lib/compression/stream_compression_gzip.c b/src/core/lib/compression/stream_compression_gzip.c new file mode 100644 index 00000000000..2d03836a0dc --- /dev/null +++ b/src/core/lib/compression/stream_compression_gzip.c @@ -0,0 +1,218 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include "src/core/lib/compression/stream_compression.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/slice/slice_internal.h" + +#define OUTPUT_BLOCK_SIZE (1024) + +typedef struct grpc_stream_compression_context_gzip { + grpc_stream_compression_context base; + + z_stream zs; + int (*flate)(z_stream *zs, int flush); +} grpc_stream_compression_context_gzip; + +static bool gzip_flate(grpc_stream_compression_context_gzip *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, int flush, + bool *end_of_context) { + GPR_ASSERT(flush == 0 || flush == Z_SYNC_FLUSH || flush == Z_FINISH); + /* Full flush is not allowed when inflating. */ + GPR_ASSERT(!(ctx->flate == inflate && (flush == Z_FINISH))); + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + int r; + bool eoc = false; + size_t original_max_output_size = max_output_size; + while (max_output_size > 0 && (in->length > 0 || flush) && !eoc) { + size_t slice_size = max_output_size < OUTPUT_BLOCK_SIZE ? max_output_size + : OUTPUT_BLOCK_SIZE; + grpc_slice slice_out = GRPC_SLICE_MALLOC(slice_size); + ctx->zs.avail_out = (uInt)slice_size; + ctx->zs.next_out = GRPC_SLICE_START_PTR(slice_out); + while (ctx->zs.avail_out > 0 && in->length > 0 && !eoc) { + grpc_slice slice = grpc_slice_buffer_take_first(in); + ctx->zs.avail_in = (uInt)GRPC_SLICE_LENGTH(slice); + ctx->zs.next_in = GRPC_SLICE_START_PTR(slice); + r = ctx->flate(&ctx->zs, Z_NO_FLUSH); + if (r < 0 && r != Z_BUF_ERROR) { + gpr_log(GPR_ERROR, "zlib error (%d)", r); + grpc_slice_unref_internal(&exec_ctx, slice_out); + grpc_exec_ctx_finish(&exec_ctx); + return false; + } else if (r == Z_STREAM_END && ctx->flate == inflate) { + eoc = true; + } + if (ctx->zs.avail_in > 0) { + grpc_slice_buffer_undo_take_first( + in, + grpc_slice_sub(slice, GRPC_SLICE_LENGTH(slice) - ctx->zs.avail_in, + GRPC_SLICE_LENGTH(slice))); + } + grpc_slice_unref_internal(&exec_ctx, slice); + } + if (flush != 0 && ctx->zs.avail_out > 0 && !eoc) { + GPR_ASSERT(in->length == 0); + r = ctx->flate(&ctx->zs, flush); + if (flush == Z_SYNC_FLUSH) { + switch (r) { + case Z_OK: + /* Maybe flush is not complete; just made some partial progress. */ + if (ctx->zs.avail_out > 0) { + flush = 0; + } + break; + case Z_BUF_ERROR: + case Z_STREAM_END: + flush = 0; + break; + default: + gpr_log(GPR_ERROR, "zlib error (%d)", r); + grpc_slice_unref_internal(&exec_ctx, slice_out); + grpc_exec_ctx_finish(&exec_ctx); + return false; + } + } else if (flush == Z_FINISH) { + switch (r) { + case Z_OK: + case Z_BUF_ERROR: + /* Wait for the next loop to assign additional output space. */ + GPR_ASSERT(ctx->zs.avail_out == 0); + break; + case Z_STREAM_END: + flush = 0; + break; + default: + gpr_log(GPR_ERROR, "zlib error (%d)", r); + grpc_slice_unref_internal(&exec_ctx, slice_out); + grpc_exec_ctx_finish(&exec_ctx); + return false; + } + } + } + + if (ctx->zs.avail_out == 0) { + grpc_slice_buffer_add(out, slice_out); + } else if (ctx->zs.avail_out < slice_size) { + slice_out.data.refcounted.length -= ctx->zs.avail_out; + grpc_slice_buffer_add(out, slice_out); + } else { + grpc_slice_unref_internal(&exec_ctx, slice_out); + } + max_output_size -= (slice_size - ctx->zs.avail_out); + } + grpc_exec_ctx_finish(&exec_ctx); + if (end_of_context) { + *end_of_context = eoc; + } + if (output_size) { + *output_size = original_max_output_size - max_output_size; + } + return true; +} + +static bool grpc_stream_compress_gzip(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + grpc_stream_compression_flush flush) { + if (ctx == NULL) { + return false; + } + grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip *)ctx; + GPR_ASSERT(gzip_ctx->flate == deflate); + int gzip_flush; + switch (flush) { + case GRPC_STREAM_COMPRESSION_FLUSH_NONE: + gzip_flush = 0; + break; + case GRPC_STREAM_COMPRESSION_FLUSH_SYNC: + gzip_flush = Z_SYNC_FLUSH; + break; + case GRPC_STREAM_COMPRESSION_FLUSH_FINISH: + gzip_flush = Z_FINISH; + break; + default: + gzip_flush = 0; + } + return gzip_flate(gzip_ctx, in, out, output_size, max_output_size, gzip_flush, + NULL); +} + +static bool grpc_stream_decompress_gzip(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context) { + if (ctx == NULL) { + return false; + } + grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip *)ctx; + GPR_ASSERT(gzip_ctx->flate == inflate); + return gzip_flate(gzip_ctx, in, out, output_size, max_output_size, Z_SYNC_FLUSH, + end_of_context); +} + +static grpc_stream_compression_context *grpc_stream_compression_context_create_gzip( + grpc_stream_compression_method method) { + GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_GZIP_COMPRESS || method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + grpc_stream_compression_context_gzip *gzip_ctx = + gpr_zalloc(sizeof(grpc_stream_compression_context_gzip)); + int r; + if (gzip_ctx == NULL) { + return NULL; + } + if (method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS) { + r = inflateInit2(&gzip_ctx->zs, 0x1F); + gzip_ctx->flate = inflate; + } else { + r = deflateInit2(&gzip_ctx->zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, + Z_DEFAULT_STRATEGY); + gzip_ctx->flate = deflate; + } + if (r != Z_OK) { + gpr_free(gzip_ctx); + return NULL; + } + + return (grpc_stream_compression_context *)gzip_ctx; +} + +static void grpc_stream_compression_context_destroy_gzip( + grpc_stream_compression_context *ctx) { + if (ctx == NULL) { + return; + } + grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip*)ctx; + if (gzip_ctx->flate == inflate) { + inflateEnd(&gzip_ctx->zs); + } else { + deflateEnd(&gzip_ctx->zs); + } + gpr_free(ctx); +} + +const grpc_stream_compression_vtable grpc_stream_compression_gzip_vtable = { + .compress = grpc_stream_compress_gzip, + .decompress = grpc_stream_decompress_gzip, + .context_create = grpc_stream_compression_context_create_gzip, + .context_destroy = grpc_stream_compression_context_destroy_gzip +}; diff --git a/src/core/lib/compression/stream_compression_gzip.h b/src/core/lib/compression/stream_compression_gzip.h new file mode 100644 index 00000000000..7cf49a0de93 --- /dev/null +++ b/src/core/lib/compression/stream_compression_gzip.h @@ -0,0 +1,26 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_GZIP_H +#define GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_GZIP_H + +#include "src/core/lib/compression/stream_compression.h" + +extern const grpc_stream_compression_vtable grpc_stream_compression_gzip_vtable; + +#endif diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c new file mode 100644 index 00000000000..1395b762d48 --- /dev/null +++ b/src/core/lib/compression/stream_compression_identity.c @@ -0,0 +1,77 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include "src/core/lib/compression/stream_compression.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/slice/slice_internal.h" + +#define OUTPUT_BLOCK_SIZE (1024) + +static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size) { + if (max_output_size >= in->length) { + *output_size = in->length; + grpc_slice_buffer_move_into(in, out); + } else { + *output_size = max_output_size; + grpc_slice_buffer_move_first(in, max_output_size, out); + } +} + +static bool grpc_stream_compress_identity(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + grpc_stream_compression_flush flush) { + if (ctx == NULL) { + return false; + } + grpc_stream_compression_pass_through(in, out, output_size, max_output_size); + return true; +} + +static bool grpc_stream_decompress_identity(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context) { + if (ctx == NULL) { + return false; + } + grpc_stream_compression_pass_through(in, out, output_size, max_output_size); + return true; +} + +static grpc_stream_compression_context *grpc_stream_compression_context_create_identity( + grpc_stream_compression_method method) { + GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS || method == GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS); + /* No context needed in this case. Use fake context instead. */ + return (grpc_stream_compression_context *)1; +} + +static void grpc_stream_compression_context_destroy_identity( + grpc_stream_compression_context *ctx) { + return; +} + +const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable = { + .compress = grpc_stream_compress_identity, + .decompress = grpc_stream_decompress_identity, + .context_create = grpc_stream_compression_context_create_identity, + .context_destroy = grpc_stream_compression_context_destroy_identity +}; diff --git a/src/core/lib/compression/stream_compression_identity.h b/src/core/lib/compression/stream_compression_identity.h new file mode 100644 index 00000000000..0d770dd32b7 --- /dev/null +++ b/src/core/lib/compression/stream_compression_identity.h @@ -0,0 +1,26 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_IDENTITY_H +#define GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_IDENTITY_H + +#include "src/core/lib/compression/stream_compression.h" + +extern const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable; + +#endif From bf5484e785f93e1183a15415239a729d3d120af1 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 1 Sep 2017 18:03:03 -0700 Subject: [PATCH 035/227] build_project --- CMakeLists.txt | 12 ++++++++++++ Makefile | 12 ++++++++++++ binding.gyp | 2 ++ build.yaml | 6 ++++-- config.m4 | 2 ++ config.w32 | 2 ++ gRPC-Core.podspec | 6 ++++++ grpc.gemspec | 4 ++++ grpc.gyp | 8 ++++++++ package.xml | 4 ++++ src/python/grpcio/grpc_core_dependencies.py | 2 ++ tools/doxygen/Doxyfile.c++.internal | 2 ++ tools/doxygen/Doxyfile.core.internal | 4 ++++ tools/run_tests/generated/sources_and_headers.json | 6 ++++++ 14 files changed, 70 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b22aa1899e8..ee4d2bcbaba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -968,6 +968,8 @@ add_library(grpc src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -1319,6 +1321,8 @@ add_library(grpc_cronet src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -1638,6 +1642,8 @@ add_library(grpc_test_util src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -1901,6 +1907,8 @@ add_library(grpc_test_util_unsecure src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -2150,6 +2158,8 @@ add_library(grpc_unsecure src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -2851,6 +2861,8 @@ add_library(grpc++_cronet src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c diff --git a/Makefile b/Makefile index 743ca41dc6c..a3db8106dae 100644 --- a/Makefile +++ b/Makefile @@ -2915,6 +2915,8 @@ LIBGRPC_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -3264,6 +3266,8 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -3580,6 +3584,8 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -3832,6 +3838,8 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -4057,6 +4065,8 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -4741,6 +4751,8 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ diff --git a/binding.gyp b/binding.gyp index 946edd8139c..0547a82512f 100644 --- a/binding.gyp +++ b/binding.gyp @@ -667,6 +667,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', diff --git a/build.yaml b/build.yaml index 0939796bcb3..0de5b9859d8 100644 --- a/build.yaml +++ b/build.yaml @@ -195,9 +195,10 @@ filegroups: - src/core/lib/compression/compression.c - src/core/lib/compression/message_compress.c - src/core/lib/compression/stream_compression.c + - src/core/lib/compression/stream_compression_gzip.c + - src/core/lib/compression/stream_compression_identity.c - src/core/lib/debug/stats.c - src/core/lib/debug/stats_data.c - - src/core/lib/compression/stream_compression_gzip.c - src/core/lib/http/format_request.c - src/core/lib/http/httpcli.c - src/core/lib/http/parser.c @@ -348,9 +349,10 @@ filegroups: - src/core/lib/compression/algorithm_metadata.h - src/core/lib/compression/message_compress.h - src/core/lib/compression/stream_compression.h + - src/core/lib/compression/stream_compression_gzip.h + - src/core/lib/compression/stream_compression_identity.h - src/core/lib/debug/stats.h - src/core/lib/debug/stats_data.h - - src/core/lib/compression/stream_compression_gzip.h - src/core/lib/http/format_request.h - src/core/lib/http/httpcli.h - src/core/lib/http/parser.h diff --git a/config.m4 b/config.m4 index d9d25cb0065..fbae82de7f6 100644 --- a/config.m4 +++ b/config.m4 @@ -96,6 +96,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ diff --git a/config.w32 b/config.w32 index 640115e8614..3dacc31e6ad 100644 --- a/config.w32 +++ b/config.w32 @@ -73,6 +73,8 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\compression\\compression.c " + "src\\core\\lib\\compression\\message_compress.c " + "src\\core\\lib\\compression\\stream_compression.c " + + "src\\core\\lib\\compression\\stream_compression_gzip.c " + + "src\\core\\lib\\compression\\stream_compression_identity.c " + "src\\core\\lib\\debug\\stats.c " + "src\\core\\lib\\debug\\stats_data.c " + "src\\core\\lib\\http\\format_request.c " + diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 66368b1b4b6..8fea17c5dba 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -329,6 +329,8 @@ Pod::Spec.new do |s| 'src/core/lib/compression/algorithm_metadata.h', 'src/core/lib/compression/message_compress.h', 'src/core/lib/compression/stream_compression.h', + 'src/core/lib/compression/stream_compression_gzip.h', + 'src/core/lib/compression/stream_compression_identity.h', 'src/core/lib/debug/stats.h', 'src/core/lib/debug/stats_data.h', 'src/core/lib/http/format_request.h', @@ -479,6 +481,8 @@ Pod::Spec.new do |s| 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', @@ -826,6 +830,8 @@ Pod::Spec.new do |s| 'src/core/lib/compression/algorithm_metadata.h', 'src/core/lib/compression/message_compress.h', 'src/core/lib/compression/stream_compression.h', + 'src/core/lib/compression/stream_compression_gzip.h', + 'src/core/lib/compression/stream_compression_identity.h', 'src/core/lib/debug/stats.h', 'src/core/lib/debug/stats_data.h', 'src/core/lib/http/format_request.h', diff --git a/grpc.gemspec b/grpc.gemspec index d3779a9991e..166c914c529 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -261,6 +261,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/compression/algorithm_metadata.h ) s.files += %w( src/core/lib/compression/message_compress.h ) s.files += %w( src/core/lib/compression/stream_compression.h ) + s.files += %w( src/core/lib/compression/stream_compression_gzip.h ) + s.files += %w( src/core/lib/compression/stream_compression_identity.h ) s.files += %w( src/core/lib/debug/stats.h ) s.files += %w( src/core/lib/debug/stats_data.h ) s.files += %w( src/core/lib/http/format_request.h ) @@ -415,6 +417,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/compression/compression.c ) s.files += %w( src/core/lib/compression/message_compress.c ) s.files += %w( src/core/lib/compression/stream_compression.c ) + s.files += %w( src/core/lib/compression/stream_compression_gzip.c ) + s.files += %w( src/core/lib/compression/stream_compression_identity.c ) s.files += %w( src/core/lib/debug/stats.c ) s.files += %w( src/core/lib/debug/stats_data.c ) s.files += %w( src/core/lib/http/format_request.c ) diff --git a/grpc.gyp b/grpc.gyp index 40938a4564a..2e2e0b7dfda 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -233,6 +233,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', @@ -533,6 +535,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', @@ -738,6 +742,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', @@ -928,6 +934,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', diff --git a/package.xml b/package.xml index 1cca4fbdddb..33b8378a35d 100644 --- a/package.xml +++ b/package.xml @@ -271,6 +271,8 @@ + + @@ -425,6 +427,8 @@ + + diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 1cbf345ab6e..9025a07ba9c 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -72,6 +72,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 91c149eec92..71f500b53ff 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -942,6 +942,8 @@ src/core/lib/channel/handshaker_registry.h \ src/core/lib/compression/algorithm_metadata.h \ src/core/lib/compression/message_compress.h \ src/core/lib/compression/stream_compression.h \ +src/core/lib/compression/stream_compression_gzip.h \ +src/core/lib/compression/stream_compression_identity.h \ src/core/lib/debug/stats.h \ src/core/lib/debug/stats_data.h \ src/core/lib/debug/trace.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 26d982acd7d..63993c808bc 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1077,6 +1077,10 @@ src/core/lib/compression/message_compress.c \ src/core/lib/compression/message_compress.h \ src/core/lib/compression/stream_compression.c \ src/core/lib/compression/stream_compression.h \ +src/core/lib/compression/stream_compression_gzip.c \ +src/core/lib/compression/stream_compression_gzip.h \ +src/core/lib/compression/stream_compression_identity.c \ +src/core/lib/compression/stream_compression_identity.h \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats.h \ src/core/lib/debug/stats_data.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index fbd47389f71..cca4cff0fb1 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7856,6 +7856,8 @@ "src/core/lib/compression/compression.c", "src/core/lib/compression/message_compress.c", "src/core/lib/compression/stream_compression.c", + "src/core/lib/compression/stream_compression_gzip.c", + "src/core/lib/compression/stream_compression_identity.c", "src/core/lib/debug/stats.c", "src/core/lib/debug/stats_data.c", "src/core/lib/http/format_request.c", @@ -8009,6 +8011,8 @@ "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", "src/core/lib/compression/stream_compression.h", + "src/core/lib/compression/stream_compression_gzip.h", + "src/core/lib/compression/stream_compression_identity.h", "src/core/lib/debug/stats.h", "src/core/lib/debug/stats_data.h", "src/core/lib/http/format_request.h", @@ -8141,6 +8145,8 @@ "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", "src/core/lib/compression/stream_compression.h", + "src/core/lib/compression/stream_compression_gzip.h", + "src/core/lib/compression/stream_compression_identity.h", "src/core/lib/debug/stats.h", "src/core/lib/debug/stats_data.h", "src/core/lib/http/format_request.h", From 8e14acc76bacdf67fa852a96e44381df238e3907 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 13:39:07 -0700 Subject: [PATCH 036/227] Remove duplicate sentences on send path --- .../chttp2/transport/chttp2_transport.c | 14 ++- .../ext/transport/chttp2/transport/internal.h | 9 +- .../ext/transport/chttp2/transport/writing.c | 89 ++++++++----------- 3 files changed, 45 insertions(+), 67 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 0e00995d11d..e1fb9dd9154 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -727,10 +727,7 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, grpc_slice_buffer_destroy_internal(exec_ctx, &s->unprocessed_incoming_frames_buffer); grpc_slice_buffer_destroy_internal(exec_ctx, &s->frame_storage); - if (s->compressed_data_buffer) { - grpc_slice_buffer_destroy_internal(exec_ctx, s->compressed_data_buffer); - gpr_free(s->compressed_data_buffer); - } + grpc_slice_buffer_destroy_internal(exec_ctx, &s->compressed_data_buffer); if (s->decompressed_data_buffer) { grpc_slice_buffer_destroy_internal(exec_ctx, s->decompressed_data_buffer); gpr_free(s->decompressed_data_buffer); @@ -1300,12 +1297,11 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE; /* Identify stream compression */ - if ((s->stream_compression_send_enabled = - (op_payload->send_initial_metadata.send_initial_metadata->idx.named - .content_encoding != NULL)) == true) { - s->compressed_data_buffer = gpr_malloc(sizeof(grpc_slice_buffer)); - grpc_slice_buffer_init(s->compressed_data_buffer); + if (op_payload->send_initial_metadata.send_initial_metadata->idx.named.content_encoding == NULL || + grpc_stream_compression_method_parse(GRPC_MDVALUE(op_payload->send_initial_metadata.send_initial_metadata->idx.named.content_encoding->md), true, &s->stream_compression_method) == 0) { + s->stream_compression_method = GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS; } + grpc_slice_buffer_init(&s->compressed_data_buffer); s->send_initial_metadata_finished = add_closure_barrier(on_complete); s->send_initial_metadata = diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 3c41a8958f6..e5e4dd33443 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -557,10 +557,11 @@ struct grpc_chttp2_stream { grpc_chttp2_write_cb *finish_after_write; size_t sending_bytes; - /** Whether stream compression send is enabled */ + /* Stream compression method to be used. */ + grpc_stream_compression_method stream_compression_method; + /* Stream decompression method to be used. */ + grpc_stream_compression_method stream_decompression_method; bool stream_compression_recv_enabled; - /** Whether stream compression recv is enabled */ - bool stream_compression_send_enabled; /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed */ bool unprocessed_incoming_frames_decompressed; @@ -570,7 +571,7 @@ struct grpc_chttp2_stream { grpc_stream_compression_context *stream_compression_ctx; /** Buffer storing data that is compressed but not sent */ - grpc_slice_buffer *compressed_data_buffer; + grpc_slice_buffer compressed_data_buffer; /** Amount of uncompressed bytes sent out when compressed_data_buffer is * emptied */ size_t uncompressed_data_size; diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index b877e9f1264..b6bc864c29d 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -288,8 +288,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (sent_initial_metadata) { /* send any body bytes, if allowed by flow control */ if (s->flow_controlled_buffer.length > 0 || - (s->stream_compression_send_enabled && - s->compressed_data_buffer->length > 0)) { + s->compressed_data_buffer.length > 0) { uint32_t stream_remote_window = (uint32_t)GPR_MAX( 0, s->flow_control.remote_window_delta + @@ -302,56 +301,40 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (max_outgoing > 0) { bool is_last_data_frame = false; bool is_last_frame = false; - if (s->stream_compression_send_enabled) { - while ((s->flow_controlled_buffer.length > 0 || - s->compressed_data_buffer->length > 0) && - max_outgoing > 0) { - if (s->compressed_data_buffer->length > 0) { - uint32_t send_bytes = (uint32_t)GPR_MIN( - max_outgoing, s->compressed_data_buffer->length); - is_last_data_frame = - (send_bytes == s->compressed_data_buffer->length && - s->flow_controlled_buffer.length == 0 && - s->fetching_send_message == NULL); - is_last_frame = - is_last_data_frame && s->send_trailing_metadata != NULL && - grpc_metadata_batch_is_empty(s->send_trailing_metadata); - grpc_chttp2_encode_data(s->id, s->compressed_data_buffer, - send_bytes, is_last_frame, - &s->stats.outgoing, &t->outbuf); - grpc_chttp2_flowctl_sent_data(&t->flow_control, - &s->flow_control, send_bytes); - max_outgoing -= send_bytes; - if (s->compressed_data_buffer->length == 0) { - s->sending_bytes += s->uncompressed_data_size; - } - } else { - if (s->stream_compression_ctx == NULL) { - s->stream_compression_ctx = - grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); - } - s->uncompressed_data_size = s->flow_controlled_buffer.length; - GPR_ASSERT(grpc_stream_compress( - s->stream_compression_ctx, &s->flow_controlled_buffer, - s->compressed_data_buffer, NULL, MAX_SIZE_T, - GRPC_STREAM_COMPRESSION_FLUSH_SYNC)); + while ((s->flow_controlled_buffer.length > 0 || + s->compressed_data_buffer.length > 0) && + max_outgoing > 0) { + if (s->compressed_data_buffer.length > 0) { + uint32_t send_bytes = (uint32_t)GPR_MIN( + max_outgoing, s->compressed_data_buffer.length); + is_last_data_frame = + (send_bytes == s->compressed_data_buffer.length && + s->flow_controlled_buffer.length == 0 && + s->fetching_send_message == NULL); + is_last_frame = + is_last_data_frame && s->send_trailing_metadata != NULL && + grpc_metadata_batch_is_empty(s->send_trailing_metadata); + grpc_chttp2_encode_data(s->id, &s->compressed_data_buffer, + send_bytes, is_last_frame, + &s->stats.outgoing, &t->outbuf); + grpc_chttp2_flowctl_sent_data(&t->flow_control, + &s->flow_control, send_bytes); + max_outgoing -= send_bytes; + if (s->compressed_data_buffer.length == 0) { + s->sending_bytes += s->uncompressed_data_size; } + } else { + if (s->stream_compression_ctx == NULL) { + s->stream_compression_ctx = + grpc_stream_compression_context_create( + s->stream_compression_method); + } + s->uncompressed_data_size = s->flow_controlled_buffer.length; + GPR_ASSERT(grpc_stream_compress( + s->stream_compression_ctx, &s->flow_controlled_buffer, + &s->compressed_data_buffer, NULL, MAX_SIZE_T, + GRPC_STREAM_COMPRESSION_FLUSH_SYNC)); } - } else { - uint32_t send_bytes = (uint32_t)GPR_MIN( - max_outgoing, s->flow_controlled_buffer.length); - is_last_data_frame = s->fetching_send_message == NULL && - send_bytes == s->flow_controlled_buffer.length; - is_last_frame = - is_last_data_frame && s->send_trailing_metadata != NULL && - grpc_metadata_batch_is_empty(s->send_trailing_metadata); - grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer, - send_bytes, is_last_frame, - &s->stats.outgoing, &t->outbuf); - grpc_chttp2_flowctl_sent_data(&t->flow_control, &s->flow_control, - send_bytes); - s->sending_bytes += send_bytes; } t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; @@ -371,8 +354,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( } now_writing = true; if (s->flow_controlled_buffer.length > 0 || - (s->stream_compression_send_enabled && - s->compressed_data_buffer->length > 0)) { + s->compressed_data_buffer.length > 0) { GRPC_CHTTP2_STREAM_REF(s, "chttp2_writing:fork"); grpc_chttp2_list_add_writable_stream(t, s); } @@ -387,8 +369,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (s->send_trailing_metadata != NULL && s->fetching_send_message == NULL && s->flow_controlled_buffer.length == 0 && - (!s->stream_compression_send_enabled || - s->compressed_data_buffer->length == 0)) { + s->compressed_data_buffer.length == 0) { GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "sending trailing_metadata")); if (grpc_metadata_batch_is_empty(s->send_trailing_metadata)) { grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer, 0, true, From 03600fc37df35399063ea9d89e9b745835a41396 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 13:42:44 -0700 Subject: [PATCH 037/227] clang-format --- .../chttp2/transport/chttp2_transport.c | 9 +++- .../ext/transport/chttp2/transport/writing.c | 4 +- src/core/lib/compression/stream_compression.c | 19 +++++--- src/core/lib/compression/stream_compression.h | 22 +++++----- .../lib/compression/stream_compression_gzip.c | 44 +++++++++++-------- .../compression/stream_compression_identity.c | 36 ++++++++------- .../compression/stream_compression_identity.h | 3 +- 7 files changed, 80 insertions(+), 57 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index e1fb9dd9154..64a5e342feb 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1297,8 +1297,13 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE; /* Identify stream compression */ - if (op_payload->send_initial_metadata.send_initial_metadata->idx.named.content_encoding == NULL || - grpc_stream_compression_method_parse(GRPC_MDVALUE(op_payload->send_initial_metadata.send_initial_metadata->idx.named.content_encoding->md), true, &s->stream_compression_method) == 0) { + if (op_payload->send_initial_metadata.send_initial_metadata->idx.named + .content_encoding == NULL || + grpc_stream_compression_method_parse( + GRPC_MDVALUE( + op_payload->send_initial_metadata.send_initial_metadata->idx + .named.content_encoding->md), + true, &s->stream_compression_method) == 0) { s->stream_compression_method = GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS; } grpc_slice_buffer_init(&s->compressed_data_buffer); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index b6bc864c29d..fff98a62016 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -317,8 +317,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( grpc_chttp2_encode_data(s->id, &s->compressed_data_buffer, send_bytes, is_last_frame, &s->stats.outgoing, &t->outbuf); - grpc_chttp2_flowctl_sent_data(&t->flow_control, - &s->flow_control, send_bytes); + grpc_chttp2_flowctl_sent_data(&t->flow_control, &s->flow_control, + send_bytes); max_outgoing -= send_bytes; if (s->compressed_data_buffer.length == 0) { s->sending_bytes += s->uncompressed_data_size; diff --git a/src/core/lib/compression/stream_compression.c b/src/core/lib/compression/stream_compression.c index 274ee100ba5..411489f0296 100644 --- a/src/core/lib/compression/stream_compression.c +++ b/src/core/lib/compression/stream_compression.c @@ -21,20 +21,23 @@ #include "src/core/lib/compression/stream_compression.h" #include "src/core/lib/compression/stream_compression_gzip.h" -extern const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable; +extern const grpc_stream_compression_vtable + grpc_stream_compression_identity_vtable; bool grpc_stream_compress(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, grpc_stream_compression_flush flush) { - return ctx->vtable->compress(ctx, in, out, output_size, max_output_size, flush); + return ctx->vtable->compress(ctx, in, out, output_size, max_output_size, + flush); } bool grpc_stream_decompress(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, bool *end_of_context) { - return ctx->vtable->decompress(ctx, in, out, output_size, max_output_size, end_of_context); + return ctx->vtable->decompress(ctx, in, out, output_size, max_output_size, + end_of_context); } grpc_stream_compression_context *grpc_stream_compression_context_create( @@ -58,15 +61,17 @@ void grpc_stream_compression_context_destroy( } int grpc_stream_compression_method_parse( - grpc_slice value, bool is_compress, grpc_stream_compression_method *method) { + grpc_slice value, bool is_compress, + grpc_stream_compression_method *method) { if (grpc_slice_eq(value, GRPC_MDSTR_IDENTITY)) { - *method = is_compress ? GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS : GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; + *method = is_compress ? GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS + : GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; return 1; } else if (grpc_slice_eq(value, GRPC_MDSTR_GZIP)) { - *method = is_compress ? GRPC_STREAM_COMPRESSION_GZIP_COMPRESS : GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS; + *method = is_compress ? GRPC_STREAM_COMPRESSION_GZIP_COMPRESS + : GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS; return 1; } else { return 0; } } - diff --git a/src/core/lib/compression/stream_compression.h b/src/core/lib/compression/stream_compression.h index fa07247c310..6d073280faa 100644 --- a/src/core/lib/compression/stream_compression.h +++ b/src/core/lib/compression/stream_compression.h @@ -49,18 +49,16 @@ typedef enum grpc_stream_compression_flush { } grpc_stream_compression_flush; struct grpc_stream_compression_vtable { -bool (*compress)(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - grpc_stream_compression_flush flush); -bool (*decompress)(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - bool *end_of_context); -grpc_stream_compression_context *(*context_create)( - grpc_stream_compression_method method); -void (*context_destroy)( - grpc_stream_compression_context *ctx); + bool (*compress)(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, + grpc_slice_buffer *out, size_t *output_size, + size_t max_output_size, grpc_stream_compression_flush flush); + bool (*decompress)(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context); + grpc_stream_compression_context *(*context_create)( + grpc_stream_compression_method method); + void (*context_destroy)(grpc_stream_compression_context *ctx); }; /** diff --git a/src/core/lib/compression/stream_compression_gzip.c b/src/core/lib/compression/stream_compression_gzip.c index 2d03836a0dc..5d21bf27ae0 100644 --- a/src/core/lib/compression/stream_compression_gzip.c +++ b/src/core/lib/compression/stream_compression_gzip.c @@ -132,13 +132,16 @@ static bool gzip_flate(grpc_stream_compression_context_gzip *ctx, } static bool grpc_stream_compress_gzip(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - grpc_stream_compression_flush flush) { + grpc_slice_buffer *in, + grpc_slice_buffer *out, + size_t *output_size, + size_t max_output_size, + grpc_stream_compression_flush flush) { if (ctx == NULL) { return false; } - grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip *)ctx; + grpc_stream_compression_context_gzip *gzip_ctx = + (grpc_stream_compression_context_gzip *)ctx; GPR_ASSERT(gzip_ctx->flate == deflate); int gzip_flush; switch (flush) { @@ -159,21 +162,26 @@ static bool grpc_stream_compress_gzip(grpc_stream_compression_context *ctx, } static bool grpc_stream_decompress_gzip(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - bool *end_of_context) { + grpc_slice_buffer *in, + grpc_slice_buffer *out, + size_t *output_size, + size_t max_output_size, + bool *end_of_context) { if (ctx == NULL) { return false; } - grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip *)ctx; + grpc_stream_compression_context_gzip *gzip_ctx = + (grpc_stream_compression_context_gzip *)ctx; GPR_ASSERT(gzip_ctx->flate == inflate); - return gzip_flate(gzip_ctx, in, out, output_size, max_output_size, Z_SYNC_FLUSH, - end_of_context); + return gzip_flate(gzip_ctx, in, out, output_size, max_output_size, + Z_SYNC_FLUSH, end_of_context); } -static grpc_stream_compression_context *grpc_stream_compression_context_create_gzip( +static grpc_stream_compression_context * +grpc_stream_compression_context_create_gzip( grpc_stream_compression_method method) { - GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_GZIP_COMPRESS || method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_GZIP_COMPRESS || + method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_stream_compression_context_gzip *gzip_ctx = gpr_zalloc(sizeof(grpc_stream_compression_context_gzip)); int r; @@ -201,7 +209,8 @@ static void grpc_stream_compression_context_destroy_gzip( if (ctx == NULL) { return; } - grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip*)ctx; + grpc_stream_compression_context_gzip *gzip_ctx = + (grpc_stream_compression_context_gzip *)ctx; if (gzip_ctx->flate == inflate) { inflateEnd(&gzip_ctx->zs); } else { @@ -211,8 +220,7 @@ static void grpc_stream_compression_context_destroy_gzip( } const grpc_stream_compression_vtable grpc_stream_compression_gzip_vtable = { - .compress = grpc_stream_compress_gzip, - .decompress = grpc_stream_decompress_gzip, - .context_create = grpc_stream_compression_context_create_gzip, - .context_destroy = grpc_stream_compression_context_destroy_gzip -}; + .compress = grpc_stream_compress_gzip, + .decompress = grpc_stream_decompress_gzip, + .context_create = grpc_stream_compression_context_create_gzip, + .context_destroy = grpc_stream_compression_context_destroy_gzip}; diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c index 1395b762d48..133c3250d27 100644 --- a/src/core/lib/compression/stream_compression_identity.c +++ b/src/core/lib/compression/stream_compression_identity.c @@ -25,7 +25,10 @@ #define OUTPUT_BLOCK_SIZE (1024) -static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size) { +static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, + grpc_slice_buffer *out, + size_t *output_size, + size_t max_output_size) { if (max_output_size >= in->length) { *output_size = in->length; grpc_slice_buffer_move_into(in, out); @@ -36,9 +39,11 @@ static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_sli } static bool grpc_stream_compress_identity(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - grpc_stream_compression_flush flush) { + grpc_slice_buffer *in, + grpc_slice_buffer *out, + size_t *output_size, + size_t max_output_size, + grpc_stream_compression_flush flush) { if (ctx == NULL) { return false; } @@ -46,10 +51,10 @@ static bool grpc_stream_compress_identity(grpc_stream_compression_context *ctx, return true; } -static bool grpc_stream_decompress_identity(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - bool *end_of_context) { +static bool grpc_stream_decompress_identity( + grpc_stream_compression_context *ctx, grpc_slice_buffer *in, + grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, + bool *end_of_context) { if (ctx == NULL) { return false; } @@ -57,9 +62,11 @@ static bool grpc_stream_decompress_identity(grpc_stream_compression_context *ctx return true; } -static grpc_stream_compression_context *grpc_stream_compression_context_create_identity( +static grpc_stream_compression_context * +grpc_stream_compression_context_create_identity( grpc_stream_compression_method method) { - GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS || method == GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS); + GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS || + method == GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS); /* No context needed in this case. Use fake context instead. */ return (grpc_stream_compression_context *)1; } @@ -70,8 +77,7 @@ static void grpc_stream_compression_context_destroy_identity( } const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable = { - .compress = grpc_stream_compress_identity, - .decompress = grpc_stream_decompress_identity, - .context_create = grpc_stream_compression_context_create_identity, - .context_destroy = grpc_stream_compression_context_destroy_identity -}; + .compress = grpc_stream_compress_identity, + .decompress = grpc_stream_decompress_identity, + .context_create = grpc_stream_compression_context_create_identity, + .context_destroy = grpc_stream_compression_context_destroy_identity}; diff --git a/src/core/lib/compression/stream_compression_identity.h b/src/core/lib/compression/stream_compression_identity.h index 0d770dd32b7..41926e949e5 100644 --- a/src/core/lib/compression/stream_compression_identity.h +++ b/src/core/lib/compression/stream_compression_identity.h @@ -21,6 +21,7 @@ #include "src/core/lib/compression/stream_compression.h" -extern const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable; +extern const grpc_stream_compression_vtable + grpc_stream_compression_identity_vtable; #endif From 68a1a1530be8573e109c4ec6754941a682f8b5dd Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 14:47:33 -0700 Subject: [PATCH 038/227] Build fix --- .../compression/stream_compression_test.c | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c index e576507aaf6..26c957ded49 100644 --- a/test/core/compression/stream_compression_test.c +++ b/test/core/compression/stream_compression_test.c @@ -59,10 +59,10 @@ static void test_stream_compression_simple_compress_decompress() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -91,10 +91,10 @@ test_stream_compression_simple_compress_decompress_with_output_size_constraint() grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -139,10 +139,10 @@ test_stream_compression_simple_compress_decompress_with_large_data() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -172,7 +172,7 @@ static void test_stream_compression_drop_context() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -181,7 +181,7 @@ static void test_stream_compression_drop_context() { grpc_stream_compression_context_destroy(compress_ctx); compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); slice = grpc_slice_from_static_string(test_str2); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -205,7 +205,7 @@ static void test_stream_compression_drop_context() { grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); bool end_of_context; size_t output_size; GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, @@ -219,7 +219,7 @@ static void test_stream_compression_drop_context() { grpc_slice_buffer_init(&sink); decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, ~(size_t)0, &end_of_context)); GPR_ASSERT(end_of_context == true); @@ -240,7 +240,7 @@ static void test_stream_compression_sync_flush() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -249,7 +249,7 @@ static void test_stream_compression_sync_flush() { grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); bool end_of_context; size_t output_size; GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, From c3229b777ce58f49b092a92a07b18155d1ae7799 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 17:57:19 -0700 Subject: [PATCH 039/227] Remove duplicate sentences on recv path --- .../chttp2/transport/chttp2_transport.c | 37 ++++++++----------- .../transport/chttp2/transport/hpack_parser.c | 13 ++----- .../ext/transport/chttp2/transport/internal.h | 9 ++--- .../compression/stream_compression_identity.c | 1 + 4 files changed, 23 insertions(+), 37 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 64a5e342feb..ad0521cc42f 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -693,6 +693,8 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_schedule_on_exec_ctx); grpc_slice_buffer_init(&s->unprocessed_incoming_frames_buffer); grpc_slice_buffer_init(&s->frame_storage); + grpc_slice_buffer_init(&s->compressed_data_buffer); + grpc_slice_buffer_init(&s->decompressed_data_buffer); s->pending_byte_stream = false; GRPC_CLOSURE_INIT(&s->reset_byte_stream, reset_byte_stream, s, grpc_combiner_scheduler(t->combiner)); @@ -728,10 +730,7 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, &s->unprocessed_incoming_frames_buffer); grpc_slice_buffer_destroy_internal(exec_ctx, &s->frame_storage); grpc_slice_buffer_destroy_internal(exec_ctx, &s->compressed_data_buffer); - if (s->decompressed_data_buffer) { - grpc_slice_buffer_destroy_internal(exec_ctx, s->decompressed_data_buffer); - gpr_free(s->decompressed_data_buffer); - } + grpc_slice_buffer_destroy_internal(exec_ctx, &s->decompressed_data_buffer); grpc_chttp2_list_remove_stalled_by_transport(t, s); grpc_chttp2_list_remove_stalled_by_stream(t, s); @@ -1306,7 +1305,6 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, true, &s->stream_compression_method) == 0) { s->stream_compression_method = GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS; } - grpc_slice_buffer_init(&s->compressed_data_buffer); s->send_initial_metadata_finished = add_closure_barrier(on_complete); s->send_initial_metadata = @@ -1725,18 +1723,17 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, &s->frame_storage); s->unprocessed_incoming_frames_decompressed = false; } - if (s->stream_compression_recv_enabled && - !s->unprocessed_incoming_frames_decompressed) { - GPR_ASSERT(s->decompressed_data_buffer->length == 0); + if (!s->unprocessed_incoming_frames_decompressed) { + GPR_ASSERT(s->decompressed_data_buffer.length == 0); bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + s->stream_decompression_method); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, - s->decompressed_data_buffer, NULL, + &s->decompressed_data_buffer, NULL, GRPC_HEADER_SIZE_IN_BYTES, &end_of_context)) { grpc_slice_buffer_reset_and_unref_internal(exec_ctx, @@ -1747,7 +1744,7 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, "Stream decompression error."); } else { error = grpc_deframe_unprocessed_incoming_frames( - exec_ctx, &s->data_parser, s, s->decompressed_data_buffer, NULL, + exec_ctx, &s->data_parser, s, &s->decompressed_data_buffer, NULL, s->recv_message); if (end_of_context) { grpc_stream_compression_context_destroy( @@ -1755,10 +1752,6 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, s->stream_decompression_ctx = NULL; } } - } else { - error = grpc_deframe_unprocessed_incoming_frames( - exec_ctx, &s->data_parser, s, - &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); } if (error != GRPC_ERROR_NONE) { s->seen_error = true; @@ -1797,7 +1790,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, } bool pending_data = s->pending_byte_stream || s->unprocessed_incoming_frames_buffer.length > 0; - if (s->stream_compression_recv_enabled && s->read_closed && + if (s->read_closed && s->frame_storage.length > 0 && !pending_data && !s->seen_error && s->recv_trailing_metadata_finished != NULL) { /* Maybe some SYNC_FLUSH data is left in frame_storage. Consume them and @@ -1805,7 +1798,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + s->stream_decompression_method); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->frame_storage, @@ -1818,6 +1811,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, } else { if (s->unprocessed_incoming_frames_buffer.length > 0) { s->unprocessed_incoming_frames_decompressed = true; + pending_data = true; } if (end_of_context) { grpc_stream_compression_context_destroy(s->stream_decompression_ctx); @@ -2690,16 +2684,15 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, grpc_error *error; if (s->unprocessed_incoming_frames_buffer.length > 0) { - if (s->stream_compression_recv_enabled && - !s->unprocessed_incoming_frames_decompressed) { + if (!s->unprocessed_incoming_frames_decompressed) { bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + s->stream_decompression_method); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, - s->decompressed_data_buffer, NULL, MAX_SIZE_T, + &s->decompressed_data_buffer, NULL, MAX_SIZE_T, &end_of_context)) { error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Stream decompression error."); @@ -2707,7 +2700,7 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, } GPR_ASSERT(s->unprocessed_incoming_frames_buffer.length == 0); grpc_slice_buffer_swap(&s->unprocessed_incoming_frames_buffer, - s->decompressed_data_buffer); + &s->decompressed_data_buffer); s->unprocessed_incoming_frames_decompressed = true; if (end_of_context) { grpc_stream_compression_context_destroy(s->stream_decompression_ctx); diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index c21d76ba710..3b338da288d 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1659,16 +1659,9 @@ static void parse_stream_compression_md(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, grpc_metadata_batch *initial_metadata) { - if (initial_metadata->idx.named.content_encoding != NULL) { - grpc_slice content_encoding = - GRPC_MDVALUE(initial_metadata->idx.named.content_encoding->md); - if (!grpc_slice_eq(content_encoding, GRPC_MDSTR_IDENTITY)) { - if (grpc_slice_eq(content_encoding, GRPC_MDSTR_GZIP)) { - s->stream_compression_recv_enabled = true; - s->decompressed_data_buffer = gpr_malloc(sizeof(grpc_slice_buffer)); - grpc_slice_buffer_init(s->decompressed_data_buffer); - } - } + if (initial_metadata->idx.named.content_encoding == NULL || + grpc_stream_compression_method_parse(GRPC_MDVALUE(initial_metadata->idx.named.content_encoding->md), false, &s->stream_decompression_method) == 0) { + s->stream_decompression_method = GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; } } diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index e5e4dd33443..33cc962f64e 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -561,10 +561,6 @@ struct grpc_chttp2_stream { grpc_stream_compression_method stream_compression_method; /* Stream decompression method to be used. */ grpc_stream_compression_method stream_decompression_method; - bool stream_compression_recv_enabled; - /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed - */ - bool unprocessed_incoming_frames_decompressed; /** Stream compression decompress context */ grpc_stream_compression_context *stream_decompression_ctx; /** Stream compression compress context */ @@ -576,7 +572,10 @@ struct grpc_chttp2_stream { * emptied */ size_t uncompressed_data_size; /** Temporary buffer storing decompressed data */ - grpc_slice_buffer *decompressed_data_buffer; + grpc_slice_buffer decompressed_data_buffer; + /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed + */ + bool unprocessed_incoming_frames_decompressed; }; /** Transport writing call flow: diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c index 133c3250d27..91ee06d7831 100644 --- a/src/core/lib/compression/stream_compression_identity.c +++ b/src/core/lib/compression/stream_compression_identity.c @@ -59,6 +59,7 @@ static bool grpc_stream_decompress_identity( return false; } grpc_stream_compression_pass_through(in, out, output_size, max_output_size); + *end_of_context = false; return true; } From 93197df3d5c5ad49c7fcbaa05b005e6ad7837560 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 19:19:44 -0700 Subject: [PATCH 040/227] Add patch to identity --- .../lib/compression/stream_compression_gzip.c | 3 ++- .../compression/stream_compression_identity.c | 21 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/core/lib/compression/stream_compression_gzip.c b/src/core/lib/compression/stream_compression_gzip.c index 5d21bf27ae0..502bf44a913 100644 --- a/src/core/lib/compression/stream_compression_gzip.c +++ b/src/core/lib/compression/stream_compression_gzip.c @@ -19,7 +19,7 @@ #include #include -#include "src/core/lib/compression/stream_compression.h" +#include "src/core/lib/compression/stream_compression_gzip.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" @@ -201,6 +201,7 @@ grpc_stream_compression_context_create_gzip( return NULL; } + gzip_ctx->base.vtable = &grpc_stream_compression_gzip_vtable; return (grpc_stream_compression_context *)gzip_ctx; } diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c index 91ee06d7831..19eb6a9a00d 100644 --- a/src/core/lib/compression/stream_compression_identity.c +++ b/src/core/lib/compression/stream_compression_identity.c @@ -19,21 +19,30 @@ #include #include -#include "src/core/lib/compression/stream_compression.h" +#include "src/core/lib/compression/stream_compression_identity.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" #define OUTPUT_BLOCK_SIZE (1024) +/* Singleton context used for all identity streams. */ +static grpc_stream_compression_context identity_ctx = { + .vtable = &grpc_stream_compression_identity_vtable +}; + static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size) { if (max_output_size >= in->length) { - *output_size = in->length; + if (output_size) { + *output_size = in->length; + } grpc_slice_buffer_move_into(in, out); } else { - *output_size = max_output_size; + if (output_size) { + *output_size = max_output_size; + } grpc_slice_buffer_move_first(in, max_output_size, out); } } @@ -59,7 +68,9 @@ static bool grpc_stream_decompress_identity( return false; } grpc_stream_compression_pass_through(in, out, output_size, max_output_size); - *end_of_context = false; + if (end_of_context) { + *end_of_context = false; + } return true; } @@ -69,7 +80,7 @@ grpc_stream_compression_context_create_identity( GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS || method == GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS); /* No context needed in this case. Use fake context instead. */ - return (grpc_stream_compression_context *)1; + return (grpc_stream_compression_context *)&identity_ctx; } static void grpc_stream_compression_context_destroy_identity( From f4c95fa4ebfa5cd8dfea81f949fce2412125b085 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 6 Sep 2017 11:58:29 -0700 Subject: [PATCH 041/227] Patch for fuzzer test failure --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 7 ++++++- src/core/ext/transport/chttp2/transport/frame_data.c | 4 ++-- src/core/ext/transport/chttp2/transport/internal.h | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index ad0521cc42f..2a26bbb0c8b 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -696,6 +696,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_slice_buffer_init(&s->compressed_data_buffer); grpc_slice_buffer_init(&s->decompressed_data_buffer); s->pending_byte_stream = false; + s->decompressed_header_bytes = 0; GRPC_CLOSURE_INIT(&s->reset_byte_stream, reset_byte_stream, s, grpc_combiner_scheduler(t->combiner)); @@ -1734,7 +1735,7 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, &s->decompressed_data_buffer, NULL, - GRPC_HEADER_SIZE_IN_BYTES, + GRPC_HEADER_SIZE_IN_BYTES - s->decompressed_header_bytes, &end_of_context)) { grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &s->frame_storage); @@ -1743,6 +1744,10 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Stream decompression error."); } else { + s->decompressed_header_bytes += s->decompressed_data_buffer.length; + if (s->decompressed_header_bytes == GRPC_HEADER_SIZE_IN_BYTES) { + s->decompressed_header_bytes = 0; + } error = grpc_deframe_unprocessed_incoming_frames( exec_ctx, &s->data_parser, s, &s->decompressed_data_buffer, NULL, s->recv_message); diff --git a/src/core/ext/transport/chttp2/transport/frame_data.c b/src/core/ext/transport/chttp2/transport/frame_data.c index 222d2177b29..73aaab18025 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.c +++ b/src/core/ext/transport/chttp2/transport/frame_data.c @@ -210,7 +210,7 @@ grpc_error *grpc_deframe_unprocessed_incoming_frames( if (cur != end) { grpc_slice_buffer_undo_take_first( - &s->unprocessed_incoming_frames_buffer, + slices, grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); } grpc_slice_unref_internal(exec_ctx, slice); @@ -277,7 +277,7 @@ grpc_error *grpc_deframe_unprocessed_incoming_frames( p->state = GRPC_CHTTP2_DATA_FH_0; cur += p->frame_size; grpc_slice_buffer_undo_take_first( - &s->unprocessed_incoming_frames_buffer, + slices, grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); grpc_slice_unref_internal(exec_ctx, slice); return GRPC_ERROR_NONE; diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 33cc962f64e..b64e9a0cd65 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -576,6 +576,8 @@ struct grpc_chttp2_stream { /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed */ bool unprocessed_incoming_frames_decompressed; + /** gRPC header bytes that are already decompressed */ + size_t decompressed_header_bytes; }; /** Transport writing call flow: From c2a61a2780e56c474a12f5570133fe166ec9840f Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 6 Sep 2017 22:07:42 -0700 Subject: [PATCH 042/227] write_buffering patch --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 2a26bbb0c8b..9b80e1d3369 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1757,6 +1757,9 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, s->stream_decompression_ctx = NULL; } } + } else { + error = grpc_deframe_unprocessed_incoming_frames( + exec_ctx, &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); } if (error != GRPC_ERROR_NONE) { s->seen_error = true; From 1e99b2388bf5e13c79143463f6d503275fd63cb7 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 7 Sep 2017 09:47:58 -0700 Subject: [PATCH 043/227] clang-format --- .../chttp2/transport/chttp2_transport.c | 27 ++++++++++--------- .../transport/chttp2/transport/hpack_parser.c | 7 +++-- .../compression/stream_compression_identity.c | 3 +-- .../compression/stream_compression_test.c | 19 ++++++++----- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 9b80e1d3369..721f694506b 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1732,11 +1732,12 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, grpc_stream_compression_context_create( s->stream_decompression_method); } - if (!grpc_stream_decompress(s->stream_decompression_ctx, - &s->unprocessed_incoming_frames_buffer, - &s->decompressed_data_buffer, NULL, - GRPC_HEADER_SIZE_IN_BYTES - s->decompressed_header_bytes, - &end_of_context)) { + if (!grpc_stream_decompress( + s->stream_decompression_ctx, + &s->unprocessed_incoming_frames_buffer, + &s->decompressed_data_buffer, NULL, + GRPC_HEADER_SIZE_IN_BYTES - s->decompressed_header_bytes, + &end_of_context)) { grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &s->frame_storage); grpc_slice_buffer_reset_and_unref_internal( @@ -1749,8 +1750,8 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, s->decompressed_header_bytes = 0; } error = grpc_deframe_unprocessed_incoming_frames( - exec_ctx, &s->data_parser, s, &s->decompressed_data_buffer, NULL, - s->recv_message); + exec_ctx, &s->data_parser, s, &s->decompressed_data_buffer, + NULL, s->recv_message); if (end_of_context) { grpc_stream_compression_context_destroy( s->stream_decompression_ctx); @@ -1759,7 +1760,8 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, } } else { error = grpc_deframe_unprocessed_incoming_frames( - exec_ctx, &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); + exec_ctx, &s->data_parser, s, + &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); } if (error != GRPC_ERROR_NONE) { s->seen_error = true; @@ -1798,9 +1800,8 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, } bool pending_data = s->pending_byte_stream || s->unprocessed_incoming_frames_buffer.length > 0; - if (s->read_closed && - s->frame_storage.length > 0 && !pending_data && !s->seen_error && - s->recv_trailing_metadata_finished != NULL) { + if (s->read_closed && s->frame_storage.length > 0 && !pending_data && + !s->seen_error && s->recv_trailing_metadata_finished != NULL) { /* Maybe some SYNC_FLUSH data is left in frame_storage. Consume them and * maybe decompress the next 5 bytes in the stream. */ bool end_of_context; @@ -2700,8 +2701,8 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, - &s->decompressed_data_buffer, NULL, MAX_SIZE_T, - &end_of_context)) { + &s->decompressed_data_buffer, NULL, + MAX_SIZE_T, &end_of_context)) { error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Stream decompression error."); return error; diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 3b338da288d..2028877cac0 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1660,8 +1660,11 @@ static void parse_stream_compression_md(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s, grpc_metadata_batch *initial_metadata) { if (initial_metadata->idx.named.content_encoding == NULL || - grpc_stream_compression_method_parse(GRPC_MDVALUE(initial_metadata->idx.named.content_encoding->md), false, &s->stream_decompression_method) == 0) { - s->stream_decompression_method = GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; + grpc_stream_compression_method_parse( + GRPC_MDVALUE(initial_metadata->idx.named.content_encoding->md), false, + &s->stream_decompression_method) == 0) { + s->stream_decompression_method = + GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; } } diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c index 19eb6a9a00d..3dfcf53b85d 100644 --- a/src/core/lib/compression/stream_compression_identity.c +++ b/src/core/lib/compression/stream_compression_identity.c @@ -27,8 +27,7 @@ /* Singleton context used for all identity streams. */ static grpc_stream_compression_context identity_ctx = { - .vtable = &grpc_stream_compression_identity_vtable -}; + .vtable = &grpc_stream_compression_identity_vtable}; static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_slice_buffer *out, diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c index 26c957ded49..afed6cd6b5c 100644 --- a/test/core/compression/stream_compression_test.c +++ b/test/core/compression/stream_compression_test.c @@ -59,7 +59,8 @@ static void test_stream_compression_simple_compress_decompress() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); @@ -91,7 +92,8 @@ test_stream_compression_simple_compress_decompress_with_output_size_constraint() grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); @@ -139,7 +141,8 @@ test_stream_compression_simple_compress_decompress_with_large_data() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); @@ -172,7 +175,8 @@ static void test_stream_compression_drop_context() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -180,8 +184,8 @@ static void test_stream_compression_drop_context() { GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); grpc_stream_compression_context_destroy(compress_ctx); - compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + compress_ctx = grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); slice = grpc_slice_from_static_string(test_str2); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -240,7 +244,8 @@ static void test_stream_compression_sync_flush() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, From f5a7f7f559ec6cbc30227cf051603ade22b556f8 Mon Sep 17 00:00:00 2001 From: Michael Bausor Date: Thu, 7 Sep 2017 09:47:50 -0700 Subject: [PATCH 044/227] Determine type name using php namespace option --- src/compiler/php_generator.cc | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/compiler/php_generator.cc b/src/compiler/php_generator.cc index 67967d0bd7c..9fa14d16840 100644 --- a/src/compiler/php_generator.cc +++ b/src/compiler/php_generator.cc @@ -33,7 +33,7 @@ using std::map; namespace grpc_php_generator { namespace { -grpc::string MessageIdentifierName(const grpc::string &name) { +grpc::string ConvertToPhpNamespace(const grpc::string &name) { std::vector tokens = grpc_generator::tokenize(name, "."); std::ostringstream oss; for (unsigned int i = 0; i < tokens.size(); i++) { @@ -43,14 +43,32 @@ grpc::string MessageIdentifierName(const grpc::string &name) { return oss.str(); } +grpc::string PackageName(const FileDescriptor *file) { + if (file->options().has_php_namespace()) { + return file->options().php_namespace(); + } else { + return ConvertToPhpNamespace(file->package()); + } +} + +grpc::string MessageIdentifierName(const grpc::string &name, const FileDescriptor *file) { + std::vector tokens = grpc_generator::tokenize(name, "."); + std::ostringstream oss; + oss << PackageName(file) + << "\\" + << grpc_generator::CapitalizeFirstLetter(tokens[tokens.size() - 1]); + return oss.str(); +} + + void PrintMethod(const MethodDescriptor *method, Printer *out) { const Descriptor *input_type = method->input_type(); const Descriptor *output_type = method->output_type(); map vars; vars["service_name"] = method->service()->full_name(); vars["name"] = method->name(); - vars["input_type_id"] = MessageIdentifierName(input_type->full_name()); - vars["output_type_id"] = MessageIdentifierName(output_type->full_name()); + vars["input_type_id"] = MessageIdentifierName(input_type->full_name(), input_type->file()); + vars["output_type_id"] = MessageIdentifierName(output_type->full_name(), output_type->file()); out->Print("/**\n"); out->Print(GetPHPComments(method, " *").c_str()); @@ -149,12 +167,7 @@ grpc::string GenerateFile(const FileDescriptor *file, } map vars; - grpc::string php_namespace; - if (file->options().has_php_namespace()) { - php_namespace = file->options().php_namespace(); - } else { - php_namespace = MessageIdentifierName(file->package()); - } + grpc::string php_namespace = PackageName(file); vars["package"] = php_namespace; out.Print(vars, "namespace $package$;\n\n"); From 9cd90f43dcfeabff95cb7b61fb029ff7e4a899b3 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 7 Sep 2017 09:51:13 -0700 Subject: [PATCH 045/227] fix bazel build --- BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/BUILD b/BUILD index 67e727bd521..e147dec4d49 100644 --- a/BUILD +++ b/BUILD @@ -574,6 +574,8 @@ grpc_cc_library( "src/core/lib/compression/compression.c", "src/core/lib/compression/message_compress.c", "src/core/lib/compression/stream_compression.c", + "src/core/lib/compression/stream_compression_gzip.c", + "src/core/lib/compression/stream_compression_identity.c", "src/core/lib/http/format_request.c", "src/core/lib/http/httpcli.c", "src/core/lib/http/parser.c", @@ -706,6 +708,8 @@ grpc_cc_library( "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", "src/core/lib/compression/stream_compression.h", + "src/core/lib/compression/stream_compression_gzip.h", + "src/core/lib/compression/stream_compression_identity.h", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", From 677395558d4f28ca3fad53d6e082552f0ce2eb5e Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 20 Jul 2017 10:02:47 -0700 Subject: [PATCH 046/227] Allow SerializationTraits to use grpc::ByteBuffer, not just grpc_byte_buffer --- BUILD | 2 + CMakeLists.txt | 15 ++ Makefile | 15 ++ build.yaml | 2 + gRPC-Core.podspec | 1 + grpc.def | 20 +-- grpc.gemspec | 1 + include/grpc++/impl/codegen/byte_buffer.h | 141 ++++++++++++++++++ include/grpc++/impl/codegen/call.h | 113 +++++++++++--- .../grpc++/impl/codegen/method_handler_impl.h | 9 +- .../grpc++/impl/codegen/rpc_service_method.h | 14 +- .../impl/codegen/serialization_traits.h | 25 +++- include/grpc++/impl/codegen/slice.h | 78 ++++++++++ include/grpc++/support/byte_buffer.h | 68 +-------- include/grpc++/support/slice.h | 80 +--------- include/grpc/byte_buffer.h | 64 +------- include/grpc/impl/codegen/byte_buffer.h | 86 +++++++++++ package.xml | 1 + src/core/lib/iomgr/resource_quota.c | 1 + .../health/default_health_check_service.cc | 1 + src/cpp/util/byte_buffer_cc.cc | 37 +++-- src/cpp/util/slice_cc.cc | 2 + src/ruby/ext/grpc/rb_grpc_imports.generated.c | 40 ++--- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 62 ++++---- .../core/surface/public_headers_must_be_c89.c | 1 + test/cpp/util/byte_buffer_test.cc | 5 +- tools/doxygen/Doxyfile.c++ | 2 + tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + .../generated/sources_and_headers.json | 4 + 31 files changed, 564 insertions(+), 330 deletions(-) create mode 100644 include/grpc++/impl/codegen/byte_buffer.h create mode 100644 include/grpc/impl/codegen/byte_buffer.h diff --git a/BUILD b/BUILD index f74db0a1c8d..70d9635bc5f 100644 --- a/BUILD +++ b/BUILD @@ -989,6 +989,7 @@ grpc_cc_library( name = "grpc_codegen", language = "c", public_hdrs = [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -1487,6 +1488,7 @@ grpc_cc_library( public_hdrs = [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index f77cd5b131d..01d6904c566 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1255,6 +1255,7 @@ target_link_libraries(grpc ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1563,6 +1564,7 @@ target_link_libraries(grpc_cronet ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1841,6 +1843,7 @@ target_link_libraries(grpc_test_util ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2103,6 +2106,7 @@ target_link_libraries(grpc_test_util_unsecure ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2400,6 +2404,7 @@ target_link_libraries(grpc_unsecure ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2715,6 +2720,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2725,6 +2731,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3149,6 +3156,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3159,6 +3167,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3529,6 +3538,7 @@ target_link_libraries(grpc++_test_util foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3556,6 +3566,7 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3669,6 +3680,7 @@ target_link_libraries(grpc++_test_util_unsecure foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3696,6 +3708,7 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3900,6 +3913,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3910,6 +3924,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h diff --git a/Makefile b/Makefile index d933c2a5ee3..02fa235146f 100644 --- a/Makefile +++ b/Makefile @@ -3165,6 +3165,7 @@ LIBGRPC_SRC = \ src/core/plugin_registry/grpc_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3471,6 +3472,7 @@ LIBGRPC_CRONET_SRC = \ src/core/plugin_registry/grpc_cronet_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3748,6 +3750,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3999,6 +4002,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4271,6 +4275,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/plugin_registry/grpc_unsecure_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4559,6 +4564,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4569,6 +4575,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5000,6 +5007,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5010,6 +5018,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5363,6 +5372,7 @@ LIBGRPC++_TEST_UTIL_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5390,6 +5400,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5478,6 +5489,7 @@ LIBGRPC++_TEST_UTIL_UNSECURE_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5505,6 +5517,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5712,6 +5725,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5722,6 +5736,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ diff --git a/build.yaml b/build.yaml index 089b8423a80..3ebaa24e255 100644 --- a/build.yaml +++ b/build.yaml @@ -502,6 +502,7 @@ filegroups: - grpc_deadline_filter - name: grpc_codegen public_headers: + - include/grpc/impl/codegen/byte_buffer.h - include/grpc/impl/codegen/byte_buffer_reader.h - include/grpc/impl/codegen/compression_types.h - include/grpc/impl/codegen/connectivity_state.h @@ -969,6 +970,7 @@ filegroups: public_headers: - include/grpc++/impl/codegen/async_stream.h - include/grpc++/impl/codegen/async_unary_call.h + - include/grpc++/impl/codegen/byte_buffer.h - include/grpc++/impl/codegen/call.h - include/grpc++/impl/codegen/call_hook.h - include/grpc++/impl/codegen/channel_interface.h diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index fdab4b3f819..8acd4380f50 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -141,6 +141,7 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_generic.h', 'include/grpc/impl/codegen/sync_posix.h', 'include/grpc/impl/codegen/sync_windows.h', + 'include/grpc/impl/codegen/byte_buffer.h', 'include/grpc/impl/codegen/byte_buffer_reader.h', 'include/grpc/impl/codegen/compression_types.h', 'include/grpc/impl/codegen/connectivity_state.h', diff --git a/grpc.def b/grpc.def index 37cc7e20f0d..30d89857faa 100644 --- a/grpc.def +++ b/grpc.def @@ -1,14 +1,4 @@ EXPORTS - grpc_raw_byte_buffer_create - grpc_raw_compressed_byte_buffer_create - grpc_byte_buffer_copy - grpc_byte_buffer_length - grpc_byte_buffer_destroy - grpc_byte_buffer_reader_init - grpc_byte_buffer_reader_destroy - grpc_byte_buffer_reader_next - grpc_byte_buffer_reader_readall - grpc_raw_byte_buffer_from_reader census_initialize census_shutdown census_supported @@ -144,6 +134,16 @@ EXPORTS grpc_server_add_secure_http2_port grpc_call_set_credentials grpc_server_credentials_set_auth_metadata_processor + grpc_raw_byte_buffer_create + grpc_raw_compressed_byte_buffer_create + grpc_byte_buffer_copy + grpc_byte_buffer_length + grpc_byte_buffer_destroy + grpc_byte_buffer_reader_init + grpc_byte_buffer_reader_destroy + grpc_byte_buffer_reader_next + grpc_byte_buffer_reader_readall + grpc_raw_byte_buffer_from_reader grpc_slice_ref grpc_slice_unref grpc_slice_copy diff --git a/grpc.gemspec b/grpc.gemspec index c4411076435..3c093dc99ec 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -145,6 +145,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/support/tmpfile_posix.c ) s.files += %w( src/core/lib/support/tmpfile_windows.c ) s.files += %w( src/core/lib/support/wrap_memcpy.c ) + s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h ) s.files += %w( include/grpc/impl/codegen/compression_types.h ) s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h new file mode 100644 index 00000000000..e6231ca2434 --- /dev/null +++ b/include/grpc++/impl/codegen/byte_buffer.h @@ -0,0 +1,141 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H +#define GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H + +#include + +#include +#include +#include +#include +#include + +#include + +namespace grpc { + +template +class CallOpRecvMessage; +class MethodHandler; +namespace internal { +template +class MessageDeserializer; +} + +/// A sequence of bytes. +class ByteBuffer final { + public: + /// Constuct an empty buffer. + ByteBuffer() : buffer_(nullptr) {} + + /// Construct buffer from \a slices, of which there are \a nslices. + ByteBuffer(const Slice* slices, size_t nslices); + + /// Constuct a byte buffer by referencing elements of existing buffer + /// \a buf. Wrapper of core function grpc_byte_buffer_copy + ByteBuffer(const ByteBuffer& buf); + + ~ByteBuffer() { + if (buffer_) { + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); + } + } + + ByteBuffer& operator=(const ByteBuffer&); + + /// Dump (read) the buffer contents into \a slices. + Status Dump(std::vector* slices) const; + + /// Remove all data. + void Clear() { + if (buffer_) { + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); + buffer_ = nullptr; + } + } + + /// Make a duplicate copy of the internals of this byte + /// buffer so that we have our own owned version of it. + /// bbuf.Duplicate(); is equivalent to bbuf=bbuf; but is actually readable + void Duplicate() { + buffer_ = g_core_codegen_interface->grpc_byte_buffer_copy(buffer_); + } + + /// Forget underlying byte buffer without destroying + /// Use this only for un-owned byte buffers + void Release() { buffer_ = nullptr; } + + /// Buffer size in bytes. + size_t Length() const; + + /// Swap the state of *this and *other. + void Swap(ByteBuffer* other); + + /// Is this ByteBuffer valid? + bool Valid() const { return (buffer_ != nullptr); } + + private: + friend class SerializationTraits; + friend class CallOpSendMessage; + template + friend class CallOpRecvMessage; + friend class CallOpGenericRecvMessage; + friend class MethodHandler; + template + friend class internal::MessageDeserializer; + + // takes ownership + void set_buffer(grpc_byte_buffer* buf) { + if (buffer_) { + Clear(); + } + buffer_ = buf; + } + + grpc_byte_buffer* c_buffer() { return buffer_; } + grpc_byte_buffer** c_buffer_ptr() { return &buffer_; } + + // DEPRECATED: Implicit conversion to transparently + // support deprecated SerializationTraits API + // No need to inline since deprecated + operator grpc_byte_buffer*(); + operator const grpc_byte_buffer*() const; + + grpc_byte_buffer* buffer_; +}; + +template <> +class SerializationTraits { + public: + static Status Deserialize(const ByteBuffer& byte_buffer, ByteBuffer* dest) { + dest->set_buffer(byte_buffer.buffer_); + return Status::OK; + } + static Status Serialize(const ByteBuffer& source, ByteBuffer* buffer, + bool* own_buffer) { + *buffer = source; + *own_buffer = true; + return Status::OK; + } +}; + +} // namespace grpc + +#endif // GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 8e70225f86a..bbd277f288f 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -39,8 +40,6 @@ #include #include -struct grpc_byte_buffer; - namespace grpc { class ByteBuffer; @@ -272,7 +271,7 @@ class CallOpSendInitialMetadata { class CallOpSendMessage { public: - CallOpSendMessage() : send_buf_(nullptr) {} + CallOpSendMessage() : send_buf_() {} /// Send \a message using \a options for the write. The \a options are cleared /// after use. @@ -285,33 +284,64 @@ class CallOpSendMessage { protected: void AddOp(grpc_op* ops, size_t* nops) { - if (send_buf_ == nullptr) return; + if (!send_buf_.Valid()) return; grpc_op* op = &ops[(*nops)++]; op->op = GRPC_OP_SEND_MESSAGE; op->flags = write_options_.flags(); op->reserved = NULL; - op->data.send_message.send_message = send_buf_; + op->data.send_message.send_message = send_buf_.c_buffer(); // Flags are per-message: clear them after use. write_options_.Clear(); } - void FinishOp(bool* status) { - g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_); - send_buf_ = nullptr; - } + void FinishOp(bool* status) { send_buf_.Clear(); } private: - grpc_byte_buffer* send_buf_; + template + class MessageSerializer; + + ByteBuffer send_buf_; WriteOptions write_options_; }; +template +class CallOpSendMessage::MessageSerializer< + M, typename std::enable_if::Serialize( + static_cast(*(static_cast(nullptr))), + static_cast(nullptr), + static_cast(nullptr)))>::value>::type> { + public: + static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, + bool* own_buf) { + return SerializationTraits::Serialize(message, bbuf->c_buffer_ptr(), + own_buf); + } +}; + +template +class CallOpSendMessage::MessageSerializer< + M, typename std::enable_if::Serialize( + static_cast(*(static_cast(nullptr))), + static_cast<::grpc::ByteBuffer*>(nullptr), + static_cast(nullptr)))>::value>::type> { + public: + static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, + bool* own_buf) { + return SerializationTraits::Serialize(message, bbuf, own_buf); + } +}; + template Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) { write_options_ = options; bool own_buf; Status result = - SerializationTraits::Serialize(message, &send_buf_, &own_buf); + MessageSerializer::SendMessageInternal(message, &send_buf_, &own_buf); if (!own_buf) { - send_buf_ = g_core_codegen_interface->grpc_byte_buffer_copy(send_buf_); + send_buf_.Duplicate(); } return result; } @@ -321,6 +351,38 @@ Status CallOpSendMessage::SendMessage(const M& message) { return SendMessage(message, WriteOptions()); } +namespace internal { +template +class MessageDeserializer; + +template +class MessageDeserializer< + M, typename std::enable_if::Deserialize( + static_cast( + *(static_cast(nullptr))), + static_cast(nullptr)))>::value>::type> { + public: + static Status Deserialize(const ByteBuffer& bbuf, M* message) { + return SerializationTraits::Deserialize(bbuf, message); + } +}; + +template +class MessageDeserializer< + M, typename std::enable_if::Deserialize( + static_cast(nullptr), + static_cast(nullptr)))>::value>::type> { + public: + static Status Deserialize(const ByteBuffer& bbuf, M* message) { + return SerializationTraits::Deserialize( + const_cast(bbuf).c_buffer(), message); + } +}; +} // namespace internal + template class CallOpRecvMessage { public: @@ -343,18 +405,20 @@ class CallOpRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = &recv_buf_; + op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); } void FinishOp(bool* status) { if (message_ == nullptr) return; - if (recv_buf_) { + if (recv_buf_.Valid()) { if (*status) { got_message = *status = - SerializationTraits::Deserialize(recv_buf_, message_).ok(); + internal::MessageDeserializer::Deserialize(recv_buf_, message_) + .ok(); + recv_buf_.Release(); } else { got_message = false; - g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); + recv_buf_.Clear(); } } else { got_message = false; @@ -367,14 +431,14 @@ class CallOpRecvMessage { private: R* message_; - grpc_byte_buffer* recv_buf_; + ByteBuffer recv_buf_; bool allow_not_getting_message_; }; namespace CallOpGenericRecvMessageHelper { class DeserializeFunc { public: - virtual Status Deserialize(grpc_byte_buffer* buf) = 0; + virtual Status Deserialize(const ByteBuffer& buf) = 0; virtual ~DeserializeFunc() {} }; @@ -382,8 +446,8 @@ template class DeserializeFuncType final : public DeserializeFunc { public: DeserializeFuncType(R* message) : message_(message) {} - Status Deserialize(grpc_byte_buffer* buf) override { - return SerializationTraits::Deserialize(buf, message_); + Status Deserialize(const ByteBuffer& buf) override { + return grpc::internal::MessageDeserializer::Deserialize(buf, message_); } ~DeserializeFuncType() override {} @@ -419,18 +483,19 @@ class CallOpGenericRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = &recv_buf_; + op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); } void FinishOp(bool* status) { if (!deserialize_) return; - if (recv_buf_) { + if (recv_buf_.Valid()) { if (*status) { got_message = true; *status = deserialize_->Deserialize(recv_buf_).ok(); + recv_buf_.Release(); } else { got_message = false; - g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); + recv_buf_.Clear(); } } else { got_message = false; @@ -443,7 +508,7 @@ class CallOpGenericRecvMessage { private: std::unique_ptr deserialize_; - grpc_byte_buffer* recv_buf_; + ByteBuffer recv_buf_; bool allow_not_getting_message_; }; diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h index 15e24bdcdcb..8125e0a6513 100644 --- a/include/grpc++/impl/codegen/method_handler_impl.h +++ b/include/grpc++/impl/codegen/method_handler_impl.h @@ -19,6 +19,7 @@ #ifndef GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H #define GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H +#include #include #include #include @@ -37,8 +38,8 @@ class RpcMethodHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = - SerializationTraits::Deserialize(param.request, &req); + Status status = internal::MessageDeserializer::Deserialize( + param.request, &req); ResponseType rsp; if (status.ok()) { status = func_(service_, param.server_context, &req, &rsp); @@ -123,8 +124,8 @@ class ServerStreamingHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = - SerializationTraits::Deserialize(param.request, &req); + Status status = internal::MessageDeserializer::Deserialize( + param.request, &req); if (status.ok()) { ServerWriter writer(param.call, param.server_context); diff --git a/include/grpc++/impl/codegen/rpc_service_method.h b/include/grpc++/impl/codegen/rpc_service_method.h index 7165774172c..d356012ad60 100644 --- a/include/grpc++/impl/codegen/rpc_service_method.h +++ b/include/grpc++/impl/codegen/rpc_service_method.h @@ -25,14 +25,11 @@ #include #include +#include #include #include #include -extern "C" { -struct grpc_byte_buffer; -} - namespace grpc { class ServerContext; class StreamContextInterface; @@ -43,11 +40,14 @@ class MethodHandler { virtual ~MethodHandler() {} struct HandlerParameter { HandlerParameter(Call* c, ServerContext* context, grpc_byte_buffer* req) - : call(c), server_context(context), request(req) {} + : call(c), server_context(context) { + request.set_buffer(req); + } + ~HandlerParameter() { request.Release(); } Call* call; ServerContext* server_context; - // Handler required to grpc_byte_buffer_destroy this - grpc_byte_buffer* request; + // Handler required to destroy these contents + ByteBuffer request; }; virtual void RunHandler(const HandlerParameter& param) = 0; }; diff --git a/include/grpc++/impl/codegen/serialization_traits.h b/include/grpc++/impl/codegen/serialization_traits.h index b72d4741267..7fc4e24172f 100644 --- a/include/grpc++/impl/codegen/serialization_traits.h +++ b/include/grpc++/impl/codegen/serialization_traits.h @@ -26,15 +26,24 @@ namespace grpc { /// Used for hooking different message serialization API's into GRPC. /// Each SerializationTraits implementation must provide the following /// functions: -/// static Status Serialize(const Message& msg, -/// grpc_byte_buffer** buffer, -/// bool* own_buffer); -/// static Status Deserialize(grpc_byte_buffer* buffer, -/// Message* msg, -/// int max_receive_message_size); +/// 1. static Status Serialize(const Message& msg, +/// ByteBuffer* buffer, +/// bool* own_buffer); +/// AND/OR +/// static Status Serialize(const Message& msg, +/// grpc_byte_buffer** buffer, +/// bool* own_buffer); +/// The former is preferred; the latter is deprecated /// -/// Serialize is required to convert message to a grpc_byte_buffer, and -/// to store a pointer to that byte buffer at *buffer. *own_buffer should +/// 2. static Status Deserialize(const ByteBuffer& buffer, +/// Message* msg); +/// AND/OR +/// static Status Deserialize(grpc_byte_buffer* buffer, +/// Message* msg); +/// The former is preferred; the latter is deprecated +/// +/// Serialize is required to convert message to a ByteBuffer, and +/// return that byte buffer through *buffer. *own_buffer should /// be set to true if the caller owns said byte buffer, or false if /// ownership is retained elsewhere. /// diff --git a/include/grpc++/impl/codegen/slice.h b/include/grpc++/impl/codegen/slice.h index e682bdef6af..c185bf4fd01 100644 --- a/include/grpc++/impl/codegen/slice.h +++ b/include/grpc++/impl/codegen/slice.h @@ -19,11 +19,89 @@ #ifndef GRPCXX_IMPL_CODEGEN_SLICE_H #define GRPCXX_IMPL_CODEGEN_SLICE_H +#include #include #include +#include + namespace grpc { +/// A wrapper around \a grpc_slice. +/// +/// A slice represents a contiguous reference counted array of bytes. +/// It is cheap to take references to a slice, and it is cheap to create a +/// slice pointing to a subset of another slice. +class Slice final { + public: + /// Construct an empty slice. + Slice(); + /// Destructor - drops one reference. + ~Slice(); + + enum AddRef { ADD_REF }; + /// Construct a slice from \a slice, adding a reference. + Slice(grpc_slice slice, AddRef); + + enum StealRef { STEAL_REF }; + /// Construct a slice from \a slice, stealing a reference. + Slice(grpc_slice slice, StealRef); + + /// Allocate a slice of specified size + Slice(size_t len); + + /// Construct a slice from a copied buffer + Slice(const void* buf, size_t len); + + /// Construct a slice from a copied string + Slice(const grpc::string& str); + + enum StaticSlice { STATIC_SLICE }; + + /// Construct a slice from a static buffer + Slice(const void* buf, size_t len, StaticSlice); + + /// Copy constructor, adds a reference. + Slice(const Slice& other); + + /// Assignment, reference count is unchanged. + Slice& operator=(Slice other) { + std::swap(slice_, other.slice_); + return *this; + } + + /// Create a slice pointing at some data. Calls malloc to allocate a refcount + /// for the object, and arranges that destroy will be called with the + /// user data pointer passed in at destruction. Can be the same as buf or + /// different (e.g., if data is part of a larger structure that must be + /// destroyed when the data is no longer needed) + Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); + + /// Specialization of above for common case where buf == user_data + Slice(void* buf, size_t len, void (*destroy)(void*)) + : Slice(buf, len, destroy, buf) {} + + /// Similar to the above but has a destroy that also takes slice length + Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); + + /// Byte size. + size_t size() const { return GRPC_SLICE_LENGTH(slice_); } + + /// Raw pointer to the beginning (first element) of the slice. + const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } + + /// Raw pointer to the end (one byte \em past the last element) of the slice. + const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } + + /// Raw C slice. Caller needs to call grpc_slice_unref when done. + grpc_slice c_slice() const; + + private: + friend class ByteBuffer; + + grpc_slice slice_; +}; + inline grpc::string_ref StringRefFromSlice(const grpc_slice* slice) { return grpc::string_ref( reinterpret_cast(GRPC_SLICE_START_PTR(*slice)), diff --git a/include/grpc++/support/byte_buffer.h b/include/grpc++/support/byte_buffer.h index df16c6a75e4..81fa3b0a184 100644 --- a/include/grpc++/support/byte_buffer.h +++ b/include/grpc++/support/byte_buffer.h @@ -19,6 +19,7 @@ #ifndef GRPCXX_SUPPORT_BYTE_BUFFER_H #define GRPCXX_SUPPORT_BYTE_BUFFER_H +#include #include #include #include @@ -27,71 +28,4 @@ #include #include -#include - -namespace grpc { - -/// A sequence of bytes. -class ByteBuffer final { - public: - /// Constuct an empty buffer. - ByteBuffer() : buffer_(nullptr) {} - - /// Construct buffer from \a slices, of which there are \a nslices. - ByteBuffer(const Slice* slices, size_t nslices); - - /// Constuct a byte buffer by referencing elements of existing buffer - /// \a buf. Wrapper of core function grpc_byte_buffer_copy - ByteBuffer(const ByteBuffer& buf); - - ~ByteBuffer(); - - ByteBuffer& operator=(const ByteBuffer&); - - /// Dump (read) the buffer contents into \a slices. - Status Dump(std::vector* slices) const; - - /// Remove all data. - void Clear(); - - /// Buffer size in bytes. - size_t Length() const; - - /// Swap the state of *this and *other. - void Swap(ByteBuffer* other); - - private: - friend class SerializationTraits; - - // takes ownership - void set_buffer(grpc_byte_buffer* buf) { - if (buffer_) { - Clear(); - } - buffer_ = buf; - } - - // For \a SerializationTraits's usage. - grpc_byte_buffer* buffer() const { return buffer_; } - - grpc_byte_buffer* buffer_; -}; - -template <> -class SerializationTraits { - public: - static Status Deserialize(grpc_byte_buffer* byte_buffer, ByteBuffer* dest) { - dest->set_buffer(byte_buffer); - return Status::OK; - } - static Status Serialize(const ByteBuffer& source, grpc_byte_buffer** buffer, - bool* own_buffer) { - *buffer = grpc_byte_buffer_copy(source.buffer()); - *own_buffer = true; - return Status::OK; - } -}; - -} // namespace grpc - #endif // GRPCXX_SUPPORT_BYTE_BUFFER_H diff --git a/include/grpc++/support/slice.h b/include/grpc++/support/slice.h index bbf97f280ef..10db10d79c3 100644 --- a/include/grpc++/support/slice.h +++ b/include/grpc++/support/slice.h @@ -19,86 +19,8 @@ #ifndef GRPCXX_SUPPORT_SLICE_H #define GRPCXX_SUPPORT_SLICE_H +#include #include #include -namespace grpc { - -/// A wrapper around \a grpc_slice. -/// -/// A slice represents a contiguous reference counted array of bytes. -/// It is cheap to take references to a slice, and it is cheap to create a -/// slice pointing to a subset of another slice. -class Slice final { - public: - /// Construct an empty slice. - Slice(); - /// Destructor - drops one reference. - ~Slice(); - - enum AddRef { ADD_REF }; - /// Construct a slice from \a slice, adding a reference. - Slice(grpc_slice slice, AddRef); - - enum StealRef { STEAL_REF }; - /// Construct a slice from \a slice, stealing a reference. - Slice(grpc_slice slice, StealRef); - - /// Allocate a slice of specified size - Slice(size_t len); - - /// Construct a slice from a copied buffer - Slice(const void* buf, size_t len); - - /// Construct a slice from a copied string - Slice(const grpc::string& str); - - enum StaticSlice { STATIC_SLICE }; - - /// Construct a slice from a static buffer - Slice(const void* buf, size_t len, StaticSlice); - - /// Copy constructor, adds a reference. - Slice(const Slice& other); - - /// Assignment, reference count is unchanged. - Slice& operator=(Slice other) { - std::swap(slice_, other.slice_); - return *this; - } - - /// Create a slice pointing at some data. Calls malloc to allocate a refcount - /// for the object, and arranges that destroy will be called with the - /// user data pointer passed in at destruction. Can be the same as buf or - /// different (e.g., if data is part of a larger structure that must be - /// destroyed when the data is no longer needed) - Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); - - /// Specialization of above for common case where buf == user_data - Slice(void* buf, size_t len, void (*destroy)(void*)) - : Slice(buf, len, destroy, buf) {} - - /// Similar to the above but has a destroy that also takes slice length - Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); - - /// Byte size. - size_t size() const { return GRPC_SLICE_LENGTH(slice_); } - - /// Raw pointer to the beginning (first element) of the slice. - const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } - - /// Raw pointer to the end (one byte \em past the last element) of the slice. - const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } - - /// Raw C slice. Caller needs to call grpc_slice_unref when done. - grpc_slice c_slice() const { return grpc_slice_ref(slice_); } - - private: - friend class ByteBuffer; - - grpc_slice slice_; -}; - -} // namespace grpc - #endif // GRPCXX_SUPPORT_SLICE_H diff --git a/include/grpc/byte_buffer.h b/include/grpc/byte_buffer.h index 55e191da315..7669582af27 100644 --- a/include/grpc/byte_buffer.h +++ b/include/grpc/byte_buffer.h @@ -19,69 +19,7 @@ #ifndef GRPC_BYTE_BUFFER_H #define GRPC_BYTE_BUFFER_H -#include +#include #include -#ifdef __cplusplus -extern "C" { -#endif - -/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, - size_t nslices); - -/** Returns a *compressed* RAW byte buffer instance over the given slices (up to - * \a nslices). The \a compression argument defines the compression algorithm - * used to generate the data in \a slices. - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( - grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); - -/** Copies input byte buffer \a bb. - * - * Increases the reference count of all the source slices. The user is - * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ -GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); - -/** Returns the size of the given byte buffer, in bytes. */ -GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); - -/** Destroys \a byte_buffer deallocating all its memory. */ -GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); - -/** Reader for byte buffers. Iterates over slices in the byte buffer */ -struct grpc_byte_buffer_reader; -typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; - -/** Initialize \a reader to read over \a buffer. - * Returns 1 upon success, 0 otherwise. */ -GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, - grpc_byte_buffer *buffer); - -/** Cleanup and destroy \a reader */ -GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); - -/** Updates \a slice with the next piece of data from from \a reader and returns - * 1. Returns 0 at the end of the stream. Caller is responsible for calling - * grpc_slice_unref on the result. */ -GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, - grpc_slice *slice); - -/** Merge all data from \a reader into single slice */ -GRPCAPI grpc_slice -grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); - -/** Returns a RAW byte buffer instance from the output of \a reader. */ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( - grpc_byte_buffer_reader *reader); - -#ifdef __cplusplus -} -#endif - #endif /* GRPC_BYTE_BUFFER_H */ diff --git a/include/grpc/impl/codegen/byte_buffer.h b/include/grpc/impl/codegen/byte_buffer.h new file mode 100644 index 00000000000..fc333057134 --- /dev/null +++ b/include/grpc/impl/codegen/byte_buffer.h @@ -0,0 +1,86 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_IMPL_CODEGEN_BYTE_BUFFER_H +#define GRPC_IMPL_CODEGEN_BYTE_BUFFER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, + size_t nslices); + +/** Returns a *compressed* RAW byte buffer instance over the given slices (up to + * \a nslices). The \a compression argument defines the compression algorithm + * used to generate the data in \a slices. + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( + grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); + +/** Copies input byte buffer \a bb. + * + * Increases the reference count of all the source slices. The user is + * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ +GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); + +/** Returns the size of the given byte buffer, in bytes. */ +GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); + +/** Destroys \a byte_buffer deallocating all its memory. */ +GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); + +/** Reader for byte buffers. Iterates over slices in the byte buffer */ +struct grpc_byte_buffer_reader; +typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; + +/** Initialize \a reader to read over \a buffer. + * Returns 1 upon success, 0 otherwise. */ +GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, + grpc_byte_buffer *buffer); + +/** Cleanup and destroy \a reader */ +GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); + +/** Updates \a slice with the next piece of data from from \a reader and returns + * 1. Returns 0 at the end of the stream. Caller is responsible for calling + * grpc_slice_unref on the result. */ +GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, + grpc_slice *slice); + +/** Merge all data from \a reader into single slice */ +GRPCAPI grpc_slice +grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); + +/** Returns a RAW byte buffer instance from the output of \a reader. */ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( + grpc_byte_buffer_reader *reader); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_IMPL_CODEGEN_BYTE_BUFFER_H */ diff --git a/package.xml b/package.xml index 4b1f42dd025..c3daa8f8478 100644 --- a/package.xml +++ b/package.xml @@ -155,6 +155,7 @@ + diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index a31d9eef93e..b096679a4cf 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index 815b6070320..d2cba6d6627 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include diff --git a/src/cpp/util/byte_buffer_cc.cc b/src/cpp/util/byte_buffer_cc.cc index b1ff25252a1..1ebe22b6a49 100644 --- a/src/cpp/util/byte_buffer_cc.cc +++ b/src/cpp/util/byte_buffer_cc.cc @@ -16,11 +16,15 @@ * */ +#include #include +#include #include namespace grpc { +static internal::GrpcLibraryInitializer g_gli_initializer; + ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { // The following assertions check that the representation of a grpc::Slice is // identical to that of a grpc_slice: it has a grpc_slice field, and nothing @@ -29,6 +33,7 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { "Slice must have same representation as grpc_slice"); static_assert(sizeof(Slice) == sizeof(grpc_slice), "Slice must have same representation as grpc_slice"); + g_gli_initializer.summon(); // Make sure that initializer linked in // The const_cast is legal if grpc_raw_byte_buffer_create() does no more // than its advertised side effect of increasing the reference count of the // slices it processes, and such an increase does not affect the semantics @@ -37,19 +42,6 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { reinterpret_cast(const_cast(slices)), nslices); } -ByteBuffer::~ByteBuffer() { - if (buffer_) { - grpc_byte_buffer_destroy(buffer_); - } -} - -void ByteBuffer::Clear() { - if (buffer_) { - grpc_byte_buffer_destroy(buffer_); - buffer_ = nullptr; - } -} - Status ByteBuffer::Dump(std::vector* slices) const { slices->clear(); if (!buffer_) { @@ -80,7 +72,9 @@ ByteBuffer::ByteBuffer(const ByteBuffer& buf) : buffer_(grpc_byte_buffer_copy(buf.buffer_)) {} ByteBuffer& ByteBuffer::operator=(const ByteBuffer& buf) { - Clear(); // first remove existing data + if (this != &buf) { + Clear(); // first remove existing data + } if (buf.buffer_) { buffer_ = grpc_byte_buffer_copy(buf.buffer_); // then copy } @@ -93,4 +87,19 @@ void ByteBuffer::Swap(ByteBuffer* other) { buffer_ = tmp; } +ByteBuffer::operator grpc_byte_buffer*() { + // The following assertions check that the representation of a ByteBuffer is + // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, + // and nothing else. + static_assert(std::is_same::value, + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + return buffer_; +} + +ByteBuffer::operator const grpc_byte_buffer*() const { return buffer_; } + } // namespace grpc diff --git a/src/cpp/util/slice_cc.cc b/src/cpp/util/slice_cc.cc index 486d0cdf0ec..3ae17e80520 100644 --- a/src/cpp/util/slice_cc.cc +++ b/src/cpp/util/slice_cc.cc @@ -50,4 +50,6 @@ Slice::Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data) Slice::Slice(void* buf, size_t len, void (*destroy)(void*, size_t)) : slice_(grpc_slice_new_with_len(buf, len, destroy)) {} +grpc_slice Slice::c_slice() const { return grpc_slice_ref(slice_); } + } // namespace grpc diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 0402ce34fb9..0d223700af3 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -22,16 +22,6 @@ #include "rb_grpc_imports.generated.h" -grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; census_initialize_type census_initialize_import; census_shutdown_type census_shutdown_import; census_supported_type census_supported_import; @@ -167,6 +157,16 @@ grpc_ssl_server_credentials_create_ex_type grpc_ssl_server_credentials_create_ex grpc_server_add_secure_http2_port_type grpc_server_add_secure_http2_port_import; grpc_call_set_credentials_type grpc_call_set_credentials_import; grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; +grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; grpc_slice_ref_type grpc_slice_ref_import; grpc_slice_unref_type grpc_slice_unref_import; grpc_slice_copy_type grpc_slice_copy_import; @@ -329,16 +329,6 @@ gpr_sleep_until_type gpr_sleep_until_import; gpr_timespec_to_micros_type gpr_timespec_to_micros_import; void grpc_rb_load_imports(HMODULE library) { - grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); - grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); - grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); - grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); - grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); - grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); - grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); - grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); - grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); - grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize"); census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown"); census_supported_import = (census_supported_type) GetProcAddress(library, "census_supported"); @@ -474,6 +464,16 @@ void grpc_rb_load_imports(HMODULE library) { grpc_server_add_secure_http2_port_import = (grpc_server_add_secure_http2_port_type) GetProcAddress(library, "grpc_server_add_secure_http2_port"); grpc_call_set_credentials_import = (grpc_call_set_credentials_type) GetProcAddress(library, "grpc_call_set_credentials"); grpc_server_credentials_set_auth_metadata_processor_import = (grpc_server_credentials_set_auth_metadata_processor_type) GetProcAddress(library, "grpc_server_credentials_set_auth_metadata_processor"); + grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); + grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); + grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); + grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); + grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); + grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); + grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); + grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); + grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); + grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); grpc_slice_ref_import = (grpc_slice_ref_type) GetProcAddress(library, "grpc_slice_ref"); grpc_slice_unref_import = (grpc_slice_unref_type) GetProcAddress(library, "grpc_slice_unref"); grpc_slice_copy_import = (grpc_slice_copy_type) GetProcAddress(library, "grpc_slice_copy"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index e3704e592b8..a13d8aa804f 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -25,12 +25,12 @@ #include -#include #include #include #include #include #include +#include #include #include #include @@ -47,36 +47,6 @@ #include #include -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); -extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); -extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import -typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -#define grpc_byte_buffer_length grpc_byte_buffer_length_import -typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); -extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import -typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); -extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import -typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import -typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); -extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import -typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); -extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; -#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef int(*census_initialize_type)(int features); extern census_initialize_type census_initialize_import; #define census_initialize census_initialize_import @@ -482,6 +452,36 @@ extern grpc_call_set_credentials_type grpc_call_set_credentials_import; typedef void(*grpc_server_credentials_set_auth_metadata_processor_type)(grpc_server_credentials *creds, grpc_auth_metadata_processor processor); extern grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; #define grpc_server_credentials_set_auth_metadata_processor grpc_server_credentials_set_auth_metadata_processor_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); +extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); +extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import +typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +#define grpc_byte_buffer_length grpc_byte_buffer_length_import +typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); +extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import +typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); +extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import +typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import +typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); +extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import +typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); +extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; +#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef grpc_slice(*grpc_slice_ref_type)(grpc_slice s); extern grpc_slice_ref_type grpc_slice_ref_import; #define grpc_slice_ref grpc_slice_ref_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 0d7f68c0add..d36d116afb8 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/test/cpp/util/byte_buffer_test.cc b/test/cpp/util/byte_buffer_test.cc index cac01a73078..8fb51bc6635 100644 --- a/test/cpp/util/byte_buffer_test.cc +++ b/test/cpp/util/byte_buffer_test.cc @@ -93,7 +93,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { std::vector slices; slices.push_back(Slice(hello, Slice::STEAL_REF)); slices.push_back(Slice(world, Slice::STEAL_REF)); - grpc_byte_buffer* send_buffer = nullptr; + ByteBuffer send_buffer; bool owned = false; ByteBuffer buffer(&slices[0], 2); slices.clear(); @@ -101,8 +101,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { buffer, &send_buffer, &owned); EXPECT_TRUE(status.ok()); EXPECT_TRUE(owned); - EXPECT_TRUE(send_buffer != nullptr); - grpc_byte_buffer_destroy(send_buffer); + EXPECT_TRUE(send_buffer.Valid()); } } // namespace diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 62f113907db..d81b7b4d115 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ +include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -881,6 +882,7 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 49919415bd5..263c6c1afbc 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ +include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -882,6 +883,7 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index e4cc1c7461d..135b8923a0c 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index c2daf900fb6..ce5800b6f86 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index eefd80e46f6..ddc9d392fa8 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8324,6 +8324,7 @@ "gpr_codegen" ], "headers": [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -8337,6 +8338,7 @@ "language": "c", "name": "grpc_codegen", "src": [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -9215,6 +9217,7 @@ "headers": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", @@ -9249,6 +9252,7 @@ "src": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", From 4657a4818b56e11d6b11654933dcd80a434f9f93 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Fri, 8 Sep 2017 14:49:38 -0700 Subject: [PATCH 047/227] Warm pip cache with libraries used by virtualenv --- tools/internal_ci/helper_scripts/prepare_build_linux_rc | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc index 91627d60cb5..2ade8dac51f 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -26,6 +26,7 @@ sudo service docker restart # Populate xdg-cache-home to workaround https://github.com/grpc/grpc/issues/11968 sudo mkdir -p /tmp/xdg-cache-home +PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install setuptools wheel PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install coverage==4.4 pylint==1.6.5 # Download Docker images from DockerHub From 9dba7eb6b8edfb8035f1166be6e501f90a5e33d0 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 8 Sep 2017 15:32:51 -0700 Subject: [PATCH 048/227] Fix error unref problem --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 721f694506b..39c2e7baaab 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -2596,7 +2596,7 @@ static void reset_byte_stream(grpc_exec_ctx *exec_ctx, void *arg, GRPC_ERROR_UNREF(s->byte_stream_error); s->byte_stream_error = GRPC_ERROR_NONE; grpc_chttp2_cancel_stream(exec_ctx, s->t, s, GRPC_ERROR_REF(error)); - s->byte_stream_error = error; + s->byte_stream_error = GRPC_ERROR_REF(error); } } From 2426163b314d9dc7f841b433aed55fe3c6f17350 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 8 Sep 2017 16:12:05 -0700 Subject: [PATCH 049/227] Switch to three threads per cq for unconstrained benchmarks --- tools/run_tests/generated/tests.json | 64 +++++++++---------- .../run_tests/performance/scenario_config.py | 2 + 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index ad39f2d6d06..1f97309a97a 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -49905,7 +49905,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -49955,7 +49955,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50005,7 +50005,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50105,7 +50105,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50205,7 +50205,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50255,7 +50255,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50305,7 +50305,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50355,7 +50355,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50759,7 +50759,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50809,7 +50809,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50859,7 +50859,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50959,7 +50959,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51059,7 +51059,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51109,7 +51109,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51159,7 +51159,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51209,7 +51209,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51884,7 +51884,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51960,7 +51960,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52036,7 +52036,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52188,7 +52188,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52340,7 +52340,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52416,7 +52416,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52492,7 +52492,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52568,7 +52568,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53180,7 +53180,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53256,7 +53256,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53332,7 +53332,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53484,7 +53484,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53636,7 +53636,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53712,7 +53712,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53788,7 +53788,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53864,7 +53864,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 86236b19bf3..b97650de67e 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -452,6 +452,8 @@ class CXXLanguage: unconstrained_client=synchronicity, secure=secure, minimal_stack=not secure, + server_threads_per_cq=3, + client_threads_per_cq=3, categories=smoketest_categories+[SCALABLE]) # TODO(vjpai): Re-enable this test. It has a lot of timeouts From eda5a4db1e88e484d0c45401c345cbc199d633fd Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Sat, 9 Sep 2017 15:52:11 +0000 Subject: [PATCH 050/227] Fix metadata memory leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gRPC Core has two styles for passing metadata: as an integer count along with a grpc_metadata* pointer, which is used for passing metadata into the core, and as a grpc_metadata_array, which is used for passing metadata out of the core. The Cython layer of gRPC Python was using a single data structure wrapping grpc_metadata_array for both purposes, but this was complex because the core manages the slices contained in grpc_metadata_array objects (at least those of which it is aware), so the Cython layer had to keep track of whether or not the core was aware of the slices it was using (and it was also defective, leaking slices). This is solved by realigning with the Cython layer’s intended design of mirroring as closely as possible in Python the gRPC Core API: we use one structure for passing metadata into the core (what is now called cygrpc.Metadata) and second, different structure for receiving metadata out of the core (what was called cygrpc.Metadata but is now cygrpc.MetadataArray, reflecting that it wraps the core’s grpc_metadata_array). All bug fixes should contain added tests preventing regression but this doesn't because I don't know at this time how to write a does-not-leak test for Python that fits well into our existing body of tests. Phooey. Thanks to Dominik Janků (djanku@email.cz) for investigation and an earlier draft of a solution. --- .../_cython/_cygrpc/completion_queue.pyx.pxi | 11 +- .../grpc/_cython/_cygrpc/credentials.pyx.pxi | 9 +- .../grpcio/grpc/_cython/_cygrpc/grpc.pxi | 1 + .../grpc/_cython/_cygrpc/records.pxd.pxi | 13 +- .../grpc/_cython/_cygrpc/records.pyx.pxi | 118 ++++++++++-------- .../grpc/_cython/_cygrpc/server.pyx.pxi | 2 +- 6 files changed, 83 insertions(+), 71 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index 28c30e5d357..237f4307990 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -41,9 +41,8 @@ cdef class CompletionQueue: cdef object user_tag = None cdef Call operation_call = None cdef CallDetails request_call_details = None - cdef Metadata request_metadata = None + cdef object request_metadata = None cdef Operations batch_operations = None - cdef Operation batch_operation = None if event.type == GRPC_QUEUE_TIMEOUT: return Event( event.type, False, None, None, None, None, False, None) @@ -63,14 +62,8 @@ cdef class CompletionQueue: operation_call = tag.operation_call request_call_details = tag.request_call_details if tag.request_metadata is not None: - request_metadata = tag.request_metadata - request_metadata._claim_slice_ownership() + request_metadata = tuple(tag.request_metadata) batch_operations = tag.batch_operations - if tag.batch_operations is not None: - for op in batch_operations.operations: - batch_operation = op - if batch_operation._received_metadata is not None: - batch_operation._received_metadata._claim_slice_ownership() if tag.is_new_request: # Stuff in the tag not explicitly handled by us needs to live through # the life of the call diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index 98d7a9820df..57816f1cab0 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -76,7 +76,7 @@ cdef class CredentialsMetadataPlugin: """ Args: plugin_callback (callable): Callback accepting a service URL (str/bytes) - and callback object (accepting a Metadata, + and callback object (accepting a MetadataArray, grpc_status_code, and a str/bytes error message). This argument when called should be non-blocking and eventually call the callback object with the appropriate status code/details and metadata (if @@ -129,8 +129,7 @@ cdef void plugin_get_metadata( def python_callback( Metadata metadata, grpc_status_code status, bytes error_details): - cb(user_data, metadata.c_metadata_array.metadata, - metadata.c_metadata_array.count, status, error_details) + cb(user_data, metadata.c_metadata, metadata.c_count, status, error_details) called_flag[0] = True cdef CredentialsMetadataPlugin self = state cdef AuthMetadataContext cy_context = AuthMetadataContext() @@ -139,8 +138,8 @@ cdef void plugin_get_metadata( self.plugin_callback(cy_context, python_callback) except Exception as error: if not called_flag[0]: - cb(user_data, Metadata([]).c_metadata_array.metadata, - 0, StatusCode.unknown, traceback.format_exc().encode()) + cb(user_data, NULL, 0, StatusCode.unknown, + traceback.format_exc().encode()) cdef void plugin_destroy_c_plugin_state(void *state) with gil: cpython.Py_DECREF(state) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 5950bfa0e6a..28cf1144510 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -59,6 +59,7 @@ cdef extern from "grpc/grpc.h": grpc_slice grpc_slice_malloc(size_t length) nogil grpc_slice grpc_slice_from_copied_string(const char *source) nogil grpc_slice grpc_slice_from_copied_buffer(const char *source, size_t len) nogil + grpc_slice grpc_slice_copy(grpc_slice s) nogil # Declare functions for function-like macros (because Cython)... void *grpc_slice_start_ptr "GRPC_SLICE_START_PTR" (grpc_slice s) nogil diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi index 8ace6aeb525..9c40ebf0c28 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi @@ -37,7 +37,7 @@ cdef class OperationTag: cdef Server shutting_down_server cdef Call operation_call cdef CallDetails request_call_details - cdef Metadata request_metadata + cdef MetadataArray request_metadata cdef Operations batch_operations cdef bint is_new_request @@ -51,7 +51,7 @@ cdef class Event: # For Server.request_call cdef readonly bint is_new_request cdef readonly CallDetails request_call_details - cdef readonly Metadata request_metadata + cdef readonly object request_metadata # For server calls cdef readonly Call operation_call @@ -92,15 +92,20 @@ cdef class Metadatum: cdef class Metadata: + cdef grpc_metadata *c_metadata + cdef readonly size_t c_count + + +cdef class MetadataArray: + cdef grpc_metadata_array c_metadata_array - cdef void _claim_slice_ownership(self) cdef class Operation: cdef grpc_op c_op cdef ByteBuffer _received_message - cdef Metadata _received_metadata + cdef MetadataArray _received_metadata cdef grpc_status_code _received_status_code cdef grpc_slice _status_details cdef int _received_cancelled diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi index 1b2ddd2469c..0a2a6eee051 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi @@ -238,7 +238,7 @@ cdef class Event: def __cinit__(self, grpc_completion_type type, bint success, object tag, Call operation_call, CallDetails request_call_details, - Metadata request_metadata, + object request_metadata, bint is_new_request, Operations batch_operations): self.type = type @@ -437,48 +437,79 @@ cdef class Metadatum: cdef class _MetadataIterator: cdef size_t i - cdef Metadata metadata + cdef size_t _length + cdef object _metadatum_indexable - def __cinit__(self, Metadata metadata not None): + def __cinit__(self, length, metadatum_indexable): + self._length = length + self._metadatum_indexable = metadatum_indexable self.i = 0 - self.metadata = metadata def __iter__(self): return self def __next__(self): - if self.i < len(self.metadata): - result = self.metadata[self.i] + if self.i < self._length: + result = self._metadatum_indexable[self.i] self.i = self.i + 1 return result else: raise StopIteration +# TODO(https://github.com/grpc/grpc/issues/7950): Eliminate this; just use an +# ordinary sequence of pairs of bytestrings all the way down to the +# grpc_call_start_batch call. cdef class Metadata: + """Metadata being passed from application to core.""" def __cinit__(self, metadata_iterable): + metadata_sequence = tuple(metadata_iterable) + cdef size_t count = len(metadata_sequence) with nogil: grpc_init() - grpc_metadata_array_init(&self.c_metadata_array) - metadata = list(metadata_iterable) - for metadatum in metadata: - if not isinstance(metadatum, Metadatum): - raise TypeError("expected list of Metadatum") - self.c_metadata_array.count = len(metadata) - self.c_metadata_array.capacity = len(metadata) + self.c_metadata = gpr_malloc( + count * sizeof(grpc_metadata)) + self.c_count = count + for index, metadatum in enumerate(metadata_sequence): + self.c_metadata[index].key = grpc_slice_copy( + (metadatum).c_metadata.key) + self.c_metadata[index].value = grpc_slice_copy( + (metadatum).c_metadata.value) + + def __dealloc__(self): + with nogil: + for index in range(self.c_count): + grpc_slice_unref(self.c_metadata[index].key) + grpc_slice_unref(self.c_metadata[index].value) + gpr_free(self.c_metadata) + grpc_shutdown() + + def __len__(self): + return self.c_count + + def __getitem__(self, size_t index): + if index < self.c_count: + key = _slice_bytes(self.c_metadata[index].key) + value = _slice_bytes(self.c_metadata[index].value) + return Metadatum(key, value) + else: + raise IndexError() + + def __iter__(self): + return _MetadataIterator(self.c_count, self) + + +cdef class MetadataArray: + """Metadata being passed from core to application.""" + + def __cinit__(self): with nogil: - self.c_metadata_array.metadata = gpr_malloc( - self.c_metadata_array.count*sizeof(grpc_metadata) - ) - for i in range(self.c_metadata_array.count): - (metadata[i])._copy_metadatum(&self.c_metadata_array.metadata[i]) + grpc_init() + grpc_metadata_array_init(&self.c_metadata_array) def __dealloc__(self): with nogil: - # this frees the allocated memory for the grpc_metadata_array (although - # it'd be nice if that were documented somewhere...) - # TODO(atash): document this in the C core grpc_metadata_array_destroy(&self.c_metadata_array) grpc_shutdown() @@ -493,21 +524,7 @@ cdef class Metadata: return Metadatum(key=key, value=value) def __iter__(self): - return _MetadataIterator(self) - - cdef void _claim_slice_ownership(self): - cdef grpc_metadata_array new_c_metadata_array - grpc_metadata_array_init(&new_c_metadata_array) - new_c_metadata_array.metadata = gpr_malloc( - self.c_metadata_array.count*sizeof(grpc_metadata)) - new_c_metadata_array.count = self.c_metadata_array.count - for i in range(self.c_metadata_array.count): - new_c_metadata_array.metadata[i].key = _copy_slice( - self.c_metadata_array.metadata[i].key) - new_c_metadata_array.metadata[i].value = _copy_slice( - self.c_metadata_array.metadata[i].value) - grpc_metadata_array_destroy(&self.c_metadata_array) - self.c_metadata_array = new_c_metadata_array + return _MetadataIterator(self.c_metadata_array.count, self) cdef class Operation: @@ -547,14 +564,13 @@ cdef class Operation: if (self.c_op.type != GRPC_OP_RECV_INITIAL_METADATA and self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT): raise TypeError("self must be an operation receiving metadata") - return self._received_metadata - - @property - def received_metadata_or_none(self): - if (self.c_op.type != GRPC_OP_RECV_INITIAL_METADATA and - self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT): - return None - return self._received_metadata + # TODO(https://github.com/grpc/grpc/issues/7950): Drop the "all Cython + # objects must be legitimate for use from Python at any time" policy in + # place today, shift the policy toward "Operation objects are only usable + # while their calls are active", and move this making-a-copy-because-this- + # data-needs-to-live-much-longer-than-the-call-from-which-it-arose to the + # lowest Python layer. + return tuple(self._received_metadata) @property def received_status_code(self): @@ -601,9 +617,8 @@ def operation_send_initial_metadata(Metadata metadata, int flags): cdef Operation op = Operation() op.c_op.type = GRPC_OP_SEND_INITIAL_METADATA op.c_op.flags = flags - op.c_op.data.send_initial_metadata.count = metadata.c_metadata_array.count - op.c_op.data.send_initial_metadata.metadata = ( - metadata.c_metadata_array.metadata) + op.c_op.data.send_initial_metadata.count = metadata.c_count + op.c_op.data.send_initial_metadata.metadata = metadata.c_metadata op.references.append(metadata) op.is_valid = True return op @@ -631,9 +646,8 @@ def operation_send_status_from_server( op.c_op.type = GRPC_OP_SEND_STATUS_FROM_SERVER op.c_op.flags = flags op.c_op.data.send_status_from_server.trailing_metadata_count = ( - metadata.c_metadata_array.count) - op.c_op.data.send_status_from_server.trailing_metadata = ( - metadata.c_metadata_array.metadata) + metadata.c_count) + op.c_op.data.send_status_from_server.trailing_metadata = metadata.c_metadata op.c_op.data.send_status_from_server.status = code grpc_slice_unref(op._status_details) op._status_details = _slice_from_bytes(details) @@ -646,7 +660,7 @@ def operation_receive_initial_metadata(int flags): cdef Operation op = Operation() op.c_op.type = GRPC_OP_RECV_INITIAL_METADATA op.c_op.flags = flags - op._received_metadata = Metadata([]) + op._received_metadata = MetadataArray() op.c_op.data.receive_initial_metadata.receive_initial_metadata = ( &op._received_metadata.c_metadata_array) op.is_valid = True @@ -669,7 +683,7 @@ def operation_receive_status_on_client(int flags): cdef Operation op = Operation() op.c_op.type = GRPC_OP_RECV_STATUS_ON_CLIENT op.c_op.flags = flags - op._received_metadata = Metadata([]) + op._received_metadata = MetadataArray() op.c_op.data.receive_status_on_client.trailing_metadata = ( &op._received_metadata.c_metadata_array) op.c_op.data.receive_status_on_client.status = ( diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index dd276fd57b5..b8db27469f9 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -44,7 +44,7 @@ cdef class Server: cdef OperationTag operation_tag = OperationTag(tag) operation_tag.operation_call = Call() operation_tag.request_call_details = CallDetails() - operation_tag.request_metadata = Metadata([]) + operation_tag.request_metadata = MetadataArray() operation_tag.references.extend([self, call_queue, server_queue]) operation_tag.is_new_request = True operation_tag.batch_operations = Operations([]) From 5a2a792509af457322b8fc63eff3ec1ee360b0a0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 24 Apr 2017 13:31:20 +0200 Subject: [PATCH 051/227] add cmake distribtest --- test/distrib/cpp/run_distrib_test_cmake.sh | 66 +++++++++++++++++++ ...test.sh => run_distrib_test_routeguide.sh} | 0 .../distribtest/cpp_jessie_x64/Dockerfile | 2 + .../artifacts/distribtest_targets.py | 12 ++-- 4 files changed, 75 insertions(+), 5 deletions(-) create mode 100755 test/distrib/cpp/run_distrib_test_cmake.sh rename test/distrib/cpp/{run_distrib_test.sh => run_distrib_test_routeguide.sh} (100%) diff --git a/test/distrib/cpp/run_distrib_test_cmake.sh b/test/distrib/cpp/run_distrib_test_cmake.sh new file mode 100755 index 00000000000..a81de57810b --- /dev/null +++ b/test/distrib/cpp/run_distrib_test_cmake.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd $(dirname $0)/../../.. + +echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +apt-get update +#apt-get install -t jessie-backports -y libc-ares-dev # we need specifically version 1.12 +apt-get install -t jessie-backports -y libssl-dev + +# Install c-ares +cd third_party/cares/cares +git fetch origin +git checkout cares-1_13_0 +mkdir -p cmake/build +cd cmake/build +cmake -DCMAKE_BUILD_TYPE=Release ../.. +make -j4 install +cd ../../../../.. + +# Install zlib +cd third_party/zlib +mkdir -p cmake/build +cd cmake/build +cmake -DCMAKE_BUILD_TYPE=Release ../.. +make -j4 install +cd ../../../.. + +# Install protobuf +cd third_party/protobuf +mkdir -p cmake/build +cd cmake/build +cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release .. +make -j4 install +cd ../../../.. + +# TODO: Install boringssl + +# Install gRPC +mkdir -p cmake/build +cd cmake/build +cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DCMAKE_BUILD_TYPE=Release ../.. +make -j4 install +cd ../.. + +# Build helloworld example using cmake +cd examples/cpp/helloworld +mkdir -p cmake/build +cd cmake/build +cmake ../.. +make + diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test_routeguide.sh similarity index 100% rename from test/distrib/cpp/run_distrib_test.sh rename to test/distrib/cpp/run_distrib_test_routeguide.sh diff --git a/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile index ff66bca0aec..0e8186d40ce 100644 --- a/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile @@ -27,4 +27,6 @@ RUN apt-get update && apt-get install -y \ pkg-config \ unzip && apt-get clean +RUN apt-get update && apt-get install -y cmake golang && apt-get clean + CMD ["bash"] diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index fa461efa85e..fb1be383cd9 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -255,12 +255,13 @@ class PHPDistribTest(object): class CppDistribTest(object): """Tests Cpp make intall by building examples.""" - def __init__(self, platform, arch, docker_suffix=None): - self.name = 'cpp_%s_%s_%s' % (platform, arch, docker_suffix) + def __init__(self, platform, arch, docker_suffix=None, testcase=None): + self.name = 'cpp_%s_%s_%s_%s' % (platform, arch, docker_suffix, testcase) self.platform = platform self.arch = arch self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix] + self.testcase = testcase + self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix, testcase] def pre_build_jobspecs(self): return [] @@ -271,7 +272,7 @@ class CppDistribTest(object): 'tools/dockerfile/distribtest/cpp_%s_%s' % ( self.docker_suffix, self.arch), - 'test/distrib/cpp/run_distrib_test.sh') + 'test/distrib/cpp/run_distrib_test_%s.sh' % self.testcase) else: raise Exception("Not supported yet.") @@ -281,7 +282,8 @@ class CppDistribTest(object): def targets(): """Gets list of supported targets""" - return [CppDistribTest('linux', 'x64', 'jessie'), + return [CppDistribTest('linux', 'x64', 'jessie', 'routeguide'), + CppDistribTest('linux', 'x64', 'jessie', 'cmake'), CSharpDistribTest('linux', 'x64', 'wheezy'), CSharpDistribTest('linux', 'x64', 'jessie'), CSharpDistribTest('linux', 'x86', 'jessie'), From 4caadb9205bcb9f745cda2e3c310b27837cfd636 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 25 Aug 2017 12:59:11 +0200 Subject: [PATCH 052/227] fix helloworld cmake build --- examples/cpp/helloworld/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/cpp/helloworld/CMakeLists.txt b/examples/cpp/helloworld/CMakeLists.txt index 8f098c91a6b..71a8db4f24f 100644 --- a/examples/cpp/helloworld/CMakeLists.txt +++ b/examples/cpp/helloworld/CMakeLists.txt @@ -2,7 +2,11 @@ cmake_minimum_required(VERSION 2.8) # Project -project(HelloWorld CXX) +project(HelloWorld C CXX) + +if(NOT MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +endif() # Protobuf set(protobuf_MODULE_COMPATIBLE TRUE) From f567ab0d2b3b761a3647126c4c5a2ec2549f4c73 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 25 Aug 2017 16:13:30 +0200 Subject: [PATCH 053/227] fix public header path for installed grpc --- CMakeLists.txt | 70 +++++++++++++++---------------- templates/CMakeLists.txt.template | 2 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5131bf39b47..a82e9ef6d03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -832,7 +832,7 @@ endif() target_include_directories(gpr - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -924,7 +924,7 @@ endif() target_include_directories(gpr_test_util - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -1219,7 +1219,7 @@ endif() target_include_directories(grpc - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -1525,7 +1525,7 @@ endif() target_include_directories(grpc_cronet - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -1803,7 +1803,7 @@ endif() target_include_directories(grpc_test_util - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2063,7 +2063,7 @@ endif() target_include_directories(grpc_test_util_unsecure - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2357,7 +2357,7 @@ endif() target_include_directories(grpc_unsecure - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2446,7 +2446,7 @@ endif() target_include_directories(reconnect_server - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2488,7 +2488,7 @@ endif() target_include_directories(test_tcp_server - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2569,7 +2569,7 @@ endif() target_include_directories(grpc++ - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2769,7 +2769,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_core_stats - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3060,7 +3060,7 @@ endif() target_include_directories(grpc++_cronet - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3259,7 +3259,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_error_details - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3324,7 +3324,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_proto_reflection_desc_db - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3385,7 +3385,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_reflection - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3443,7 +3443,7 @@ endif() target_include_directories(grpc++_test_config - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3521,7 +3521,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_test_util - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3659,7 +3659,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_test_util_unsecure - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3799,7 +3799,7 @@ endif() target_include_directories(grpc++_unsecure - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3989,7 +3989,7 @@ endif() target_include_directories(grpc_benchmark - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4048,7 +4048,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc_cli_libs - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4108,7 +4108,7 @@ endif() target_include_directories(grpc_plugin_support - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4186,7 +4186,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(http2_client_main - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4241,7 +4241,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(interop_client_helper - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4311,7 +4311,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(interop_client_main - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4362,7 +4362,7 @@ endif() target_include_directories(interop_server_helper - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4431,7 +4431,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(interop_server_lib - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4482,7 +4482,7 @@ endif() target_include_directories(interop_server_main - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4570,7 +4570,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(qps - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4617,7 +4617,7 @@ endif() target_include_directories(grpc_csharp_ext - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4712,7 +4712,7 @@ endif() target_include_directories(ares - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4750,7 +4750,7 @@ endif() target_include_directories(bad_client_test - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4791,7 +4791,7 @@ endif() target_include_directories(bad_ssl_test_server - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4892,7 +4892,7 @@ endif() target_include_directories(end2end_tests - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4993,7 +4993,7 @@ endif() target_include_directories(end2end_nosec_tests - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 6adff889fef..ad79873d562 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -515,7 +515,7 @@ % endfor target_include_directories(${lib.name} - PUBLIC <%text>$ $ + PUBLIC <%text>$ $ PRIVATE <%text>${CMAKE_CURRENT_SOURCE_DIR} PRIVATE <%text>${BORINGSSL_ROOT_DIR}/include PRIVATE <%text>${PROTOBUF_ROOT_DIR}/src From b2cf73e1f81a9c9eeff6dab8f8980c1711784796 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 28 Aug 2017 12:18:45 +0200 Subject: [PATCH 054/227] prevent submodule headers from influencing the build --- test/distrib/cpp/run_distrib_test_cmake.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/distrib/cpp/run_distrib_test_cmake.sh b/test/distrib/cpp/run_distrib_test_cmake.sh index a81de57810b..ead8cc10bc7 100755 --- a/test/distrib/cpp/run_distrib_test_cmake.sh +++ b/test/distrib/cpp/run_distrib_test_cmake.sh @@ -31,6 +31,7 @@ cd cmake/build cmake -DCMAKE_BUILD_TYPE=Release ../.. make -j4 install cd ../../../../.. +rm -rf third_party/cares/cares # wipe out to prevent influencing the grpc build # Install zlib cd third_party/zlib @@ -39,6 +40,7 @@ cd cmake/build cmake -DCMAKE_BUILD_TYPE=Release ../.. make -j4 install cd ../../../.. +rm -rf third_party/zlib # wipe out to prevent influencing the grpc build # Install protobuf cd third_party/protobuf @@ -47,8 +49,7 @@ cd cmake/build cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release .. make -j4 install cd ../../../.. - -# TODO: Install boringssl +rm -rf third_party/protobuf # wipe out to prevent influencing the grpc build # Install gRPC mkdir -p cmake/build From ae6fd66cef94bf9d4d89bcdcf64a7463ef0e77f2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 28 Aug 2017 12:21:46 +0200 Subject: [PATCH 055/227] fix zlib and openssl package mode --- CMakeLists.txt | 18 +++++++----------- templates/CMakeLists.txt.template | 18 +++++++----------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a82e9ef6d03..dd68016be86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,10 +123,8 @@ if("${gRPC_ZLIB_PROVIDER}" STREQUAL "module") set(gRPC_INSTALL FALSE) endif() elseif("${gRPC_ZLIB_PROVIDER}" STREQUAL "package") - find_package(ZLIB) - if(TARGET ZLIB::ZLIB) - set(_gRPC_ZLIB_LIBRARIES ZLIB::ZLIB) - endif() + find_package(ZLIB REQUIRED) + set(_gRPC_ZLIB_LIBRARIES ${ZLIB_LIBRARIES}) set(_gRPC_FIND_ZLIB "if(NOT ZLIB_FOUND)\n find_package(ZLIB)\nendif()") endif() @@ -145,7 +143,7 @@ if("${gRPC_CARES_PROVIDER}" STREQUAL "module") set(gRPC_INSTALL FALSE) endif() elseif("${gRPC_CARES_PROVIDER}" STREQUAL "package") - find_package(c-ares CONFIG) + find_package(c-ares REQUIRED CONFIG) if(TARGET c-ares::cares) set(_gRPC_CARES_LIBRARIES c-ares::cares) endif() @@ -189,7 +187,7 @@ if("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "module") set(gRPC_INSTALL FALSE) endif() elseif("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "package") - find_package(Protobuf ${gRPC_PROTOBUF_PACKAGE_TYPE}) + find_package(Protobuf REQUIRED ${gRPC_PROTOBUF_PACKAGE_TYPE}) if(Protobuf_FOUND OR PROTOBUF_FOUND) if(TARGET protobuf::${_gRPC_PROTOBUF_LIBRARY_NAME}) set(_gRPC_PROTOBUF_LIBRARIES protobuf::${_gRPC_PROTOBUF_LIBRARY_NAME}) @@ -234,11 +232,9 @@ if("${gRPC_SSL_PROVIDER}" STREQUAL "module") set(gRPC_INSTALL FALSE) endif() elseif("${gRPC_SSL_PROVIDER}" STREQUAL "package") - find_package(OpenSSL) - if(TARGET OpenSSL::SSL) - set(_gRPC_SSL_LIBRARIES OpenSSL::SSL) - endif() - set(_gRPC_FIND_SSL "if(NOT OpenSSL_FOUND)\n find_package(OpenSSL)\nendif()") + find_package(OpenSSL REQUIRED) + set(_gRPC_SSL_LIBRARIES ${OPENSSL_LIBRARIES}) + set(_gRPC_FIND_SSL "if(NOT OPENSSL_FOUND)\n find_package(OpenSSL)\nendif()") endif() if("${gRPC_GFLAGS_PROVIDER}" STREQUAL "module") diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index ad79873d562..f2ce9b5a90c 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -168,10 +168,8 @@ set(gRPC_INSTALL FALSE) endif() elseif("<%text>${gRPC_ZLIB_PROVIDER}" STREQUAL "package") - find_package(ZLIB) - if(TARGET ZLIB::ZLIB) - set(_gRPC_ZLIB_LIBRARIES ZLIB::ZLIB) - endif() + find_package(ZLIB REQUIRED) + set(_gRPC_ZLIB_LIBRARIES <%text>${ZLIB_LIBRARIES}) set(_gRPC_FIND_ZLIB "if(NOT ZLIB_FOUND)\n find_package(ZLIB)\nendif()") endif() @@ -190,7 +188,7 @@ set(gRPC_INSTALL FALSE) endif() elseif("<%text>${gRPC_CARES_PROVIDER}" STREQUAL "package") - find_package(c-ares CONFIG) + find_package(c-ares REQUIRED CONFIG) if(TARGET c-ares::cares) set(_gRPC_CARES_LIBRARIES c-ares::cares) endif() @@ -234,7 +232,7 @@ set(gRPC_INSTALL FALSE) endif() elseif("<%text>${gRPC_PROTOBUF_PROVIDER}" STREQUAL "package") - find_package(Protobuf <%text>${gRPC_PROTOBUF_PACKAGE_TYPE}) + find_package(Protobuf REQUIRED <%text>${gRPC_PROTOBUF_PACKAGE_TYPE}) if(Protobuf_FOUND OR PROTOBUF_FOUND) if(TARGET protobuf::<%text>${_gRPC_PROTOBUF_LIBRARY_NAME}) set(_gRPC_PROTOBUF_LIBRARIES protobuf::<%text>${_gRPC_PROTOBUF_LIBRARY_NAME}) @@ -279,11 +277,9 @@ set(gRPC_INSTALL FALSE) endif() elseif("<%text>${gRPC_SSL_PROVIDER}" STREQUAL "package") - find_package(OpenSSL) - if(TARGET OpenSSL::SSL) - set(_gRPC_SSL_LIBRARIES OpenSSL::SSL) - endif() - set(_gRPC_FIND_SSL "if(NOT OpenSSL_FOUND)\n find_package(OpenSSL)\nendif()") + find_package(OpenSSL REQUIRED) + set(_gRPC_SSL_LIBRARIES <%text>${OPENSSL_LIBRARIES}) + set(_gRPC_FIND_SSL "if(NOT OPENSSL_FOUND)\n find_package(OpenSSL)\nendif()") endif() if("<%text>${gRPC_GFLAGS_PROVIDER}" STREQUAL "module") From aba0a0a54412fe59bc2090334ae26c99bb148097 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Sun, 10 Sep 2017 18:28:03 -0700 Subject: [PATCH 056/227] Add fallback (use backends from resolver if can't reach balancer) to grpclb. --- include/grpc++/support/channel_arguments.h | 6 + include/grpc/impl/codegen/grpc_types.h | 6 +- .../client_channel/lb_policy/grpclb/grpclb.c | 197 ++++++++++++++---- .../client_channel/lb_policy_factory.c | 2 +- .../client_channel/lb_policy_factory.h | 2 +- src/cpp/common/channel_arguments.cc | 4 + test/cpp/end2end/grpclb_end2end_test.cc | 74 ++++++- 7 files changed, 244 insertions(+), 47 deletions(-) diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h index 7b6befeaf1e..9dc505f0082 100644 --- a/include/grpc++/support/channel_arguments.h +++ b/include/grpc++/support/channel_arguments.h @@ -64,6 +64,12 @@ class ChannelArguments { /// Set the compression algorithm for the channel. void SetCompressionAlgorithm(grpc_compression_algorithm algorithm); + /// Set the grpclb fallback timeout (in ms) for the channel. If this amount + /// of time has passed but we have not gotten any non-empty \a serverlist from + /// the balancer, we will fall back to use the backend address(es) returned by + /// the resolver. + void SetGrpclbFallbackTimeout(int fallback_timeout); + /// Set the socket mutator for the channel. void SetSocketMutator(grpc_socket_mutator* mutator); diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 59b90af03a3..4745fddd71c 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -287,7 +287,11 @@ typedef struct { "grpc.experimental.tcp_max_read_chunk_size" /* Timeout in milliseconds to use for calls to the grpclb load balancer. If 0 or unset, the balancer calls will have no deadline. */ -#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_timeout_ms" +#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms" +/* Timeout in milliseconds to wait for the serverlist from the grpclb load + balancer before using fallback backend addresses from the resolver. + If 0, fallback will never be used. */ +#define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms" /** If non-zero, grpc server's cronet compression workaround will be enabled */ #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \ "grpc.workaround.cronet_compression" diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 5aafed1374d..18979829bd2 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -122,6 +122,7 @@ #define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6 #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 +#define GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS 10000 grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false, "glb"); @@ -298,6 +299,10 @@ typedef struct glb_lb_policy { /** timeout in milliseconds for the LB call. 0 means no deadline. */ int lb_call_timeout_ms; + /** timeout in milliseconds for before using fallback backend addresses. + * 0 means not using fallback. */ + int lb_fallback_timeout_ms; + /** for communicating with the LB server */ grpc_channel *lb_channel; @@ -324,6 +329,9 @@ typedef struct glb_lb_policy { * Otherwise, we delegate to the RR policy. */ size_t serverlist_index; + /** stores the backend addresses from the resolver */ + grpc_lb_addresses *fallback_backend_addresses; + /** list of picks that are waiting on RR's policy connectivity */ pending_pick *pending_picks; @@ -344,6 +352,9 @@ typedef struct glb_lb_policy { /** is \a lb_call_retry_timer active? */ bool retry_timer_active; + /** is \a lb_fallback_timer active? */ + bool fallback_timer_active; + /** called upon changes to the LB channel's connectivity. */ grpc_closure lb_channel_on_connectivity_changed; @@ -366,6 +377,9 @@ typedef struct glb_lb_policy { /* LB call retry timer callback. */ grpc_closure lb_on_call_retry; + /* LB fallback timer callback. */ + grpc_closure lb_on_fallback; + grpc_call *lb_call; /* streaming call to the LB server, */ grpc_metadata_array lb_initial_metadata_recv; /* initial MD from LB server */ @@ -389,6 +403,9 @@ typedef struct glb_lb_policy { /** LB call retry timer */ grpc_timer lb_call_retry_timer; + /** LB fallback timer */ + grpc_timer lb_fallback_timer; + bool initial_request_sent; bool seen_initial_response; @@ -535,6 +552,32 @@ static grpc_lb_addresses *process_serverlist_locked( return lb_addresses; } +/* Returns the backend addresses extracted from the given addresses */ +static grpc_lb_addresses *extract_backend_addresses_locked( + grpc_exec_ctx *exec_ctx, const grpc_lb_addresses *addresses) { + /* first pass: count the number of backend addresses */ + size_t num_backends = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (!addresses->addresses[i].is_balancer) { + ++num_backends; + } + } + /* second pass: actually populate the addresses and (empty) LB tokens */ + grpc_lb_addresses *backend_addresses = + grpc_lb_addresses_create(num_backends, &lb_token_vtable); + size_t num_copied = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (addresses->addresses[i].is_balancer) continue; + const grpc_resolved_address *addr = &addresses->addresses[i].address; + grpc_lb_addresses_set_address(backend_addresses, num_copied, &addr->addr, + addr->len, false /* is_balancer */, + NULL /* balancer_name */, + (void *)GRPC_MDELEM_LB_TOKEN_EMPTY.payload); + ++num_copied; + } + return backend_addresses; +} + static void update_lb_connectivity_status_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, grpc_connectivity_state rr_state, grpc_error *rr_state_error) { @@ -602,35 +645,38 @@ static bool pick_from_internal_rr_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, const grpc_lb_policy_pick_args *pick_args, bool force_async, grpc_connected_subchannel **target, wrapped_rr_closure_arg *wc_arg) { - // Look at the index into the serverlist to see if we should drop this call. - grpc_grpclb_server *server = - glb_policy->serverlist->servers[glb_policy->serverlist_index++]; - if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { - glb_policy->serverlist_index = 0; // Wrap-around. - } - if (server->drop) { - // Not using the RR policy, so unref it. - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", - (intptr_t)wc_arg->rr_policy); + // Check for drops if we are not using fallback backend addresses. + if (glb_policy->serverlist != NULL) { + // Look at the index into the serverlist to see if we should drop this call. + grpc_grpclb_server *server = + glb_policy->serverlist->servers[glb_policy->serverlist_index++]; + if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { + glb_policy->serverlist_index = 0; // Wrap-around. } - GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); - // Update client load reporting stats to indicate the number of - // dropped calls. Note that we have to do this here instead of in - // the client_load_reporting filter, because we do not create a - // subchannel call (and therefore no client_load_reporting filter) - // for dropped calls. - grpc_grpclb_client_stats_add_call_dropped_locked(server->load_balance_token, - wc_arg->client_stats); - grpc_grpclb_client_stats_unref(wc_arg->client_stats); - if (force_async) { - GPR_ASSERT(wc_arg->wrapped_closure != NULL); - GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); + if (server->drop) { + // Not using the RR policy, so unref it. + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", + (intptr_t)wc_arg->rr_policy); + } + GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); + // Update client load reporting stats to indicate the number of + // dropped calls. Note that we have to do this here instead of in + // the client_load_reporting filter, because we do not create a + // subchannel call (and therefore no client_load_reporting filter) + // for dropped calls. + grpc_grpclb_client_stats_add_call_dropped_locked( + server->load_balance_token, wc_arg->client_stats); + grpc_grpclb_client_stats_unref(wc_arg->client_stats); + if (force_async) { + GPR_ASSERT(wc_arg->wrapped_closure != NULL); + GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); + gpr_free(wc_arg->free_when_done); + return false; + } gpr_free(wc_arg->free_when_done); - return false; + return true; } - gpr_free(wc_arg->free_when_done); - return true; } // Pick via the RR policy. const bool pick_done = grpc_lb_policy_pick_locked( @@ -668,8 +714,18 @@ static bool pick_from_internal_rr_locked( static grpc_lb_policy_args *lb_policy_args_create(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - grpc_lb_addresses *addresses = - process_serverlist_locked(exec_ctx, glb_policy->serverlist); + grpc_lb_addresses *addresses; + if (glb_policy->serverlist != NULL) { + GPR_ASSERT(glb_policy->serverlist->num_servers > 0); + addresses = process_serverlist_locked(exec_ctx, glb_policy->serverlist); + } else { + // If rr_handover_locked() is invoked when we haven't received any + // serverlist from the balancer, we use the fallback backends returned by + // the resolver. Note that the fallback backend list may be empty, in which + // case the new round_robin policy will keep the requested picks pending. + GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); + addresses = grpc_lb_addresses_copy(glb_policy->fallback_backend_addresses); + } GPR_ASSERT(addresses != NULL); grpc_lb_policy_args *args = (grpc_lb_policy_args *)gpr_zalloc(sizeof(*args)); args->client_channel_factory = glb_policy->cc_factory; @@ -775,8 +831,6 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, /* glb_policy->rr_policy may be NULL (initial handover) */ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - GPR_ASSERT(glb_policy->serverlist != NULL && - glb_policy->serverlist->num_servers > 0); if (glb_policy->shutting_down) return; grpc_lb_policy_args *args = lb_policy_args_create(exec_ctx, glb_policy); GPR_ASSERT(args != NULL); @@ -917,13 +971,7 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. - * TODO(roth): For now, we ignore non-balancer addresses, but in the - * future, we may change the behavior such that we fall back to using - * the non-balancer addresses if we cannot reach any balancers. In the - * fallback case, we should use the LB policy indicated by - * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is - * unset, we should default to pick_first). */ + /* Count the number of gRPC-LB addresses. There must be at least one. */ const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); if (arg == NULL || arg->type != GRPC_ARG_POINTER) { @@ -959,6 +1007,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->lb_call_timeout_ms = grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); + arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); + glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( + arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, + INT_MAX}); + // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. grpc_arg new_arg = @@ -967,6 +1020,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); + /* Extract the backend addresses (may be empty) from the resolver for + * fallback. */ + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + /* Create a client channel over them to communicate with a LB service */ glb_policy->response_generator = grpc_fake_resolver_response_generator_create(); @@ -1010,6 +1068,9 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { if (glb_policy->serverlist != NULL) { grpc_grpclb_destroy_serverlist(glb_policy->serverlist); } + if (glb_policy->fallback_backend_addresses != NULL) { + grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); + } grpc_fake_resolver_response_generator_unref(glb_policy->response_generator); if (glb_policy->pending_update_args != NULL) { grpc_channel_args_destroy(exec_ctx, glb_policy->pending_update_args->args); @@ -1150,10 +1211,28 @@ static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } +static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error); static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy); static void start_picking_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { + /* start a timer to fall back */ + if (glb_policy->lb_fallback_timeout_ms > 0 && + glb_policy->serverlist == NULL) { + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec deadline = gpr_time_add( + now, + gpr_time_from_millis(glb_policy->lb_fallback_timeout_ms, GPR_TIMESPAN)); + GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer"); + GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked, + glb_policy, + grpc_combiner_scheduler(glb_policy->base.combiner)); + glb_policy->fallback_timer_active = true; + grpc_timer_init(exec_ctx, &glb_policy->lb_fallback_timer, deadline, + &glb_policy->lb_on_fallback, now); + } + glb_policy->started_picking = true; gpr_backoff_reset(&glb_policy->lb_call_backoff_state); query_for_backends_locked(exec_ctx, glb_policy); @@ -1607,6 +1686,15 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, if (glb_policy->serverlist != NULL) { /* dispose of the old serverlist */ grpc_grpclb_destroy_serverlist(glb_policy->serverlist); + } else { + /* or dispose of the fallback */ + grpc_lb_addresses_destroy(exec_ctx, + glb_policy->fallback_backend_addresses); + glb_policy->fallback_backend_addresses = NULL; + if (glb_policy->fallback_timer_active) { + grpc_timer_cancel(exec_ctx, &glb_policy->lb_fallback_timer); + glb_policy->fallback_timer_active = false; + } } /* and update the copy in the glb_lb_policy instance. This * serverlist instance will be destroyed either upon the next @@ -1617,9 +1705,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, } } else { if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, - "Received empty server list. Picks will stay pending until " - "a response with > 0 servers is received"); + gpr_log(GPR_INFO, "Received empty server list, ignoring."); } grpc_grpclb_destroy_serverlist(serverlist); } @@ -1666,6 +1752,26 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer"); } +static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + glb_lb_policy *glb_policy = arg; + /* If we receive a serverlist after the timer fires but before this callback + * actually runs, don't do anything. */ + if (glb_policy->serverlist != NULL) return; + glb_policy->fallback_timer_active = false; + if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, + "Falling back to use backends from resolver (grpclb %p)", + (void *)glb_policy); + } + GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); + rr_handover_locked(exec_ctx, glb_policy); + } + GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, + "grpclb_fallback_timer"); +} + static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { glb_lb_policy *glb_policy = (glb_lb_policy *)arg; @@ -1786,6 +1892,17 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, &glb_policy->lb_channel_connectivity, &glb_policy->lb_channel_on_connectivity_changed, NULL); } + + // Propagate update to fallback_backend_addresses if a non-empty serverlist + // hasn't been received from the balancer. + if (glb_policy->serverlist == NULL) { + grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + if (glb_policy->rr_policy != NULL) { + rr_handover_locked(exec_ctx, glb_policy); + } + } } // Invoked as part of the update process. It continues watching the LB channel diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index cdcaf17544b..918bab745c6 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -56,7 +56,7 @@ grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses) { } void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index, - void* address, size_t address_len, + const void* address, size_t address_len, bool is_balancer, const char* balancer_name, void* user_data) { GPR_ASSERT(index < addresses->num_addresses); diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.h b/src/core/ext/filters/client_channel/lb_policy_factory.h index 9d9fb143df5..cf0f8cb6157 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.h +++ b/src/core/ext/filters/client_channel/lb_policy_factory.h @@ -73,7 +73,7 @@ grpc_lb_addresses *grpc_lb_addresses_copy(const grpc_lb_addresses *addresses); * \a address is a socket address of length \a address_len. * Takes ownership of \a balancer_name. */ void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index, - void *address, size_t address_len, + const void *address, size_t address_len, bool is_balancer, const char *balancer_name, void *user_data); diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index f130aecd4b5..f89f5f1f03d 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -86,6 +86,10 @@ void ChannelArguments::SetCompressionAlgorithm( SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, algorithm); } +void ChannelArguments::SetGrpclbFallbackTimeout(int fallback_timeout) { + SetInt(GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS, fallback_timeout); +} + void ChannelArguments::SetSocketMutator(grpc_socket_mutator* mutator) { if (!mutator) { return; diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 570a3d10677..17a094f7a21 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -368,8 +368,9 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_fake_resolver_response_generator_unref(response_generator_); } - void ResetStub() { + void ResetStub(int fallback_timeout = 0) { ChannelArguments args; + args.SetGrpclbFallbackTimeout(fallback_timeout); args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR, response_generator_); std::ostringstream uri; @@ -441,10 +442,10 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_exec_ctx_finish(&exec_ctx); } - const std::vector GetBackendPorts() const { + const std::vector GetBackendPorts(const size_t start_index = 0) const { std::vector backend_ports; - for (const auto& bs : backend_servers_) { - backend_ports.push_back(bs.port_); + for (size_t i = start_index; i < backend_servers_.size(); ++i) { + backend_ports.push_back(backend_servers_[i].port_); } return backend_ports; } @@ -615,6 +616,71 @@ TEST_F(SingleBalancerTest, InitiallyEmptyServerlist) { EXPECT_EQ("grpclb", channel_->GetLoadBalancingPolicyName()); } +TEST_F(SingleBalancerTest, Fallback) { + const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); + const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor(); + const size_t kNumBackendInResolution = backends_.size() / 2; + + ResetStub(kFallbackTimeoutMs); + std::vector addresses; + addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""}); + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""}); + } + SetNextResolution(addresses); + + // Send non-empty serverlist only after kServerlistDelayMs + ScheduleResponseForBalancer( + 0, BalancerServiceImpl::BuildResponseForBackends( + GetBackendPorts(kNumBackendInResolution /* start_index */), {}), + kServerlistDelayMs); + + // The first request. The client will block while it's still trying to + // contact the balancer. + gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); + CheckRpcSendOk(kNumBackendInResolution); + gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); + + // Fallback is used: each backend returned by the resolver should have + // gotten one request. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + + // Wait until update has been processed, as signaled by the backend returned + // by the balancer receiving a request. + do { + CheckRpcSendOk(1); + } while ( + backend_servers_[kNumBackendInResolution].service_->request_count() == 0); + for (size_t i = 0; i < backends_.size(); ++i) { + backend_servers_[i].service_->ResetCounters(); + } + + // Send out the second request. + gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); + CheckRpcSendOk(backends_.size() - kNumBackendInResolution); + gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); + + // Serverlist is used: each backend returned by the balancer should + // have gotten one request. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { + EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); + } + + balancers_[0]->NotifyDoneWithServerlists(); + // The balancer got a single request. + EXPECT_EQ(1U, balancer_servers_[0].service_->request_count()); + // and sent a single response. + EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); +} + TEST_F(SingleBalancerTest, BackendsRestart) { const size_t kNumRpcsPerAddress = 100; ScheduleResponseForBalancer( From b29296042a308060574d5cd119d6b05c00660624 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 11 Sep 2017 09:31:11 -0700 Subject: [PATCH 057/227] More cleanup in client_channel code. --- .../filters/client_channel/client_channel.c | 314 +++++++++--------- 1 file changed, 153 insertions(+), 161 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index dad5c4fce5b..e5f4a8a813c 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -1016,13 +1016,11 @@ static void create_subchannel_call_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } -static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_error *error) { +// Invoked when a pick is completed, on both success or failure. +static void pick_done_locked(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_error *error) { call_data *calld = (call_data *)elem->call_data; channel_data *chand = (channel_data *)elem->channel_data; - grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent, - chand->interested_parties); if (calld->connected_subchannel == NULL) { // Failed to create subchannel. GRPC_ERROR_UNREF(calld->error); @@ -1044,12 +1042,116 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } -/** Return true if subchannel is available immediately (in which case - subchannel_ready_locked() should not be called), or false otherwise (in - which case subchannel_ready_locked() should be called when the subchannel - is available). */ -static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem); +// A wrapper around pick_done_locked() that is used in cases where +// either (a) the pick was deferred pending a resolver result or (b) the +// pick was done asynchronously. Removes the call's polling entity from +// chand->interested_parties before invoking pick_done_locked(). +static void async_pick_done_locked(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, grpc_error *error) { + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; + grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent, + chand->interested_parties); + pick_done_locked(exec_ctx, elem, error); +} + +// Note: This runs under the client_channel combiner, but will NOT be +// holding the call combiner. +static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_call_element *elem = (grpc_call_element *)arg; + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; + if (calld->lb_policy != NULL) { + if (GRPC_TRACER_ON(grpc_client_channel_trace)) { + gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p", + chand, calld, calld->lb_policy); + } + grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy, + &calld->connected_subchannel, + GRPC_ERROR_REF(error)); + } + GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_callback_cancel"); +} + +// Callback invoked by grpc_lb_policy_pick_locked() for async picks. +// Unrefs the LB policy and invokes async_pick_done_locked(). +static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_call_element *elem = (grpc_call_element *)arg; + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; + if (GRPC_TRACER_ON(grpc_client_channel_trace)) { + gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously", + chand, calld); + } + GPR_ASSERT(calld->lb_policy != NULL); + GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel"); + calld->lb_policy = NULL; + async_pick_done_locked(exec_ctx, elem, GRPC_ERROR_REF(error)); +} + +// Takes a ref to chand->lb_policy and calls grpc_lb_policy_pick_locked(). +// If the pick was completed synchronously, unrefs the LB policy and +// returns true. +static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem) { + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; + if (GRPC_TRACER_ON(grpc_client_channel_trace)) { + gpr_log(GPR_DEBUG, "chand=%p calld=%p: starting pick on lb_policy=%p", + chand, calld, chand->lb_policy); + } + apply_service_config_to_call_locked(exec_ctx, elem); + // If the application explicitly set wait_for_ready, use that. + // Otherwise, if the service config specified a value for this + // method, use that. + uint32_t initial_metadata_flags = + calld->initial_metadata_batch->payload->send_initial_metadata + .send_initial_metadata_flags; + const bool wait_for_ready_set_from_api = + initial_metadata_flags & + GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET; + const bool wait_for_ready_set_from_service_config = + calld->method_params != NULL && + calld->method_params->wait_for_ready != WAIT_FOR_READY_UNSET; + if (!wait_for_ready_set_from_api && wait_for_ready_set_from_service_config) { + if (calld->method_params->wait_for_ready == WAIT_FOR_READY_TRUE) { + initial_metadata_flags |= GRPC_INITIAL_METADATA_WAIT_FOR_READY; + } else { + initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY; + } + } + const grpc_lb_policy_pick_args inputs = { + calld->initial_metadata_batch->payload->send_initial_metadata + .send_initial_metadata, + initial_metadata_flags, &calld->lb_token_mdelem}; + // Keep a ref to the LB policy in calld while the pick is pending. + GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel"); + calld->lb_policy = chand->lb_policy; + GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem, + grpc_combiner_scheduler(chand->combiner)); + const bool pick_done = grpc_lb_policy_pick_locked( + exec_ctx, chand->lb_policy, &inputs, &calld->connected_subchannel, + calld->subchannel_call_context, NULL, &calld->lb_pick_closure); + if (pick_done) { + /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */ + if (GRPC_TRACER_ON(grpc_client_channel_trace)) { + gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously", + chand, calld); + } + GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel"); + calld->lb_policy = NULL; + } else { + GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel"); + grpc_call_combiner_set_notify_on_cancel( + exec_ctx, calld->call_combiner, + GRPC_CLOSURE_INIT(&calld->lb_pick_cancel_closure, + pick_callback_cancel_locked, elem, + grpc_combiner_scheduler(chand->combiner))); + } + return pick_done; +} typedef struct { grpc_call_element *elem; @@ -1069,17 +1171,17 @@ static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx, gpr_free(args); return; } - args->finished = true; - grpc_call_element *elem = args->elem; - channel_data *chand = (channel_data *)elem->channel_data; - call_data *calld = (call_data *)elem->call_data; // If we don't yet have a resolver result, then a closure for // pick_after_resolver_result_done_locked() will have been added to // chand->waiting_for_resolver_result_closures, and it may not be invoked // until after this call has been destroyed. We mark the operation as // finished, so that when pick_after_resolver_result_done_locked() // is called, it will be a no-op. We also immediately invoke - // subchannel_ready_locked() to propagate the error back to the caller. + // async_pick_done_locked() to propagate the error back to the caller. + args->finished = true; + grpc_call_element *elem = args->elem; + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; if (GRPC_TRACER_ON(grpc_client_channel_trace)) { gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick waiting for resolver result", @@ -1087,12 +1189,12 @@ static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx, } // Note: Although we are not in the call combiner here, we are // basically stealing the call combiner from the pending pick, so - // it's safe to call subchannel_ready_locked() here -- we are + // it's safe to call async_pick_done_locked() here -- we are // essentially calling it here instead of calling it in // pick_after_resolver_result_done_locked(). - subchannel_ready_locked(exec_ctx, elem, - GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "Pick cancelled", &error, 1)); + async_pick_done_locked(exec_ctx, elem, + GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "Pick cancelled", &error, 1)); } static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx, @@ -1117,14 +1219,19 @@ static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx, gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver failed to return data", chand, calld); } - subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error)); + async_pick_done_locked(exec_ctx, elem, GRPC_ERROR_REF(error)); } else { if (GRPC_TRACER_ON(grpc_client_channel_trace)) { gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver returned, doing pick", chand, calld); } - if (pick_subchannel_locked(exec_ctx, elem)) { - subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_NONE); + if (pick_callback_start_locked(exec_ctx, elem)) { + // Even if the LB policy returns a result synchronously, we have + // already added our polling entity to chand->interested_parties + // in order to wait for the resolver result, so we need to + // remove it here. Therefore, we call async_pick_done_locked() + // instead of pick_done_locked(). + async_pick_done_locked(exec_ctx, elem, GRPC_ERROR_NONE); } } } @@ -1152,154 +1259,38 @@ static void pick_after_resolver_result_start_locked(grpc_exec_ctx *exec_ctx, grpc_combiner_scheduler(chand->combiner))); } -// Note: This runs under the client_channel combiner, but will NOT be -// holding the call combiner. -static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - grpc_call_element *elem = (grpc_call_element *)arg; - channel_data *chand = (channel_data *)elem->channel_data; - call_data *calld = (call_data *)elem->call_data; - if (error != GRPC_ERROR_NONE && calld->lb_policy != NULL) { - if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p", - chand, calld, calld->lb_policy); - } - grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy, - &calld->connected_subchannel, - GRPC_ERROR_REF(error)); - } - GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_callback_cancel"); -} - -// Callback invoked by grpc_lb_policy_pick_locked() for async picks. -// Unrefs the LB policy and invokes subchannel_ready_locked(). -static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { +static void start_pick_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *ignored) { grpc_call_element *elem = (grpc_call_element *)arg; - channel_data *chand = (channel_data *)elem->channel_data; call_data *calld = (call_data *)elem->call_data; - if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously", - chand, calld); - } - GPR_ASSERT(calld->lb_policy != NULL); - GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel"); - calld->lb_policy = NULL; - subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error)); -} - -// Takes a ref to chand->lb_policy and calls grpc_lb_policy_pick_locked(). -// If the pick was completed synchronously, unrefs the LB policy and -// returns true. -static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - const grpc_lb_policy_pick_args *inputs) { channel_data *chand = (channel_data *)elem->channel_data; - call_data *calld = (call_data *)elem->call_data; - if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: starting pick on lb_policy=%p", - chand, calld, chand->lb_policy); - } - // Keep a ref to the LB policy in calld while the pick is pending. - GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel"); - calld->lb_policy = chand->lb_policy; - GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem, - grpc_combiner_scheduler(chand->combiner)); - const bool pick_done = grpc_lb_policy_pick_locked( - exec_ctx, chand->lb_policy, inputs, &calld->connected_subchannel, - calld->subchannel_call_context, NULL, &calld->lb_pick_closure); - if (pick_done) { - /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */ - if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously", - chand, calld); + GPR_ASSERT(calld->connected_subchannel == NULL); + if (chand->lb_policy != NULL) { + // We already have an LB policy, so ask it for a pick. + if (pick_callback_start_locked(exec_ctx, elem)) { + // Pick completed synchronously. + pick_done_locked(exec_ctx, elem, GRPC_ERROR_NONE); + return; } - GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel"); - calld->lb_policy = NULL; } else { - GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel"); - grpc_call_combiner_set_notify_on_cancel( - exec_ctx, calld->call_combiner, - GRPC_CLOSURE_INIT(&calld->lb_pick_cancel_closure, - pick_callback_cancel_locked, elem, - grpc_combiner_scheduler(chand->combiner))); - } - return pick_done; -} - -static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem) { - GPR_TIMER_BEGIN("pick_subchannel", 0); - channel_data *chand = (channel_data *)elem->channel_data; - call_data *calld = (call_data *)elem->call_data; - bool pick_done = false; - if (chand->lb_policy != NULL) { - apply_service_config_to_call_locked(exec_ctx, elem); - // If the application explicitly set wait_for_ready, use that. - // Otherwise, if the service config specified a value for this - // method, use that. - uint32_t initial_metadata_flags = - calld->initial_metadata_batch->payload->send_initial_metadata - .send_initial_metadata_flags; - const bool wait_for_ready_set_from_api = - initial_metadata_flags & - GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET; - const bool wait_for_ready_set_from_service_config = - calld->method_params != NULL && - calld->method_params->wait_for_ready != WAIT_FOR_READY_UNSET; - if (!wait_for_ready_set_from_api && - wait_for_ready_set_from_service_config) { - if (calld->method_params->wait_for_ready == WAIT_FOR_READY_TRUE) { - initial_metadata_flags |= GRPC_INITIAL_METADATA_WAIT_FOR_READY; - } else { - initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY; - } + // We do not yet have an LB policy, so wait for a resolver result. + if (chand->resolver == NULL) { + pick_done_locked(exec_ctx, elem, + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Disconnected")); + return; } - const grpc_lb_policy_pick_args inputs = { - calld->initial_metadata_batch->payload->send_initial_metadata - .send_initial_metadata, - initial_metadata_flags, &calld->lb_token_mdelem}; - pick_done = pick_callback_start_locked(exec_ctx, elem, &inputs); - } else if (chand->resolver != NULL) { if (!chand->started_resolving) { start_resolving_locked(exec_ctx, chand); } pick_after_resolver_result_start_locked(exec_ctx, elem); - } else { - subchannel_ready_locked( - exec_ctx, elem, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Disconnected")); - } - GPR_TIMER_END("pick_subchannel", 0); - return pick_done; -} - -static void start_pick_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error_ignored) { - GPR_TIMER_BEGIN("start_pick_locked", 0); - grpc_call_element *elem = (grpc_call_element *)arg; - call_data *calld = (call_data *)elem->call_data; - channel_data *chand = (channel_data *)elem->channel_data; - GPR_ASSERT(calld->connected_subchannel == NULL); - if (pick_subchannel_locked(exec_ctx, elem)) { - // Pick was returned synchronously. - if (calld->connected_subchannel == NULL) { - GRPC_ERROR_UNREF(calld->error); - calld->error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Call dropped by load balancing policy"); - waiting_for_pick_batches_fail(exec_ctx, elem, - GRPC_ERROR_REF(calld->error)); - } else { - // Create subchannel call. - create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_NONE); - } - } else { - // Pick will be done asynchronously. Add the call's polling entity to - // the channel's interested_parties, so that I/O for the resolver - // and LB policy can be done under it. - grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent, - chand->interested_parties); } - GPR_TIMER_END("start_pick_locked", 0); + // We need to wait for either a resolver result or for an async result + // from the LB policy. Add the polling entity from call_data to the + // channel_data's interested_parties, so that the I/O of the LB policy + // and resolver can be done under it. The polling entity will be + // removed in async_pick_done_locked(). + grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent, + chand->interested_parties); } static void on_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { @@ -1394,7 +1385,8 @@ static void cc_start_transport_stream_op_batch( // combiner to start a pick. if (batch->send_initial_metadata) { if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering combiner", chand, calld); + gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering client_channel combiner", + chand, calld); } GRPC_CLOSURE_SCHED( exec_ctx, From 1c32292c7be078303f4bb59e635319773c5012c2 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 11 Sep 2017 09:05:59 -0700 Subject: [PATCH 058/227] Address reviewer comments --- include/grpc++/impl/codegen/call.h | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index bbd277f288f..0bf33cb88f1 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -303,14 +303,18 @@ class CallOpSendMessage { WriteOptions write_options_; }; +namespace internal { +template +T Example(); +} // namespace internal + template class CallOpSendMessage::MessageSerializer< M, typename std::enable_if::Serialize( - static_cast(*(static_cast(nullptr))), - static_cast(nullptr), - static_cast(nullptr)))>::value>::type> { + ::grpc::Status, decltype(SerializationTraits::Serialize( + internal::Example(), + internal::Example(), + internal::Example()))>::value>::type> { public: static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, bool* own_buf) { @@ -322,11 +326,10 @@ class CallOpSendMessage::MessageSerializer< template class CallOpSendMessage::MessageSerializer< M, typename std::enable_if::Serialize( - static_cast(*(static_cast(nullptr))), - static_cast<::grpc::ByteBuffer*>(nullptr), - static_cast(nullptr)))>::value>::type> { + ::grpc::Status, decltype(SerializationTraits::Serialize( + internal::Example(), + internal::Example<::grpc::ByteBuffer*>(), + internal::Example()))>::value>::type> { public: static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, bool* own_buf) { From c75ae78b0806f8ecea3ddfedbf4c2497331bbca5 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 11 Sep 2017 19:21:34 +0000 Subject: [PATCH 059/227] Add more Cython-layer tests I wrote these in the course of a bug hunt. I haven't yet caught and fixed the bug, but that's no reason not to check in perfectly good tests. :-) --- src/python/grpcio_tests/tests/tests.json | 2 + .../tests/unit/_cython/_common.py | 118 ++++++++++++++++ ...s_server_completion_queue_per_call_test.py | 131 ++++++++++++++++++ ...ges_single_server_completion_queue_test.py | 126 +++++++++++++++++ 4 files changed, 377 insertions(+) create mode 100644 src/python/grpcio_tests/tests/unit/_cython/_common.py create mode 100644 src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py create mode 100644 src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index d61297b9187..4c078e6c22b 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -26,6 +26,8 @@ "unit._credentials_test.CredentialsTest", "unit._cython._cancel_many_calls_test.CancelManyCallsTest", "unit._cython._channel_test.ChannelTest", + "unit._cython._no_messages_server_completion_queue_per_call_test.Test", + "unit._cython._no_messages_single_server_completion_queue_test.Test", "unit._cython._read_some_but_not_all_responses_test.ReadSomeButNotAllResponsesTest", "unit._cython.cygrpc_test.InsecureServerInsecureClient", "unit._cython.cygrpc_test.SecureServerSecureClient", diff --git a/src/python/grpcio_tests/tests/unit/_cython/_common.py b/src/python/grpcio_tests/tests/unit/_cython/_common.py new file mode 100644 index 00000000000..ac66d1db3d8 --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_cython/_common.py @@ -0,0 +1,118 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common utilities for tests of the Cython layer of gRPC Python.""" + +import collections +import threading + +from grpc._cython import cygrpc + +RPC_COUNT = 4000 + +INFINITE_FUTURE = cygrpc.Timespec(float('+inf')) +EMPTY_FLAGS = 0 + +INVOCATION_METADATA = cygrpc.Metadata( + (cygrpc.Metadatum(b'client-md-key', b'client-md-key'), + cygrpc.Metadatum(b'client-md-key-bin', b'\x00\x01' * 3000),)) + +INITIAL_METADATA = cygrpc.Metadata( + (cygrpc.Metadatum(b'server-initial-md-key', b'server-initial-md-value'), + cygrpc.Metadatum(b'server-initial-md-key-bin', b'\x00\x02' * 3000),)) + +TRAILING_METADATA = cygrpc.Metadata( + (cygrpc.Metadatum(b'server-trailing-md-key', b'server-trailing-md-value'), + cygrpc.Metadatum(b'server-trailing-md-key-bin', b'\x00\x03' * 3000),)) + + +class QueueDriver(object): + + def __init__(self, condition, completion_queue): + self._condition = condition + self._completion_queue = completion_queue + self._due = collections.defaultdict(int) + self._events = collections.defaultdict(list) + + def add_due(self, tags): + if not self._due: + + def in_thread(): + while True: + event = self._completion_queue.poll() + with self._condition: + self._events[event.tag].append(event) + self._due[event.tag] -= 1 + self._condition.notify_all() + if self._due[event.tag] <= 0: + self._due.pop(event.tag) + if not self._due: + return + + thread = threading.Thread(target=in_thread) + thread.start() + for tag in tags: + self._due[tag] += 1 + + def event_with_tag(self, tag): + with self._condition: + while True: + if self._events[tag]: + return self._events[tag].pop(0) + else: + self._condition.wait() + + +def execute_many_times(behavior): + return tuple(behavior() for _ in range(RPC_COUNT)) + + +class OperationResult( + collections.namedtuple('OperationResult', ( + 'start_batch_result', 'completion_type', 'success',))): + pass + + +SUCCESSFUL_OPERATION_RESULT = OperationResult( + cygrpc.CallError.ok, cygrpc.CompletionType.operation_complete, True) + + +class RpcTest(object): + + def setUp(self): + self.server_completion_queue = cygrpc.CompletionQueue() + self.server = cygrpc.Server(cygrpc.ChannelArgs([])) + self.server.register_completion_queue(self.server_completion_queue) + port = self.server.add_http2_port(b'[::]:0') + self.server.start() + self.channel = cygrpc.Channel('localhost:{}'.format(port).encode(), + cygrpc.ChannelArgs([])) + + self._server_shutdown_tag = 'server_shutdown_tag' + self.server_condition = threading.Condition() + self.server_driver = QueueDriver(self.server_condition, + self.server_completion_queue) + with self.server_condition: + self.server_driver.add_due({ + self._server_shutdown_tag, + }) + + self.client_condition = threading.Condition() + self.client_completion_queue = cygrpc.CompletionQueue() + self.client_driver = QueueDriver(self.client_condition, + self.client_completion_queue) + + def tearDown(self): + self.server.shutdown(self.server_completion_queue, + self._server_shutdown_tag) + self.server.cancel_all_calls() diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py new file mode 100644 index 00000000000..14cc66675c3 --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py @@ -0,0 +1,131 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test a corner-case at the level of the Cython API.""" + +import threading +import unittest + +from grpc._cython import cygrpc + +from tests.unit._cython import _common + + +class Test(_common.RpcTest, unittest.TestCase): + + def _do_rpcs(self): + server_call_condition = threading.Condition() + server_call_completion_queue = cygrpc.CompletionQueue() + server_call_driver = _common.QueueDriver(server_call_condition, + server_call_completion_queue) + + server_request_call_tag = 'server_request_call_tag' + server_send_initial_metadata_tag = 'server_send_initial_metadata_tag' + server_complete_rpc_tag = 'server_complete_rpc_tag' + + with self.server_condition: + server_request_call_start_batch_result = self.server.request_call( + server_call_completion_queue, self.server_completion_queue, + server_request_call_tag) + self.server_driver.add_due({ + server_request_call_tag, + }) + + client_call = self.channel.create_call( + None, _common.EMPTY_FLAGS, self.client_completion_queue, + b'/twinkies', None, _common.INFINITE_FUTURE) + client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag' + client_complete_rpc_tag = 'client_complete_rpc_tag' + with self.client_condition: + client_receive_initial_metadata_start_batch_result = ( + client_call.start_client_batch( + cygrpc.Operations([ + cygrpc.operation_receive_initial_metadata( + _common.EMPTY_FLAGS), + ]), client_receive_initial_metadata_tag)) + client_complete_rpc_start_batch_result = client_call.start_client_batch( + cygrpc.Operations([ + cygrpc.operation_send_initial_metadata( + _common.INVOCATION_METADATA, _common.EMPTY_FLAGS), + cygrpc.operation_send_close_from_client( + _common.EMPTY_FLAGS), + cygrpc.operation_receive_status_on_client( + _common.EMPTY_FLAGS), + ]), client_complete_rpc_tag) + self.client_driver.add_due({ + client_receive_initial_metadata_tag, + client_complete_rpc_tag, + }) + + server_request_call_event = self.server_driver.event_with_tag( + server_request_call_tag) + + with server_call_condition: + server_send_initial_metadata_start_batch_result = ( + server_request_call_event.operation_call.start_server_batch([ + cygrpc.operation_send_initial_metadata( + _common.INITIAL_METADATA, _common.EMPTY_FLAGS), + ], server_send_initial_metadata_tag)) + server_call_driver.add_due({ + server_send_initial_metadata_tag, + }) + server_send_initial_metadata_event = server_call_driver.event_with_tag( + server_send_initial_metadata_tag) + + with server_call_condition: + server_complete_rpc_start_batch_result = ( + server_request_call_event.operation_call.start_server_batch([ + cygrpc.operation_receive_close_on_server( + _common.EMPTY_FLAGS), + cygrpc.operation_send_status_from_server( + _common.TRAILING_METADATA, cygrpc.StatusCode.ok, + b'test details', _common.EMPTY_FLAGS), + ], server_complete_rpc_tag)) + server_call_driver.add_due({ + server_complete_rpc_tag, + }) + server_complete_rpc_event = server_call_driver.event_with_tag( + server_complete_rpc_tag) + + client_receive_initial_metadata_event = self.client_driver.event_with_tag( + client_receive_initial_metadata_tag) + client_complete_rpc_event = self.client_driver.event_with_tag( + client_complete_rpc_tag) + + return (_common.OperationResult(server_request_call_start_batch_result, + server_request_call_event.type, + server_request_call_event.success), + _common.OperationResult( + client_receive_initial_metadata_start_batch_result, + client_receive_initial_metadata_event.type, + client_receive_initial_metadata_event.success), + _common.OperationResult(client_complete_rpc_start_batch_result, + client_complete_rpc_event.type, + client_complete_rpc_event.success), + _common.OperationResult( + server_send_initial_metadata_start_batch_result, + server_send_initial_metadata_event.type, + server_send_initial_metadata_event.success), + _common.OperationResult(server_complete_rpc_start_batch_result, + server_complete_rpc_event.type, + server_complete_rpc_event.success),) + + def test_rpcs(self): + expecteds = [(_common.SUCCESSFUL_OPERATION_RESULT,) * + 5] * _common.RPC_COUNT + actuallys = _common.execute_many_times(self._do_rpcs) + self.assertSequenceEqual(expecteds, actuallys) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py new file mode 100644 index 00000000000..1e44bcc4dc6 --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py @@ -0,0 +1,126 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test a corner-case at the level of the Cython API.""" + +import threading +import unittest + +from grpc._cython import cygrpc + +from tests.unit._cython import _common + + +class Test(_common.RpcTest, unittest.TestCase): + + def _do_rpcs(self): + server_request_call_tag = 'server_request_call_tag' + server_send_initial_metadata_tag = 'server_send_initial_metadata_tag' + server_complete_rpc_tag = 'server_complete_rpc_tag' + + with self.server_condition: + server_request_call_start_batch_result = self.server.request_call( + self.server_completion_queue, self.server_completion_queue, + server_request_call_tag) + self.server_driver.add_due({ + server_request_call_tag, + }) + + client_call = self.channel.create_call( + None, _common.EMPTY_FLAGS, self.client_completion_queue, + b'/twinkies', None, _common.INFINITE_FUTURE) + client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag' + client_complete_rpc_tag = 'client_complete_rpc_tag' + with self.client_condition: + client_receive_initial_metadata_start_batch_result = ( + client_call.start_client_batch( + cygrpc.Operations([ + cygrpc.operation_receive_initial_metadata( + _common.EMPTY_FLAGS), + ]), client_receive_initial_metadata_tag)) + client_complete_rpc_start_batch_result = client_call.start_client_batch( + cygrpc.Operations([ + cygrpc.operation_send_initial_metadata( + _common.INVOCATION_METADATA, _common.EMPTY_FLAGS), + cygrpc.operation_send_close_from_client( + _common.EMPTY_FLAGS), + cygrpc.operation_receive_status_on_client( + _common.EMPTY_FLAGS), + ]), client_complete_rpc_tag) + self.client_driver.add_due({ + client_receive_initial_metadata_tag, + client_complete_rpc_tag, + }) + + server_request_call_event = self.server_driver.event_with_tag( + server_request_call_tag) + + with self.server_condition: + server_send_initial_metadata_start_batch_result = ( + server_request_call_event.operation_call.start_server_batch([ + cygrpc.operation_send_initial_metadata( + _common.INITIAL_METADATA, _common.EMPTY_FLAGS), + ], server_send_initial_metadata_tag)) + self.server_driver.add_due({ + server_send_initial_metadata_tag, + }) + server_send_initial_metadata_event = self.server_driver.event_with_tag( + server_send_initial_metadata_tag) + + with self.server_condition: + server_complete_rpc_start_batch_result = ( + server_request_call_event.operation_call.start_server_batch([ + cygrpc.operation_receive_close_on_server( + _common.EMPTY_FLAGS), + cygrpc.operation_send_status_from_server( + _common.TRAILING_METADATA, cygrpc.StatusCode.ok, + b'test details', _common.EMPTY_FLAGS), + ], server_complete_rpc_tag)) + self.server_driver.add_due({ + server_complete_rpc_tag, + }) + server_complete_rpc_event = self.server_driver.event_with_tag( + server_complete_rpc_tag) + + client_receive_initial_metadata_event = self.client_driver.event_with_tag( + client_receive_initial_metadata_tag) + client_complete_rpc_event = self.client_driver.event_with_tag( + client_complete_rpc_tag) + + return (_common.OperationResult(server_request_call_start_batch_result, + server_request_call_event.type, + server_request_call_event.success), + _common.OperationResult( + client_receive_initial_metadata_start_batch_result, + client_receive_initial_metadata_event.type, + client_receive_initial_metadata_event.success), + _common.OperationResult(client_complete_rpc_start_batch_result, + client_complete_rpc_event.type, + client_complete_rpc_event.success), + _common.OperationResult( + server_send_initial_metadata_start_batch_result, + server_send_initial_metadata_event.type, + server_send_initial_metadata_event.success), + _common.OperationResult(server_complete_rpc_start_batch_result, + server_complete_rpc_event.type, + server_complete_rpc_event.success),) + + def test_rpcs(self): + expecteds = [(_common.SUCCESSFUL_OPERATION_RESULT,) * + 5] * _common.RPC_COUNT + actuallys = _common.execute_many_times(self._do_rpcs) + self.assertSequenceEqual(expecteds, actuallys) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From afa47005015295e65b932403142938f9bb9b72fc Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 11 Sep 2017 12:22:49 -0700 Subject: [PATCH 060/227] Use Apple internal type for timeout --- src/objective-c/GRPCClient/GRPCCall.h | 2 +- src/objective-c/GRPCClient/private/GRPCChannel.h | 2 +- src/objective-c/GRPCClient/private/GRPCChannel.m | 4 ++-- src/objective-c/GRPCClient/private/GRPCHost.h | 2 +- src/objective-c/GRPCClient/private/GRPCHost.m | 2 +- src/objective-c/GRPCClient/private/GRPCWrappedCall.h | 2 +- src/objective-c/GRPCClient/private/GRPCWrappedCall.m | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 26c920b973a..03552779d87 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -172,7 +172,7 @@ extern id const kGRPCTrailersKey; /** * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. */ -@property UInt64 timeout; +@property NSTimeInterval timeout; /** * The container of the request headers of an RPC conforms to this protocol, which is a subset of diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h index 9dff86c271f..d37182f754c 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.h +++ b/src/objective-c/GRPCClient/private/GRPCChannel.h @@ -63,6 +63,6 @@ struct grpc_channel_credentials; - (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path serverName:(nonnull NSString *)serverName - timeout:(UInt64)timeout + timeout:(NSTimeInterval)timeout completionQueue:(nonnull GRPCCompletionQueue *)queue; @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 39b4e6c1a7e..2eb02115504 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -182,7 +182,7 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { - (grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - timeout:(UInt64)timeout + timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue { grpc_slice host_slice; if (serverName) { @@ -193,7 +193,7 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { gpr_inf_future(GPR_CLOCK_REALTIME) : gpr_time_add( gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_millis(timeout, GPR_TIMESPAN)); + gpr_time_from_millis((int64_t)(timeout * 1000), GPR_TIMESPAN)); grpc_call *call = grpc_channel_create_call(_unmanagedChannel, NULL, GRPC_PROPAGATE_DEFAULTS, queue.unmanagedQueue, diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index a9b3032c9c6..58171211b0e 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -55,7 +55,7 @@ struct grpc_channel_credentials; /** Create a grpc_call object to the provided path on this host. */ - (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - timeout:(UInt64)timeout + timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue; // TODO: There's a race when a new RPC is coming through just as an existing one is getting diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index a0791827db3..f73e9cbc507 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -121,7 +121,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - timeout:(UInt64)timeout + timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue { GRPCChannel *channel; // This is racing -[GRPCHost disconnect]. diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index c0a6c71c72f..1cd9da8f3ea 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -77,7 +77,7 @@ - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName path:(NSString *)path - timeout:(UInt64)timeout NS_DESIGNATED_INITIALIZER; + timeout:(NSTimeInterval)timeout NS_DESIGNATED_INITIALIZER; - (void)startBatchWithOperations:(NSArray *)ops errorHandler:(void(^)())errorHandler; diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index 7366f319312..b0b1223b64c 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -244,7 +244,7 @@ - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName path:(NSString *)path - timeout:(UInt64)timeout { + timeout:(NSTimeInterval)timeout { if (!path || !host) { [NSException raise:NSInvalidArgumentException format:@"path and host cannot be nil."]; From 54f3c2eefe65a2f5586b8c1ee8b7c9981502fe85 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 11 Sep 2017 13:12:04 -0700 Subject: [PATCH 061/227] Automatically fetch schema from BigQuery --- tools/profiling/microbenchmarks/bm2bq.py | 43 +++++------------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 0797f3e21d3..6c5040de56d 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -22,42 +22,15 @@ import sys import json import csv import bm_json +import json +import subprocess + +columns = [] -columns = [ - ('jenkins_build', 'integer'), - ('jenkins_job', 'string'), - ('date', 'timestamp'), - ('cpu_scaling_enabled', 'boolean'), - ('num_cpus', 'integer'), - ('mhz_per_cpu', 'integer'), - ('library_build_type', 'string'), - ('name', 'string'), - ('fixture', 'string'), - ('client_mutator', 'string'), - ('server_mutator', 'string'), - ('request_size', 'integer'), - ('response_size', 'integer'), - ('request_count', 'integer'), - ('iterations', 'integer'), - ('time_unit', 'string'), - ('real_time', 'integer'), - ('cpu_time', 'integer'), - ('bytes_per_second', 'float'), - ('allocs_per_iteration', 'float'), - ('locks_per_iteration', 'float'), - ('writes_per_iteration', 'float'), - ('bandwidth_kilobits', 'integer'), - ('cli_transport_stalls_per_iteration', 'float'), - ('cli_stream_stalls_per_iteration', 'float'), - ('svr_transport_stalls_per_iteration', 'float'), - ('svr_stream_stalls_per_iteration', 'float'), - ('atm_cas_per_iteration', 'float'), - ('atm_add_per_iteration', 'float'), - ('end_of_stream', 'boolean'), - ('header_bytes_per_iteration', 'float'), - ('framing_bytes_per_iteration', 'float'), - ('nows_per_iteration', 'float'), -] +for row in json.loads( + subprocess.check_output([ + 'bq','--format=json','show','microbenchmarks.microbenchmarks']))['schema']['fields']: + columns.append((row['name'], row['type'])) SANITIZE = { 'integer': int, From 4b7fe94a9728686065a0cd4275ca64e24dc32c3c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 11 Sep 2017 13:30:13 -0700 Subject: [PATCH 062/227] Automatically generate BigQuery schema details for all counters --- src/core/lib/debug/stats_data_bq_schema.sql | 33 +++++++++++++++++++++ tools/codegen/core/gen_stats_data.py | 6 ++++ 2 files changed, 39 insertions(+) create mode 100644 src/core/lib/debug/stats_data_bq_schema.sql diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql new file mode 100644 index 00000000000..b669555fa0d --- /dev/null +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -0,0 +1,33 @@ +client_calls_created_per_iteration:INTEGER, +server_calls_created_per_iteration:INTEGER, +syscall_poll_per_iteration:INTEGER, +syscall_wait_per_iteration:INTEGER, +histogram_slow_lookups_per_iteration:INTEGER, +syscall_write_per_iteration:INTEGER, +syscall_read_per_iteration:INTEGER, +tcp_backup_pollers_created_per_iteration:INTEGER, +tcp_backup_poller_polls_per_iteration:INTEGER, +http2_op_batches_per_iteration:INTEGER, +http2_op_cancel_per_iteration:INTEGER, +http2_op_send_initial_metadata_per_iteration:INTEGER, +http2_op_send_message_per_iteration:INTEGER, +http2_op_send_trailing_metadata_per_iteration:INTEGER, +http2_op_recv_initial_metadata_per_iteration:INTEGER, +http2_op_recv_message_per_iteration:INTEGER, +http2_op_recv_trailing_metadata_per_iteration:INTEGER, +http2_settings_writes_per_iteration:INTEGER, +http2_pings_sent_per_iteration:INTEGER, +http2_writes_begun_per_iteration:INTEGER, +http2_writes_offloaded_per_iteration:INTEGER, +http2_writes_continued_per_iteration:INTEGER, +http2_partial_writes_per_iteration:INTEGER, +combiner_locks_initiated_per_iteration:INTEGER, +combiner_locks_scheduled_items_per_iteration:INTEGER, +combiner_locks_scheduled_final_items_per_iteration:INTEGER, +combiner_locks_offloaded_per_iteration:INTEGER, +executor_scheduled_short_items_per_iteration:INTEGER, +executor_scheduled_long_items_per_iteration:INTEGER, +executor_scheduled_to_self_per_iteration:INTEGER, +executor_wakeup_initiated_per_iteration:INTEGER, +executor_queue_drained_per_iteration:INTEGER, +executor_push_retries_per_iteration:INTEGER diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8e4ef594af2..cb01321ed3a 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -313,3 +313,9 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % ( len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + +with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: + columns = [] + for counter in inst_map['Counter']: + columns.append(('%s_per_iteration' % counter.name, 'INTEGER')) + print >>S, ',\n'.join('%s:%s' % x for x in columns) From 8e87c74fe95e445dac4dc66d4f464d572fcbaa5e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 11 Sep 2017 14:06:09 -0700 Subject: [PATCH 063/227] Fix compilation for C++ --- test/cpp/microbenchmarks/bm_cq_multiple_threads.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index 9d71d3990d4..5c9405f583d 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -59,7 +59,8 @@ static void pollset_destroy(grpc_exec_ctx* exec_ctx, grpc_pollset* ps) { gpr_mu_destroy(&ps->mu); } -static grpc_error* pollset_kick(grpc_pollset* p, grpc_pollset_worker* worker) { +static grpc_error* pollset_kick(grpc_exec_ctx* exec_ctx, grpc_pollset* p, + grpc_pollset_worker* worker) { return GRPC_ERROR_NONE; } From 4433973242e21db18291263a5b41ee292002eb6f Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 11 Sep 2017 14:26:34 -0700 Subject: [PATCH 064/227] Address review comments --- include/grpc++/impl/codegen/byte_buffer.h | 2 +- include/grpc++/impl/codegen/call.h | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h index e6231ca2434..87d390c688a 100644 --- a/include/grpc++/impl/codegen/byte_buffer.h +++ b/include/grpc++/impl/codegen/byte_buffer.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015 gRPC authors. + * Copyright 2017 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 0bf33cb88f1..0652d69d12f 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -361,11 +361,9 @@ class MessageDeserializer; template class MessageDeserializer< M, typename std::enable_if::Deserialize( - static_cast( - *(static_cast(nullptr))), - static_cast(nullptr)))>::value>::type> { + ::grpc::Status, decltype(SerializationTraits::Deserialize( + internal::Example(), + internal::Example()))>::value>::type> { public: static Status Deserialize(const ByteBuffer& bbuf, M* message) { return SerializationTraits::Deserialize(bbuf, message); @@ -376,8 +374,8 @@ template class MessageDeserializer< M, typename std::enable_if::Deserialize( - static_cast(nullptr), - static_cast(nullptr)))>::value>::type> { + internal::Example(), + internal::Example()))>::value>::type> { public: static Status Deserialize(const ByteBuffer& bbuf, M* message) { return SerializationTraits::Deserialize( From 20104ce596b8ccbe4cd167e245654f56e8d86edc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 11 Sep 2017 15:26:57 -0700 Subject: [PATCH 065/227] Add a clamp to remove ubsan failure --- src/core/ext/transport/chttp2/transport/flow_control.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/flow_control.c b/src/core/ext/transport/chttp2/transport/flow_control.c index cec99f6fb69..39aa521029b 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.c +++ b/src/core/ext/transport/chttp2/transport/flow_control.c @@ -483,7 +483,8 @@ grpc_chttp2_flowctl_action grpc_chttp2_flowctl_get_bdp_action( if (grpc_bdp_estimator_get_bw(&tfc->bdp_estimator, &bw_dbl)) { // we target the max of BDP or bandwidth in microseconds. int32_t frame_size = (int32_t)GPR_CLAMP( - GPR_MAX((int32_t)bw_dbl / 1000, bdp), 16384, 16777215); + GPR_MAX((int32_t)GPR_CLAMP(bw_dbl, 0, INT_MAX) / 1000, bdp), 16384, + 16777215); grpc_chttp2_flowctl_urgency frame_size_urgency = delta_is_significant( tfc, frame_size, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE); if (frame_size_urgency != GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED) { From f6e61bf7686bfce40d4f3912fd9eeb2ba6987e62 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 11 Sep 2017 15:48:51 -0700 Subject: [PATCH 066/227] Add comment for timeout; assert on negative timeout value; fix timeout test --- src/objective-c/GRPCClient/GRPCCall.h | 4 +++- src/objective-c/GRPCClient/private/GRPCChannel.m | 4 ++++ src/objective-c/tests/GRPCClientTests.m | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 03552779d87..3b0ef56c3af 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -170,7 +170,9 @@ extern id const kGRPCTrailersKey; @property (atomic, copy, readwrite) NSString *serverName; /** - * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. + * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. If set to + * positive, the gRPC call returns with status GRPCErrorCodeDeadlineExceeded. A negative value is + * not allowed. */ @property NSTimeInterval timeout; diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 2eb02115504..b78b14f2afd 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -184,6 +184,10 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { serverName:(NSString *)serverName timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue { + GPR_ASSERT(timeout >= 0); + if (timeout < 0) { + timeout = 0; + } grpc_slice host_slice; if (serverName) { host_slice = grpc_slice_from_copied_string(serverName.UTF8String); diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index d050319533a..4be3d1c5e87 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -437,7 +437,7 @@ static GRPCProtoMethod *kUnaryCallMethod; [completion fulfill]; }]; - call.timeout = 1; + call.timeout = 0.001; [call startWithWriteable:responsesWriteable]; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; From 41388890860f29cd6e147bde163f57322a408a2f Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 11 Sep 2017 22:47:38 +0000 Subject: [PATCH 067/227] Ignore all Python 3 virtual environments How soon could Python 4 happen anyway? :-P --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5e38f5fa01b..5ccad2e4f2f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ htmlcov/ dist/ *.egg py27/ -py34/ +py3[0-9]*/ # Node installation output node_modules From a6721a0979029a2492631e075946556ce7c0eaf2 Mon Sep 17 00:00:00 2001 From: "K.K. Yap" Date: Mon, 11 Sep 2017 16:20:04 -0700 Subject: [PATCH 068/227] Comment that IPv6 Any accepts both IPv4 and IPv6 connections. --- include/grpc++/server_builder.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index 21ae70d13ad..bbf45b3e74c 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -136,8 +136,10 @@ class ServerBuilder { /// It can be invoked multiple times. /// /// \param addr_uri The address to try to bind to the server in URI form. If - /// the scheme name is omitted, "dns:///" is assumed. Valid values include - /// dns:///localhost:1234, / 192.168.1.1:31416, dns:///[::1]:27182, etc.). + /// the scheme name is omitted, "dns:///" is assumed. To bind to any address, + /// please use IPv6 any, i.e., [::]:, which also accepts IPv4 + /// connections. Valid values include dns:///localhost:1234, / + /// 192.168.1.1:31416, dns:///[::1]:27182, etc.). /// \params creds The credentials associated with the server. /// \param selected_port[out] If not `nullptr`, gets populated with the port /// number bound to the \a grpc::Server for the corresponding endpoint after From 6d11e9092667565a181d051932e04c50ab1db099 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 23 Aug 2017 11:43:40 -0700 Subject: [PATCH 069/227] Add Kokoro test filtering for MacOS/Windows --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 7 +++++++ .../internal_ci/helper_scripts/prepare_build_windows.bat | 8 ++++++++ tools/run_tests/python_utils/filter_pull_request_tests.py | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index bd250957f43..2a1225683de 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -32,6 +32,13 @@ ulimit -a sudo easy_install --upgrade google-api-python-client export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json +# If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then + brew install jq + ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) + export RUN_TESTS_FLAGS="$RUN_TESTS_FLAGS --filter_pr_tests --base_branch origin/$ghprbTargetBranch" +fi + # required to build protobuf brew install gflags diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index 69e087e1db1..f625755db9f 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -18,6 +18,14 @@ set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% bash tools/internal_ci/helper_scripts/gen_report_index.sh +@rem If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests +if defined KOKORO_GITHUB_PULL_REQUEST_NUMBER if defined RUN_TESTS_FLAGS ( + chocolatey install -y jq + for /f "usebackq delims=" %%x in (`curl -s https://api.github.com/repos/grpc/grpc/pulls/%KOKORO_GITHUB_PULL_REQUEST_NUMBER% ^| jq -r .base.ref`) do ( + set RUN_TESTS_FLAGS=%RUN_TESTS_FLAGS% --filter_pr_tests --base_branch origin/%%x + ) +) + @rem Update DNS settings to: @rem 1. allow resolving metadata.google.internal hostname @rem 2. make fetching default GCE credential by oauth2client work diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 032c0701e2e..4ad981237b7 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -128,7 +128,7 @@ def _get_changed_files(base_branch): # Get file changes between branch and merge-base of specified branch # Not combined to be Windows friendly base_commit = check_output(["git", "merge-base", base_branch, "HEAD"]).rstrip() - return check_output(["git", "diff", base_commit, "--name-only"]).splitlines() + return check_output(["git", "diff", base_commit, "--name-only", "HEAD"]).splitlines() def _can_skip_tests(file_names, triggers): From bfcbad3b86c7912968dc8e64f2121c920dad4dfb Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Tue, 12 Sep 2017 11:52:16 +0800 Subject: [PATCH 070/227] cmake: fix cares configuration --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5131bf39b47..187c7002335 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,7 +134,8 @@ if("${gRPC_CARES_PROVIDER}" STREQUAL "module") if(NOT CARES_ROOT_DIR) set(CARES_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares) endif() - set(CARES_STATIC ON) + set(CARES_SHARED OFF CACHE BOOL "disable shared library") + set(CARES_STATIC ON CACHE BOOL "link cares statically") set(CARES_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares") add_subdirectory(third_party/cares/cares) if(TARGET c-ares) From 4b7ef4d52546e05a1b33aac1eeb218409c4b0a3a Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 11 Sep 2017 23:09:22 -0700 Subject: [PATCH 071/227] Build a wall and make 'u' pay for it --- src/core/lib/iomgr/ev_epoll1_linux.c | 143 +++++++++++++-------------- 1 file changed, 71 insertions(+), 72 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 5bc7e878de0..4efd705fa81 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -160,18 +160,18 @@ struct grpc_pollset_worker { (worker)->kick_state_mutator = __LINE__; \ } while (false) -#define MAX_NEIGHBOURHOODS 1024 +#define MAX_NEIGHBORHOODS 1024 -typedef struct pollset_neighbourhood { +typedef struct pollset_neighborhood { gpr_mu mu; grpc_pollset *active_root; char pad[GPR_CACHELINE_SIZE]; -} pollset_neighbourhood; +} pollset_neighborhood; struct grpc_pollset { gpr_mu mu; - pollset_neighbourhood *neighbourhood; - bool reassigning_neighbourhood; + pollset_neighborhood *neighborhood; + bool reassigning_neighborhood; grpc_pollset_worker *root_worker; bool kicked_without_poller; @@ -384,8 +384,8 @@ GPR_TLS_DECL(g_current_thread_worker); /* The designated poller */ static gpr_atm g_active_poller; -static pollset_neighbourhood *g_neighbourhoods; -static size_t g_num_neighbourhoods; +static pollset_neighborhood *g_neighborhoods; +static size_t g_num_neighborhoods; /* Return true if first in list */ static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { @@ -424,8 +424,8 @@ static worker_remove_result worker_remove(grpc_pollset *pollset, } } -static size_t choose_neighbourhood(void) { - return (size_t)gpr_cpu_current_cpu() % g_num_neighbourhoods; +static size_t choose_neighborhood(void) { + return (size_t)gpr_cpu_current_cpu() % g_num_neighborhoods; } static grpc_error *pollset_global_init(void) { @@ -441,11 +441,11 @@ static grpc_error *pollset_global_init(void) { &ev) != 0) { return GRPC_OS_ERROR(errno, "epoll_ctl"); } - g_num_neighbourhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBOURHOODS); - g_neighbourhoods = (pollset_neighbourhood *)gpr_zalloc( - sizeof(*g_neighbourhoods) * g_num_neighbourhoods); - for (size_t i = 0; i < g_num_neighbourhoods; i++) { - gpr_mu_init(&g_neighbourhoods[i].mu); + g_num_neighborhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBORHOODS); + g_neighborhoods = (pollset_neighborhood *)gpr_zalloc( + sizeof(*g_neighborhoods) * g_num_neighborhoods); + for (size_t i = 0; i < g_num_neighborhoods; i++) { + gpr_mu_init(&g_neighborhoods[i].mu); } return GRPC_ERROR_NONE; } @@ -454,17 +454,17 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); - for (size_t i = 0; i < g_num_neighbourhoods; i++) { - gpr_mu_destroy(&g_neighbourhoods[i].mu); + for (size_t i = 0; i < g_num_neighborhoods; i++) { + gpr_mu_destroy(&g_neighborhoods[i].mu); } - gpr_free(g_neighbourhoods); + gpr_free(g_neighborhoods); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; - pollset->reassigning_neighbourhood = false; + pollset->neighborhood = &g_neighborhoods[choose_neighborhood()]; + pollset->reassigning_neighborhood = false; pollset->root_worker = NULL; pollset->kicked_without_poller = false; pollset->seen_inactive = true; @@ -477,26 +477,26 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + pollset_neighborhood *neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); - retry_lock_neighbourhood: - gpr_mu_lock(&neighbourhood->mu); + retry_lock_neighborhood: + gpr_mu_lock(&neighborhood->mu); gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { - if (pollset->neighbourhood != neighbourhood) { - gpr_mu_unlock(&neighbourhood->mu); - neighbourhood = pollset->neighbourhood; + if (pollset->neighborhood != neighborhood) { + gpr_mu_unlock(&neighborhood->mu); + neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); - goto retry_lock_neighbourhood; + goto retry_lock_neighborhood; } pollset->prev->next = pollset->next; pollset->next->prev = pollset->prev; - if (pollset == pollset->neighbourhood->active_root) { - pollset->neighbourhood->active_root = + if (pollset == pollset->neighborhood->active_root) { + pollset->neighborhood->active_root = pollset->next == pollset ? NULL : pollset->next; } } - gpr_mu_unlock(&pollset->neighbourhood->mu); + gpr_mu_unlock(&pollset->neighborhood->mu); } gpr_mu_unlock(&pollset->mu); gpr_mu_destroy(&pollset->mu); @@ -675,16 +675,16 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, // pollset has been observed to be inactive, we need to move back to the // active list bool is_reassigning = false; - if (!pollset->reassigning_neighbourhood) { + if (!pollset->reassigning_neighborhood) { is_reassigning = true; - pollset->reassigning_neighbourhood = true; - pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; + pollset->reassigning_neighborhood = true; + pollset->neighborhood = &g_neighborhoods[choose_neighborhood()]; } - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + pollset_neighborhood *neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); // pollset unlocked: state may change (even worker->kick_state) - retry_lock_neighbourhood: - gpr_mu_lock(&neighbourhood->mu); + retry_lock_neighborhood: + gpr_mu_lock(&neighborhood->mu); gpr_mu_lock(&pollset->mu); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d", @@ -692,17 +692,17 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, is_reassigning); } if (pollset->seen_inactive) { - if (neighbourhood != pollset->neighbourhood) { - gpr_mu_unlock(&neighbourhood->mu); - neighbourhood = pollset->neighbourhood; + if (neighborhood != pollset->neighborhood) { + gpr_mu_unlock(&neighborhood->mu); + neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); - goto retry_lock_neighbourhood; + goto retry_lock_neighborhood; } /* In the brief time we released the pollset locks above, the worker MAY have been kicked. In this case, the worker should get out of this pollset ASAP and hence this should neither add the pollset to - neighbourhood nor mark the pollset as active. + neighborhood nor mark the pollset as active. On a side note, the only way a worker's kick state could have changed at this point is if it were "kicked specifically". Since the worker has @@ -710,25 +710,25 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, not visible in the "kick any" path yet */ if (worker->kick_state == UNKICKED) { pollset->seen_inactive = false; - if (neighbourhood->active_root == NULL) { - neighbourhood->active_root = pollset->next = pollset->prev = pollset; + if (neighborhood->active_root == NULL) { + neighborhood->active_root = pollset->next = pollset->prev = pollset; /* Make this the designated poller if there isn't one already */ if (worker->kick_state == UNKICKED && gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { SET_KICK_STATE(worker, DESIGNATED_POLLER); } } else { - pollset->next = neighbourhood->active_root; + pollset->next = neighborhood->active_root; pollset->prev = pollset->next->prev; pollset->next->prev = pollset->prev->next = pollset; } } } if (is_reassigning) { - GPR_ASSERT(pollset->reassigning_neighbourhood); - pollset->reassigning_neighbourhood = false; + GPR_ASSERT(pollset->reassigning_neighborhood); + pollset->reassigning_neighborhood = false; } - gpr_mu_unlock(&neighbourhood->mu); + gpr_mu_unlock(&neighborhood->mu); } worker_insert(pollset, worker); @@ -763,7 +763,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } /* We release pollset lock in this function at a couple of places: - * 1. Briefly when assigning pollset to a neighbourhood + * 1. Briefly when assigning pollset to a neighborhood * 2. When doing gpr_cv_wait() * It is possible that 'kicked_without_poller' was set to true during (1) and * 'shutting_down' is set to true during (1) or (2). If either of them is @@ -781,12 +781,12 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down; } -static bool check_neighbourhood_for_available_poller( - pollset_neighbourhood *neighbourhood) { - GPR_TIMER_BEGIN("check_neighbourhood_for_available_poller", 0); +static bool check_neighborhood_for_available_poller( + pollset_neighborhood *neighborhood) { + GPR_TIMER_BEGIN("check_neighborhood_for_available_poller", 0); bool found_worker = false; do { - grpc_pollset *inspect = neighbourhood->active_root; + grpc_pollset *inspect = neighborhood->active_root; if (inspect == NULL) { break; } @@ -831,8 +831,8 @@ static bool check_neighbourhood_for_available_poller( gpr_log(GPR_DEBUG, " .. mark pollset %p inactive", inspect); } inspect->seen_inactive = true; - if (inspect == neighbourhood->active_root) { - neighbourhood->active_root = + if (inspect == neighborhood->active_root) { + neighborhood->active_root = inspect->next == inspect ? NULL : inspect->next; } inspect->next->prev = inspect->prev; @@ -841,7 +841,7 @@ static bool check_neighbourhood_for_available_poller( } gpr_mu_unlock(&inspect->mu); } while (!found_worker); - GPR_TIMER_END("check_neighbourhood_for_available_poller", 0); + GPR_TIMER_END("check_neighborhood_for_available_poller", 0); return found_worker; } @@ -873,32 +873,31 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } else { gpr_atm_no_barrier_store(&g_active_poller, 0); - size_t poller_neighbourhood_idx = - (size_t)(pollset->neighbourhood - g_neighbourhoods); + size_t poller_neighborhood_idx = + (size_t)(pollset->neighborhood - g_neighborhoods); gpr_mu_unlock(&pollset->mu); bool found_worker = false; - bool scan_state[MAX_NEIGHBOURHOODS]; - for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { - pollset_neighbourhood *neighbourhood = - &g_neighbourhoods[(poller_neighbourhood_idx + i) % - g_num_neighbourhoods]; - if (gpr_mu_trylock(&neighbourhood->mu)) { - found_worker = - check_neighbourhood_for_available_poller(neighbourhood); - gpr_mu_unlock(&neighbourhood->mu); + bool scan_state[MAX_NEIGHBORHOODS]; + for (size_t i = 0; !found_worker && i < g_num_neighborhoods; i++) { + pollset_neighborhood *neighborhood = + &g_neighborhoods[(poller_neighborhood_idx + i) % + g_num_neighborhoods]; + if (gpr_mu_trylock(&neighborhood->mu)) { + found_worker = check_neighborhood_for_available_poller(neighborhood); + gpr_mu_unlock(&neighborhood->mu); scan_state[i] = true; } else { scan_state[i] = false; } } - for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + for (size_t i = 0; !found_worker && i < g_num_neighborhoods; i++) { if (scan_state[i]) continue; - pollset_neighbourhood *neighbourhood = - &g_neighbourhoods[(poller_neighbourhood_idx + i) % - g_num_neighbourhoods]; - gpr_mu_lock(&neighbourhood->mu); - found_worker = check_neighbourhood_for_available_poller(neighbourhood); - gpr_mu_unlock(&neighbourhood->mu); + pollset_neighborhood *neighborhood = + &g_neighborhoods[(poller_neighborhood_idx + i) % + g_num_neighborhoods]; + gpr_mu_lock(&neighborhood->mu); + found_worker = check_neighborhood_for_available_poller(neighborhood); + gpr_mu_unlock(&neighborhood->mu); } grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); From 4fa33649753f64283c82f73f5ca59daaa8895671 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 12 Sep 2017 12:56:45 +0200 Subject: [PATCH 072/227] bump version to 1.6.1 --- BUILD | 2 +- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 31 files changed, 36 insertions(+), 36 deletions(-) diff --git a/BUILD b/BUILD index 4d1121a79d5..cbd7e46b970 100644 --- a/BUILD +++ b/BUILD @@ -38,7 +38,7 @@ g_stands_for = "garcia" core_version = "4.0.0" -version = "1.6.0" +version = "1.6.1" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index ebbaa6a9ec7..5225c6e34e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.6.0") +set(PACKAGE_VERSION "1.6.1") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 801f8a7d6ed..5feff15071e 100644 --- a/Makefile +++ b/Makefile @@ -411,8 +411,8 @@ Q = @ endif CORE_VERSION = 4.0.0 -CPP_VERSION = 1.6.0 -CSHARP_VERSION = 1.6.0 +CPP_VERSION = 1.6.1 +CSHARP_VERSION = 1.6.1 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 723c8fe4460..954d6a3ecd7 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 4.0.0 g_stands_for: garcia - version: 1.6.0 + version: 1.6.1 filegroups: - name: census public_headers: diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 222d3e4f0f5..1b90beffc46 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.6.0' + version = '1.6.1' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 2d2f31d3239..f060274b5f7 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.6.0' + version = '1.6.1' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 38a159404fd..76bf4004248 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.6.0' + version = '1.6.1' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 12591cda336..3c310c8cd9d 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.6.0' + version = '1.6.1' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.json b/package.json index 94e6391a8d8..dd383548f38 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.6.0", + "version": "1.6.1", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "https://grpc.io/", diff --git a/package.xml b/package.xml index 30018620dd7..0735c02944c 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2017-08-24 - 1.6.0 - 1.6.0 + 1.6.1 + 1.6.1 beta diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index c65185c77ee..35c1ec6f5e7 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.6.0"; } +grpc::string Version() { return "1.6.1"; } } diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 23f6e409091..616f390f888 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.6.0 + 1.6.1 3.3.0 diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index d402c323c1f..52f8e2642aa 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "1.6.0.0"; + public const string CurrentAssemblyFileVersion = "1.6.1.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.6.0"; + public const string CurrentVersion = "1.6.1"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index b7aa22b2928..a6581ac7336 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.6.0 +set VERSION=1.6.1 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 455b930525b..76d6544c07b 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -39,7 +39,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.6.0" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.6.0" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.6.1" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.6.1" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 66ab2a292fe..92e748e72f0 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.6.0", + "version": "1.6.1", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.6.0", + "grpc": "^1.6.1", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index b0b1e8679a5..023cb1e4640 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.6.0", + "version": "1.6.1", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "https://grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 0e180a47113..32f6c39e6cb 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.6.0' + v = '1.6.1' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 35215375787..64e08f9151b 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -23,4 +23,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.6.0" +#define GRPC_OBJC_VERSION_STRING @"1.6.1" diff --git a/src/php/composer.json b/src/php/composer.json index 5d492ccb86d..df3dff39228 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.6.0", + "version": "1.6.1", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 1a46570117f..67f0c4b5ee8 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.6.0" +#define PHP_GRPC_VERSION "1.6.1" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 28f5a1a8a26..8707a5a9a62 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.6.0""" +__version__ = """1.6.1""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index ce35bda91c4..5a76a66e6cc 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index fb5c5f5f651..41f7e344baf 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 73e7d2e4977..63865b1bb37 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 99510bbe092..ea0278c7e52 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 66834d20e16..680e5d1454d 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.6.0' + VERSION = '1.6.1' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 73fa15f85f8..2d2db885756 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.6.0' + VERSION = '1.6.1' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index db3b39ae683..0630445fd48 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index f9225688ad6..cefce74f98e 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.6.0 +PROJECT_NUMBER = 1.6.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 1b41266c715..c9bdd1454d6 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.6.0 +PROJECT_NUMBER = 1.6.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 29828c58d33fbbc6503784f0d03064ced6a22841 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 12 Sep 2017 16:42:55 +0200 Subject: [PATCH 073/227] differentiate run_tests suite runtime --- tools/run_tests/run_tests_matrix.py | 50 ++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 00680b02d3c..957e7b569e2 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -29,9 +29,11 @@ from python_utils.filter_pull_request_tests import filter_tests _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) +_DEFAULT_RUNTESTS_TIMEOUT = 1*60*60 + # Set the timeout high to allow enough time for sanitizers and pre-building # clang docker. -_RUNTESTS_TIMEOUT = 4*60*60 +_CPP_RUNTESTS_TIMEOUT = 4*60*60 # Number of jobs assigned to each run_tests.py instance _DEFAULT_INNER_JOBS = 2 @@ -51,8 +53,11 @@ def _report_filename_internal_ci(name): def _docker_jobspec(name, runtests_args=[], runtests_envs={}, - inner_jobs=_DEFAULT_INNER_JOBS): + inner_jobs=_DEFAULT_INNER_JOBS, + timeout_seconds=None): """Run a single instance of run_tests.py in a docker container""" + if not timeout_seconds: + timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT test_job = jobset.JobSpec( cmdline=['python', 'tools/run_tests/run_tests.py', '--use_docker', @@ -62,15 +67,18 @@ def _docker_jobspec(name, runtests_args=[], runtests_envs={}, '--report_suite_name', '%s' % name] + runtests_args, environ=runtests_envs, shortname='run_tests_%s' % name, - timeout_seconds=_RUNTESTS_TIMEOUT) + timeout_seconds=timeout_seconds) return test_job def _workspace_jobspec(name, runtests_args=[], workspace_name=None, - runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS): + runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS, + timeout_seconds=None): """Run a single instance of run_tests.py in a separate workspace""" if not workspace_name: workspace_name = 'workspace_%s' % name + if not timeout_seconds: + timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT env = {'WORKSPACE_NAME': workspace_name} env.update(runtests_envs) test_job = jobset.JobSpec( @@ -82,14 +90,15 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None, '--report_suite_name', '%s' % name] + runtests_args, environ=env, shortname='run_tests_%s' % name, - timeout_seconds=_RUNTESTS_TIMEOUT) + timeout_seconds=timeout_seconds) return test_job def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', arch=None, compiler=None, labels=[], extra_args=[], extra_envs={}, - inner_jobs=_DEFAULT_INNER_JOBS): + inner_jobs=_DEFAULT_INNER_JOBS, + timeout_seconds=None): result = [] for language in languages: for platform in platforms: @@ -110,10 +119,12 @@ def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', runtests_args += extra_args if platform == 'linux': job = _docker_jobspec(name=name, runtests_args=runtests_args, - runtests_envs=extra_envs, inner_jobs=inner_jobs) + runtests_envs=extra_envs, inner_jobs=inner_jobs, + timeout_seconds=timeout_seconds) else: job = _workspace_jobspec(name=name, runtests_args=runtests_args, - runtests_envs=extra_envs, inner_jobs=inner_jobs) + runtests_envs=extra_envs, inner_jobs=inner_jobs, + timeout_seconds=timeout_seconds) job.labels = [platform, config, language, iomgr_platform] + labels result.append(job) @@ -136,7 +147,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): platforms=['linux', 'macos', 'windows'], labels=['basictests', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['csharp', 'node', 'python'], configs=['dbg', 'opt'], @@ -151,7 +163,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): platforms=['linux', 'macos'], labels=['basictests', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['ruby', 'php'], configs=['dbg', 'opt'], @@ -174,13 +187,15 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): platforms=['linux'], labels=['sanitizers', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['c++'], configs=['asan', 'tsan'], platforms=['linux'], labels=['sanitizers', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) return test_jobs @@ -207,7 +222,8 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) compiler=compiler, labels=['portability', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) # portability C on Windows 64-bit (x86 is the default) test_jobs += _generate_jobs(languages=['c'], @@ -246,10 +262,11 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) configs=['dbg'], platforms=['linux'], labels=['portability', 'corelang'], extra_args=extra_args, - extra_envs={'GRPC_DNS_RESOLVER': 'ares'}) + extra_envs={'GRPC_DNS_RESOLVER': 'ares'}, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) # TODO(zyc): Turn on this test after adding c-ares support on windows. - # C with the c-ares DNS resolver on Windonws + # C with the c-ares DNS resolver on Windows # test_jobs += _generate_jobs(languages=['c'], # configs=['dbg'], platforms=['windows'], # labels=['portability', 'corelang'], @@ -292,7 +309,8 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) iomgr_platform='uv', labels=['portability', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['node'], configs=['dbg'], From 44a2563cab51ea20cf729a98400e2f8287c953fb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 12 Sep 2017 16:55:23 +0200 Subject: [PATCH 074/227] add missing template for grpcio_testing versining --- .../grpcio_testing/grpc_version.py.template | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 templates/src/python/grpcio_testing/grpc_version.py.template diff --git a/templates/src/python/grpcio_testing/grpc_version.py.template b/templates/src/python/grpcio_testing/grpc_version.py.template new file mode 100644 index 00000000000..74db811d60e --- /dev/null +++ b/templates/src/python/grpcio_testing/grpc_version.py.template @@ -0,0 +1,19 @@ +%YAML 1.2 +--- | + # Copyright 2017 gRPC authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! + + VERSION='${settings.python_version.pep440()}' From ea7807876cb7a4c53e610f516d6dbc09d1ef2d2e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 12 Sep 2017 16:56:05 +0200 Subject: [PATCH 075/227] regenerate --- src/python/grpcio_testing/grpc_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 41a75d46f68..1638b38c2ef 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! +# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.5.0.dev0' +VERSION='1.6.1' From d6c4e937ce3f13151ec070ace6d73712f6d11f26 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 13 Sep 2017 00:56:08 +0800 Subject: [PATCH 076/227] update template --- templates/CMakeLists.txt.template | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 6adff889fef..cb6a82c3802 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -179,7 +179,8 @@ if(NOT CARES_ROOT_DIR) set(CARES_ROOT_DIR <%text>${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares) endif() - set(CARES_STATIC ON) + set(CARES_SHARED OFF CACHE BOOL "disable shared library") + set(CARES_STATIC ON CACHE BOOL "link cares statically") set(CARES_INCLUDE_DIR "<%text>${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares") add_subdirectory(third_party/cares/cares) if(TARGET c-ares) From f11a40d23b982129a56fc5e167ebc213afe52e6d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 10:54:20 -0700 Subject: [PATCH 077/227] Upload core stats to qps_test --- tools/codegen/core/gen_stats_data.py | 87 ++ .../run_tests/performance/bq_upload_result.py | 2 + .../performance/massage_qps_stats.py | 105 ++ .../performance/massage_qps_stats_helpers.py | 57 + .../performance/scenario_result_schema.json | 1296 ++++++++++++++--- 5 files changed, 1314 insertions(+), 233 deletions(-) create mode 100644 tools/run_tests/performance/massage_qps_stats.py create mode 100644 tools/run_tests/performance/massage_qps_stats_helpers.py diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8e4ef594af2..f665b5808e6 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -313,3 +313,90 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % ( len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + +# patch qps_test bigquery schema +RECORD_EXPLICIT_PERCENTILES = [50, 95, 99] + +with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f: + qps_schema = json.loads(f.read()) + +def FindNamed(js, name): + for el in js: + if el['name'] == name: + return el + +def RemoveCoreFields(js): + new_fields = [] + for field in js['fields']: + if not field['name'].startswith('core_'): + new_fields.append(field) + js['fields'] = new_fields + +RemoveCoreFields(FindNamed(qps_schema, 'clientStats')) +RemoveCoreFields(FindNamed(qps_schema, 'serverStats')) + +def AddCoreFields(js): + for counter in inst_map['Counter']: + js['fields'].append({ + 'name': 'core_%s' % counter.name, + 'type': 'INTEGER', + 'mode': 'NULLABLE' + }) + for histogram in inst_map['Histogram']: + js['fields'].append({ + 'name': 'core_%s' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + js['fields'].append({ + 'name': 'core_%s_bkts' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + for pctl in RECORD_EXPLICIT_PERCENTILES: + js['fields'].append({ + 'name': 'core_%s_%dp' % (histogram.name, pctl), + 'type': 'FLOAT', + 'mode': 'NULLABLE' + }) + +AddCoreFields(FindNamed(qps_schema, 'clientStats')) +AddCoreFields(FindNamed(qps_schema, 'serverStats')) + +with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f: + f.write(json.dumps(qps_schema, indent=2, sort_keys=True)) + +# and generate a helper script to massage scenario results into the format we'd +# like to query +with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: + with open(sys.argv[0]) as my_source: + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + print >>P, line.rstrip() + break + for line in my_source: + if line[0] != '#': + break + print >>P, line.rstrip() + + print >>P + print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py' + print >>P + + print >>P, 'import massage_qps_stats_helpers' + + print >>P, 'def massage_qps_stats(scenario_result):' + print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:' + print >>P, ' if "coreStats" not in stats: return' + print >>P, ' core_stats = stats["coreStats"]' + print >>P, ' del stats["coreStats"]' + for counter in inst_map['Counter']: + print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) + for i, histogram in enumerate(inst_map['Histogram']): + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + for pctl in RECORD_EXPLICIT_PERCENTILES: + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( + histogram.name, pctl, histogram.name, pctl, histogram.name) diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 630ac231962..31819d6159c 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -24,6 +24,7 @@ import os import sys import time import uuid +import massage_qps_stats gcp_utils_dir = os.path.abspath(os.path.join( @@ -117,6 +118,7 @@ def _flatten_result_inplace(scenario_result): scenario_result['serverCpuUsage'] = scenario_result['summary'].pop('serverCpuUsage', None) scenario_result['summary'].pop('successfulRequestsPerSecond', None) scenario_result['summary'].pop('failedRequestsPerSecond', None) + massage_qps_stats.massage_qps_stats(scenario_result) def _populate_metadata_inplace(scenario_result): diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py new file mode 100644 index 00000000000..1ed5be731de --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -0,0 +1,105 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated by tools/codegen/core/gen_stats_data.py + +import massage_qps_stats_helpers +def massage_qps_stats(scenario_result): + for stats in scenario_result["serverStats"] + scenario_result["clientStats"]: + if "coreStats" not in stats: return + core_stats = stats["coreStats"] + del stats["coreStats"] + stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created") + stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") + stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") + stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") + stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") + stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write") + stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read") + stats["core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_pollers_created") + stats["core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_poller_polls") + stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_batches") + stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_cancel") + stats["core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_initial_metadata") + stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_message") + stats["core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_trailing_metadata") + stats["core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_initial_metadata") + stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_message") + stats["core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_trailing_metadata") + stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_settings_writes") + stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_pings_sent") + stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_begun") + stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded") + stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued") + stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes") + stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") + stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") + stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") + stats["core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_offloaded") + stats["core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_short_items") + stats["core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_long_items") + stats["core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_to_self") + stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") + stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") + stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py new file mode 100644 index 00000000000..c5d884344ff --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -0,0 +1,57 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +def threshold_for_count_below(buckets, boundaries, count_below): + count_so_far = 0 + for lower_idx in range(0, len(buckets)): + count_so_far += buckets[lower_idx] + if count_so_far >= count_below: + break + if count_so_far == count_below: + # this bucket hits the threshold exactly... we should be midway through + # any run of zero values following the bucket + for upper_idx in range(lower_idx + 1, num_buckets): + if buckets[upper_idx] != 0: + break + return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0 + else: + # treat values as uniform throughout the bucket, and find where this value + # should lie + lower_bound = boundaries[lower_idx] + upper_bound = boundaries[lower_idx + 1] + return (upper_bound - + (upper_bound - lower_bound) * (count_so_far - count_below) / + float(buckets[lower_idx])) + +def percentile(buckets, pctl, boundaries): + return threshold_for_count_below( + buckets, boundaries, sum(buckets) * pctl / 100.0) + +def counter(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + return int(stat.get('count', 0)) + +Histogram = collections.namedtuple('Histogram', 'buckets boundaries') +def histogram(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + buckets = [] + boundaries = [] + for b in stat['histogram']['buckets']: + buckets.append(int(b.get('count', 0))) + boundaries.append(int(b.get('start', 0))) + return Histogram(buckets=buckets, boundaries=boundaries) diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index d7e2e292a7f..a28d27fece2 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -1,269 +1,1099 @@ [ { - "name": "metadata", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "buildNumber", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "buildUrl", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "jobName", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitActualCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "created", - "type": "TIMESTAMP", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "buildNumber", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "buildUrl", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "jobName", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitActualCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "created", + "type": "TIMESTAMP" } - ] - }, + ], + "mode": "NULLABLE", + "name": "metadata", + "type": "RECORD" + }, { - "name": "scenario", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "clientConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numClients", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "serverConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numServers", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "warmupSeconds", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "benchmarkSeconds", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "clientConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numClients", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "serverConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numServers", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "warmupSeconds", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "benchmarkSeconds", + "type": "INTEGER" } - ] - }, + ], + "mode": "NULLABLE", + "name": "scenario", + "type": "RECORD" + }, { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, { - "name": "clientStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "clientStats", + "type": "RECORD" + }, { - "name": "serverStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverStats", + "type": "RECORD" + }, { - "name": "serverCores", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverCores", + "type": "STRING" + }, { - "name": "summary", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "qps", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "qpsPerServerCore", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency50", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency90", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency95", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency99", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency999", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "qps", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "qpsPerServerCore", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency50", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency90", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency95", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency99", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency999", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverQueriesPerCpuSec", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientQueriesPerCpuSec", + "type": "FLOAT" } - ] - }, + ], + "mode": "NULLABLE", + "name": "summary", + "type": "RECORD" + }, { - "name": "clientSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "clientSuccess", + "type": "STRING" + }, { - "name": "serverSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverSuccess", + "type": "STRING" + }, { - "name": "requestResults", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "requestResults", + "type": "STRING" + }, { - "name": "serverCpuStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "totalCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "totalCpuTime", + "type": "INTEGER" + }, { - "name": "idleCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "idleCpuTime", + "type": "INTEGER" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverCpuStats", + "type": "RECORD" + }, { - "name": "serverCpuUsage", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "serverCpuUsage", + "type": "FLOAT" } -] +] \ No newline at end of file From f9da3a7bc40171c249fd1ffe56ed09d330520b38 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 8 Mar 2017 00:12:50 -0800 Subject: [PATCH 078/227] Start setting up files for autorun of php_to_cpp qps tests --- tools/gce/linux_performance_worker_init.sh | 6 +++ tools/run_tests/performance/run_worker_php.sh | 37 +++++++++++++++++++ .../run_tests/performance/scenario_config.py | 28 ++++++++++++++ 3 files changed, 71 insertions(+) create mode 100755 tools/run_tests/performance/run_worker_php.sh diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 8f0a0f65c34..04844506c9d 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -128,6 +128,12 @@ ruby -v # Install bundler (prerequisite for gRPC Ruby) gem install bundler +# PHP dependencies + +sudo apt-get install -y php5 php5-dev phpunit php-pear zlib1g-dev +curl -sS https://getcomposer.org/installer | php +sudo mv composer.phar /usr/local/bin/composer + # Java dependencies - nothing as we already have Java JDK 8 # Go dependencies diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh new file mode 100755 index 00000000000..b4d1a1cccd1 --- /dev/null +++ b/tools/run_tests/performance/run_worker_php.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +source ~/.rvm/scripts/rvm +set -ex + +cd $(dirname $0)/../../.. + +# The proxy worker for PHP is implemented in Ruby +ruby src/ruby/qps/proxy-worker.rb $@ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index cbcc4d0ecc4..15bce80fa4b 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -798,6 +798,33 @@ class RubyLanguage: return 'ruby' +class PhpLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_php.sh'] + + def worker_port_offset(self): + return 800 + + def scenarios(self): + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + def __str__(self): + return 'ruby' + + class JavaLanguage: def __init__(self): @@ -995,6 +1022,7 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), + 'php' : PhpLanguage(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), From 649c95eadc313e211b54f1abb7f9826f15a2ac36 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 13 Mar 2017 03:40:02 -0700 Subject: [PATCH 079/227] Make sure to get all PHP dependences set up properly --- tools/gce/linux_performance_worker_init.sh | 2 +- tools/run_tests/performance/run_worker_php.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 04844506c9d..b952fd1b35c 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -130,7 +130,7 @@ gem install bundler # PHP dependencies -sudo apt-get install -y php5 php5-dev phpunit php-pear zlib1g-dev +sudo apt-get install -y php5 php5-dev phpunit php-pear unzip zlib1g-dev curl -sS https://getcomposer.org/installer | php sudo mv composer.phar /usr/local/bin/composer diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index b4d1a1cccd1..ebedae8795d 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -31,7 +31,14 @@ source ~/.rvm/scripts/rvm set -ex -cd $(dirname $0)/../../.. +repo=$(dirname $0)/../../.. + +# First set up all dependences needed for PHP QPS test +cd $repo +cd src/php/tests/qps +curl -sS https://getcomposer.org/installer | php +php composer.phar install # The proxy worker for PHP is implemented in Ruby +cd $repo ruby src/ruby/qps/proxy-worker.rb $@ From 644676fb12f756a7efa34c27dd42809da8cbf311 Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Sun, 10 Sep 2017 19:06:56 -0700 Subject: [PATCH 080/227] vjpai's PR --- tools/gce/linux_performance_worker_init.sh | 5 +++ tools/run_tests/performance/run_worker_php.sh | 45 +++++++++++++++++++ .../run_tests/performance/scenario_config.py | 28 ++++++++++++ 3 files changed, 78 insertions(+) create mode 100755 tools/run_tests/performance/run_worker_php.sh diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 8f0a0f65c34..88d8de74028 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -128,6 +128,11 @@ ruby -v # Install bundler (prerequisite for gRPC Ruby) gem install bundler +# PHP dependencies +sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev +curl -sS https://getcomposer.org/installer | php +sudo mv composer.phar /usr/local/bin/composer + # Java dependencies - nothing as we already have Java JDK 8 # Go dependencies diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh new file mode 100755 index 00000000000..ba26f38a77a --- /dev/null +++ b/tools/run_tests/performance/run_worker_php.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +source ~/.rvm/scripts/rvm +set -ex + +repo=$(dirname $0)/../../.. + +# First set up all dependences needed for PHP QPS test +cd $repo +cd src/php/tests/qps +curl -sS https://getcomposer.org/installer | php +php composer.phar install + +# The proxy worker for PHP is implemented in Ruby +cd $repo +ruby src/ruby/qps/proxy-worker.rb $@ + diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 86236b19bf3..329b9f8c470 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -797,6 +797,33 @@ class RubyLanguage: return 'ruby' +class PhpLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_php.sh'] + + def worker_port_offset(self): + return 800 + + def scenarios(self): + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + def __str__(self): + return 'php' + + class JavaLanguage: def __init__(self): @@ -994,6 +1021,7 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), + 'php' : PhpLanguage(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), From 3c77ee8ad1e1c864c86d51126d9a74a62899133e Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Tue, 12 Sep 2017 11:45:03 -0700 Subject: [PATCH 081/227] dir bug --- tools/run_tests/performance/run_worker_php.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index ba26f38a77a..1da74cd0dbd 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -36,10 +36,8 @@ repo=$(dirname $0)/../../.. # First set up all dependences needed for PHP QPS test cd $repo cd src/php/tests/qps -curl -sS https://getcomposer.org/installer | php -php composer.phar install - +composer install # The proxy worker for PHP is implemented in Ruby -cd $repo +cd ../../../.. ruby src/ruby/qps/proxy-worker.rb $@ From 9b2a91306beafe5829ecfcf658988657e0eff1af Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 13 Mar 2017 04:05:33 -0700 Subject: [PATCH 082/227] php7 is new default for ubuntu 16.04 --- tools/gce/linux_performance_worker_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index b952fd1b35c..210bbbd3a26 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -130,7 +130,7 @@ gem install bundler # PHP dependencies -sudo apt-get install -y php5 php5-dev phpunit php-pear unzip zlib1g-dev +sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev curl -sS https://getcomposer.org/installer | php sudo mv composer.phar /usr/local/bin/composer From 11b48c766e00290bd7481132fd20806f5317d2fe Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 13 Mar 2017 04:26:19 -0700 Subject: [PATCH 083/227] typo --- tools/run_tests/performance/scenario_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 15bce80fa4b..262001d23eb 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -822,7 +822,7 @@ class PhpLanguage: server_language='c++', async_server_threads=1) def __str__(self): - return 'ruby' + return 'php' class JavaLanguage: From d5e390b509221ef1574b6b02d91155efd459c30b Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Tue, 12 Sep 2017 11:56:59 -0700 Subject: [PATCH 084/227] No newline at the end of file --- tools/run_tests/performance/run_worker_php.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index 15cee151f88..1da74cd0dbd 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -39,4 +39,5 @@ cd src/php/tests/qps composer install # The proxy worker for PHP is implemented in Ruby cd ../../../.. -ruby src/ruby/qps/proxy-worker.rb $@ \ No newline at end of file +ruby src/ruby/qps/proxy-worker.rb $@ + From 3a24364b48dcce2f7d84a1c8e8b755cf8925e997 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 12 Sep 2017 12:06:27 -0700 Subject: [PATCH 085/227] Fix a memory leak in cares wrapper --- .../client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 0d71f3560e0..2e2b411ab8d 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -174,7 +174,7 @@ static void on_hostbyname_done_cb(void *arg, int status, int timeouts, grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, hr->is_balancer /* is_balancer */, - hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */, + hr->is_balancer ? hr->host : NULL /* balancer_name */, NULL /* user_data */); char output[INET6_ADDRSTRLEN]; ares_inet_ntop(AF_INET6, &addr.sin6_addr, output, INET6_ADDRSTRLEN); @@ -195,7 +195,7 @@ static void on_hostbyname_done_cb(void *arg, int status, int timeouts, grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, hr->is_balancer /* is_balancer */, - hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */, + hr->is_balancer ? hr->host : NULL /* balancer_name */, NULL /* user_data */); char output[INET_ADDRSTRLEN]; ares_inet_ntop(AF_INET, &addr.sin_addr, output, INET_ADDRSTRLEN); From 0852acd8b3700b052676b21b096fc641fc4194aa Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 13:15:03 -0700 Subject: [PATCH 086/227] Revert "Let alarms trigger at end of jobset.py instead of clearing them" --- tools/run_tests/python_utils/jobset.py | 22 ++++++++++++++++------ tools/run_tests/run_performance_tests.py | 8 ++++---- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 6151a7276a9..08d652ae3f3 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -71,8 +71,10 @@ def platform_string(): if platform_string() == 'windows': pass else: + have_alarm = False def alarm_handler(unused_signum, unused_frame): - pass + global have_alarm + have_alarm = False signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None) signal.signal(signal.SIGALRM, alarm_handler) @@ -365,9 +367,10 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, add_env, quiet_success, max_time): + stop_on_failure, add_env, quiet_success, max_time, clear_alarms): self._running = set() self._check_cancelled = check_cancelled + self._clear_alarms = clear_alarms self._cancelled = False self._failures = 0 self._completed = 0 @@ -452,7 +455,10 @@ class Jobset(object): if platform_string() == 'windows': time.sleep(0.1) else: - signal.alarm(10) + global have_alarm + if not have_alarm: + have_alarm = True + signal.alarm(10) signal.pause() def cancelled(self): @@ -468,7 +474,10 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - if platform_string() != 'windows': + # Clear the alarms when finished to avoid a race condition causing job + # failures. Don't do this when running multi-VM tests because clearing + # the alarms causes the test to stall + if platform_string() != 'windows' and self._clear_alarms: signal.alarm(0) return not self.cancelled() and self._failures == 0 @@ -498,7 +507,8 @@ def run(cmdlines, add_env={}, skip_jobs=False, quiet_success=False, - max_time=-1): + max_time=-1, + clear_alarms=True): if skip_jobs: resultset = {} skipped_job_result = JobResult() @@ -510,7 +520,7 @@ def run(cmdlines, js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, - quiet_success, max_time) + quiet_success, max_time, clear_alarms) for cmdline, remaining in tag_remaining(cmdlines): if not js.start(cmdline): break diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 3bfd736c51a..9b20fae78fd 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -183,7 +183,7 @@ def archive_repo(languages): jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( - [archive_job], newline_on_success=True, maxjobs=1) + [archive_job], newline_on_success=True, maxjobs=1, clear_alarms=False) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository created successfully.', @@ -215,7 +215,7 @@ def prepare_remote_hosts(hosts, prepare_local=False): timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( - prepare_jobs, newline_on_success=True, maxjobs=10) + prepare_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', @@ -248,7 +248,7 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), bui timeout_seconds=build_timeout)) jobset.message('START', 'Building.', do_newline=True) num_failures, _ = jobset.run( - build_jobs, newline_on_success=True, maxjobs=10) + build_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) if num_failures == 0: jobset.message('SUCCESS', 'Built successfully.', @@ -414,7 +414,7 @@ def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name, flame_gra perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename, flame_graph_reports)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) - failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) + failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures From 5d6156033e2fff01abd6b01b49626fed95a6d81e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 13:15:58 -0700 Subject: [PATCH 087/227] Review feedback --- tools/codegen/core/gen_stats_data.py | 9 +- .../performance/massage_qps_stats.py | 118 ++++++++++-------- .../performance/massage_qps_stats_helpers.py | 4 +- .../performance/scenario_result_schema.json | 70 +++++++++++ 4 files changed, 145 insertions(+), 56 deletions(-) diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index f665b5808e6..985c1f51c9b 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -395,8 +395,9 @@ with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: for counter in inst_map['Counter']: print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) for i, histogram in enumerate(inst_map['Histogram']): - print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) - print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + print >>P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name for pctl in RECORD_EXPLICIT_PERCENTILES: - print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( - histogram.name, pctl, histogram.name, pctl, histogram.name) + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % ( + histogram.name, pctl, pctl) diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 1ed5be731de..30a94ca7cdf 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -53,53 +53,71 @@ def massage_qps_stats(scenario_result): stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") - stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) - stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) - stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) - stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) - stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) - stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) - stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) - stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) - stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) - stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) - stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls") + stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued") + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size") + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size") + stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer") + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size") + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size") + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write") + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write") + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write") + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write") + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked") + stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets) + stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py index c5d884344ff..400a0c82bfd 100644 --- a/tools/run_tests/performance/massage_qps_stats_helpers.py +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -14,7 +14,7 @@ import collections -def threshold_for_count_below(buckets, boundaries, count_below): +def _threshold_for_count_below(buckets, boundaries, count_below): count_so_far = 0 for lower_idx in range(0, len(buckets)): count_so_far += buckets[lower_idx] @@ -37,7 +37,7 @@ def threshold_for_count_below(buckets, boundaries, count_below): float(buckets[lower_idx])) def percentile(buckets, pctl, boundaries): - return threshold_for_count_below( + return _threshold_for_count_below( buckets, boundaries, sum(buckets) * pctl / 100.0) def counter(core_stats, name): diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index a28d27fece2..84b3cb7c0a3 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -275,6 +275,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -524,6 +534,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", @@ -717,6 +752,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -966,6 +1011,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", From 3c8f6dbffd715d0dd9ff03555b15a20c12c3f3ae Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 12 Sep 2017 13:19:49 -0700 Subject: [PATCH 088/227] Remove leftover instance of clear_alarms --- tools/run_tests/run_performance_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 9b20fae78fd..6e407b4a410 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -556,7 +556,7 @@ def main(): jobs = [scenario.jobspec] if scenario.workers: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) - scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset), six.iteritems(resultset))) From 69ce380ccc018a5618d35f5942f102c61b1e91f9 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 12 Sep 2017 13:30:51 -0700 Subject: [PATCH 089/227] Revert "Merge pull request #12513 from grpc/revert-12289-sig_hand" This reverts commit 81ad19d4be0e12d519b026a37982f14ff87a0847, reversing changes made to ed63aac1952b8f19c7f1b4186cae139613c77603. --- tools/run_tests/python_utils/jobset.py | 22 ++++++---------------- tools/run_tests/run_performance_tests.py | 8 ++++---- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 08d652ae3f3..6151a7276a9 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -71,10 +71,8 @@ def platform_string(): if platform_string() == 'windows': pass else: - have_alarm = False def alarm_handler(unused_signum, unused_frame): - global have_alarm - have_alarm = False + pass signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None) signal.signal(signal.SIGALRM, alarm_handler) @@ -367,10 +365,9 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, add_env, quiet_success, max_time, clear_alarms): + stop_on_failure, add_env, quiet_success, max_time): self._running = set() self._check_cancelled = check_cancelled - self._clear_alarms = clear_alarms self._cancelled = False self._failures = 0 self._completed = 0 @@ -455,10 +452,7 @@ class Jobset(object): if platform_string() == 'windows': time.sleep(0.1) else: - global have_alarm - if not have_alarm: - have_alarm = True - signal.alarm(10) + signal.alarm(10) signal.pause() def cancelled(self): @@ -474,10 +468,7 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - # Clear the alarms when finished to avoid a race condition causing job - # failures. Don't do this when running multi-VM tests because clearing - # the alarms causes the test to stall - if platform_string() != 'windows' and self._clear_alarms: + if platform_string() != 'windows': signal.alarm(0) return not self.cancelled() and self._failures == 0 @@ -507,8 +498,7 @@ def run(cmdlines, add_env={}, skip_jobs=False, quiet_success=False, - max_time=-1, - clear_alarms=True): + max_time=-1): if skip_jobs: resultset = {} skipped_job_result = JobResult() @@ -520,7 +510,7 @@ def run(cmdlines, js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, - quiet_success, max_time, clear_alarms) + quiet_success, max_time) for cmdline, remaining in tag_remaining(cmdlines): if not js.start(cmdline): break diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 6e407b4a410..1bbab9e894f 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -183,7 +183,7 @@ def archive_repo(languages): jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( - [archive_job], newline_on_success=True, maxjobs=1, clear_alarms=False) + [archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository created successfully.', @@ -215,7 +215,7 @@ def prepare_remote_hosts(hosts, prepare_local=False): timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( - prepare_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', @@ -248,7 +248,7 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), bui timeout_seconds=build_timeout)) jobset.message('START', 'Building.', do_newline=True) num_failures, _ = jobset.run( - build_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + build_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Built successfully.', @@ -414,7 +414,7 @@ def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name, flame_gra perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename, flame_graph_reports)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) - failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures From 40c346ced94ecdb5a5d818d160f86e1be1c77046 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Tue, 12 Sep 2017 22:39:30 +0200 Subject: [PATCH 090/227] Adding --base to generate_projects.py --- tools/buildgen/generate_projects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index f885b266972..559ea1624bc 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -34,6 +34,7 @@ argp.add_argument('build_files', nargs='+', default=[]) argp.add_argument('--templates', nargs='+', default=[]) argp.add_argument('--output_merged', default=None, type=str) argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) +argp.add_argument('--base', default='.', type=str) args = argp.parse_args() json = args.build_files @@ -69,7 +70,7 @@ jobs = [] for template in reversed(sorted(templates)): root, f = os.path.split(template) if os.path.splitext(f)[1] == '.template': - out_dir = '.' + root[len('templates'):] + out_dir = args.base + root[len('templates'):] out = out_dir + '/' + os.path.splitext(f)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) From dd2c3cf23afe0c1e11770157a8fdedb3c4146d2b Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 12 Sep 2017 13:54:37 -0700 Subject: [PATCH 091/227] fix handling of service config choosing when percentage is zero --- .../client_channel/resolver/dns/c_ares/dns_resolver_ares.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index b87a3b70820..371c59b8cf3 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -204,7 +204,7 @@ static char *choose_service_config(char *service_config_choice_json) { int random_pct = rand() % 100; int percentage; if (sscanf(field->value, "%d", &percentage) != 1 || - random_pct > percentage) { + random_pct > percentage || percentage == 0) { service_config_json = NULL; break; } From 73a61708965d35e3b96d7a4f8feeba85601d5aa2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 10:54:20 -0700 Subject: [PATCH 092/227] Upload core stats to qps_test --- tools/codegen/core/gen_stats_data.py | 87 ++ .../run_tests/performance/bq_upload_result.py | 2 + .../performance/massage_qps_stats.py | 105 ++ .../performance/massage_qps_stats_helpers.py | 57 + .../performance/scenario_result_schema.json | 1296 ++++++++++++++--- 5 files changed, 1314 insertions(+), 233 deletions(-) create mode 100644 tools/run_tests/performance/massage_qps_stats.py create mode 100644 tools/run_tests/performance/massage_qps_stats_helpers.py diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8e4ef594af2..f665b5808e6 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -313,3 +313,90 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % ( len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + +# patch qps_test bigquery schema +RECORD_EXPLICIT_PERCENTILES = [50, 95, 99] + +with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f: + qps_schema = json.loads(f.read()) + +def FindNamed(js, name): + for el in js: + if el['name'] == name: + return el + +def RemoveCoreFields(js): + new_fields = [] + for field in js['fields']: + if not field['name'].startswith('core_'): + new_fields.append(field) + js['fields'] = new_fields + +RemoveCoreFields(FindNamed(qps_schema, 'clientStats')) +RemoveCoreFields(FindNamed(qps_schema, 'serverStats')) + +def AddCoreFields(js): + for counter in inst_map['Counter']: + js['fields'].append({ + 'name': 'core_%s' % counter.name, + 'type': 'INTEGER', + 'mode': 'NULLABLE' + }) + for histogram in inst_map['Histogram']: + js['fields'].append({ + 'name': 'core_%s' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + js['fields'].append({ + 'name': 'core_%s_bkts' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + for pctl in RECORD_EXPLICIT_PERCENTILES: + js['fields'].append({ + 'name': 'core_%s_%dp' % (histogram.name, pctl), + 'type': 'FLOAT', + 'mode': 'NULLABLE' + }) + +AddCoreFields(FindNamed(qps_schema, 'clientStats')) +AddCoreFields(FindNamed(qps_schema, 'serverStats')) + +with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f: + f.write(json.dumps(qps_schema, indent=2, sort_keys=True)) + +# and generate a helper script to massage scenario results into the format we'd +# like to query +with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: + with open(sys.argv[0]) as my_source: + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + print >>P, line.rstrip() + break + for line in my_source: + if line[0] != '#': + break + print >>P, line.rstrip() + + print >>P + print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py' + print >>P + + print >>P, 'import massage_qps_stats_helpers' + + print >>P, 'def massage_qps_stats(scenario_result):' + print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:' + print >>P, ' if "coreStats" not in stats: return' + print >>P, ' core_stats = stats["coreStats"]' + print >>P, ' del stats["coreStats"]' + for counter in inst_map['Counter']: + print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) + for i, histogram in enumerate(inst_map['Histogram']): + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + for pctl in RECORD_EXPLICIT_PERCENTILES: + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( + histogram.name, pctl, histogram.name, pctl, histogram.name) diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 630ac231962..31819d6159c 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -24,6 +24,7 @@ import os import sys import time import uuid +import massage_qps_stats gcp_utils_dir = os.path.abspath(os.path.join( @@ -117,6 +118,7 @@ def _flatten_result_inplace(scenario_result): scenario_result['serverCpuUsage'] = scenario_result['summary'].pop('serverCpuUsage', None) scenario_result['summary'].pop('successfulRequestsPerSecond', None) scenario_result['summary'].pop('failedRequestsPerSecond', None) + massage_qps_stats.massage_qps_stats(scenario_result) def _populate_metadata_inplace(scenario_result): diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py new file mode 100644 index 00000000000..1ed5be731de --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -0,0 +1,105 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated by tools/codegen/core/gen_stats_data.py + +import massage_qps_stats_helpers +def massage_qps_stats(scenario_result): + for stats in scenario_result["serverStats"] + scenario_result["clientStats"]: + if "coreStats" not in stats: return + core_stats = stats["coreStats"] + del stats["coreStats"] + stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created") + stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") + stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") + stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") + stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") + stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write") + stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read") + stats["core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_pollers_created") + stats["core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_poller_polls") + stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_batches") + stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_cancel") + stats["core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_initial_metadata") + stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_message") + stats["core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_trailing_metadata") + stats["core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_initial_metadata") + stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_message") + stats["core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_trailing_metadata") + stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_settings_writes") + stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_pings_sent") + stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_begun") + stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded") + stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued") + stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes") + stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") + stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") + stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") + stats["core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_offloaded") + stats["core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_short_items") + stats["core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_long_items") + stats["core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_to_self") + stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") + stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") + stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py new file mode 100644 index 00000000000..c5d884344ff --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -0,0 +1,57 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +def threshold_for_count_below(buckets, boundaries, count_below): + count_so_far = 0 + for lower_idx in range(0, len(buckets)): + count_so_far += buckets[lower_idx] + if count_so_far >= count_below: + break + if count_so_far == count_below: + # this bucket hits the threshold exactly... we should be midway through + # any run of zero values following the bucket + for upper_idx in range(lower_idx + 1, num_buckets): + if buckets[upper_idx] != 0: + break + return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0 + else: + # treat values as uniform throughout the bucket, and find where this value + # should lie + lower_bound = boundaries[lower_idx] + upper_bound = boundaries[lower_idx + 1] + return (upper_bound - + (upper_bound - lower_bound) * (count_so_far - count_below) / + float(buckets[lower_idx])) + +def percentile(buckets, pctl, boundaries): + return threshold_for_count_below( + buckets, boundaries, sum(buckets) * pctl / 100.0) + +def counter(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + return int(stat.get('count', 0)) + +Histogram = collections.namedtuple('Histogram', 'buckets boundaries') +def histogram(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + buckets = [] + boundaries = [] + for b in stat['histogram']['buckets']: + buckets.append(int(b.get('count', 0))) + boundaries.append(int(b.get('start', 0))) + return Histogram(buckets=buckets, boundaries=boundaries) diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index d7e2e292a7f..a28d27fece2 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -1,269 +1,1099 @@ [ { - "name": "metadata", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "buildNumber", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "buildUrl", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "jobName", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitActualCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "created", - "type": "TIMESTAMP", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "buildNumber", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "buildUrl", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "jobName", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitActualCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "created", + "type": "TIMESTAMP" } - ] - }, + ], + "mode": "NULLABLE", + "name": "metadata", + "type": "RECORD" + }, { - "name": "scenario", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "clientConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numClients", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "serverConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numServers", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "warmupSeconds", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "benchmarkSeconds", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "clientConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numClients", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "serverConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numServers", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "warmupSeconds", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "benchmarkSeconds", + "type": "INTEGER" } - ] - }, + ], + "mode": "NULLABLE", + "name": "scenario", + "type": "RECORD" + }, { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, { - "name": "clientStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "clientStats", + "type": "RECORD" + }, { - "name": "serverStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverStats", + "type": "RECORD" + }, { - "name": "serverCores", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverCores", + "type": "STRING" + }, { - "name": "summary", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "qps", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "qpsPerServerCore", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency50", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency90", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency95", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency99", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency999", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "qps", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "qpsPerServerCore", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency50", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency90", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency95", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency99", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency999", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverQueriesPerCpuSec", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientQueriesPerCpuSec", + "type": "FLOAT" } - ] - }, + ], + "mode": "NULLABLE", + "name": "summary", + "type": "RECORD" + }, { - "name": "clientSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "clientSuccess", + "type": "STRING" + }, { - "name": "serverSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverSuccess", + "type": "STRING" + }, { - "name": "requestResults", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "requestResults", + "type": "STRING" + }, { - "name": "serverCpuStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "totalCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "totalCpuTime", + "type": "INTEGER" + }, { - "name": "idleCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "idleCpuTime", + "type": "INTEGER" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverCpuStats", + "type": "RECORD" + }, { - "name": "serverCpuUsage", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "serverCpuUsage", + "type": "FLOAT" } -] +] \ No newline at end of file From 27147b85f8eeede4f918705e42a91093d450757c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 13:15:58 -0700 Subject: [PATCH 093/227] Review feedback --- tools/codegen/core/gen_stats_data.py | 9 +- .../performance/massage_qps_stats.py | 118 ++++++++++-------- .../performance/massage_qps_stats_helpers.py | 4 +- .../performance/scenario_result_schema.json | 70 +++++++++++ 4 files changed, 145 insertions(+), 56 deletions(-) diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index f665b5808e6..985c1f51c9b 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -395,8 +395,9 @@ with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: for counter in inst_map['Counter']: print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) for i, histogram in enumerate(inst_map['Histogram']): - print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) - print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + print >>P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name for pctl in RECORD_EXPLICIT_PERCENTILES: - print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( - histogram.name, pctl, histogram.name, pctl, histogram.name) + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % ( + histogram.name, pctl, pctl) diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 1ed5be731de..30a94ca7cdf 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -53,53 +53,71 @@ def massage_qps_stats(scenario_result): stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") - stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) - stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) - stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) - stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) - stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) - stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) - stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) - stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) - stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) - stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) - stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls") + stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued") + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size") + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size") + stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer") + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size") + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size") + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write") + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write") + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write") + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write") + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked") + stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets) + stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py index c5d884344ff..400a0c82bfd 100644 --- a/tools/run_tests/performance/massage_qps_stats_helpers.py +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -14,7 +14,7 @@ import collections -def threshold_for_count_below(buckets, boundaries, count_below): +def _threshold_for_count_below(buckets, boundaries, count_below): count_so_far = 0 for lower_idx in range(0, len(buckets)): count_so_far += buckets[lower_idx] @@ -37,7 +37,7 @@ def threshold_for_count_below(buckets, boundaries, count_below): float(buckets[lower_idx])) def percentile(buckets, pctl, boundaries): - return threshold_for_count_below( + return _threshold_for_count_below( buckets, boundaries, sum(buckets) * pctl / 100.0) def counter(core_stats, name): diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index a28d27fece2..84b3cb7c0a3 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -275,6 +275,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -524,6 +534,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", @@ -717,6 +752,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -966,6 +1011,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", From 52778c4729ae4e59eadae3bb49728967f397eb6f Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 11 Sep 2017 15:00:11 -0700 Subject: [PATCH 094/227] Adding pointer conversions. Renaming a few variables and type names to avoid C++ compilation issues --- src/core/lib/channel/channel_args.c | 4 +- src/core/lib/channel/channel_stack_builder.c | 16 ++--- src/core/lib/compression/stream_compression.c | 3 +- src/core/lib/iomgr/combiner.c | 3 +- src/core/lib/iomgr/error.c | 34 +++++------ src/core/lib/iomgr/ev_epoll1_linux.c | 6 +- src/core/lib/iomgr/ev_epollex_linux.c | 45 +++++++------- src/core/lib/iomgr/polling_entity.h | 4 +- src/core/lib/iomgr/resource_quota.c | 6 +- src/core/lib/iomgr/socket_factory_posix.c | 4 +- src/core/lib/iomgr/socket_mutator.c | 4 +- src/core/lib/iomgr/timer_manager.c | 2 +- src/core/lib/iomgr/udp_server.c | 2 +- src/core/lib/slice/slice.c | 4 +- src/core/lib/surface/call.c | 58 ++++++++++--------- 15 files changed, 101 insertions(+), 94 deletions(-) diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 16d0737b405..9215707f1c3 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -212,7 +212,7 @@ void grpc_channel_args_destroy(grpc_exec_ctx *exec_ctx, grpc_channel_args *a) { grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( const grpc_channel_args *a) { size_t i; - if (a == NULL) return 0; + if (a == NULL) return GRPC_COMPRESS_NONE; for (i = 0; i < a->num_args; ++i) { if (a->args[i].type == GRPC_ARG_INTEGER && !strcmp(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, a->args[i].key)) { @@ -226,7 +226,7 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( grpc_stream_compression_algorithm grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args *a) { size_t i; - if (a == NULL) return 0; + if (a == NULL) return GRPC_STREAM_COMPRESS_NONE; for (i = 0; i < a->num_args; ++i) { if (a->args[i].type == GRPC_ARG_INTEGER && !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c index df45f13c813..b663ebfb522 100644 --- a/src/core/lib/channel/channel_stack_builder.c +++ b/src/core/lib/channel/channel_stack_builder.c @@ -214,13 +214,13 @@ bool grpc_channel_stack_builder_prepend_filter( static void add_after(filter_node *before, const grpc_channel_filter *filter, grpc_post_filter_create_init_func post_init_func, void *user_data) { - filter_node *new = (filter_node *)gpr_malloc(sizeof(*new)); - new->next = before->next; - new->prev = before; - new->next->prev = new->prev->next = new; - new->filter = filter; - new->init = post_init_func; - new->init_arg = user_data; + filter_node *new_node = (filter_node *)gpr_malloc(sizeof(*new_node)); + new_node->next = before->next; + new_node->prev = before; + new_node->next->prev = new_node->prev->next = new_node; + new_node->filter = filter; + new_node->init = post_init_func; + new_node->init_arg = user_data; } bool grpc_channel_stack_builder_add_filter_before( @@ -268,7 +268,7 @@ grpc_error *grpc_channel_stack_builder_finish( // create an array of filters const grpc_channel_filter **filters = - gpr_malloc(sizeof(*filters) * num_filters); + (const grpc_channel_filter **)gpr_malloc(sizeof(*filters) * num_filters); size_t i = 0; for (filter_node *p = builder->begin.next; p != &builder->end; p = p->next) { filters[i++] = p->filter; diff --git a/src/core/lib/compression/stream_compression.c b/src/core/lib/compression/stream_compression.c index df13d53e06f..ba9302794f7 100644 --- a/src/core/lib/compression/stream_compression.c +++ b/src/core/lib/compression/stream_compression.c @@ -159,7 +159,8 @@ bool grpc_stream_decompress(grpc_stream_compression_context *ctx, grpc_stream_compression_context *grpc_stream_compression_context_create( grpc_stream_compression_method method) { grpc_stream_compression_context *ctx = - gpr_zalloc(sizeof(grpc_stream_compression_context)); + (grpc_stream_compression_context *)gpr_zalloc( + sizeof(grpc_stream_compression_context)); int r; if (ctx == NULL) { return NULL; diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index 360967f3ba2..f899b25f105 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -356,7 +356,8 @@ static void combiner_finally_exec(grpc_exec_ctx *exec_ctx, static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure, grpc_error *error) { - combiner_finally_exec(exec_ctx, closure, GRPC_ERROR_REF(error)); + combiner_finally_exec(exec_ctx, (grpc_closure *)closure, + GRPC_ERROR_REF(error)); } grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *combiner) { diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index dcd175a2e1a..d5e00a8f643 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -278,13 +278,13 @@ static void internal_set_time(grpc_error **err, grpc_error_times which, memcpy((*err)->arena + slot, &value, sizeof(value)); } -static void internal_add_error(grpc_error **err, grpc_error *new) { - grpc_linked_error new_last = {new, UINT8_MAX}; +static void internal_add_error(grpc_error **err, grpc_error *new_err) { + grpc_linked_error new_last = {new_err, UINT8_MAX}; uint8_t slot = get_placement(err, sizeof(grpc_linked_error)); if (slot == UINT8_MAX) { - gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err, new, - grpc_error_string(new)); - GRPC_ERROR_UNREF(new); + gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err, + new_err, grpc_error_string(new_err)); + GRPC_ERROR_UNREF(new_err); return; } if ((*err)->first_err == UINT8_MAX) { @@ -321,8 +321,8 @@ grpc_error *grpc_error_create(const char *file, int line, grpc_slice desc, uint8_t initial_arena_capacity = (uint8_t)( DEFAULT_ERROR_CAPACITY + (uint8_t)(num_referencing * SLOTS_PER_LINKED_ERROR) + SURPLUS_CAPACITY); - grpc_error *err = - gpr_malloc(sizeof(*err) + initial_arena_capacity * sizeof(intptr_t)); + grpc_error *err = (grpc_error *)gpr_malloc( + sizeof(*err) + initial_arena_capacity * sizeof(intptr_t)); if (err == NULL) { // TODO(ctiller): make gpr_malloc return NULL return GRPC_ERROR_OOM; } @@ -432,10 +432,10 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { grpc_error *grpc_error_set_int(grpc_error *src, grpc_error_ints which, intptr_t value) { GPR_TIMER_BEGIN("grpc_error_set_int", 0); - grpc_error *new = copy_error_and_unref(src); - internal_set_int(&new, which, value); + grpc_error *new_err = copy_error_and_unref(src); + internal_set_int(&new_err, which, value); GPR_TIMER_END("grpc_error_set_int", 0); - return new; + return new_err; } typedef struct { @@ -477,10 +477,10 @@ bool grpc_error_get_int(grpc_error *err, grpc_error_ints which, intptr_t *p) { grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which, grpc_slice str) { GPR_TIMER_BEGIN("grpc_error_set_str", 0); - grpc_error *new = copy_error_and_unref(src); - internal_set_str(&new, which, str); + grpc_error *new_err = copy_error_and_unref(src); + internal_set_str(&new_err, which, str); GPR_TIMER_END("grpc_error_set_str", 0); - return new; + return new_err; } bool grpc_error_get_str(grpc_error *err, grpc_error_strs which, @@ -507,10 +507,10 @@ bool grpc_error_get_str(grpc_error *err, grpc_error_strs which, grpc_error *grpc_error_add_child(grpc_error *src, grpc_error *child) { GPR_TIMER_BEGIN("grpc_error_add_child", 0); - grpc_error *new = copy_error_and_unref(src); - internal_add_error(&new, child); + grpc_error *new_err = copy_error_and_unref(src); + internal_add_error(&new_err, child); GPR_TIMER_END("grpc_error_add_child", 0); - return new; + return new_err; } static const char *no_error_string = "\"No Error\""; @@ -733,7 +733,7 @@ const char *grpc_error_string(grpc_error *err) { void *p = (void *)gpr_atm_acq_load(&err->atomics.error_string); if (p != NULL) { GPR_TIMER_END("grpc_error_string", 0); - return p; + return (const char *)p; } kv_pairs kvs; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 5bc7e878de0..01d28220c71 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -130,9 +130,9 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state; +typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state_t; -static const char *kick_state_string(kick_state st) { +static const char *kick_state_string(kick_state_t st) { switch (st) { case UNKICKED: return "UNKICKED"; @@ -145,7 +145,7 @@ static const char *kick_state_string(kick_state st) { } struct grpc_pollset_worker { - kick_state kick_state; + kick_state_t kick_state; int kick_state_mutator; // which line of code last changed kick state bool initialized_cv; grpc_pollset_worker *next; diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 277347ac704..df69025f1a3 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -97,12 +97,12 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, * pollable Declarations */ -typedef struct pollable { +typedef struct pollable_t { polling_obj po; int epfd; grpc_wakeup_fd wakeup; grpc_pollset_worker *root_worker; -} pollable; +} pollable_t; static const char *polling_obj_type_string(polling_obj_type t) { switch (t) { @@ -122,7 +122,7 @@ static const char *polling_obj_type_string(polling_obj_type t) { return ""; } -static char *pollable_desc(pollable *p) { +static char *pollable_desc(pollable_t *p) { char *out; gpr_asprintf(&out, "type=%s group=%p epfd=%d wakeup=%d", polling_obj_type_string(p->po.type), p->po.group, p->epfd, @@ -130,19 +130,19 @@ static char *pollable_desc(pollable *p) { return out; } -static pollable g_empty_pollable; +static pollable_t g_empty_pollable; -static void pollable_init(pollable *p, polling_obj_type type); -static void pollable_destroy(pollable *p); +static void pollable_init(pollable_t *p, polling_obj_type type); +static void pollable_destroy(pollable_t *p); /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable *p); +static grpc_error *pollable_materialize(pollable_t *p); /******************************************************************************* * Fd Declarations */ struct grpc_fd { - pollable pollable; + pollable_t pollable; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -193,15 +193,15 @@ struct grpc_pollset_worker { pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; gpr_cv cv; grpc_pollset *pollset; - pollable *pollable; + pollable_t *pollable; }; #define MAX_EPOLL_EVENTS 100 #define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 5 struct grpc_pollset { - pollable pollable; - pollable *current_pollable; + pollable_t pollable; + pollable_t *current_pollable; int kick_alls_pending; bool kicked_without_poller; grpc_closure *shutdown_closure; @@ -451,13 +451,13 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, * Pollable Definitions */ -static void pollable_init(pollable *p, polling_obj_type type) { +static void pollable_init(pollable_t *p, polling_obj_type type) { po_init(&p->po, type); p->root_worker = NULL; p->epfd = -1; } -static void pollable_destroy(pollable *p) { +static void pollable_destroy(pollable_t *p) { po_destroy(&p->po); if (p->epfd != -1) { close(p->epfd); @@ -466,7 +466,7 @@ static void pollable_destroy(pollable *p) { } /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable *p) { +static grpc_error *pollable_materialize(pollable_t *p) { if (p->epfd == -1) { int new_epfd = epoll_create1(EPOLL_CLOEXEC); if (new_epfd < 0) { @@ -492,7 +492,7 @@ static grpc_error *pollable_materialize(pollable *p) { } /* pollable must be materialized */ -static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { +static grpc_error *pollable_add_fd(pollable_t *p, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "pollable_add_fd"; const int epfd = p->epfd; @@ -599,7 +599,7 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -664,7 +664,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - pollable *p = pollset->current_pollable; + pollable_t *p = pollset->current_pollable; if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } @@ -744,7 +744,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { +static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable_t *p) { return p != &g_empty_pollable && p != &pollset->pollable; } @@ -762,8 +762,9 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p got pollset_wakeup %p", pollset, data_ptr); } - append_error(&error, grpc_wakeup_fd_consume_wakeup( - (void *)((~(intptr_t)1) & (intptr_t)data_ptr)), + append_error(&error, + grpc_wakeup_fd_consume_wakeup( + (grpc_wakeup_fd *)((~(intptr_t)1) & (intptr_t)data_ptr)), err_desc); } else { grpc_fd *fd = (grpc_fd *)data_ptr; @@ -800,7 +801,7 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { } static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - pollable *p, gpr_timespec now, + pollable_t *p, gpr_timespec now, gpr_timespec deadline) { int timeout = poll_deadline_to_millis_timeout(deadline, now); @@ -1028,7 +1029,7 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, "PS:%p add fd %p; transition pollable from empty to fd", pollset, fd); } - /* empty pollable --> single fd pollable */ + /* empty pollable --> single fd pollable_t */ pollset_kick_all(exec_ctx, pollset); pollset->current_pollable = &fd->pollable; if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu); diff --git a/src/core/lib/iomgr/polling_entity.h b/src/core/lib/iomgr/polling_entity.h index 971fd88b428..940f10f152d 100644 --- a/src/core/lib/iomgr/polling_entity.h +++ b/src/core/lib/iomgr/polling_entity.h @@ -22,6 +22,8 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" +typedef enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } pops_tag; + /* A grpc_polling_entity is a pollset-or-pollset_set container. It allows * functions that accept a pollset XOR a pollset_set to do so through an * abstract interface. No ownership is taken. */ @@ -31,7 +33,7 @@ typedef struct grpc_polling_entity { grpc_pollset *pollset; grpc_pollset_set *pollset_set; } pollent; - enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } tag; + pops_tag tag; } grpc_polling_entity; grpc_polling_entity grpc_polling_entity_create_from_pollset_set( diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 6c58986b534..4895e0d1c92 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -656,7 +656,7 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args( if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { if (channel_args->args[i].type == GRPC_ARG_POINTER) { return grpc_resource_quota_ref_internal( - channel_args->args[i].value.pointer.p); + (grpc_resource_quota *)channel_args->args[i].value.pointer.p); } else { gpr_log(GPR_DEBUG, GRPC_ARG_RESOURCE_QUOTA " should be a pointer"); } @@ -666,12 +666,12 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args( } static void *rq_copy(void *rq) { - grpc_resource_quota_ref(rq); + grpc_resource_quota_ref((grpc_resource_quota *)rq); return rq; } static void rq_destroy(grpc_exec_ctx *exec_ctx, void *rq) { - grpc_resource_quota_unref_internal(exec_ctx, rq); + grpc_resource_quota_unref_internal(exec_ctx, (grpc_resource_quota *)rq); } static int rq_cmp(void *a, void *b) { return GPR_ICMP(a, b); } diff --git a/src/core/lib/iomgr/socket_factory_posix.c b/src/core/lib/iomgr/socket_factory_posix.c index 0f82dea570f..c81566575ef 100644 --- a/src/core/lib/iomgr/socket_factory_posix.c +++ b/src/core/lib/iomgr/socket_factory_posix.c @@ -69,11 +69,11 @@ void grpc_socket_factory_unref(grpc_socket_factory *factory) { } static void *socket_factory_arg_copy(void *p) { - return grpc_socket_factory_ref(p); + return grpc_socket_factory_ref((grpc_socket_factory *)p); } static void socket_factory_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) { - grpc_socket_factory_unref(p); + grpc_socket_factory_unref((grpc_socket_factory *)p); } static int socket_factory_cmp(void *a, void *b) { diff --git a/src/core/lib/iomgr/socket_mutator.c b/src/core/lib/iomgr/socket_mutator.c index 5d6c2c400eb..300ac75b387 100644 --- a/src/core/lib/iomgr/socket_mutator.c +++ b/src/core/lib/iomgr/socket_mutator.c @@ -60,11 +60,11 @@ void grpc_socket_mutator_unref(grpc_socket_mutator *mutator) { } static void *socket_mutator_arg_copy(void *p) { - return grpc_socket_mutator_ref(p); + return grpc_socket_mutator_ref((grpc_socket_mutator *)p); } static void socket_mutator_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) { - grpc_socket_mutator_unref(p); + grpc_socket_mutator_unref((grpc_socket_mutator *)p); } static int socket_mutator_cmp(void *a, void *b) { diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index ae2c0bf0ae6..04ca44563d6 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -276,7 +276,7 @@ static void timer_thread(void *completed_thread_ptr) { GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); timer_main_loop(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); - timer_thread_cleanup(completed_thread_ptr); + timer_thread_cleanup((completed_thread *)completed_thread_ptr); } static void start_threads(void) { diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index 9a02c1d1bba..00b2e68bb5e 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -118,7 +118,7 @@ static grpc_socket_factory *get_socket_factory(const grpc_channel_args *args) { const grpc_arg *arg = grpc_channel_args_find(args, GRPC_ARG_SOCKET_FACTORY); if (arg) { GPR_ASSERT(arg->type == GRPC_ARG_POINTER); - return arg->value.pointer.p; + return (grpc_socket_factory *)arg->value.pointer.p; } } return NULL; diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 321a21a10b7..0764eda0527 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -174,8 +174,8 @@ static const grpc_slice_refcount_vtable new_with_len_vtable = { grpc_slice grpc_slice_new_with_len(void *p, size_t len, void (*destroy)(void *, size_t)) { grpc_slice slice; - new_with_len_slice_refcount *rc = - gpr_malloc(sizeof(new_with_len_slice_refcount)); + new_with_len_slice_refcount *rc = (new_with_len_slice_refcount *)gpr_malloc( + sizeof(new_with_len_slice_refcount)); gpr_ref_init(&rc->refs, 1); rc->rc.vtable = &new_with_len_vtable; rc->rc.sub_refcount = &rc->rc; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 93c512df694..f4e11025b1b 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -135,7 +135,7 @@ typedef struct batch_control { typedef struct { gpr_mu child_list_mu; grpc_call *first_child; -} parent_call; +} parent_call_t; typedef struct { grpc_call *parent; @@ -144,7 +144,7 @@ typedef struct { parent->mu */ grpc_call *sibling_next; grpc_call *sibling_prev; -} child_call; +} child_call_t; #define RECV_NONE ((gpr_atm)0) #define RECV_INITIAL_METADATA_FIRST ((gpr_atm)1) @@ -157,8 +157,8 @@ struct grpc_call { grpc_polling_entity pollent; grpc_channel *channel; gpr_timespec start_time; - /* parent_call* */ gpr_atm parent_call_atm; - child_call *child_call; + /* parent_call_t* */ gpr_atm parent_call_atm; + child_call_t *child_call; /* client or server call */ bool is_client; @@ -293,32 +293,32 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, batch_control *bctl); static void add_batch_error(grpc_exec_ctx *exec_ctx, batch_control *bctl, grpc_error *error, bool has_cancelled); -static void add_init_error(grpc_error **composite, grpc_error *new) { - if (new == GRPC_ERROR_NONE) return; +static void add_init_error(grpc_error **composite, grpc_error *new_err) { + if (new_err == GRPC_ERROR_NONE) return; if (*composite == GRPC_ERROR_NONE) *composite = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Call creation failed"); - *composite = grpc_error_add_child(*composite, new); + *composite = grpc_error_add_child(*composite, new_err); } void *grpc_call_arena_alloc(grpc_call *call, size_t size) { return gpr_arena_alloc(call->arena, size); } -static parent_call *get_or_create_parent_call(grpc_call *call) { - parent_call *p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call_t *get_or_create_parent_call(grpc_call *call) { + parent_call_t *p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); if (p == NULL) { - p = (parent_call *)gpr_arena_alloc(call->arena, sizeof(*p)); + p = (parent_call_t *)gpr_arena_alloc(call->arena, sizeof(*p)); gpr_mu_init(&p->child_list_mu); if (!gpr_atm_rel_cas(&call->parent_call_atm, (gpr_atm)NULL, (gpr_atm)p)) { gpr_mu_destroy(&p->child_list_mu); - p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); + p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); } } return p; } -static parent_call *get_parent_call(grpc_call *call) { - return (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call_t *get_parent_call(grpc_call *call) { + return (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); } grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, @@ -378,15 +378,15 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, bool immediately_cancel = false; if (args->parent_call != NULL) { - child_call *cc = call->child_call = - gpr_arena_alloc(arena, sizeof(child_call)); + child_call_t *cc = call->child_call = + (child_call_t *)gpr_arena_alloc(arena, sizeof(child_call_t)); call->child_call->parent = args->parent_call; GRPC_CALL_INTERNAL_REF(args->parent_call, "child"); GPR_ASSERT(call->is_client); GPR_ASSERT(!args->parent_call->is_client); - parent_call *pc = get_or_create_parent_call(args->parent_call); + parent_call_t *pc = get_or_create_parent_call(args->parent_call); gpr_mu_lock(&pc->child_list_mu); @@ -533,7 +533,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - parent_call *pc = get_parent_call(c); + parent_call_t *pc = get_parent_call(c); if (pc != NULL) { gpr_mu_destroy(&pc->child_list_mu); } @@ -549,8 +549,8 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind"); } - get_final_status(call, set_status_value_directly, &c->final_info.final_status, - NULL); + get_final_status((grpc_call *)call, set_status_value_directly, + &c->final_info.final_status, NULL); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); @@ -570,14 +570,14 @@ void grpc_call_ref(grpc_call *c) { gpr_ref(&c->ext_ref); } void grpc_call_unref(grpc_call *c) { if (!gpr_unref(&c->ext_ref)) return; - child_call *cc = c->child_call; + child_call_t *cc = c->child_call; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_call_unref", 0); GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (c)); if (cc) { - parent_call *pc = get_parent_call(cc->parent); + parent_call_t *pc = get_parent_call(cc->parent); gpr_mu_lock(&pc->child_list_mu); if (c == pc->first_child) { pc->first_child = cc->sibling_next; @@ -1309,7 +1309,7 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, /* propagate cancellation to any interested children */ gpr_atm_rel_store(&call->received_final_op_atm, 1); - parent_call *pc = get_parent_call(call); + parent_call_t *pc = get_parent_call(call); if (pc != NULL) { grpc_call *child; gpr_mu_lock(&pc->child_list_mu); @@ -1345,7 +1345,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, if (bctl->completion_data.notify_tag.is_closure) { /* unrefs bctl->error */ bctl->call = NULL; - GRPC_CLOSURE_RUN(exec_ctx, bctl->completion_data.notify_tag.tag, error); + GRPC_CLOSURE_RUN( + exec_ctx, (grpc_closure *)bctl->completion_data.notify_tag.tag, error); GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completion"); } else { /* unrefs bctl->error */ @@ -1474,7 +1475,7 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, * acq_load is in receiving_initial_metadata_ready() */ if (error != GRPC_ERROR_NONE || call->receiving_stream == NULL || !gpr_atm_rel_cas(&call->recv_state, RECV_NONE, (gpr_atm)bctlp)) { - process_data_after_md(exec_ctx, bctlp); + process_data_after_md(exec_ctx, (batch_control *)bctlp); } } @@ -1679,11 +1680,12 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, if (nops == 0) { if (!is_notify_tag_closure) { GPR_ASSERT(grpc_cq_begin_op(call->cq, notify_tag)); - grpc_cq_end_op(exec_ctx, call->cq, notify_tag, GRPC_ERROR_NONE, - free_no_op_completion, NULL, - gpr_malloc(sizeof(grpc_cq_completion))); + grpc_cq_end_op( + exec_ctx, call->cq, notify_tag, GRPC_ERROR_NONE, + free_no_op_completion, NULL, + (grpc_cq_completion *)gpr_malloc(sizeof(grpc_cq_completion))); } else { - GRPC_CLOSURE_SCHED(exec_ctx, notify_tag, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(exec_ctx, (grpc_closure *)notify_tag, GRPC_ERROR_NONE); } error = GRPC_CALL_OK; goto done; From 090aca55bda549a1c80bae8f4770de330b88b4bd Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 12 Sep 2017 12:14:13 -0700 Subject: [PATCH 095/227] Using already type casted variables --- src/core/lib/surface/call.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index f4e11025b1b..de5aa7b6bec 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -549,8 +549,8 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind"); } - get_final_status((grpc_call *)call, set_status_value_directly, - &c->final_info.final_status, NULL); + get_final_status(c, set_status_value_directly, &c->final_info.final_status, + NULL); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); @@ -1475,7 +1475,7 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, * acq_load is in receiving_initial_metadata_ready() */ if (error != GRPC_ERROR_NONE || call->receiving_stream == NULL || !gpr_atm_rel_cas(&call->recv_state, RECV_NONE, (gpr_atm)bctlp)) { - process_data_after_md(exec_ctx, (batch_control *)bctlp); + process_data_after_md(exec_ctx, bctl); } } From 97dab51a3f356a1cb324c9d7e01ed12ba84e5ed4 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 12 Sep 2017 14:27:46 -0700 Subject: [PATCH 096/227] Renaming pops_tag enum --- src/core/lib/iomgr/polling_entity.c | 18 +++++++++--------- src/core/lib/iomgr/polling_entity.h | 8 ++++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/core/lib/iomgr/polling_entity.c b/src/core/lib/iomgr/polling_entity.c index 74d8794af50..8591a5518e3 100644 --- a/src/core/lib/iomgr/polling_entity.c +++ b/src/core/lib/iomgr/polling_entity.c @@ -25,7 +25,7 @@ grpc_polling_entity grpc_polling_entity_create_from_pollset_set( grpc_pollset_set *pollset_set) { grpc_polling_entity pollent; pollent.pollent.pollset_set = pollset_set; - pollent.tag = POPS_POLLSET_SET; + pollent.tag = GRPC_POLLS_POLLSET_SET; return pollent; } @@ -33,12 +33,12 @@ grpc_polling_entity grpc_polling_entity_create_from_pollset( grpc_pollset *pollset) { grpc_polling_entity pollent; pollent.pollent.pollset = pollset; - pollent.tag = POPS_POLLSET; + pollent.tag = GRPC_POLLS_POLLSET; return pollent; } grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) { - if (pollent->tag == POPS_POLLSET) { + if (pollent->tag == GRPC_POLLS_POLLSET) { return pollent->pollent.pollset; } return NULL; @@ -46,23 +46,23 @@ grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) { grpc_pollset_set *grpc_polling_entity_pollset_set( grpc_polling_entity *pollent) { - if (pollent->tag == POPS_POLLSET_SET) { + if (pollent->tag == GRPC_POLLS_POLLSET_SET) { return pollent->pollent.pollset_set; } return NULL; } bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent) { - return pollent->tag == POPS_NONE; + return pollent->tag == GRPC_POLLS_NONE; } void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_polling_entity *pollent, grpc_pollset_set *pss_dst) { - if (pollent->tag == POPS_POLLSET) { + if (pollent->tag == GRPC_POLLS_POLLSET) { GPR_ASSERT(pollent->pollent.pollset != NULL); grpc_pollset_set_add_pollset(exec_ctx, pss_dst, pollent->pollent.pollset); - } else if (pollent->tag == POPS_POLLSET_SET) { + } else if (pollent->tag == GRPC_POLLS_POLLSET_SET) { GPR_ASSERT(pollent->pollent.pollset_set != NULL); grpc_pollset_set_add_pollset_set(exec_ctx, pss_dst, pollent->pollent.pollset_set); @@ -75,10 +75,10 @@ void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx, void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx, grpc_polling_entity *pollent, grpc_pollset_set *pss_dst) { - if (pollent->tag == POPS_POLLSET) { + if (pollent->tag == GRPC_POLLS_POLLSET) { GPR_ASSERT(pollent->pollent.pollset != NULL); grpc_pollset_set_del_pollset(exec_ctx, pss_dst, pollent->pollent.pollset); - } else if (pollent->tag == POPS_POLLSET_SET) { + } else if (pollent->tag == GRPC_POLLS_POLLSET_SET) { GPR_ASSERT(pollent->pollent.pollset_set != NULL); grpc_pollset_set_del_pollset_set(exec_ctx, pss_dst, pollent->pollent.pollset_set); diff --git a/src/core/lib/iomgr/polling_entity.h b/src/core/lib/iomgr/polling_entity.h index 940f10f152d..a161e1fea6f 100644 --- a/src/core/lib/iomgr/polling_entity.h +++ b/src/core/lib/iomgr/polling_entity.h @@ -22,7 +22,11 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" -typedef enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } pops_tag; +typedef enum grpc_pollset_tag { + GRPC_POLLS_NONE, + GRPC_POLLS_POLLSET, + GRPC_POLLS_POLLSET_SET +} grpc_pollset_tag; /* A grpc_polling_entity is a pollset-or-pollset_set container. It allows * functions that accept a pollset XOR a pollset_set to do so through an @@ -33,7 +37,7 @@ typedef struct grpc_polling_entity { grpc_pollset *pollset; grpc_pollset_set *pollset_set; } pollent; - pops_tag tag; + grpc_pollset_tag tag; } grpc_polling_entity; grpc_polling_entity grpc_polling_entity_create_from_pollset_set( From 72b36736ceaa7a267722cbcfe99a94bb879e3204 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Wed, 13 Sep 2017 00:05:29 +0200 Subject: [PATCH 097/227] The input files also need to follow the base argument... --- tools/buildgen/generate_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 559ea1624bc..d29cd02152f 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -85,7 +85,7 @@ for template in reversed(sorted(templates)): test[out] = tf[1] os.close(tf[0]) cmd.append(test[out]) - cmd.append(root + '/' + f) + cmd.append(args.base + '/' + root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) jobset.run(pre_jobs, maxjobs=args.jobs) From 49119a4caf48c20ed3655e0e5637129c9ed94481 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 8 Sep 2017 15:29:19 -0700 Subject: [PATCH 098/227] Fix for max_concurrent_streams issue - Call mark_stream_closed before sending trailing metadata --- .../ext/transport/chttp2/transport/chttp2_transport.c | 8 +++++++- .../ext/transport/chttp2/transport/frame_rst_stream.c | 3 +++ src/core/ext/transport/chttp2/transport/hpack_parser.c | 1 + src/core/ext/transport/chttp2/transport/writing.c | 8 ++++---- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 3fd701fe2f1..5fb943d3ca8 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1974,6 +1974,10 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { s->seen_error = true; } + if (!s->write_closed) { + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, + GRPC_ERROR_REF(due_to_error)); + } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, due_to_error); } @@ -2103,7 +2107,6 @@ void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx, if (close_writes && !s->write_closed) { s->write_closed_error = GRPC_ERROR_REF(error); s->write_closed = true; - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); } if (s->read_closed && s->write_closed) { became_closed = true; @@ -2288,6 +2291,9 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); + if (!s->write_closed) { + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); + } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); grpc_chttp2_initiate_write(exec_ctx, t, "close_from_api"); } diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index 0133b6efa25..d77097811a9 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -103,6 +103,9 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); gpr_free(message); } + if (!s->write_closed) { + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); + } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, error); } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 82ff2c8e2cf..e4f2ae84759 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1650,6 +1650,7 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index fa224a49a45..7512fe122d5 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -386,6 +386,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (is_last_frame) { s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_rst_stream_create( s->id, GRPC_HTTP2_NO_ERROR, @@ -444,6 +446,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( } s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_rst_stream_create( @@ -519,10 +523,6 @@ void grpc_chttp2_end_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ERROR_REF(error)); s->sending_bytes = 0; } - if (s->sent_trailing_metadata) { - grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, - GRPC_ERROR_REF(error)); - } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:end"); } grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &t->outbuf); From 6988dbb45696f385cb3945e3c10b5ba67872ec35 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 11 Sep 2017 15:34:09 -0700 Subject: [PATCH 099/227] There is no need to remove fail_pending_writes from mark_stream_closed anymore --- .../ext/transport/chttp2/transport/chttp2_transport.c | 8 +------- .../ext/transport/chttp2/transport/frame_rst_stream.c | 3 --- src/core/ext/transport/chttp2/transport/hpack_parser.c | 2 +- src/core/ext/transport/chttp2/transport/writing.c | 8 ++++---- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 5fb943d3ca8..3fd701fe2f1 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1974,10 +1974,6 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { s->seen_error = true; } - if (!s->write_closed) { - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, - GRPC_ERROR_REF(due_to_error)); - } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, due_to_error); } @@ -2107,6 +2103,7 @@ void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx, if (close_writes && !s->write_closed) { s->write_closed_error = GRPC_ERROR_REF(error); s->write_closed = true; + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); } if (s->read_closed && s->write_closed) { became_closed = true; @@ -2291,9 +2288,6 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); - if (!s->write_closed) { - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); - } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); grpc_chttp2_initiate_write(exec_ctx, t, "close_from_api"); } diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index d77097811a9..0133b6efa25 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -103,9 +103,6 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); gpr_free(message); } - if (!s->write_closed) { - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); - } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, error); } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index e4f2ae84759..35d52963e22 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1650,7 +1650,7 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); + // grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 7512fe122d5..3fa38db4db6 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -386,13 +386,13 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (is_last_frame) { s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; - grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, - GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_rst_stream_create( s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); } + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); } result.early_results_scheduled |= update_list(exec_ctx, t, s, @@ -446,13 +446,13 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( } s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; - grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, - GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_rst_stream_create( s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); } + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); now_writing = true; result.early_results_scheduled = true; grpc_chttp2_complete_closure_step( From a6ede3bea1cde14ffd5fc7f9b80a27dcdee345d9 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 11 Sep 2017 15:37:27 -0700 Subject: [PATCH 100/227] Remove unnecessary comment --- src/core/ext/transport/chttp2/transport/hpack_parser.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 35d52963e22..82ff2c8e2cf 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1650,7 +1650,6 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); - // grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); From 2be06b43e7532ef2e9d1e5709ceb162957e6a926 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 15:24:49 -0700 Subject: [PATCH 101/227] Make unmeasured CPU costs be 1, not 0 Allows capturing 0 as a CPU cost, which we'll be able to leverage to increase concurrency (once data has caught up in a few days) --- tools/run_tests/python_utils/jobset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 08d652ae3f3..80e79444ea8 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -224,7 +224,7 @@ class JobResult(object): self.retries = 0 self.message = '' self.cpu_estimated = 1 - self.cpu_measured = 0 + self.cpu_measured = 1 def read_from_start(f): From b2f490c565e87822245c486ce5b04b17adacd52d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 12 Sep 2017 16:25:39 -0700 Subject: [PATCH 102/227] Include limits.h since we're using INT_MAX (affects some compilers) --- src/core/ext/transport/chttp2/transport/flow_control.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/ext/transport/chttp2/transport/flow_control.c b/src/core/ext/transport/chttp2/transport/flow_control.c index 39aa521029b..0f078e79e90 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.c +++ b/src/core/ext/transport/chttp2/transport/flow_control.c @@ -18,6 +18,7 @@ #include "src/core/ext/transport/chttp2/transport/internal.h" +#include #include #include From 707145c78a7e1d65ddf14836adbbb7032172a3a0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 13 Sep 2017 10:02:58 +0200 Subject: [PATCH 103/227] mingw already installed on win kokoro workers --- tools/internal_ci/windows/grpc_build_artifacts.bat | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 17d9571d43e..c6e523697ca 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -19,8 +19,6 @@ rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits -pacman -S --noconfirm mingw64/mingw-w64-x86_64-gcc mingw32/mingw-w64-i686-gcc - @rem enter repo root cd /d %~dp0\..\..\.. From e97bc51d3966aaa6d8bbcd6ec9238063bba8dad2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 13 Sep 2017 12:26:32 +0200 Subject: [PATCH 104/227] already installed on mac workers --- .../helper_scripts/prepare_build_macos_rc | 13 +------------ tools/internal_ci/macos/grpc_build_artifacts.sh | 12 ------------ 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index f7fbec93ffa..dd2741e595b 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -31,23 +31,16 @@ ulimit -a pip install google-api-python-client --user python export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json -# required to build protobuf -brew install gflags - set +ex # rvm script is very verbose and exits with errorcode source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose -rvm install ruby-2.4 +rvm use ruby-2.4 rvm osx-ssl-certs status all rvm osx-ssl-certs update all set -ex -gem install bundler - # cocoapods export LANG=en_US.UTF-8 -gem install cocoapods -gem install xcpretty pod repo update # needed by python # python @@ -56,10 +49,6 @@ pip install virtualenv --user python pip install -U six tox setuptools --user python export PYTHONPATH=/Library/Python/3.4/site-packages -# python 3.4 -wget -q https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg -sudo installer -pkg python-3.4.4-macosx10.6.pkg -target / - # set xcode version for Obj-C tests sudo xcode-select -switch /Applications/Xcode_8.2.1.app/Contents/Developer diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 603c15f210f..09784e3bb47 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -20,25 +20,13 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_macos_rc -# python 3.5 -wget -q https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg -sudo installer -pkg python-3.5.2-macosx10.6.pkg -target / - # install cython for all python versions python2.7 -m pip install cython setuptools wheel python3.4 -m pip install cython setuptools wheel python3.5 -m pip install cython setuptools wheel python3.6 -m pip install cython setuptools wheel -# node-gyp (needed for node artifacts) -npm install -g node-gyp - -# php dependencies: pecl -curl -O http://pear.php.net/go-pear.phar -sudo php -d detect_unicode=0 go-pear.phar - # needed to build ruby artifacts -gem install rake-compiler wget https://raw.githubusercontent.com/grpc/grpc/master/tools/distrib/build_ruby_environment_macos.sh bash build_ruby_environment_macos.sh From e6d2e8f16bd2ebbfd112b09e589205191211a2c8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 06:10:18 -0700 Subject: [PATCH 105/227] Fix case on internal type names Should allow microbenchmarks to get running again --- tools/profiling/microbenchmarks/bm2bq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 6c5040de56d..9f9b672f759 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -30,7 +30,7 @@ columns = [] for row in json.loads( subprocess.check_output([ 'bq','--format=json','show','microbenchmarks.microbenchmarks']))['schema']['fields']: - columns.append((row['name'], row['type'])) + columns.append((row['name'], row['type'].lower())) SANITIZE = { 'integer': int, From 70bbb6b1176626346efc7c7549fb027896f8d8c2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:04:30 -0700 Subject: [PATCH 106/227] Fix schema generation --- src/core/lib/debug/stats_data_bq_schema.sql | 70 ++++++++++----------- tools/codegen/core/gen_stats_data.py | 2 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index 7291bbf0709..53547692918 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -1,35 +1,35 @@ -client_calls_created_per_iteration:INTEGER, -server_calls_created_per_iteration:INTEGER, -syscall_poll_per_iteration:INTEGER, -syscall_wait_per_iteration:INTEGER, -histogram_slow_lookups_per_iteration:INTEGER, -syscall_write_per_iteration:INTEGER, -syscall_read_per_iteration:INTEGER, -tcp_backup_pollers_created_per_iteration:INTEGER, -tcp_backup_poller_polls_per_iteration:INTEGER, -http2_op_batches_per_iteration:INTEGER, -http2_op_cancel_per_iteration:INTEGER, -http2_op_send_initial_metadata_per_iteration:INTEGER, -http2_op_send_message_per_iteration:INTEGER, -http2_op_send_trailing_metadata_per_iteration:INTEGER, -http2_op_recv_initial_metadata_per_iteration:INTEGER, -http2_op_recv_message_per_iteration:INTEGER, -http2_op_recv_trailing_metadata_per_iteration:INTEGER, -http2_settings_writes_per_iteration:INTEGER, -http2_pings_sent_per_iteration:INTEGER, -http2_writes_begun_per_iteration:INTEGER, -http2_writes_offloaded_per_iteration:INTEGER, -http2_writes_continued_per_iteration:INTEGER, -http2_partial_writes_per_iteration:INTEGER, -combiner_locks_initiated_per_iteration:INTEGER, -combiner_locks_scheduled_items_per_iteration:INTEGER, -combiner_locks_scheduled_final_items_per_iteration:INTEGER, -combiner_locks_offloaded_per_iteration:INTEGER, -executor_scheduled_short_items_per_iteration:INTEGER, -executor_scheduled_long_items_per_iteration:INTEGER, -executor_scheduled_to_self_per_iteration:INTEGER, -executor_wakeup_initiated_per_iteration:INTEGER, -executor_queue_drained_per_iteration:INTEGER, -executor_push_retries_per_iteration:INTEGER, -server_requested_calls_per_iteration:INTEGER, -server_slowpath_requests_queued_per_iteration:INTEGER +client_calls_created_per_iteration:FLOAT, +server_calls_created_per_iteration:FLOAT, +syscall_poll_per_iteration:FLOAT, +syscall_wait_per_iteration:FLOAT, +histogram_slow_lookups_per_iteration:FLOAT, +syscall_write_per_iteration:FLOAT, +syscall_read_per_iteration:FLOAT, +tcp_backup_pollers_created_per_iteration:FLOAT, +tcp_backup_poller_polls_per_iteration:FLOAT, +http2_op_batches_per_iteration:FLOAT, +http2_op_cancel_per_iteration:FLOAT, +http2_op_send_initial_metadata_per_iteration:FLOAT, +http2_op_send_message_per_iteration:FLOAT, +http2_op_send_trailing_metadata_per_iteration:FLOAT, +http2_op_recv_initial_metadata_per_iteration:FLOAT, +http2_op_recv_message_per_iteration:FLOAT, +http2_op_recv_trailing_metadata_per_iteration:FLOAT, +http2_settings_writes_per_iteration:FLOAT, +http2_pings_sent_per_iteration:FLOAT, +http2_writes_begun_per_iteration:FLOAT, +http2_writes_offloaded_per_iteration:FLOAT, +http2_writes_continued_per_iteration:FLOAT, +http2_partial_writes_per_iteration:FLOAT, +combiner_locks_initiated_per_iteration:FLOAT, +combiner_locks_scheduled_items_per_iteration:FLOAT, +combiner_locks_scheduled_final_items_per_iteration:FLOAT, +combiner_locks_offloaded_per_iteration:FLOAT, +executor_scheduled_short_items_per_iteration:FLOAT, +executor_scheduled_long_items_per_iteration:FLOAT, +executor_scheduled_to_self_per_iteration:FLOAT, +executor_wakeup_initiated_per_iteration:FLOAT, +executor_queue_drained_per_iteration:FLOAT, +executor_push_retries_per_iteration:FLOAT, +server_requested_calls_per_iteration:FLOAT, +server_slowpath_requests_queued_per_iteration:FLOAT diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 877a5d9d430..8359734c848 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -405,6 +405,6 @@ with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: columns = [] for counter in inst_map['Counter']: - columns.append(('%s_per_iteration' % counter.name, 'INTEGER')) + columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) print >>S, ',\n'.join('%s:%s' % x for x in columns) From f2cbd6c81eab267fb8c04121ff76ffbbe689dfa1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:31:16 -0700 Subject: [PATCH 107/227] Fix qps_test histogram python code --- tools/run_tests/performance/massage_qps_stats_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py index 400a0c82bfd..a2fe4ae6c34 100644 --- a/tools/run_tests/performance/massage_qps_stats_helpers.py +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -23,7 +23,7 @@ def _threshold_for_count_below(buckets, boundaries, count_below): if count_so_far == count_below: # this bucket hits the threshold exactly... we should be midway through # any run of zero values following the bucket - for upper_idx in range(lower_idx + 1, num_buckets): + for upper_idx in range(lower_idx + 1, len(buckets)): if buckets[upper_idx] != 0: break return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0 From 480f5d861264f00ee2226b2aa000bbeb268f0a29 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:36:07 -0700 Subject: [PATCH 108/227] Add missing increments --- src/core/lib/iomgr/ev_epoll1_linux.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 322fa160d08..8a984e24eb8 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -512,14 +512,17 @@ static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_POLLSET_KICK(exec_ctx); switch (worker->kick_state) { case KICKED: + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); break; case UNKICKED: + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); SET_KICK_STATE(worker, KICKED); if (worker->initialized_cv) { gpr_cv_signal(&worker->cv); } break; case DESIGNATED_POLLER: + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); SET_KICK_STATE(worker, KICKED); append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd), "pollset_kick_all"); From f7617bb8f6bd4b6993d6cf5bf70a14a5819f5fd6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:47:28 -0700 Subject: [PATCH 109/227] Really use measured CPU costs (with a small delta) for run_tests --- tools/run_tests/run_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index b66c5f7f71f..591a0bebbde 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -80,7 +80,7 @@ def get_bqtest_data(limit=None): SELECT filtered_test_name, SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, - MAX(cpu_measured) as cpu + MAX(cpu_measured) + 0.01 as cpu FROM ( SELECT REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, @@ -92,9 +92,7 @@ SELECT AND platform = '"""+platform_string()+"""' AND NOT REGEXP_MATCH(job_name, '.*portability.*') ) GROUP BY - filtered_test_name -HAVING - flaky OR cpu > 0""" + filtered_test_name""" if limit: query += " limit {}".format(limit) query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) From 5160155005c7d0f1ee9b855dfbe6aa45d8602bf1 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 12 Sep 2017 09:31:07 -0700 Subject: [PATCH 110/227] Upload Kokoro MacOS master results --- tools/internal_ci/macos/grpc_basictests_dbg.cfg | 2 +- tools/internal_ci/macos/grpc_basictests_opt.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/internal_ci/macos/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_dbg.cfg index f058f0c7e47..53bda1ff0a6 100644 --- a/tools/internal_ci/macos/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/grpc_basictests_dbg.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" } diff --git a/tools/internal_ci/macos/grpc_basictests_opt.cfg b/tools/internal_ci/macos/grpc_basictests_opt.cfg index 5048baaf483..d359eb601a0 100644 --- a/tools/internal_ci/macos/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/grpc_basictests_opt.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" } From b0ed7374b3a1a653612f0f5b5548964c56a93f35 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 12 Sep 2017 16:24:42 -0700 Subject: [PATCH 111/227] Enable filter if either AGE or IDLE is specified; don't require both. --- src/core/ext/filters/max_age/max_age_filter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/filters/max_age/max_age_filter.c b/src/core/ext/filters/max_age/max_age_filter.c index 450f67746ff..0ac803ed41f 100644 --- a/src/core/ext/filters/max_age/max_age_filter.c +++ b/src/core/ext/filters/max_age/max_age_filter.c @@ -402,7 +402,7 @@ static bool maybe_add_max_age_filter(grpc_exec_ctx* exec_ctx, bool enable = grpc_channel_arg_get_integer( grpc_channel_args_find(channel_args, GRPC_ARG_MAX_CONNECTION_AGE_MS), - MAX_CONNECTION_AGE_INTEGER_OPTIONS) != INT_MAX && + MAX_CONNECTION_AGE_INTEGER_OPTIONS) != INT_MAX || grpc_channel_arg_get_integer( grpc_channel_args_find(channel_args, GRPC_ARG_MAX_CONNECTION_IDLE_MS), MAX_CONNECTION_IDLE_INTEGER_OPTIONS) != INT_MAX; From 4a7fca5b655f5e9114c81f48b3615847c5cb783d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 12 Sep 2017 14:31:11 -0700 Subject: [PATCH 112/227] Give guidance on prefixing struct/enum/#define names --- doc/c-style-guide.md | 14 +++++++++++--- src/core/lib/iomgr/ev_poll_posix.c | 6 +++--- src/core/lib/iomgr/wakeup_fd_cv.c | 14 +++++++------- src/core/lib/iomgr/wakeup_fd_cv.h | 4 ++-- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/doc/c-style-guide.md b/doc/c-style-guide.md index 369bd56a463..2cfa41dd9ec 100644 --- a/doc/c-style-guide.md +++ b/doc/c-style-guide.md @@ -32,14 +32,14 @@ Header Files # endif ``` - Header files should be self-contained and end in .h. -- All header files should have a #define guard to prevent multiple inclusion. +- All header files should have a `#define` guard to prevent multiple inclusion. To guarantee uniqueness they should be based on the file's path. For public headers: `include/grpc/grpc.h` → `GRPC_GRPC_H` For private headers: - `src/core/channel/channel_stack.h` → - `GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_STACK_H` + `src/core/lib/channel/channel_stack.h` → + `GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_H` Variable Initialization ----------------------- @@ -72,8 +72,16 @@ Symbol Names - Non-static functions must be prefixed by `grpc_` - Static functions must *not* be prefixed by `grpc_` +- Typenames of `struct`s , `union`s, and `enum`s must be prefixed by `grpc_` if + they are declared in a header file. They must not be prefixed by `grpc_` if + they are declared in a source file. - Enumeration values and `#define` names must be uppercase. All other values must be lowercase. +- Enumeration values or `#define` names defined in a header file must be + prefixed with `GRPC_` (except for `#define` macros that are being used to + substitute functions; those should follow the general rules for + functions). Enumeration values or `#define`s defined in source files must not + be prefixed with `GRPC_`. - Multiple word identifiers use underscore as a delimiter, *never* camel case. E.g. `variable_name`. diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index bcf1d9001bb..7f44eda1382 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -1539,7 +1539,7 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) { for (i = 0; i < nfds; i++) { fds[i].revents = 0; if (fds[i].fd < 0 && (fds[i].events & POLLIN)) { - idx = FD_TO_IDX(fds[i].fd); + idx = GRPC_FD_TO_IDX(fds[i].fd); fd_cvs[i].cv = &pollcv_cv; fd_cvs[i].prev = NULL; fd_cvs[i].next = g_cvfds.cvfds[idx].cvs; @@ -1602,8 +1602,8 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) { idx = 0; for (i = 0; i < nfds; i++) { if (fds[i].fd < 0 && (fds[i].events & POLLIN)) { - remove_cvn(&g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs, &(fd_cvs[i])); - if (g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].is_set) { + remove_cvn(&g_cvfds.cvfds[GRPC_FD_TO_IDX(fds[i].fd)].cvs, &(fd_cvs[i])); + if (g_cvfds.cvfds[GRPC_FD_TO_IDX(fds[i].fd)].is_set) { fds[i].revents = POLLIN; if (res >= 0) res++; } diff --git a/src/core/lib/iomgr/wakeup_fd_cv.c b/src/core/lib/iomgr/wakeup_fd_cv.c index 5e0b1d1704c..268e0175ddb 100644 --- a/src/core/lib/iomgr/wakeup_fd_cv.c +++ b/src/core/lib/iomgr/wakeup_fd_cv.c @@ -57,7 +57,7 @@ static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) { g_cvfds.free_fds = g_cvfds.free_fds->next_free; g_cvfds.cvfds[idx].cvs = NULL; g_cvfds.cvfds[idx].is_set = 0; - fd_info->read_fd = IDX_TO_FD(idx); + fd_info->read_fd = GRPC_IDX_TO_FD(idx); fd_info->write_fd = -1; gpr_mu_unlock(&g_cvfds.mu); return GRPC_ERROR_NONE; @@ -66,8 +66,8 @@ static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) { static grpc_error* cv_fd_wakeup(grpc_wakeup_fd* fd_info) { cv_node* cvn; gpr_mu_lock(&g_cvfds.mu); - g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].is_set = 1; - cvn = g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].cvs; + g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].is_set = 1; + cvn = g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].cvs; while (cvn) { gpr_cv_signal(cvn->cv); cvn = cvn->next; @@ -78,7 +78,7 @@ static grpc_error* cv_fd_wakeup(grpc_wakeup_fd* fd_info) { static grpc_error* cv_fd_consume(grpc_wakeup_fd* fd_info) { gpr_mu_lock(&g_cvfds.mu); - g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].is_set = 0; + g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].is_set = 0; gpr_mu_unlock(&g_cvfds.mu); return GRPC_ERROR_NONE; } @@ -89,9 +89,9 @@ static void cv_fd_destroy(grpc_wakeup_fd* fd_info) { } gpr_mu_lock(&g_cvfds.mu); // Assert that there are no active pollers - GPR_ASSERT(!g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].cvs); - g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].next_free = g_cvfds.free_fds; - g_cvfds.free_fds = &g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)]; + GPR_ASSERT(!g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].cvs); + g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].next_free = g_cvfds.free_fds; + g_cvfds.free_fds = &g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)]; gpr_mu_unlock(&g_cvfds.mu); } diff --git a/src/core/lib/iomgr/wakeup_fd_cv.h b/src/core/lib/iomgr/wakeup_fd_cv.h index 46e84f58437..dc170ad5b43 100644 --- a/src/core/lib/iomgr/wakeup_fd_cv.h +++ b/src/core/lib/iomgr/wakeup_fd_cv.h @@ -37,8 +37,8 @@ #include "src/core/lib/iomgr/ev_posix.h" -#define FD_TO_IDX(fd) (-(fd)-1) -#define IDX_TO_FD(idx) (-(idx)-1) +#define GRPC_FD_TO_IDX(fd) (-(fd)-1) +#define GRPC_IDX_TO_FD(idx) (-(idx)-1) typedef struct cv_node { gpr_cv* cv; From ebacb2f20d5b9fbf73d1f90d06eee7553c93aee6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 12:32:33 -0700 Subject: [PATCH 113/227] Fixes --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- src/core/lib/iomgr/ev_epollex_linux.c | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index bf080fff0c0..f7f6cec9ef6 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -515,9 +515,9 @@ static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); break; case UNKICKED: - GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); SET_KICK_STATE(worker, KICKED); if (worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&worker->cv); } break; diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index f1a3b97f4da..4e5bed5016b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -666,12 +666,8 @@ static grpc_error *pollset_kick_inner(grpc_exec_ctx *exec_ctx, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { -<<<<<<< HEAD - pollable *p = pollset->current_pollable; - GRPC_STATS_INC_POLLSET_KICK(exec_ctx); -======= pollable_t *p = pollset->current_pollable; ->>>>>>> d1fefe89dacf198a721053f2b1a3e173f1f24948 + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } From cf34fa58f476aae77c4e2dca0ff866542dd3af28 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 12:37:01 -0700 Subject: [PATCH 114/227] Document only epoll1 for these stats, add missing spot --- src/core/lib/debug/stats_data.c | 14 ++++++++------ src/core/lib/debug/stats_data.yaml | 6 ++++++ src/core/lib/iomgr/ev_epoll1_linux.c | 8 +++++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index fc1ac223de3..1a020a15c38 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -70,16 +70,18 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of server side calls created by this process", "Number of polling syscalls (epoll_wait, poll, etc) made by this process", "Number of sleeping syscalls made by this process", - "How many polling wakeups were performed by the process", - "How many times was a polling wakeup requested without an active poller", + "How many polling wakeups were performed by the process (only valid for " + "epoll1 right now)", + "How many times was a polling wakeup requested without an active poller " + "(only valid for epoll1 right now)", "How many times was the same polling worker awoken repeatedly before " - "waking up", + "waking up (only valid for epoll1 right now)", "How many times was an eventfd used as the wakeup vector for a polling " - "wakeup", + "wakeup (only valid for epoll1 right now)", "How many times was a condition variable used as the wakeup vector for a " - "polling wakeup", + "polling wakeup (only valid for epoll1 right now)", "How many times could a polling wakeup be satisfied by keeping the waking " - "thread awake?", + "thread awake? (only valid for epoll1 right now)", "Number of times histogram increments went through the slow (binary " "search) path", "Number of write syscalls (or equivalent - eg sendmsg) made by this " diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 26c8e1b11d2..b43c466d4f0 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -35,20 +35,26 @@ doc: How many events are called for each syscall_poll - counter: pollset_kick doc: How many polling wakeups were performed by the process + (only valid for epoll1 right now) - counter: pollset_kicked_without_poller doc: How many times was a polling wakeup requested without an active poller + (only valid for epoll1 right now) - counter: pollset_kicked_again doc: How many times was the same polling worker awoken repeatedly before waking up + (only valid for epoll1 right now) - counter: pollset_kick_wakeup_fd doc: How many times was an eventfd used as the wakeup vector for a polling wakeup + (only valid for epoll1 right now) - counter: pollset_kick_wakeup_cv doc: How many times was a condition variable used as the wakeup vector for a polling wakeup + (only valid for epoll1 right now) - counter: pollset_kick_own_thread doc: How many times could a polling wakeup be satisfied by keeping the waking thread awake? + (only valid for epoll1 right now) # stats system - counter: histogram_slow_lookups doc: Number of times histogram increments went through the slow diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index f7f6cec9ef6..8f661583696 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -789,7 +789,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } static bool check_neighbourhood_for_available_poller( - pollset_neighbourhood *neighbourhood) { + grpc_exec_ctx *exec_ctx, pollset_neighbourhood *neighbourhood) { GPR_TIMER_BEGIN("check_neighbourhood_for_available_poller", 0); bool found_worker = false; do { @@ -813,6 +813,7 @@ static bool check_neighbourhood_for_available_poller( SET_KICK_STATE(inspect_worker, DESIGNATED_POLLER); if (inspect_worker->initialized_cv) { GPR_TIMER_MARK("signal worker", 0); + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&inspect_worker->cv); } } else { @@ -891,7 +892,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_num_neighbourhoods]; if (gpr_mu_trylock(&neighbourhood->mu)) { found_worker = - check_neighbourhood_for_available_poller(neighbourhood); + check_neighbourhood_for_available_poller(exec_ctx, neighbourhood); gpr_mu_unlock(&neighbourhood->mu); scan_state[i] = true; } else { @@ -904,7 +905,8 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, &g_neighbourhoods[(poller_neighbourhood_idx + i) % g_num_neighbourhoods]; gpr_mu_lock(&neighbourhood->mu); - found_worker = check_neighbourhood_for_available_poller(neighbourhood); + found_worker = + check_neighbourhood_for_available_poller(exec_ctx, neighbourhood); gpr_mu_unlock(&neighbourhood->mu); } grpc_exec_ctx_flush(exec_ctx); From bc130daf5f78bac986cc5cdede8476896dc5f463 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 12 Sep 2017 22:44:08 -0700 Subject: [PATCH 115/227] Adding more pointer conversions and type renaming --- .../client_channel/channel_connectivity.c | 10 +- .../filters/client_channel/client_channel.c | 5 +- .../client_channel/client_channel_factory.c | 5 +- .../grpclb/client_load_reporting_filter.c | 3 +- .../client_channel/lb_policy/grpclb/grpclb.c | 67 ++++++++---- .../lb_policy/grpclb/load_balancer_api.c | 6 +- .../lb_policy/pick_first/pick_first.c | 6 +- .../client_channel/lb_policy_factory.c | 9 +- .../dns/c_ares/grpc_ares_ev_driver_posix.c | 37 +++---- .../resolver/dns/c_ares/grpc_ares_wrapper.c | 7 +- .../filters/client_channel/retry_throttle.c | 9 +- .../filters/client_channel/subchannel_index.c | 12 ++- .../filters/http/server/http_server_filter.c | 6 +- .../chttp2/client/chttp2_connector.c | 2 +- .../transport/chttp2/server/chttp2_server.c | 18 ++-- .../chttp2/transport/chttp2_transport.c | 3 +- .../transport/chttp2/transport/flow_control.c | 16 +-- .../chttp2/transport/frame_settings.c | 17 +-- .../chttp2/transport/incoming_metadata.c | 5 +- .../transport/chttp2/transport/stream_map.c | 6 +- .../ext/transport/inproc/inproc_transport.c | 101 +++++++++--------- src/core/lib/surface/completion_queue.c | 28 ++--- src/core/lib/surface/server.c | 60 ++++++----- src/core/lib/transport/metadata_batch.c | 32 +++--- src/core/lib/transport/transport.c | 5 +- .../end2end/tests/resource_quota_server.c | 12 +-- 26 files changed, 266 insertions(+), 221 deletions(-) diff --git a/src/core/ext/filters/client_channel/channel_connectivity.c b/src/core/ext/filters/client_channel/channel_connectivity.c index e5f6fa76ae7..3844b980215 100644 --- a/src/core/ext/filters/client_channel/channel_connectivity.c +++ b/src/core/ext/filters/client_channel/channel_connectivity.c @@ -86,7 +86,7 @@ static void delete_state_watcher(grpc_exec_ctx *exec_ctx, state_watcher *w) { static void finished_completion(grpc_exec_ctx *exec_ctx, void *pw, grpc_cq_completion *ignored) { - int delete = 0; + bool should_delete = false; state_watcher *w = (state_watcher *)pw; gpr_mu_lock(&w->mu); switch (w->phase) { @@ -94,12 +94,12 @@ static void finished_completion(grpc_exec_ctx *exec_ctx, void *pw, case READY_TO_CALL_BACK: GPR_UNREACHABLE_CODE(return ); case CALLING_BACK_AND_FINISHED: - delete = 1; + should_delete = true; break; } gpr_mu_unlock(&w->mu); - if (delete) { + if (should_delete) { delete_state_watcher(exec_ctx, w); } } @@ -161,12 +161,12 @@ static void partly_done(grpc_exec_ctx *exec_ctx, state_watcher *w, static void watch_complete(grpc_exec_ctx *exec_ctx, void *pw, grpc_error *error) { - partly_done(exec_ctx, pw, true, GRPC_ERROR_REF(error)); + partly_done(exec_ctx, (state_watcher *)pw, true, GRPC_ERROR_REF(error)); } static void timeout_complete(grpc_exec_ctx *exec_ctx, void *pw, grpc_error *error) { - partly_done(exec_ctx, pw, false, GRPC_ERROR_REF(error)); + partly_done(exec_ctx, (state_watcher *)pw, false, GRPC_ERROR_REF(error)); } int grpc_channel_num_external_connectivity_watchers(grpc_channel *channel) { diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index e5f4a8a813c..129d0f368b7 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -85,7 +85,7 @@ static void method_parameters_unref(method_parameters *method_params) { } static void method_parameters_free(grpc_exec_ctx *exec_ctx, void *value) { - method_parameters_unref(value); + method_parameters_unref((method_parameters *)value); } static bool parse_wait_for_ready(grpc_json *field, @@ -717,7 +717,8 @@ static grpc_error *cc_init_channel_elem(grpc_exec_ctx *exec_ctx, return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "client channel factory arg must be a pointer"); } - grpc_client_channel_factory_ref(arg->value.pointer.p); + grpc_client_channel_factory_ref( + (grpc_client_channel_factory *)arg->value.pointer.p); chand->client_channel_factory = (grpc_client_channel_factory *)arg->value.pointer.p; // Get server name to resolve, using proxy mapper if needed. diff --git a/src/core/ext/filters/client_channel/client_channel_factory.c b/src/core/ext/filters/client_channel/client_channel_factory.c index 7220a8639e8..a1ba31e5cff 100644 --- a/src/core/ext/filters/client_channel/client_channel_factory.c +++ b/src/core/ext/filters/client_channel/client_channel_factory.c @@ -43,14 +43,15 @@ grpc_channel* grpc_client_channel_factory_create_channel( } static void* factory_arg_copy(void* factory) { - grpc_client_channel_factory_ref(factory); + grpc_client_channel_factory_ref((grpc_client_channel_factory*)factory); return factory; } static void factory_arg_destroy(grpc_exec_ctx* exec_ctx, void* factory) { // TODO(roth): Remove local exec_ctx when // https://github.com/grpc/grpc/pull/8705 is merged. - grpc_client_channel_factory_unref(exec_ctx, factory); + grpc_client_channel_factory_unref(exec_ctx, + (grpc_client_channel_factory*)factory); } static int factory_arg_cmp(void* factory1, void* factory2) { diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c index bd290464c84..7ad322902bd 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c @@ -75,7 +75,8 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(args->context != NULL); GPR_ASSERT(args->context[GRPC_GRPCLB_CLIENT_STATS].value != NULL); calld->client_stats = grpc_grpclb_client_stats_ref( - args->context[GRPC_GRPCLB_CLIENT_STATS].value); + (grpc_grpclb_client_stats *)args->context[GRPC_GRPCLB_CLIENT_STATS] + .value); // Record call started. grpc_grpclb_client_stats_add_call_started(calld->client_stats); return GRPC_ERROR_NONE; diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 18979829bd2..df4a00406e3 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -138,7 +138,7 @@ static grpc_error *initial_metadata_add_lb_token( } static void destroy_client_stats(void *arg) { - grpc_grpclb_client_stats_unref(arg); + grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats *)arg); } typedef struct wrapped_rr_closure_arg { @@ -286,7 +286,49 @@ static void add_pending_ping(pending_ping **root, grpc_closure *notify) { * glb_lb_policy */ typedef struct rr_connectivity_data rr_connectivity_data; -static const grpc_lb_policy_vtable glb_lb_policy_vtable; + +/* Forward declare functions referred in glb_lb_policy_vtable */ +static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); +static void glb_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); +static int glb_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + const grpc_lb_policy_pick_args *pick_args, + grpc_connected_subchannel **target, + grpc_call_context_element *context, void **user_data, + grpc_closure *on_complete); +static void glb_cancel_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + grpc_connected_subchannel **target, + grpc_error *error); +static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, + grpc_lb_policy *pol, + uint32_t initial_metadata_flags_mask, + uint32_t initial_metadata_flags_eq, + grpc_error *error); +static void glb_ping_one_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + grpc_closure *closure); +static void glb_exit_idle_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); +static grpc_connectivity_state glb_check_connectivity_locked( + grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + grpc_error **connectivity_error); +static void glb_notify_on_state_change_locked(grpc_exec_ctx *exec_ctx, + grpc_lb_policy *pol, + grpc_connectivity_state *current, + grpc_closure *notify); +static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, + const grpc_lb_policy_args *args); + +/* Code wiring the policy with the rest of the core */ +static const grpc_lb_policy_vtable glb_lb_policy_vtable = { + glb_destroy, + glb_shutdown_locked, + glb_pick_locked, + glb_cancel_pick_locked, + glb_cancel_picks_locked, + glb_ping_one_locked, + glb_exit_idle_locked, + glb_check_connectivity_locked, + glb_notify_on_state_change_locked, + glb_update_locked}; + typedef struct glb_lb_policy { /** base policy: must be first */ grpc_lb_policy base; @@ -783,7 +825,7 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, /* Allocate the data for the tracking of the new RR policy's connectivity. * It'll be deallocated in glb_rr_connectivity_changed() */ rr_connectivity_data *rr_connectivity = - gpr_zalloc(sizeof(rr_connectivity_data)); + (rr_connectivity_data *)gpr_zalloc(sizeof(rr_connectivity_data)); GRPC_CLOSURE_INIT(&rr_connectivity->on_change, glb_rr_connectivity_changed_locked, rr_connectivity, grpc_combiner_scheduler(glb_policy->base.combiner)); @@ -923,7 +965,8 @@ static grpc_channel_args *build_lb_channel_args( grpc_lb_addresses *lb_addresses = grpc_lb_addresses_create(num_grpclb_addrs, NULL); grpc_slice_hash_table_entry *targets_info_entries = - gpr_zalloc(sizeof(*targets_info_entries) * num_grpclb_addrs); + (grpc_slice_hash_table_entry *)gpr_zalloc(sizeof(*targets_info_entries) * + num_grpclb_addrs); size_t lb_addresses_idx = 0; for (size_t i = 0; i < addresses->num_addresses; ++i) { @@ -1382,7 +1425,8 @@ static void do_send_client_load_report_locked(grpc_exec_ctx *exec_ctx, static bool load_report_counters_are_zero(grpc_grpclb_request *request) { grpc_grpclb_dropped_call_counts *drop_entries = - request->client_stats.calls_finished_with_drop.arg; + (grpc_grpclb_dropped_call_counts *) + request->client_stats.calls_finished_with_drop.arg; return request->client_stats.num_calls_started == 0 && request->client_stats.num_calls_finished == 0 && request->client_stats.num_calls_finished_with_client_failed_to_send == @@ -1969,19 +2013,6 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, } } -/* Code wiring the policy with the rest of the core */ -static const grpc_lb_policy_vtable glb_lb_policy_vtable = { - glb_destroy, - glb_shutdown_locked, - glb_pick_locked, - glb_cancel_pick_locked, - glb_cancel_picks_locked, - glb_ping_one_locked, - glb_exit_idle_locked, - glb_check_connectivity_locked, - glb_notify_on_state_change_locked, - glb_update_locked}; - static void glb_factory_ref(grpc_lb_policy_factory *factory) {} static void glb_factory_unref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c index 407bd18adb0..8ef6dfc6f40 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c @@ -148,7 +148,8 @@ grpc_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request) { void grpc_grpclb_request_destroy(grpc_grpclb_request *request) { if (request->has_client_stats) { grpc_grpclb_dropped_call_counts *drop_entries = - request->client_stats.calls_finished_with_drop.arg; + (grpc_grpclb_dropped_call_counts *) + request->client_stats.calls_finished_with_drop.arg; grpc_grpclb_dropped_call_counts_destroy(drop_entries); } gpr_free(request); @@ -170,7 +171,8 @@ grpc_grpclb_initial_response *grpc_grpclb_initial_response_parse( if (!res.has_initial_response) return NULL; grpc_grpclb_initial_response *initial_res = - gpr_malloc(sizeof(grpc_grpclb_initial_response)); + (grpc_grpclb_initial_response *)gpr_malloc( + sizeof(grpc_grpclb_initial_response)); memcpy(initial_res, &res.initial_response, sizeof(grpc_grpclb_initial_response)); diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c index fab3073eb92..7029f71bf37 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c @@ -330,8 +330,8 @@ static void pf_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, gpr_log(GPR_INFO, "Pick First %p received update with %lu addresses", (void *)p, (unsigned long)addresses->num_addresses); } - grpc_subchannel_args *sc_args = - gpr_zalloc(sizeof(*sc_args) * addresses->num_addresses); + grpc_subchannel_args *sc_args = (grpc_subchannel_args *)gpr_zalloc( + sizeof(*sc_args) * addresses->num_addresses); /* We remove the following keys in order for subchannel keys belonging to * subchannels point to the same address to match. */ static const char *keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS, @@ -403,7 +403,7 @@ static void pf_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, } /* Create the subchannels for the new subchannel args/addresses. */ grpc_subchannel **new_subchannels = - gpr_zalloc(sizeof(*new_subchannels) * sc_args_count); + (grpc_subchannel **)gpr_zalloc(sizeof(*new_subchannels) * sc_args_count); size_t num_new_subchannels = 0; for (size_t i = 0; i < sc_args_count; i++) { grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index 918bab745c6..a9a00f5b2b4 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -126,13 +126,14 @@ void grpc_lb_addresses_destroy(grpc_exec_ctx* exec_ctx, } static void* lb_addresses_copy(void* addresses) { - return grpc_lb_addresses_copy(addresses); + return grpc_lb_addresses_copy((grpc_lb_addresses*)addresses); } static void lb_addresses_destroy(grpc_exec_ctx* exec_ctx, void* addresses) { - grpc_lb_addresses_destroy(exec_ctx, addresses); + grpc_lb_addresses_destroy(exec_ctx, (grpc_lb_addresses*)addresses); } static int lb_addresses_cmp(void* addresses1, void* addresses2) { - return grpc_lb_addresses_cmp(addresses1, addresses2); + return grpc_lb_addresses_cmp((grpc_lb_addresses*)addresses1, + (grpc_lb_addresses*)addresses2); } static const grpc_arg_pointer_vtable lb_addresses_arg_vtable = { lb_addresses_copy, lb_addresses_destroy, lb_addresses_cmp}; @@ -149,7 +150,7 @@ grpc_lb_addresses* grpc_lb_addresses_find_channel_arg( grpc_channel_args_find(channel_args, GRPC_ARG_LB_ADDRESSES); if (lb_addresses_arg == NULL || lb_addresses_arg->type != GRPC_ARG_POINTER) return NULL; - return lb_addresses_arg->value.pointer.p; + return (grpc_lb_addresses*)lb_addresses_arg->value.pointer.p; } void grpc_lb_policy_factory_ref(grpc_lb_policy_factory* factory) { diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c index 9747d39a161..7f1f57259a9 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c @@ -38,7 +38,7 @@ typedef struct fd_node { /** the owner of this fd node */ grpc_ares_ev_driver *ev_driver; /** the grpc_fd owned by this fd node */ - grpc_fd *grpc_fd; + grpc_fd *fd; /** a closure wrapping on_readable_cb, which should be invoked when the grpc_fd in this node becomes readable. */ grpc_closure read_closure; @@ -96,15 +96,15 @@ static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver *ev_driver) { } static void fd_node_destroy(grpc_exec_ctx *exec_ctx, fd_node *fdn) { - gpr_log(GPR_DEBUG, "delete fd: %d", grpc_fd_wrapped_fd(fdn->grpc_fd)); + gpr_log(GPR_DEBUG, "delete fd: %d", grpc_fd_wrapped_fd(fdn->fd)); GPR_ASSERT(!fdn->readable_registered); GPR_ASSERT(!fdn->writable_registered); gpr_mu_destroy(&fdn->mu); - grpc_pollset_set_del_fd(exec_ctx, fdn->ev_driver->pollset_set, fdn->grpc_fd); + grpc_pollset_set_del_fd(exec_ctx, fdn->ev_driver->pollset_set, fdn->fd); /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up immediately by another thread, and should not be closed by the following grpc_fd_orphan. */ - grpc_fd_orphan(exec_ctx, fdn->grpc_fd, NULL, NULL, true /* already_closed */, + grpc_fd_orphan(exec_ctx, fdn->fd, NULL, NULL, true /* already_closed */, "c-ares query finished"); gpr_free(fdn); } @@ -150,9 +150,8 @@ void grpc_ares_ev_driver_shutdown(grpc_exec_ctx *exec_ctx, ev_driver->shutting_down = true; fd_node *fn = ev_driver->fds; while (fn != NULL) { - grpc_fd_shutdown( - exec_ctx, fn->grpc_fd, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("grpc_ares_ev_driver_shutdown")); + grpc_fd_shutdown(exec_ctx, fn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "grpc_ares_ev_driver_shutdown")); fn = fn->next; } gpr_mu_unlock(&ev_driver->mu); @@ -165,7 +164,7 @@ static fd_node *pop_fd_node(fd_node **head, int fd) { dummy_head.next = *head; fd_node *node = &dummy_head; while (node->next != NULL) { - if (grpc_fd_wrapped_fd(node->next->grpc_fd) == fd) { + if (grpc_fd_wrapped_fd(node->next->fd) == fd) { fd_node *ret = node->next; node->next = node->next->next; *head = dummy_head.next; @@ -184,9 +183,9 @@ static void on_readable_cb(grpc_exec_ctx *exec_ctx, void *arg, fdn->readable_registered = false; gpr_mu_unlock(&fdn->mu); - gpr_log(GPR_DEBUG, "readable on %d", grpc_fd_wrapped_fd(fdn->grpc_fd)); + gpr_log(GPR_DEBUG, "readable on %d", grpc_fd_wrapped_fd(fdn->fd)); if (error == GRPC_ERROR_NONE) { - ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->grpc_fd), + ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->fd), ARES_SOCKET_BAD); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or @@ -211,10 +210,10 @@ static void on_writable_cb(grpc_exec_ctx *exec_ctx, void *arg, fdn->writable_registered = false; gpr_mu_unlock(&fdn->mu); - gpr_log(GPR_DEBUG, "writable on %d", grpc_fd_wrapped_fd(fdn->grpc_fd)); + gpr_log(GPR_DEBUG, "writable on %d", grpc_fd_wrapped_fd(fdn->fd)); if (error == GRPC_ERROR_NONE) { ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, - grpc_fd_wrapped_fd(fdn->grpc_fd)); + grpc_fd_wrapped_fd(fdn->fd)); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or // timed out. The pending lookups made on this ev_driver will be cancelled @@ -253,7 +252,7 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, gpr_asprintf(&fd_name, "ares_ev_driver-%" PRIuPTR, i); fdn = (fd_node *)gpr_malloc(sizeof(fd_node)); gpr_log(GPR_DEBUG, "new fd: %d", socks[i]); - fdn->grpc_fd = grpc_fd_create(socks[i], fd_name); + fdn->fd = grpc_fd_create(socks[i], fd_name); fdn->ev_driver = ev_driver; fdn->readable_registered = false; fdn->writable_registered = false; @@ -262,8 +261,7 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, grpc_schedule_on_exec_ctx); GRPC_CLOSURE_INIT(&fdn->write_closure, on_writable_cb, fdn, grpc_schedule_on_exec_ctx); - grpc_pollset_set_add_fd(exec_ctx, ev_driver->pollset_set, - fdn->grpc_fd); + grpc_pollset_set_add_fd(exec_ctx, ev_driver->pollset_set, fdn->fd); gpr_free(fd_name); } fdn->next = new_list; @@ -274,9 +272,8 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, if (ARES_GETSOCK_READABLE(socks_bitmask, i) && !fdn->readable_registered) { grpc_ares_ev_driver_ref(ev_driver); - gpr_log(GPR_DEBUG, "notify read on: %d", - grpc_fd_wrapped_fd(fdn->grpc_fd)); - grpc_fd_notify_on_read(exec_ctx, fdn->grpc_fd, &fdn->read_closure); + gpr_log(GPR_DEBUG, "notify read on: %d", grpc_fd_wrapped_fd(fdn->fd)); + grpc_fd_notify_on_read(exec_ctx, fdn->fd, &fdn->read_closure); fdn->readable_registered = true; } // Register write_closure if the socket is writable and write_closure @@ -284,9 +281,9 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, if (ARES_GETSOCK_WRITABLE(socks_bitmask, i) && !fdn->writable_registered) { gpr_log(GPR_DEBUG, "notify write on: %d", - grpc_fd_wrapped_fd(fdn->grpc_fd)); + grpc_fd_wrapped_fd(fdn->fd)); grpc_ares_ev_driver_ref(ev_driver); - grpc_fd_notify_on_write(exec_ctx, fdn->grpc_fd, &fdn->write_closure); + grpc_fd_notify_on_write(exec_ctx, fdn->fd, &fdn->write_closure); fdn->writable_registered = true; } gpr_mu_unlock(&fdn->mu); diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 2e2b411ab8d..0ffb38518a2 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -123,8 +123,8 @@ static void grpc_ares_request_unref(grpc_exec_ctx *exec_ctx, static grpc_ares_hostbyname_request *create_hostbyname_request( grpc_ares_request *parent_request, char *host, uint16_t port, bool is_balancer) { - grpc_ares_hostbyname_request *hr = - gpr_zalloc(sizeof(grpc_ares_hostbyname_request)); + grpc_ares_hostbyname_request *hr = (grpc_ares_hostbyname_request *)gpr_zalloc( + sizeof(grpc_ares_hostbyname_request)); hr->parent_request = parent_request; hr->host = gpr_strdup(host); hr->port = port; @@ -527,7 +527,8 @@ static void grpc_resolve_address_ares_impl(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, grpc_resolved_addresses **addrs) { grpc_resolve_address_ares_request *r = - gpr_zalloc(sizeof(grpc_resolve_address_ares_request)); + (grpc_resolve_address_ares_request *)gpr_zalloc( + sizeof(grpc_resolve_address_ares_request)); r->addrs_out = addrs; r->on_resolve_address_done = on_done; GRPC_CLOSURE_INIT(&r->on_dns_lookup_done, on_dns_lookup_done_cb, r, diff --git a/src/core/ext/filters/client_channel/retry_throttle.c b/src/core/ext/filters/client_channel/retry_throttle.c index 6cd6654b6f7..09dcade0890 100644 --- a/src/core/ext/filters/client_channel/retry_throttle.c +++ b/src/core/ext/filters/client_channel/retry_throttle.c @@ -99,7 +99,7 @@ static grpc_server_retry_throttle_data* grpc_server_retry_throttle_data_create( int max_milli_tokens, int milli_token_ratio, grpc_server_retry_throttle_data* old_throttle_data) { grpc_server_retry_throttle_data* throttle_data = - gpr_malloc(sizeof(*throttle_data)); + (grpc_server_retry_throttle_data*)gpr_malloc(sizeof(*throttle_data)); memset(throttle_data, 0, sizeof(*throttle_data)); gpr_ref_init(&throttle_data->refs, 1); throttle_data->max_milli_tokens = max_milli_tokens; @@ -131,11 +131,11 @@ static grpc_server_retry_throttle_data* grpc_server_retry_throttle_data_create( // static void* copy_server_name(void* key, void* unused) { - return gpr_strdup(key); + return gpr_strdup((const char*)key); } static long compare_server_name(void* key1, void* key2, void* unused) { - return strcmp(key1, key2); + return strcmp((const char*)key1, (const char*)key2); } static void destroy_server_retry_throttle_data(void* value, void* unused) { @@ -177,7 +177,8 @@ grpc_server_retry_throttle_data* grpc_retry_throttle_map_get_data_for_server( const char* server_name, int max_milli_tokens, int milli_token_ratio) { gpr_mu_lock(&g_mu); grpc_server_retry_throttle_data* throttle_data = - gpr_avl_get(g_avl, (char*)server_name, NULL); + (grpc_server_retry_throttle_data*)gpr_avl_get(g_avl, (char*)server_name, + NULL); if (throttle_data == NULL) { // Entry not found. Create a new one. throttle_data = grpc_server_retry_throttle_data_create( diff --git a/src/core/ext/filters/client_channel/subchannel_index.c b/src/core/ext/filters/client_channel/subchannel_index.c index f57b631c410..e05ab246224 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.c +++ b/src/core/ext/filters/client_channel/subchannel_index.c @@ -88,24 +88,26 @@ void grpc_subchannel_key_destroy(grpc_exec_ctx *exec_ctx, static void sck_avl_destroy(void *p, void *user_data) { grpc_exec_ctx *exec_ctx = (grpc_exec_ctx *)user_data; - grpc_subchannel_key_destroy(exec_ctx, p); + grpc_subchannel_key_destroy(exec_ctx, (grpc_subchannel_key *)p); } static void *sck_avl_copy(void *p, void *unused) { - return subchannel_key_copy(p); + return subchannel_key_copy((grpc_subchannel_key *)p); } static long sck_avl_compare(void *a, void *b, void *unused) { - return grpc_subchannel_key_compare(a, b); + return grpc_subchannel_key_compare((grpc_subchannel_key *)a, + (grpc_subchannel_key *)b); } static void scv_avl_destroy(void *p, void *user_data) { grpc_exec_ctx *exec_ctx = (grpc_exec_ctx *)user_data; - GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, p, "subchannel_index"); + GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, (grpc_subchannel *)p, + "subchannel_index"); } static void *scv_avl_copy(void *p, void *unused) { - GRPC_SUBCHANNEL_WEAK_REF(p, "subchannel_index"); + GRPC_SUBCHANNEL_WEAK_REF((grpc_subchannel *)p, "subchannel_index"); return p; } diff --git a/src/core/ext/filters/http/server/http_server_filter.c b/src/core/ext/filters/http/server/http_server_filter.c index 554a7f530d7..03958136b44 100644 --- a/src/core/ext/filters/http/server/http_server_filter.c +++ b/src/core/ext/filters/http/server/http_server_filter.c @@ -83,12 +83,12 @@ static grpc_error *server_filter_outgoing_metadata(grpc_exec_ctx *exec_ctx, } static void add_error(const char *error_name, grpc_error **cumulative, - grpc_error *new) { - if (new == GRPC_ERROR_NONE) return; + grpc_error *new_err) { + if (new_err == GRPC_ERROR_NONE) return; if (*cumulative == GRPC_ERROR_NONE) { *cumulative = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_name); } - *cumulative = grpc_error_add_child(*cumulative, new); + *cumulative = grpc_error_add_child(*cumulative, new_err); } static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, diff --git a/src/core/ext/transport/chttp2/client/chttp2_connector.c b/src/core/ext/transport/chttp2/client/chttp2_connector.c index 0ec9353c040..202bcd47f5c 100644 --- a/src/core/ext/transport/chttp2/client/chttp2_connector.c +++ b/src/core/ext/transport/chttp2/client/chttp2_connector.c @@ -161,7 +161,7 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_endpoint_shutdown(exec_ctx, c->endpoint, GRPC_ERROR_REF(error)); } gpr_mu_unlock(&c->mu); - chttp2_connector_unref(exec_ctx, arg); + chttp2_connector_unref(exec_ctx, (grpc_connector *)arg); } else { GPR_ASSERT(c->endpoint != NULL); start_handshake_locked(exec_ctx, c); diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.c b/src/core/ext/transport/chttp2/server/chttp2_server.c index d7add0538b0..45e5b908239 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.c +++ b/src/core/ext/transport/chttp2/server/chttp2_server.c @@ -49,10 +49,10 @@ typedef struct { grpc_closure tcp_server_shutdown_complete; grpc_closure *server_destroy_listener_done; grpc_handshake_manager *pending_handshake_mgrs; -} server_state; +} server_state_t; typedef struct { - server_state *server_state; + server_state_t *server_state; grpc_pollset *accepting_pollset; grpc_tcp_server_acceptor *acceptor; grpc_handshake_manager *handshake_mgr; @@ -109,7 +109,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_pollset *accepting_pollset, grpc_tcp_server_acceptor *acceptor) { - server_state *state = (server_state *)arg; + server_state_t *state = (server_state_t *)arg; gpr_mu_lock(&state->mu); if (state->shutdown) { gpr_mu_unlock(&state->mu); @@ -124,7 +124,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_mu_unlock(&state->mu); grpc_tcp_server_ref(state->tcp_server); server_connection_state *connection_state = - gpr_malloc(sizeof(*connection_state)); + (server_connection_state *)gpr_malloc(sizeof(*connection_state)); connection_state->server_state = state; connection_state->accepting_pollset = accepting_pollset; connection_state->acceptor = acceptor; @@ -144,7 +144,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, void *arg, grpc_pollset **pollsets, size_t pollset_count) { - server_state *state = (server_state *)arg; + server_state_t *state = (server_state_t *)arg; gpr_mu_lock(&state->mu); state->shutdown = false; gpr_mu_unlock(&state->mu); @@ -154,7 +154,7 @@ static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { - server_state *state = (server_state *)arg; + server_state_t *state = (server_state_t *)arg; /* ensure all threads have unlocked */ gpr_mu_lock(&state->mu); grpc_closure *destroy_done = state->server_destroy_listener_done; @@ -179,7 +179,7 @@ static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg, static void server_destroy_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, void *arg, grpc_closure *destroy_done) { - server_state *state = (server_state *)arg; + server_state_t *state = (server_state_t *)arg; gpr_mu_lock(&state->mu); state->shutdown = true; state->server_destroy_listener_done = destroy_done; @@ -199,7 +199,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, size_t count = 0; int port_temp; grpc_error *err = GRPC_ERROR_NONE; - server_state *state = NULL; + server_state_t *state = NULL; grpc_error **errors = NULL; *port_num = -1; @@ -209,7 +209,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, if (err != GRPC_ERROR_NONE) { goto error; } - state = (server_state *)gpr_zalloc(sizeof(*state)); + state = (server_state_t *)gpr_zalloc(sizeof(*state)); GRPC_CLOSURE_INIT(&state->tcp_server_shutdown_complete, tcp_server_shutdown_complete, state, grpc_schedule_on_exec_ctx); diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 3fd701fe2f1..930fa805a09 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -2912,7 +2912,8 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, uint32_t frame_size, uint32_t flags) { grpc_chttp2_incoming_byte_stream *incoming_byte_stream = - gpr_malloc(sizeof(*incoming_byte_stream)); + (grpc_chttp2_incoming_byte_stream *)gpr_malloc( + sizeof(*incoming_byte_stream)); incoming_byte_stream->base.length = frame_size; incoming_byte_stream->remaining_bytes = frame_size; incoming_byte_stream->base.flags = flags; diff --git a/src/core/ext/transport/chttp2/transport/flow_control.c b/src/core/ext/transport/chttp2/transport/flow_control.c index 0f078e79e90..569a6349d3d 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.c +++ b/src/core/ext/transport/chttp2/transport/flow_control.c @@ -60,24 +60,24 @@ static void pretrace(shadow_flow_control* shadow_fc, #define TRACE_PADDING 30 -static char* fmt_int64_diff_str(int64_t old, int64_t new) { +static char* fmt_int64_diff_str(int64_t old_val, int64_t new_val) { char* str; - if (old != new) { - gpr_asprintf(&str, "%" PRId64 " -> %" PRId64 "", old, new); + if (old_val != new_val) { + gpr_asprintf(&str, "%" PRId64 " -> %" PRId64 "", old_val, new_val); } else { - gpr_asprintf(&str, "%" PRId64 "", old); + gpr_asprintf(&str, "%" PRId64 "", old_val); } char* str_lp = gpr_leftpad(str, ' ', TRACE_PADDING); gpr_free(str); return str_lp; } -static char* fmt_uint32_diff_str(uint32_t old, uint32_t new) { +static char* fmt_uint32_diff_str(uint32_t old_val, uint32_t new_val) { char* str; - if (new > 0 && old != new) { - gpr_asprintf(&str, "%" PRIu32 " -> %" PRIu32 "", old, new); + if (new_val > 0 && old_val != new_val) { + gpr_asprintf(&str, "%" PRIu32 " -> %" PRIu32 "", old_val, new_val); } else { - gpr_asprintf(&str, "%" PRIu32 "", old); + gpr_asprintf(&str, "%" PRIu32 "", old_val); } char* str_lp = gpr_leftpad(str, ' ', TRACE_PADDING); gpr_free(str); diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c index 806100adaab..2995bf73102 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.c +++ b/src/core/ext/transport/chttp2/transport/frame_settings.c @@ -44,7 +44,8 @@ static uint8_t *fill_header(uint8_t *out, uint32_t length, uint8_t flags) { return out; } -grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, +grpc_slice grpc_chttp2_settings_create(uint32_t *old_settings, + const uint32_t *new_settings, uint32_t force_mask, size_t count) { size_t i; uint32_t n = 0; @@ -52,21 +53,21 @@ grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, uint8_t *p; for (i = 0; i < count; i++) { - n += (new[i] != old[i] || (force_mask & (1u << i)) != 0); + n += (new_settings[i] != old_settings[i] || (force_mask & (1u << i)) != 0); } output = GRPC_SLICE_MALLOC(9 + 6 * n); p = fill_header(GRPC_SLICE_START_PTR(output), 6 * n, 0); for (i = 0; i < count; i++) { - if (new[i] != old[i] || (force_mask & (1u << i)) != 0) { + if (new_settings[i] != old_settings[i] || (force_mask & (1u << i)) != 0) { *p++ = (uint8_t)(grpc_setting_id_to_wire_id[i] >> 8); *p++ = (uint8_t)(grpc_setting_id_to_wire_id[i]); - *p++ = (uint8_t)(new[i] >> 24); - *p++ = (uint8_t)(new[i] >> 16); - *p++ = (uint8_t)(new[i] >> 8); - *p++ = (uint8_t)(new[i]); - old[i] = new[i]; + *p++ = (uint8_t)(new_settings[i] >> 24); + *p++ = (uint8_t)(new_settings[i] >> 16); + *p++ = (uint8_t)(new_settings[i] >> 8); + *p++ = (uint8_t)(new_settings[i]); + old_settings[i] = new_settings[i]; } } diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.c b/src/core/ext/transport/chttp2/transport/incoming_metadata.c index cf0a9ca9201..ba680a89db1 100644 --- a/src/core/ext/transport/chttp2/transport/incoming_metadata.c +++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.c @@ -42,8 +42,9 @@ grpc_error *grpc_chttp2_incoming_metadata_buffer_add( grpc_mdelem elem) { buffer->size += GRPC_MDELEM_LENGTH(elem); return grpc_metadata_batch_add_tail( - exec_ctx, &buffer->batch, - gpr_arena_alloc(buffer->arena, sizeof(grpc_linked_mdelem)), elem); + exec_ctx, &buffer->batch, (grpc_linked_mdelem *)gpr_arena_alloc( + buffer->arena, sizeof(grpc_linked_mdelem)), + elem); } grpc_error *grpc_chttp2_incoming_metadata_buffer_replace_or_add( diff --git a/src/core/ext/transport/chttp2/transport/stream_map.c b/src/core/ext/transport/chttp2/transport/stream_map.c index 650090d8f04..d6079a9a33e 100644 --- a/src/core/ext/transport/chttp2/transport/stream_map.c +++ b/src/core/ext/transport/chttp2/transport/stream_map.c @@ -72,8 +72,10 @@ void grpc_chttp2_stream_map_add(grpc_chttp2_stream_map *map, uint32_t key, /* resize when less than 25% of the table is free, because compaction won't help much */ map->capacity = capacity = 3 * capacity / 2; - map->keys = keys = gpr_realloc(keys, capacity * sizeof(uint32_t)); - map->values = values = gpr_realloc(values, capacity * sizeof(void *)); + map->keys = keys = + (uint32_t *)gpr_realloc(keys, capacity * sizeof(uint32_t)); + map->values = values = + (void **)gpr_realloc(values, capacity * sizeof(void *)); } } diff --git a/src/core/ext/transport/inproc/inproc_transport.c b/src/core/ext/transport/inproc/inproc_transport.c index 036853a53ba..cd3e76a0b5b 100644 --- a/src/core/ext/transport/inproc/inproc_transport.c +++ b/src/core/ext/transport/inproc/inproc_transport.c @@ -37,7 +37,6 @@ if (GRPC_TRACER_ON(grpc_inproc_trace)) gpr_log(__VA_ARGS__); \ } while (0) -static const grpc_transport_vtable inproc_vtable; static grpc_slice g_empty_slice; static grpc_slice g_fake_path_key; static grpc_slice g_fake_path_value; @@ -1166,6 +1165,55 @@ static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { unref_transport(exec_ctx, t); } +/******************************************************************************* + * INTEGRATION GLUE + */ + +static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_pollset *pollset) { + // Nothing to do here +} + +static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_pollset_set *pollset_set) { + // Nothing to do here +} + +static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) { + return NULL; +} + +/******************************************************************************* + * GLOBAL INIT AND DESTROY + */ +static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} + +void grpc_inproc_transport_init(void) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GRPC_CLOSURE_INIT(&do_nothing_closure, do_nothing, NULL, + grpc_schedule_on_exec_ctx); + g_empty_slice = grpc_slice_from_static_buffer(NULL, 0); + + grpc_slice key_tmp = grpc_slice_from_static_string(":path"); + g_fake_path_key = grpc_slice_intern(key_tmp); + grpc_slice_unref_internal(&exec_ctx, key_tmp); + + g_fake_path_value = grpc_slice_from_static_string("/"); + + grpc_slice auth_tmp = grpc_slice_from_static_string(":authority"); + g_fake_auth_key = grpc_slice_intern(auth_tmp); + grpc_slice_unref_internal(&exec_ctx, auth_tmp); + + g_fake_auth_value = grpc_slice_from_static_string("inproc-fail"); + grpc_exec_ctx_finish(&exec_ctx); +} + +static const grpc_transport_vtable inproc_vtable = { + sizeof(inproc_stream), "inproc", init_stream, + set_pollset, set_pollset_set, perform_stream_op, + perform_transport_op, destroy_stream, destroy_transport, + get_endpoint}; + /******************************************************************************* * Main inproc transport functions */ @@ -1178,7 +1226,7 @@ static void inproc_transports_create(grpc_exec_ctx *exec_ctx, inproc_transport *st = (inproc_transport *)gpr_zalloc(sizeof(*st)); inproc_transport *ct = (inproc_transport *)gpr_zalloc(sizeof(*ct)); // Share one lock between both sides since both sides get affected - st->mu = ct->mu = gpr_malloc(sizeof(*st->mu)); + st->mu = ct->mu = (shared_mu *)gpr_malloc(sizeof(*st->mu)); gpr_mu_init(&st->mu->mu); gpr_ref_init(&st->mu->refs, 2); st->base.vtable = &inproc_vtable; @@ -1240,55 +1288,6 @@ grpc_channel *grpc_inproc_channel_create(grpc_server *server, return channel; } -/******************************************************************************* - * INTEGRATION GLUE - */ - -static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_pollset *pollset) { - // Nothing to do here -} - -static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_pollset_set *pollset_set) { - // Nothing to do here -} - -static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) { - return NULL; -} - -static const grpc_transport_vtable inproc_vtable = { - sizeof(inproc_stream), "inproc", init_stream, - set_pollset, set_pollset_set, perform_stream_op, - perform_transport_op, destroy_stream, destroy_transport, - get_endpoint}; - -/******************************************************************************* - * GLOBAL INIT AND DESTROY - */ -static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} - -void grpc_inproc_transport_init(void) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - GRPC_CLOSURE_INIT(&do_nothing_closure, do_nothing, NULL, - grpc_schedule_on_exec_ctx); - g_empty_slice = grpc_slice_from_static_buffer(NULL, 0); - - grpc_slice key_tmp = grpc_slice_from_static_string(":path"); - g_fake_path_key = grpc_slice_intern(key_tmp); - grpc_slice_unref_internal(&exec_ctx, key_tmp); - - g_fake_path_value = grpc_slice_from_static_string("/"); - - grpc_slice auth_tmp = grpc_slice_from_static_string(":authority"); - g_fake_auth_key = grpc_slice_intern(auth_tmp); - grpc_slice_unref_internal(&exec_ctx, auth_tmp); - - g_fake_auth_value = grpc_slice_from_static_string("inproc-fail"); - grpc_exec_ctx_finish(&exec_ctx); -} - void grpc_inproc_transport_shutdown(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_slice_unref_internal(&exec_ctx, g_empty_slice); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 47265039946..ab964c345c2 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -575,12 +575,12 @@ static bool atm_inc_if_nonzero(gpr_atm *counter) { } static bool cq_begin_op_for_next(grpc_completion_queue *cq, void *tag) { - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); return atm_inc_if_nonzero(&cqd->pending_events); } static bool cq_begin_op_for_pluck(grpc_completion_queue *cq, void *tag) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); return atm_inc_if_nonzero(&cqd->pending_events); } @@ -625,7 +625,7 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, } } - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); int is_success = (error == GRPC_ERROR_NONE); storage->tag = tag; @@ -686,7 +686,7 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx, void *done_arg, grpc_cq_completion *storage), void *done_arg, grpc_cq_completion *storage) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); int is_success = (error == GRPC_ERROR_NONE); GPR_TIMER_BEGIN("cq_end_op_for_pluck", 0); @@ -769,7 +769,7 @@ typedef struct { static bool cq_is_next_finished(grpc_exec_ctx *exec_ctx, void *arg) { cq_is_finished_arg *a = (cq_is_finished_arg *)arg; grpc_completion_queue *cq = a->cq; - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); GPR_ASSERT(a->stolen_completion == NULL); gpr_atm current_last_seen_things_queued_ever = @@ -820,7 +820,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, void *reserved) { grpc_event ret; gpr_timespec now; - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); GPR_TIMER_BEGIN("grpc_completion_queue_next", 0); @@ -953,7 +953,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, this function */ static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); GPR_ASSERT(cqd->shutdown_called); GPR_ASSERT(gpr_atm_no_barrier_load(&cqd->pending_events) == 0); @@ -964,7 +964,7 @@ static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx, static void cq_shutdown_next(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); /* Need an extra ref for cq here because: * We call cq_finish_shutdown_next() below, that would call pollset shutdown. @@ -994,7 +994,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cq, static int add_plucker(grpc_completion_queue *cq, void *tag, grpc_pollset_worker **worker) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); if (cqd->num_pluckers == GRPC_MAX_COMPLETION_QUEUE_PLUCKERS) { return 0; } @@ -1006,7 +1006,7 @@ static int add_plucker(grpc_completion_queue *cq, void *tag, static void del_plucker(grpc_completion_queue *cq, void *tag, grpc_pollset_worker **worker) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); for (int i = 0; i < cqd->num_pluckers; i++) { if (cqd->pluckers[i].tag == tag && cqd->pluckers[i].worker == worker) { cqd->num_pluckers--; @@ -1020,7 +1020,7 @@ static void del_plucker(grpc_completion_queue *cq, void *tag, static bool cq_is_pluck_finished(grpc_exec_ctx *exec_ctx, void *arg) { cq_is_finished_arg *a = (cq_is_finished_arg *)arg; grpc_completion_queue *cq = a->cq; - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); GPR_ASSERT(a->stolen_completion == NULL); gpr_atm current_last_seen_things_queued_ever = @@ -1057,7 +1057,7 @@ static grpc_event cq_pluck(grpc_completion_queue *cq, void *tag, grpc_cq_completion *prev; grpc_pollset_worker *worker = NULL; gpr_timespec now; - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); GPR_TIMER_BEGIN("grpc_completion_queue_pluck", 0); @@ -1181,7 +1181,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cq, void *tag, static void cq_finish_shutdown_pluck(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); GPR_ASSERT(cqd->shutdown_called); GPR_ASSERT(!gpr_atm_no_barrier_load(&cqd->shutdown)); @@ -1195,7 +1195,7 @@ static void cq_finish_shutdown_pluck(grpc_exec_ctx *exec_ctx, * merging them is a bit tricky and probably not worth it */ static void cq_shutdown_pluck(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); /* Need an extra ref for cq here because: * We call cq_finish_shutdown_pluck() below, that would call pollset shutdown. diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 13ecc915ecb..0bc21737136 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -55,7 +55,7 @@ typedef struct listener { typedef struct call_data call_data; typedef struct channel_data channel_data; -typedef struct registered_method registered_method; +typedef struct registered_method_t registered_method_t; typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; @@ -76,7 +76,7 @@ typedef struct requested_call { grpc_call_details *details; } batch; struct { - registered_method *registered_method; + registered_method_t *registered_method; gpr_timespec *deadline; grpc_byte_buffer **optional_payload; } registered; @@ -84,7 +84,7 @@ typedef struct requested_call { } requested_call; typedef struct channel_registered_method { - registered_method *server_registered_method; + registered_method_t *server_registered_method; uint32_t flags; bool has_host; grpc_slice method; @@ -123,7 +123,7 @@ typedef enum { ZOMBIED } call_state; -typedef struct request_matcher request_matcher; +typedef struct request_matcher_t request_matcher_t; struct call_data { grpc_call *call; @@ -145,7 +145,7 @@ struct call_data { uint32_t recv_initial_metadata_flags; grpc_metadata_array initial_metadata; - request_matcher *request_matcher; + request_matcher_t *request_matcher; grpc_byte_buffer *payload; grpc_closure got_initial_metadata; @@ -158,21 +158,21 @@ struct call_data { call_data *pending_next; }; -struct request_matcher { +struct request_matcher_t { grpc_server *server; call_data *pending_head; call_data *pending_tail; gpr_stack_lockfree **requests_per_cq; }; -struct registered_method { +struct registered_method_t { char *method; char *host; grpc_server_register_method_payload_handling payload_handling; uint32_t flags; /* one request matcher per method */ - request_matcher request_matcher; - registered_method *next; + request_matcher_t request_matcher; + registered_method_t *next; }; typedef struct { @@ -204,9 +204,9 @@ struct grpc_server { bool starting; gpr_cv starting_cv; - registered_method *registered_methods; + registered_method_t *registered_methods; /** one request matcher for unregistered methods */ - request_matcher unregistered_request_matcher; + request_matcher_t unregistered_request_matcher; /** free list of available requested_calls_per_cq indices */ gpr_stack_lockfree **request_freelist_per_cq; /** requested call backing data */ @@ -313,7 +313,7 @@ static void channel_broadcaster_shutdown(grpc_exec_ctx *exec_ctx, * request_matcher */ -static void request_matcher_init(request_matcher *rm, size_t entries, +static void request_matcher_init(request_matcher_t *rm, size_t entries, grpc_server *server) { memset(rm, 0, sizeof(*rm)); rm->server = server; @@ -324,7 +324,7 @@ static void request_matcher_init(request_matcher *rm, size_t entries, } } -static void request_matcher_destroy(request_matcher *rm) { +static void request_matcher_destroy(request_matcher_t *rm) { for (size_t i = 0; i < rm->server->cq_count; i++) { GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1); gpr_stack_lockfree_destroy(rm->requests_per_cq[i]); @@ -334,11 +334,11 @@ static void request_matcher_destroy(request_matcher *rm) { static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, grpc_error *error) { - grpc_call_unref(grpc_call_from_top_element(elem)); + grpc_call_unref(grpc_call_from_top_element((grpc_call_element *)elem)); } static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, - request_matcher *rm) { + request_matcher_t *rm) { while (rm->pending_head) { call_data *calld = rm->pending_head; rm->pending_head = calld->pending_next; @@ -355,7 +355,7 @@ static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx, grpc_server *server, - request_matcher *rm, + request_matcher_t *rm, grpc_error *error) { int request_id; for (size_t i = 0; i < server->cq_count; i++) { @@ -378,7 +378,7 @@ static void server_ref(grpc_server *server) { } static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { - registered_method *rm; + registered_method_t *rm; size_t i; grpc_channel_args_destroy(exec_ctx, server->channel_args); gpr_mu_destroy(&server->mu_global); @@ -519,7 +519,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, grpc_call_element *call_elem = (grpc_call_element *)arg; call_data *calld = (call_data *)call_elem->call_data; channel_data *chand = (channel_data *)call_elem->channel_data; - request_matcher *rm = calld->request_matcher; + request_matcher_t *rm = calld->request_matcher; grpc_server *server = rm->server; if (error != GRPC_ERROR_NONE || gpr_atm_acq_load(&server->shutdown_flag)) { @@ -569,7 +569,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, static void finish_start_new_rpc( grpc_exec_ctx *exec_ctx, grpc_server *server, grpc_call_element *elem, - request_matcher *rm, + request_matcher_t *rm, grpc_server_register_method_payload_handling payload_handling) { call_data *calld = (call_data *)elem->call_data; @@ -668,7 +668,7 @@ static int num_listeners(grpc_server *server) { static void done_shutdown_event(grpc_exec_ctx *exec_ctx, void *server, grpc_cq_completion *completion) { - server_unref(exec_ctx, server); + server_unref(exec_ctx, (grpc_server *)server); } static int num_channels(grpc_server *server) { @@ -689,7 +689,7 @@ static void kill_pending_work_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_REF(error)); request_matcher_zombify_all_pending_calls( exec_ctx, &server->unregistered_request_matcher); - for (registered_method *rm = server->registered_methods; rm; + for (registered_method_t *rm = server->registered_methods; rm; rm = rm->next) { request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher, GRPC_ERROR_REF(error)); @@ -1036,7 +1036,7 @@ void *grpc_server_register_method( grpc_server *server, const char *method, const char *host, grpc_server_register_method_payload_handling payload_handling, uint32_t flags) { - registered_method *m; + registered_method_t *m; GRPC_API_TRACE( "grpc_server_register_method(server=%p, method=%s, host=%s, " "flags=0x%08x)", @@ -1058,7 +1058,7 @@ void *grpc_server_register_method( flags); return NULL; } - m = (registered_method *)gpr_zalloc(sizeof(registered_method)); + m = (registered_method_t *)gpr_zalloc(sizeof(registered_method_t)); m->method = gpr_strdup(method); m->host = gpr_strdup(host); m->next = server->registered_methods; @@ -1113,7 +1113,8 @@ void grpc_server_start(grpc_server *server) { } request_matcher_init(&server->unregistered_request_matcher, (size_t)server->max_requested_calls_per_cq, server); - for (registered_method *rm = server->registered_methods; rm; rm = rm->next) { + for (registered_method_t *rm = server->registered_methods; rm; + rm = rm->next) { request_matcher_init(&rm->request_matcher, (size_t)server->max_requested_calls_per_cq, server); } @@ -1141,7 +1142,7 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s, const grpc_channel_args *args) { size_t num_registered_methods; size_t alloc; - registered_method *rm; + registered_method_t *rm; channel_registered_method *crm; grpc_channel *channel; channel_data *chand; @@ -1267,8 +1268,9 @@ void grpc_server_shutdown_and_notify(grpc_server *server, /* stay locked, and gather up some stuff to do */ GPR_ASSERT(grpc_cq_begin_op(cq, tag)); if (server->shutdown_published) { - grpc_cq_end_op(&exec_ctx, cq, tag, GRPC_ERROR_NONE, done_published_shutdown, - NULL, gpr_malloc(sizeof(grpc_cq_completion))); + grpc_cq_end_op( + &exec_ctx, cq, tag, GRPC_ERROR_NONE, done_published_shutdown, NULL, + (grpc_cq_completion *)gpr_malloc(sizeof(grpc_cq_completion))); gpr_mu_unlock(&server->mu_global); goto done; } @@ -1369,7 +1371,7 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx, grpc_server *server, size_t cq_idx, requested_call *rc) { call_data *calld = NULL; - request_matcher *rm = NULL; + request_matcher_t *rm = NULL; int request_id; if (gpr_atm_acq_load(&server->shutdown_flag)) { fail_call(exec_ctx, server, cq_idx, rc, @@ -1481,7 +1483,7 @@ grpc_call_error grpc_server_request_registered_call( grpc_call_error error; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; requested_call *rc = (requested_call *)gpr_malloc(sizeof(*rc)); - registered_method *rm = (registered_method *)rmp; + registered_method_t *rm = (registered_method_t *)rmp; GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx); GRPC_API_TRACE( "grpc_server_request_registered_call(" diff --git a/src/core/lib/transport/metadata_batch.c b/src/core/lib/transport/metadata_batch.c index a0770525614..54388bdcda1 100644 --- a/src/core/lib/transport/metadata_batch.c +++ b/src/core/lib/transport/metadata_batch.c @@ -233,32 +233,32 @@ void grpc_metadata_batch_remove(grpc_exec_ctx *exec_ctx, void grpc_metadata_batch_set_value(grpc_exec_ctx *exec_ctx, grpc_linked_mdelem *storage, grpc_slice value) { - grpc_mdelem old = storage->md; - grpc_mdelem new = grpc_mdelem_from_slices( - exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(old)), value); - storage->md = new; - GRPC_MDELEM_UNREF(exec_ctx, old); + grpc_mdelem old_mdelem = storage->md; + grpc_mdelem new_mdelem = grpc_mdelem_from_slices( + exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(old_mdelem)), value); + storage->md = new_mdelem; + GRPC_MDELEM_UNREF(exec_ctx, old_mdelem); } grpc_error *grpc_metadata_batch_substitute(grpc_exec_ctx *exec_ctx, grpc_metadata_batch *batch, grpc_linked_mdelem *storage, - grpc_mdelem new) { + grpc_mdelem new_mdelem) { assert_valid_callouts(exec_ctx, batch); grpc_error *error = GRPC_ERROR_NONE; - grpc_mdelem old = storage->md; - if (!grpc_slice_eq(GRPC_MDKEY(new), GRPC_MDKEY(old))) { + grpc_mdelem old_mdelem = storage->md; + if (!grpc_slice_eq(GRPC_MDKEY(new_mdelem), GRPC_MDKEY(old_mdelem))) { maybe_unlink_callout(batch, storage); - storage->md = new; + storage->md = new_mdelem; error = maybe_link_callout(batch, storage); if (error != GRPC_ERROR_NONE) { unlink_storage(&batch->list, storage); GRPC_MDELEM_UNREF(exec_ctx, storage->md); } } else { - storage->md = new; + storage->md = new_mdelem; } - GRPC_MDELEM_UNREF(exec_ctx, old); + GRPC_MDELEM_UNREF(exec_ctx, old_mdelem); assert_valid_callouts(exec_ctx, batch); return error; } @@ -302,12 +302,12 @@ grpc_error *grpc_metadata_batch_filter(grpc_exec_ctx *exec_ctx, grpc_error *error = GRPC_ERROR_NONE; while (l) { grpc_linked_mdelem *next = l->next; - grpc_filtered_mdelem new = func(exec_ctx, user_data, l->md); - add_error(&error, new.error, composite_error_string); - if (GRPC_MDISNULL(new.md)) { + grpc_filtered_mdelem new_mdelem = func(exec_ctx, user_data, l->md); + add_error(&error, new_mdelem.error, composite_error_string); + if (GRPC_MDISNULL(new_mdelem.md)) { grpc_metadata_batch_remove(exec_ctx, batch, l); - } else if (new.md.payload != l->md.payload) { - grpc_metadata_batch_substitute(exec_ctx, batch, l, new.md); + } else if (new_mdelem.md.payload != l->md.payload) { + grpc_metadata_batch_substitute(exec_ctx, batch, l, new_mdelem.md); } l = next; } diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index caa11a956ea..ae705195f3c 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -102,8 +102,9 @@ static void slice_stream_unref(grpc_exec_ctx *exec_ctx, void *p) { grpc_slice grpc_slice_from_stream_owned_buffer(grpc_stream_refcount *refcount, void *buffer, size_t length) { slice_stream_ref(&refcount->slice_refcount); - return (grpc_slice){.refcount = &refcount->slice_refcount, - .data.refcounted = {.bytes = buffer, .length = length}}; + return (grpc_slice){ + .refcount = &refcount->slice_refcount, + .data.refcounted = {.bytes = (uint8_t *)buffer, .length = length}}; } static const grpc_slice_refcount_vtable stream_ref_slice_vtable = { diff --git a/test/core/end2end/tests/resource_quota_server.c b/test/core/end2end/tests/resource_quota_server.c index 03169207623..93befeee0c6 100644 --- a/test/core/end2end/tests/resource_quota_server.c +++ b/test/core/end2end/tests/resource_quota_server.c @@ -136,20 +136,20 @@ void resource_quota_server(grpc_end2end_test_config config) { grpc_call **server_calls = (grpc_call **)malloc(sizeof(grpc_call *) * NUM_CALLS); grpc_metadata_array *initial_metadata_recv = - malloc(sizeof(grpc_metadata_array) * NUM_CALLS); + (grpc_metadata_array *)malloc(sizeof(grpc_metadata_array) * NUM_CALLS); grpc_metadata_array *trailing_metadata_recv = - malloc(sizeof(grpc_metadata_array) * NUM_CALLS); + (grpc_metadata_array *)malloc(sizeof(grpc_metadata_array) * NUM_CALLS); grpc_metadata_array *request_metadata_recv = - malloc(sizeof(grpc_metadata_array) * NUM_CALLS); + (grpc_metadata_array *)malloc(sizeof(grpc_metadata_array) * NUM_CALLS); grpc_call_details *call_details = - malloc(sizeof(grpc_call_details) * NUM_CALLS); + (grpc_call_details *)malloc(sizeof(grpc_call_details) * NUM_CALLS); grpc_status_code *status = (grpc_status_code *)malloc(sizeof(grpc_status_code) * NUM_CALLS); grpc_slice *details = (grpc_slice *)malloc(sizeof(grpc_slice) * NUM_CALLS); grpc_byte_buffer **request_payload = - malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS); + (grpc_byte_buffer **)malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS); grpc_byte_buffer **request_payload_recv = - malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS); + (grpc_byte_buffer **)malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS); int *was_cancelled = (int *)malloc(sizeof(int) * NUM_CALLS); grpc_call_error error; int pending_client_calls = 0; From 6502ecc812cba929453115a0475804304db243a6 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Wed, 13 Sep 2017 13:10:54 -0700 Subject: [PATCH 116/227] Add refcount to subchannel_index. --- .../client_channel/lb_policy/grpclb/grpclb.c | 3 +++ .../lb_policy/pick_first/pick_first.c | 2 ++ .../lb_policy/round_robin/round_robin.c | 3 +++ .../filters/client_channel/subchannel_index.c | 22 +++++++++++++++---- .../filters/client_channel/subchannel_index.h | 7 ++++++ src/core/lib/surface/init.c | 9 +++++--- 6 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 5aafed1374d..734a328446e 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -101,6 +101,7 @@ #include "src/core/ext/filters/client_channel/lb_policy_registry.h" #include "src/core/ext/filters/client_channel/parse_address.h" #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h" +#include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/iomgr/combiner.h" @@ -988,6 +989,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, gpr_free(glb_policy); return NULL; } + grpc_subchannel_index_ref(); GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed, glb_lb_channel_on_connectivity_changed_cb, glb_policy, grpc_combiner_scheduler(args->combiner)); @@ -1011,6 +1013,7 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { grpc_grpclb_destroy_serverlist(glb_policy->serverlist); } grpc_fake_resolver_response_generator_unref(glb_policy->response_generator); + grpc_subchannel_index_unref(); if (glb_policy->pending_update_args != NULL) { grpc_channel_args_destroy(exec_ctx, glb_policy->pending_update_args->args); gpr_free(glb_policy->pending_update_args); diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c index fab3073eb92..50b02016a16 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c @@ -89,6 +89,7 @@ static void pf_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { "picked_first_destroy"); } grpc_connectivity_state_destroy(exec_ctx, &p->state_tracker); + grpc_subchannel_index_unref(); if (p->pending_update_args != NULL) { grpc_channel_args_destroy(exec_ctx, p->pending_update_args->args); gpr_free(p->pending_update_args); @@ -686,6 +687,7 @@ static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx, } pf_update_locked(exec_ctx, &p->base, args); grpc_lb_policy_init(&p->base, &pick_first_lb_policy_vtable, args->combiner); + grpc_subchannel_index_ref(); GRPC_CLOSURE_INIT(&p->connectivity_changed, pf_connectivity_changed_locked, p, grpc_combiner_scheduler(args->combiner)); return &p->base; diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c index be91d3d6511..8ac1a46abdf 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c @@ -30,6 +30,7 @@ #include "src/core/ext/filters/client_channel/lb_policy_registry.h" #include "src/core/ext/filters/client_channel/subchannel.h" +#include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/combiner.h" @@ -310,6 +311,7 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { (void *)pol, (void *)pol); } grpc_connectivity_state_destroy(exec_ctx, &p->state_tracker); + grpc_subchannel_index_unref(); gpr_free(p); } @@ -890,6 +892,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, GPR_ASSERT(args->client_channel_factory != NULL); round_robin_lb_policy *p = (round_robin_lb_policy *)gpr_zalloc(sizeof(*p)); grpc_lb_policy_init(&p->base, &round_robin_lb_policy_vtable, args->combiner); + grpc_subchannel_index_ref(); grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE, "round_robin"); rr_update_locked(exec_ctx, &p->base, args); diff --git a/src/core/ext/filters/client_channel/subchannel_index.c b/src/core/ext/filters/client_channel/subchannel_index.c index f57b631c410..000ee70be7f 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.c +++ b/src/core/ext/filters/client_channel/subchannel_index.c @@ -34,6 +34,8 @@ static gpr_avl g_subchannel_index; static gpr_mu g_mu; +static gpr_refcount g_refcount; + struct grpc_subchannel_key { grpc_subchannel_args args; }; @@ -119,15 +121,27 @@ static const gpr_avl_vtable subchannel_avl_vtable = { void grpc_subchannel_index_init(void) { g_subchannel_index = gpr_avl_create(&subchannel_avl_vtable); gpr_mu_init(&g_mu); + gpr_ref_init(&g_refcount, 1); } void grpc_subchannel_index_shutdown(void) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - gpr_mu_destroy(&g_mu); - gpr_avl_unref(g_subchannel_index, &exec_ctx); - grpc_exec_ctx_finish(&exec_ctx); + // TODO(juanlishen): This refcounting mechanism may lead to memory leackage. + // To solve that, we should force polling to flush any pending callbacks, then + // shutdown safely. + grpc_subchannel_index_unref(); +} + +void grpc_subchannel_index_unref(void) { + if (gpr_unref(&g_refcount)) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_mu_destroy(&g_mu); + gpr_avl_unref(g_subchannel_index, &exec_ctx); + grpc_exec_ctx_finish(&exec_ctx); + } } +void grpc_subchannel_index_ref(void) { gpr_ref_non_zero(&g_refcount); } + grpc_subchannel *grpc_subchannel_index_find(grpc_exec_ctx *exec_ctx, grpc_subchannel_key *key) { // Lock, and take a reference to the subchannel index. diff --git a/src/core/ext/filters/client_channel/subchannel_index.h b/src/core/ext/filters/client_channel/subchannel_index.h index 98d882a453f..92e36d5283c 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.h +++ b/src/core/ext/filters/client_channel/subchannel_index.h @@ -59,6 +59,13 @@ void grpc_subchannel_index_init(void); /** Shutdown the subchannel index (global) */ void grpc_subchannel_index_shutdown(void); +/** Increment the refcount (non-zero) of subchannel index (global). */ +void grpc_subchannel_index_ref(void); + +/** Decrement the refcount of subchannel index (global). If the refcount drops + to zero, unref the subchannel index and destroy its mutex. */ +void grpc_subchannel_index_unref(void); + /** \em TEST ONLY. * If \a force_creation is true, all key comparisons will be false, resulting in * new subchannels always being created. Otherwise, the keys will be compared as diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index 280315036ff..b089da2c54f 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -36,6 +36,7 @@ #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/iomgr/resource_quota.h" +#include "src/core/lib/iomgr/timer_manager.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/alarm_internal.h" @@ -179,14 +180,16 @@ void grpc_shutdown(void) { GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); gpr_mu_lock(&g_init_mu); if (--g_initializations == 0) { - grpc_iomgr_shutdown(&exec_ctx); - gpr_timers_global_destroy(); - grpc_tracer_shutdown(); + grpc_executor_shutdown(&exec_ctx); + grpc_timer_manager_set_threading(false); // shutdown timer_manager thread for (i = g_number_of_plugins; i >= 0; i--) { if (g_all_of_the_plugins[i].destroy != NULL) { g_all_of_the_plugins[i].destroy(); } } + grpc_iomgr_shutdown(&exec_ctx); + gpr_timers_global_destroy(); + grpc_tracer_shutdown(); grpc_mdctx_global_shutdown(&exec_ctx); grpc_handshaker_factory_registry_shutdown(&exec_ctx); grpc_slice_intern_shutdown(); From 09a02ec04038e9eb45aae249d65bdf66f305d9d4 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 13 Sep 2017 20:05:34 +0000 Subject: [PATCH 117/227] Kill (grpcio_tests/)commands.BuildProtoModules In 1ff429da2a94bc79300ebce3f8aae7efb10e9a75 it appears to have been copied out of src/python/grpcio/commands.py and not used, and it seems to have remained without use since. --- src/python/grpcio_tests/commands.py | 49 ----------------------------- 1 file changed, 49 deletions(-) diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index 162200112af..93f84572b76 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -67,55 +67,6 @@ class GatherProto(setuptools.Command): open(path, 'a').close() -class BuildProtoModules(setuptools.Command): - """Command to generate project *_pb2.py modules from proto files.""" - - description = 'build protobuf modules' - user_options = [ - ('include=', None, 'path patterns to include in protobuf generation'), - ('exclude=', None, 'path patterns to exclude from protobuf generation') - ] - - def initialize_options(self): - self.exclude = None - self.include = r'.*\.proto$' - - def finalize_options(self): - pass - - def run(self): - import grpc_tools.protoc as protoc - - include_regex = re.compile(self.include) - exclude_regex = re.compile(self.exclude) if self.exclude else None - paths = [] - for walk_root, directories, filenames in os.walk(PROTO_STEM): - for filename in filenames: - path = os.path.join(walk_root, filename) - if include_regex.match(path) and not ( - exclude_regex and exclude_regex.match(path)): - paths.append(path) - - # TODO(kpayson): It would be nice to do this in a batch command, - # but we currently have name conflicts in src/proto - for path in paths: - command = [ - 'grpc_tools.protoc', - '-I {}'.format(PROTO_STEM), - '--python_out={}'.format(PROTO_STEM), - '--grpc_python_out={}'.format(PROTO_STEM), - ] + [path] - if protoc.main(command) != 0: - sys.stderr.write( - 'warning: Command:\n{}\nFailed'.format(command)) - - # Generated proto directories dont include __init__.py, but - # these are needed for python package resolution - for walk_root, _, _ in os.walk(PROTO_STEM): - path = os.path.join(walk_root, '__init__.py') - open(path, 'a').close() - - class BuildPy(build_py.build_py): """Custom project build command.""" From d01fd422ece32c875abc9ad3cd022b3727cce2f4 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 13 Sep 2017 20:13:32 +0000 Subject: [PATCH 118/227] grpc_1_0 flag for gRPC Python Protoc Plug-In We'll soon be flipping the default behavior of gRPC Python Protoc Plug-In to the later-than-gRPC-1.0 "code elements only in a separate _pb2_grpc.py file and no Beta-API-using generated code elements" behavior; this flag will be the supported (supported for a while, though not exactly sure how long) means of using the older "also put gRPC-using code elements in the generated _pb2.py file" behavior that has been in place since gRPC 1.0. --- src/compiler/python_generator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 31b177c28e6..a60b528e3b5 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -769,7 +769,7 @@ bool PythonGrpcGenerator::Generate(const FileDescriptor* file, PrivateGenerator generator(config_, &pbfile); if (parameter == "grpc_2_0") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true); - } else if (parameter == "") { + } else if (parameter == "grpc_1_0" || parameter == "") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true) && GenerateGrpc(context, generator, pb2_file_name, false); } else { From a495220f3128c94cf2f59a258de6d0ba8f4b2871 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 10:53:28 -0700 Subject: [PATCH 119/227] removing a completed TODO, removing forward declarations and rearranging code instead, removing _t suffixes with other recommendations --- .../client_channel/client_channel_factory.c | 2 - .../client_channel/lb_policy/grpclb/grpclb.c | 235 +++++++----------- .../transport/chttp2/server/chttp2_server.c | 30 +-- src/core/lib/surface/server.c | 69 +++-- 4 files changed, 145 insertions(+), 191 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel_factory.c b/src/core/ext/filters/client_channel/client_channel_factory.c index a1ba31e5cff..e8aa4cda290 100644 --- a/src/core/ext/filters/client_channel/client_channel_factory.c +++ b/src/core/ext/filters/client_channel/client_channel_factory.c @@ -48,8 +48,6 @@ static void* factory_arg_copy(void* factory) { } static void factory_arg_destroy(grpc_exec_ctx* exec_ctx, void* factory) { - // TODO(roth): Remove local exec_ctx when - // https://github.com/grpc/grpc/pull/8705 is merged. grpc_client_channel_factory_unref(exec_ctx, (grpc_client_channel_factory*)factory); } diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index df4a00406e3..8bc6095712f 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -287,48 +287,6 @@ static void add_pending_ping(pending_ping **root, grpc_closure *notify) { */ typedef struct rr_connectivity_data rr_connectivity_data; -/* Forward declare functions referred in glb_lb_policy_vtable */ -static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); -static void glb_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); -static int glb_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - const grpc_lb_policy_pick_args *pick_args, - grpc_connected_subchannel **target, - grpc_call_context_element *context, void **user_data, - grpc_closure *on_complete); -static void glb_cancel_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_connected_subchannel **target, - grpc_error *error); -static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, - grpc_lb_policy *pol, - uint32_t initial_metadata_flags_mask, - uint32_t initial_metadata_flags_eq, - grpc_error *error); -static void glb_ping_one_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_closure *closure); -static void glb_exit_idle_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); -static grpc_connectivity_state glb_check_connectivity_locked( - grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_error **connectivity_error); -static void glb_notify_on_state_change_locked(grpc_exec_ctx *exec_ctx, - grpc_lb_policy *pol, - grpc_connectivity_state *current, - grpc_closure *notify); -static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, - const grpc_lb_policy_args *args); - -/* Code wiring the policy with the rest of the core */ -static const grpc_lb_policy_vtable glb_lb_policy_vtable = { - glb_destroy, - glb_shutdown_locked, - glb_pick_locked, - glb_cancel_pick_locked, - glb_cancel_picks_locked, - glb_ping_one_locked, - glb_exit_idle_locked, - glb_check_connectivity_locked, - glb_notify_on_state_change_locked, - glb_update_locked}; - typedef struct glb_lb_policy { /** base policy: must be first */ grpc_lb_policy base; @@ -1011,92 +969,6 @@ static grpc_channel_args *build_lb_channel_args( static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); -static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, - grpc_lb_policy_factory *factory, - grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. */ - const grpc_arg *arg = - grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); - if (arg == NULL || arg->type != GRPC_ARG_POINTER) { - return NULL; - } - grpc_lb_addresses *addresses = (grpc_lb_addresses *)arg->value.pointer.p; - size_t num_grpclb_addrs = 0; - for (size_t i = 0; i < addresses->num_addresses; ++i) { - if (addresses->addresses[i].is_balancer) ++num_grpclb_addrs; - } - if (num_grpclb_addrs == 0) return NULL; - - glb_lb_policy *glb_policy = (glb_lb_policy *)gpr_zalloc(sizeof(*glb_policy)); - - /* Get server name. */ - arg = grpc_channel_args_find(args->args, GRPC_ARG_SERVER_URI); - GPR_ASSERT(arg != NULL); - GPR_ASSERT(arg->type == GRPC_ARG_STRING); - grpc_uri *uri = grpc_uri_parse(exec_ctx, arg->value.string, true); - GPR_ASSERT(uri->path[0] != '\0'); - glb_policy->server_name = - gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path); - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Will use '%s' as the server name for LB request.", - glb_policy->server_name); - } - grpc_uri_destroy(uri); - - glb_policy->cc_factory = args->client_channel_factory; - GPR_ASSERT(glb_policy->cc_factory != NULL); - - arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS); - glb_policy->lb_call_timeout_ms = - grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); - - arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); - glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( - arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, - INT_MAX}); - - // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, - // since we use this to trigger the client_load_reporting filter. - grpc_arg new_arg = - grpc_channel_arg_string_create(GRPC_ARG_LB_POLICY_NAME, "grpclb"); - static const char *args_to_remove[] = {GRPC_ARG_LB_POLICY_NAME}; - glb_policy->args = grpc_channel_args_copy_and_add_and_remove( - args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); - - /* Extract the backend addresses (may be empty) from the resolver for - * fallback. */ - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - - /* Create a client channel over them to communicate with a LB service */ - glb_policy->response_generator = - grpc_fake_resolver_response_generator_create(); - grpc_channel_args *lb_channel_args = build_lb_channel_args( - exec_ctx, addresses, glb_policy->response_generator, args->args); - char *uri_str; - gpr_asprintf(&uri_str, "fake:///%s", glb_policy->server_name); - glb_policy->lb_channel = grpc_lb_policy_grpclb_create_lb_channel( - exec_ctx, uri_str, args->client_channel_factory, lb_channel_args); - - /* Propagate initial resolution */ - grpc_fake_resolver_response_generator_set_response( - exec_ctx, glb_policy->response_generator, lb_channel_args); - grpc_channel_args_destroy(exec_ctx, lb_channel_args); - gpr_free(uri_str); - if (glb_policy->lb_channel == NULL) { - gpr_free((void *)glb_policy->server_name); - grpc_channel_args_destroy(exec_ctx, glb_policy->args); - gpr_free(glb_policy); - return NULL; - } - GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed, - glb_lb_channel_on_connectivity_changed_cb, glb_policy, - grpc_combiner_scheduler(args->combiner)); - grpc_lb_policy_init(&glb_policy->base, &glb_lb_policy_vtable, args->combiner); - grpc_connectivity_state_init(&glb_policy->state_tracker, GRPC_CHANNEL_IDLE, - "grpclb"); - return &glb_policy->base; -} static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { glb_lb_policy *glb_policy = (glb_lb_policy *)pol; @@ -1936,17 +1808,6 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, &glb_policy->lb_channel_connectivity, &glb_policy->lb_channel_on_connectivity_changed, NULL); } - - // Propagate update to fallback_backend_addresses if a non-empty serverlist - // hasn't been received from the balancer. - if (glb_policy->serverlist == NULL) { - grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - if (glb_policy->rr_policy != NULL) { - rr_handover_locked(exec_ctx, glb_policy); - } - } } // Invoked as part of the update process. It continues watching the LB channel @@ -2013,6 +1874,102 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, } } +/* Code wiring the policy with the rest of the core */ +static const grpc_lb_policy_vtable glb_lb_policy_vtable = { + glb_destroy, + glb_shutdown_locked, + glb_pick_locked, + glb_cancel_pick_locked, + glb_cancel_picks_locked, + glb_ping_one_locked, + glb_exit_idle_locked, + glb_check_connectivity_locked, + glb_notify_on_state_change_locked, + glb_update_locked}; + +static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, + grpc_lb_policy_factory *factory, + grpc_lb_policy_args *args) { + /* Count the number of gRPC-LB addresses. There must be at least one. + * TODO(roth): For now, we ignore non-balancer addresses, but in the + * future, we may change the behavior such that we fall back to using + * the non-balancer addresses if we cannot reach any balancers. In the + * fallback case, we should use the LB policy indicated by + * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is + * unset, we should default to pick_first). */ + const grpc_arg *arg = + grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); + if (arg == NULL || arg->type != GRPC_ARG_POINTER) { + return NULL; + } + grpc_lb_addresses *addresses = (grpc_lb_addresses *)arg->value.pointer.p; + size_t num_grpclb_addrs = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (addresses->addresses[i].is_balancer) ++num_grpclb_addrs; + } + if (num_grpclb_addrs == 0) return NULL; + + glb_lb_policy *glb_policy = (glb_lb_policy *)gpr_zalloc(sizeof(*glb_policy)); + + /* Get server name. */ + arg = grpc_channel_args_find(args->args, GRPC_ARG_SERVER_URI); + GPR_ASSERT(arg != NULL); + GPR_ASSERT(arg->type == GRPC_ARG_STRING); + grpc_uri *uri = grpc_uri_parse(exec_ctx, arg->value.string, true); + GPR_ASSERT(uri->path[0] != '\0'); + glb_policy->server_name = + gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path); + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, "Will use '%s' as the server name for LB request.", + glb_policy->server_name); + } + grpc_uri_destroy(uri); + + glb_policy->cc_factory = args->client_channel_factory; + GPR_ASSERT(glb_policy->cc_factory != NULL); + + arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS); + glb_policy->lb_call_timeout_ms = + grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); + + // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, + // since we use this to trigger the client_load_reporting filter. + grpc_arg new_arg = + grpc_channel_arg_string_create(GRPC_ARG_LB_POLICY_NAME, "grpclb"); + static const char *args_to_remove[] = {GRPC_ARG_LB_POLICY_NAME}; + glb_policy->args = grpc_channel_args_copy_and_add_and_remove( + args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); + + /* Create a client channel over them to communicate with a LB service */ + glb_policy->response_generator = + grpc_fake_resolver_response_generator_create(); + grpc_channel_args *lb_channel_args = build_lb_channel_args( + exec_ctx, addresses, glb_policy->response_generator, args->args); + char *uri_str; + gpr_asprintf(&uri_str, "fake:///%s", glb_policy->server_name); + glb_policy->lb_channel = grpc_lb_policy_grpclb_create_lb_channel( + exec_ctx, uri_str, args->client_channel_factory, lb_channel_args); + + /* Propagate initial resolution */ + grpc_fake_resolver_response_generator_set_response( + exec_ctx, glb_policy->response_generator, lb_channel_args); + grpc_channel_args_destroy(exec_ctx, lb_channel_args); + gpr_free(uri_str); + if (glb_policy->lb_channel == NULL) { + gpr_free((void *)glb_policy->server_name); + grpc_channel_args_destroy(exec_ctx, glb_policy->args); + gpr_free(glb_policy); + return NULL; + } + GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed, + glb_lb_channel_on_connectivity_changed_cb, glb_policy, + grpc_combiner_scheduler(args->combiner)); + grpc_lb_policy_init(&glb_policy->base, &glb_lb_policy_vtable, args->combiner); + grpc_connectivity_state_init(&glb_policy->state_tracker, GRPC_CHANNEL_IDLE, + "grpclb"); + return &glb_policy->base; +} + static void glb_factory_ref(grpc_lb_policy_factory *factory) {} static void glb_factory_unref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.c b/src/core/ext/transport/chttp2/server/chttp2_server.c index 45e5b908239..f5a409a403b 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.c +++ b/src/core/ext/transport/chttp2/server/chttp2_server.c @@ -49,10 +49,10 @@ typedef struct { grpc_closure tcp_server_shutdown_complete; grpc_closure *server_destroy_listener_done; grpc_handshake_manager *pending_handshake_mgrs; -} server_state_t; +} server_state; typedef struct { - server_state_t *server_state; + server_state *svr_state; grpc_pollset *accepting_pollset; grpc_tcp_server_acceptor *acceptor; grpc_handshake_manager *handshake_mgr; @@ -63,8 +63,8 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_handshaker_args *args = (grpc_handshaker_args *)arg; server_connection_state *connection_state = (server_connection_state *)args->user_data; - gpr_mu_lock(&connection_state->server_state->mu); - if (error != GRPC_ERROR_NONE || connection_state->server_state->shutdown) { + gpr_mu_lock(&connection_state->svr_state->mu); + if (error != GRPC_ERROR_NONE || connection_state->svr_state->shutdown) { const char *error_str = grpc_error_string(error); gpr_log(GPR_DEBUG, "Handshaking failed: %s", error_str); @@ -89,7 +89,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_transport *transport = grpc_create_chttp2_transport(exec_ctx, args->args, args->endpoint, 0); grpc_server_setup_transport( - exec_ctx, connection_state->server_state->server, transport, + exec_ctx, connection_state->svr_state->server, transport, connection_state->accepting_pollset, args->args); grpc_chttp2_transport_start_reading(exec_ctx, transport, args->read_buffer); @@ -97,11 +97,11 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, } } grpc_handshake_manager_pending_list_remove( - &connection_state->server_state->pending_handshake_mgrs, + &connection_state->svr_state->pending_handshake_mgrs, connection_state->handshake_mgr); - gpr_mu_unlock(&connection_state->server_state->mu); + gpr_mu_unlock(&connection_state->svr_state->mu); grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr); - grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp_server); + grpc_tcp_server_unref(exec_ctx, connection_state->svr_state->tcp_server); gpr_free(connection_state->acceptor); gpr_free(connection_state); } @@ -109,7 +109,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_pollset *accepting_pollset, grpc_tcp_server_acceptor *acceptor) { - server_state_t *state = (server_state_t *)arg; + server_state *state = (server_state *)arg; gpr_mu_lock(&state->mu); if (state->shutdown) { gpr_mu_unlock(&state->mu); @@ -125,7 +125,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_tcp_server_ref(state->tcp_server); server_connection_state *connection_state = (server_connection_state *)gpr_malloc(sizeof(*connection_state)); - connection_state->server_state = state; + connection_state->svr_state = state; connection_state->accepting_pollset = accepting_pollset; connection_state->acceptor = acceptor; connection_state->handshake_mgr = handshake_mgr; @@ -144,7 +144,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, void *arg, grpc_pollset **pollsets, size_t pollset_count) { - server_state_t *state = (server_state_t *)arg; + server_state *state = (server_state *)arg; gpr_mu_lock(&state->mu); state->shutdown = false; gpr_mu_unlock(&state->mu); @@ -154,7 +154,7 @@ static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { - server_state_t *state = (server_state_t *)arg; + server_state *state = (server_state *)arg; /* ensure all threads have unlocked */ gpr_mu_lock(&state->mu); grpc_closure *destroy_done = state->server_destroy_listener_done; @@ -179,7 +179,7 @@ static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg, static void server_destroy_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, void *arg, grpc_closure *destroy_done) { - server_state_t *state = (server_state_t *)arg; + server_state *state = (server_state *)arg; gpr_mu_lock(&state->mu); state->shutdown = true; state->server_destroy_listener_done = destroy_done; @@ -199,7 +199,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, size_t count = 0; int port_temp; grpc_error *err = GRPC_ERROR_NONE; - server_state_t *state = NULL; + server_state *state = NULL; grpc_error **errors = NULL; *port_num = -1; @@ -209,7 +209,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, if (err != GRPC_ERROR_NONE) { goto error; } - state = (server_state_t *)gpr_zalloc(sizeof(*state)); + state = (server_state *)gpr_zalloc(sizeof(*state)); GRPC_CLOSURE_INIT(&state->tcp_server_shutdown_complete, tcp_server_shutdown_complete, state, grpc_schedule_on_exec_ctx); diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 0bc21737136..1d0fd472d07 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -55,7 +55,7 @@ typedef struct listener { typedef struct call_data call_data; typedef struct channel_data channel_data; -typedef struct registered_method_t registered_method_t; +typedef struct registered_method registered_method; typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; @@ -76,7 +76,7 @@ typedef struct requested_call { grpc_call_details *details; } batch; struct { - registered_method_t *registered_method; + registered_method *method; gpr_timespec *deadline; grpc_byte_buffer **optional_payload; } registered; @@ -84,7 +84,7 @@ typedef struct requested_call { } requested_call; typedef struct channel_registered_method { - registered_method_t *server_registered_method; + registered_method *server_registered_method; uint32_t flags; bool has_host; grpc_slice method; @@ -123,7 +123,7 @@ typedef enum { ZOMBIED } call_state; -typedef struct request_matcher_t request_matcher_t; +typedef struct request_matcher request_matcher; struct call_data { grpc_call *call; @@ -145,7 +145,7 @@ struct call_data { uint32_t recv_initial_metadata_flags; grpc_metadata_array initial_metadata; - request_matcher_t *request_matcher; + request_matcher *matcher; grpc_byte_buffer *payload; grpc_closure got_initial_metadata; @@ -158,21 +158,21 @@ struct call_data { call_data *pending_next; }; -struct request_matcher_t { +struct request_matcher { grpc_server *server; call_data *pending_head; call_data *pending_tail; gpr_stack_lockfree **requests_per_cq; }; -struct registered_method_t { +struct registered_method { char *method; char *host; grpc_server_register_method_payload_handling payload_handling; uint32_t flags; /* one request matcher per method */ - request_matcher_t request_matcher; - registered_method_t *next; + request_matcher matcher; + registered_method *next; }; typedef struct { @@ -204,9 +204,9 @@ struct grpc_server { bool starting; gpr_cv starting_cv; - registered_method_t *registered_methods; + registered_method *registered_methods; /** one request matcher for unregistered methods */ - request_matcher_t unregistered_request_matcher; + request_matcher unregistered_request_matcher; /** free list of available requested_calls_per_cq indices */ gpr_stack_lockfree **request_freelist_per_cq; /** requested call backing data */ @@ -313,7 +313,7 @@ static void channel_broadcaster_shutdown(grpc_exec_ctx *exec_ctx, * request_matcher */ -static void request_matcher_init(request_matcher_t *rm, size_t entries, +static void request_matcher_init(request_matcher *rm, size_t entries, grpc_server *server) { memset(rm, 0, sizeof(*rm)); rm->server = server; @@ -324,7 +324,7 @@ static void request_matcher_init(request_matcher_t *rm, size_t entries, } } -static void request_matcher_destroy(request_matcher_t *rm) { +static void request_matcher_destroy(request_matcher *rm) { for (size_t i = 0; i < rm->server->cq_count; i++) { GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1); gpr_stack_lockfree_destroy(rm->requests_per_cq[i]); @@ -338,7 +338,7 @@ static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, } static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, - request_matcher_t *rm) { + request_matcher *rm) { while (rm->pending_head) { call_data *calld = rm->pending_head; rm->pending_head = calld->pending_next; @@ -355,7 +355,7 @@ static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx, grpc_server *server, - request_matcher_t *rm, + request_matcher *rm, grpc_error *error) { int request_id; for (size_t i = 0; i < server->cq_count; i++) { @@ -378,7 +378,7 @@ static void server_ref(grpc_server *server) { } static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { - registered_method_t *rm; + registered_method *rm; size_t i; grpc_channel_args_destroy(exec_ctx, server->channel_args); gpr_mu_destroy(&server->mu_global); @@ -387,7 +387,7 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { while ((rm = server->registered_methods) != NULL) { server->registered_methods = rm->next; if (server->started) { - request_matcher_destroy(&rm->request_matcher); + request_matcher_destroy(&rm->matcher); } gpr_free(rm->method); gpr_free(rm->host); @@ -519,7 +519,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, grpc_call_element *call_elem = (grpc_call_element *)arg; call_data *calld = (call_data *)call_elem->call_data; channel_data *chand = (channel_data *)call_elem->channel_data; - request_matcher_t *rm = calld->request_matcher; + request_matcher *rm = calld->matcher; grpc_server *server = rm->server; if (error != GRPC_ERROR_NONE || gpr_atm_acq_load(&server->shutdown_flag)) { @@ -569,7 +569,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, static void finish_start_new_rpc( grpc_exec_ctx *exec_ctx, grpc_server *server, grpc_call_element *elem, - request_matcher_t *rm, + request_matcher *rm, grpc_server_register_method_payload_handling payload_handling) { call_data *calld = (call_data *)elem->call_data; @@ -583,7 +583,7 @@ static void finish_start_new_rpc( return; } - calld->request_matcher = rm; + calld->matcher = rm; switch (payload_handling) { case GRPC_SRM_PAYLOAD_NONE: @@ -629,7 +629,7 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { continue; } finish_start_new_rpc(exec_ctx, server, elem, - &rm->server_registered_method->request_matcher, + &rm->server_registered_method->matcher, rm->server_registered_method->payload_handling); return; } @@ -647,7 +647,7 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { continue; } finish_start_new_rpc(exec_ctx, server, elem, - &rm->server_registered_method->request_matcher, + &rm->server_registered_method->matcher, rm->server_registered_method->payload_handling); return; } @@ -689,11 +689,11 @@ static void kill_pending_work_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_REF(error)); request_matcher_zombify_all_pending_calls( exec_ctx, &server->unregistered_request_matcher); - for (registered_method_t *rm = server->registered_methods; rm; + for (registered_method *rm = server->registered_methods; rm; rm = rm->next) { - request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher, + request_matcher_kill_requests(exec_ctx, server, &rm->matcher, GRPC_ERROR_REF(error)); - request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher); + request_matcher_zombify_all_pending_calls(exec_ctx, &rm->matcher); } } GRPC_ERROR_UNREF(error); @@ -1036,7 +1036,7 @@ void *grpc_server_register_method( grpc_server *server, const char *method, const char *host, grpc_server_register_method_payload_handling payload_handling, uint32_t flags) { - registered_method_t *m; + registered_method *m; GRPC_API_TRACE( "grpc_server_register_method(server=%p, method=%s, host=%s, " "flags=0x%08x)", @@ -1058,7 +1058,7 @@ void *grpc_server_register_method( flags); return NULL; } - m = (registered_method_t *)gpr_zalloc(sizeof(registered_method_t)); + m = (registered_method *)gpr_zalloc(sizeof(registered_method)); m->method = gpr_strdup(method); m->host = gpr_strdup(host); m->next = server->registered_methods; @@ -1113,9 +1113,8 @@ void grpc_server_start(grpc_server *server) { } request_matcher_init(&server->unregistered_request_matcher, (size_t)server->max_requested_calls_per_cq, server); - for (registered_method_t *rm = server->registered_methods; rm; - rm = rm->next) { - request_matcher_init(&rm->request_matcher, + for (registered_method *rm = server->registered_methods; rm; rm = rm->next) { + request_matcher_init(&rm->matcher, (size_t)server->max_requested_calls_per_cq, server); } @@ -1142,7 +1141,7 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s, const grpc_channel_args *args) { size_t num_registered_methods; size_t alloc; - registered_method_t *rm; + registered_method *rm; channel_registered_method *crm; grpc_channel *channel; channel_data *chand; @@ -1371,7 +1370,7 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx, grpc_server *server, size_t cq_idx, requested_call *rc) { call_data *calld = NULL; - request_matcher_t *rm = NULL; + request_matcher *rm = NULL; int request_id; if (gpr_atm_acq_load(&server->shutdown_flag)) { fail_call(exec_ctx, server, cq_idx, rc, @@ -1392,7 +1391,7 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx, rm = &server->unregistered_request_matcher; break; case REGISTERED_CALL: - rm = &rc->data.registered.registered_method->request_matcher; + rm = &rc->data.registered.method->matcher; break; } server->requested_calls_per_cq[cq_idx][request_id] = *rc; @@ -1483,7 +1482,7 @@ grpc_call_error grpc_server_request_registered_call( grpc_call_error error; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; requested_call *rc = (requested_call *)gpr_malloc(sizeof(*rc)); - registered_method_t *rm = (registered_method_t *)rmp; + registered_method *rm = (registered_method *)rmp; GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx); GRPC_API_TRACE( "grpc_server_request_registered_call(" @@ -1521,7 +1520,7 @@ grpc_call_error grpc_server_request_registered_call( rc->tag = tag; rc->cq_bound_to_call = cq_bound_to_call; rc->call = call; - rc->data.registered.registered_method = rm; + rc->data.registered.method = rm; rc->data.registered.deadline = deadline; rc->initial_metadata = initial_metadata; rc->data.registered.optional_payload = optional_payload; From 2ed5d80ddac6b233ca4b7b4b6caa8a9115ea0560 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 13:07:58 -0700 Subject: [PATCH 120/227] Removing forward declaration --- src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 8bc6095712f..ef16151b012 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -966,10 +966,6 @@ static grpc_channel_args *build_lb_channel_args( return result; } -static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, - void *arg, - grpc_error *error); - static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { glb_lb_policy *glb_policy = (glb_lb_policy *)pol; GPR_ASSERT(glb_policy->pending_picks == NULL); From 2e2bf250fbf72ea22172cae7db2d3c985363958b Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 13:32:48 -0700 Subject: [PATCH 121/227] Merging with master --- .../client_channel/lb_policy/grpclb/grpclb.c | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index ef16151b012..7e4f1b34870 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -1666,7 +1666,7 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { - glb_lb_policy *glb_policy = arg; + glb_lb_policy *glb_policy = (glb_lb_policy *)arg; /* If we receive a serverlist after the timer fires but before this callback * actually runs, don't do anything. */ if (glb_policy->serverlist != NULL) return; @@ -1804,6 +1804,17 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, &glb_policy->lb_channel_connectivity, &glb_policy->lb_channel_on_connectivity_changed, NULL); } + + // Propagate update to fallback_backend_addresses if a non-empty serverlist + // hasn't been received from the balancer. + if (glb_policy->serverlist == NULL) { + grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + if (glb_policy->rr_policy != NULL) { + rr_handover_locked(exec_ctx, glb_policy); + } + } } // Invoked as part of the update process. It continues watching the LB channel @@ -1886,13 +1897,7 @@ static const grpc_lb_policy_vtable glb_lb_policy_vtable = { static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. - * TODO(roth): For now, we ignore non-balancer addresses, but in the - * future, we may change the behavior such that we fall back to using - * the non-balancer addresses if we cannot reach any balancers. In the - * fallback case, we should use the LB policy indicated by - * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is - * unset, we should default to pick_first). */ + /* Count the number of gRPC-LB addresses. There must be at least one. */ const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); if (arg == NULL || arg->type != GRPC_ARG_POINTER) { @@ -1928,6 +1933,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->lb_call_timeout_ms = grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); + arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); + glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( + arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, + INT_MAX}); + // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. grpc_arg new_arg = @@ -1936,6 +1946,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); + /* Extract the backend addresses (may be empty) from the resolver for + * fallback. */ + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + /* Create a client channel over them to communicate with a LB service */ glb_policy->response_generator = grpc_fake_resolver_response_generator_create(); From 6b84005651517c9136a32cec49db8333a5b40ee9 Mon Sep 17 00:00:00 2001 From: Alok Kumar Date: Wed, 13 Sep 2017 11:49:18 -0700 Subject: [PATCH 122/227] Add independent library targets for newly added .h files for microbenchmarks. Also fix includes in fullstack_fixtures.h --- test/cpp/microbenchmarks/BUILD | 34 ++++++++++++++++--- .../microbenchmarks/bm_fullstack_trickle.cc | 1 - test/cpp/microbenchmarks/fullstack_fixtures.h | 1 + .../fullstack_streaming_ping_pong.h | 1 - .../fullstack_streaming_pump.h | 1 - .../fullstack_unary_ping_pong.h | 1 - 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index 985a335f1b3..0b69e9ba9aa 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -73,12 +73,29 @@ grpc_cc_binary( deps = [":helpers"], ) +grpc_cc_library( + name = "fullstack_streaming_ping_pong_h", + testonly = 1, + hdrs = [ + "fullstack_streaming_ping_pong.h", + ], + deps = [":helpers"], +) + grpc_cc_binary( name = "bm_fullstack_streaming_ping_pong", testonly = 1, srcs = [ "bm_fullstack_streaming_ping_pong.cc", - "fullstack_streaming_ping_pong.h", + ], + deps = [":fullstack_streaming_ping_pong_h"], +) + +grpc_cc_library( + name = "fullstack_streaming_pump_h", + testonly = 1, + hdrs = [ + "fullstack_streaming_pump.h", ], deps = [":helpers"], ) @@ -88,9 +105,8 @@ grpc_cc_binary( testonly = 1, srcs = [ "bm_fullstack_streaming_pump.cc", - "fullstack_streaming_pump.h", ], - deps = [":helpers"], + deps = [":fullstack_streaming_pump_h"], ) grpc_cc_binary( @@ -103,14 +119,22 @@ grpc_cc_binary( ], ) +grpc_cc_library( + name = "fullstack_unary_ping_pong_h", + testonly = 1, + hdrs = [ + "fullstack_unary_ping_pong.h", + ], + deps = [":helpers"], +) + grpc_cc_binary( name = "bm_fullstack_unary_ping_pong", testonly = 1, srcs = [ "bm_fullstack_unary_ping_pong.cc", - "fullstack_unary_ping_pong.h", ], - deps = [":helpers"], + deps = [":fullstack_unary_ping_pong_h"], ) grpc_cc_binary( diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc index 59fb29dd600..2656566a50b 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -22,7 +22,6 @@ #include #include #include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" #include "test/cpp/microbenchmarks/fullstack_fixtures.h" diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h index 5477b860b4f..ecd28c3f8ac 100644 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -41,6 +41,7 @@ extern "C" { #include "test/core/util/port.h" } +#include "src/cpp/client/create_channel_internal.h" #include "test/cpp/microbenchmarks/helpers.h" namespace grpc { diff --git a/test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h b/test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h index ff1f9667536..6df044f3440 100644 --- a/test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h +++ b/test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h @@ -24,7 +24,6 @@ #include #include #include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" #include "test/cpp/microbenchmarks/fullstack_fixtures.h" diff --git a/test/cpp/microbenchmarks/fullstack_streaming_pump.h b/test/cpp/microbenchmarks/fullstack_streaming_pump.h index f9db212b02c..9e826091ec6 100644 --- a/test/cpp/microbenchmarks/fullstack_streaming_pump.h +++ b/test/cpp/microbenchmarks/fullstack_streaming_pump.h @@ -24,7 +24,6 @@ #include #include #include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" #include "test/cpp/microbenchmarks/fullstack_fixtures.h" diff --git a/test/cpp/microbenchmarks/fullstack_unary_ping_pong.h b/test/cpp/microbenchmarks/fullstack_unary_ping_pong.h index 76d278b2a00..d4489382952 100644 --- a/test/cpp/microbenchmarks/fullstack_unary_ping_pong.h +++ b/test/cpp/microbenchmarks/fullstack_unary_ping_pong.h @@ -24,7 +24,6 @@ #include #include #include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" #include "test/cpp/microbenchmarks/fullstack_fixtures.h" From f1dc9c36a71ce11e0ebfda8553b5deb383d25107 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 14:21:27 -0700 Subject: [PATCH 123/227] Count api object creations --- src/core/ext/filters/client_channel/subchannel.c | 2 ++ src/core/lib/debug/stats_data.c | 6 ++++++ src/core/lib/debug/stats_data.h | 13 +++++++++++++ src/core/lib/debug/stats_data.yaml | 8 ++++++++ src/core/lib/surface/channel.c | 6 ++++++ src/core/lib/surface/completion_queue.c | 5 +++++ 6 files changed, 40 insertions(+) diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c index 05c55aaa893..bc9c3cc782f 100644 --- a/src/core/ext/filters/client_channel/subchannel.c +++ b/src/core/ext/filters/client_channel/subchannel.c @@ -32,6 +32,7 @@ #include "src/core/ext/filters/client_channel/uri_parser.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/connected_channel.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/profiling/timers.h" @@ -290,6 +291,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, return c; } + GRPC_STATS_INC_CLIENT_SUBCHANNELS_CREATED(exec_ctx); c = (grpc_subchannel *)gpr_zalloc(sizeof(*c)); c->key = key; gpr_atm_no_barrier_store(&c->ref_pair, 1 << INTERNAL_REF_BITS); diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 3fd8ee38ef8..1e60a333581 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -25,6 +25,10 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "client_calls_created", "server_calls_created", + "cqs_created", + "client_channels_created", + "client_subchannels_created", + "server_channels_created", "syscall_poll", "syscall_wait", "histogram_slow_lookups", @@ -60,6 +64,8 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of client side calls created by this process", "Number of server side calls created by this process", + "Number of completion queues created", "Number of client channels created", + "Number of client subchannels created", "Number of server channels created", "Number of polling syscalls (epoll_wait, poll, etc) made by this process", "Number of sleeping syscalls made by this process", "Number of times histogram increments went through the slow (binary " diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index b7c15c08a5e..ec4cbcce6c7 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -27,6 +27,10 @@ typedef enum { GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED, GRPC_STATS_COUNTER_SERVER_CALLS_CREATED, + GRPC_STATS_COUNTER_CQS_CREATED, + GRPC_STATS_COUNTER_CLIENT_CHANNELS_CREATED, + GRPC_STATS_COUNTER_CLIENT_SUBCHANNELS_CREATED, + GRPC_STATS_COUNTER_SERVER_CHANNELS_CREATED, GRPC_STATS_COUNTER_SYSCALL_POLL, GRPC_STATS_COUNTER_SYSCALL_WAIT, GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS, @@ -104,6 +108,15 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) #define GRPC_STATS_INC_SERVER_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_CALLS_CREATED) +#define GRPC_STATS_INC_CQS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CQS_CREATED) +#define GRPC_STATS_INC_CLIENT_CHANNELS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CHANNELS_CREATED) +#define GRPC_STATS_INC_CLIENT_SUBCHANNELS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_CLIENT_SUBCHANNELS_CREATED) +#define GRPC_STATS_INC_SERVER_CHANNELS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_CHANNELS_CREATED) #define GRPC_STATS_INC_SYSCALL_POLL(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL) #define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index a9d71f4fcb3..7bb1d2e2edf 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -20,6 +20,14 @@ doc: Number of client side calls created by this process - counter: server_calls_created doc: Number of server side calls created by this process +- counter: cqs_created + doc: Number of completion queues created +- counter: client_channels_created + doc: Number of client channels created +- counter: client_subchannels_created + doc: Number of client subchannels created +- counter: server_channels_created + doc: Number of server channels created # polling - counter: syscall_poll doc: Number of polling syscalls (epoll_wait, poll, etc) made by this process diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 34548dac269..2f9b677c85e 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -27,6 +27,7 @@ #include #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/support/string.h" @@ -77,6 +78,11 @@ grpc_channel *grpc_channel_create_with_builder( grpc_channel_args *args = grpc_channel_args_copy( grpc_channel_stack_builder_get_channel_arguments(builder)); grpc_channel *channel; + if (channel_stack_type == GRPC_SERVER_CHANNEL) { + GRPC_STATS_INC_SERVER_CHANNELS_CREATED(exec_ctx); + } else { + GRPC_STATS_INC_CLIENT_CHANNELS_CREATED(exec_ctx); + } grpc_error *error = grpc_channel_stack_builder_finish( exec_ctx, builder, sizeof(grpc_channel), 1, destroy_channel, NULL, (void **)&channel); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 47265039946..3205ddeadf3 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -26,6 +26,7 @@ #include #include +#include "src/core/lib/debug/stats.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/profiling/timers.h" @@ -420,6 +421,10 @@ grpc_completion_queue *grpc_completion_queue_create_internal( const cq_poller_vtable *poller_vtable = &g_poller_vtable_by_poller_type[polling_type]; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GRPC_STATS_INC_CQS_CREATED(&exec_ctx); + grpc_exec_ctx_finish(&exec_ctx); + cq = (grpc_completion_queue *)gpr_zalloc(sizeof(grpc_completion_queue) + vtable->data_size + poller_vtable->size()); From 1a012bbe8fceacfff9a2fc49528ad07de75743a2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 14:29:00 -0700 Subject: [PATCH 124/227] Review feedback --- src/core/lib/iomgr/ev_epoll1_linux.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 8f855a7b6ec..aa861d3cc19 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -873,6 +873,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); SET_KICK_STATE(worker->next, DESIGNATED_POLLER); + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&worker->next->cv); if (grpc_exec_ctx_has_work(exec_ctx)) { gpr_mu_unlock(&pollset->mu); @@ -1098,7 +1099,7 @@ static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, goto done; } } else { - GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); + GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked while waking up"); } From 324ad6a6c239ebb6dfafbe00a825a90a409daa11 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Wed, 13 Sep 2017 15:10:08 -0700 Subject: [PATCH 125/227] Update service_config doc The current grpclb fallback implementation is to continue using the internal round_robin policy but feed it with the backends from the resolver. We didn't fall back to the policy indicated in service config (or client API) because that's too complex and has more different behavior from the normal situation where we do connect to the balancer. --- doc/service_config.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/service_config.md b/doc/service_config.md index 99d985f3bfe..0abbd7f324d 100644 --- a/doc/service_config.md +++ b/doc/service_config.md @@ -24,10 +24,7 @@ The service config is a JSON string of the following form: // opposed to backend addresses), gRPC will use grpclb (see // https://github.com/grpc/grpc/blob/master/doc/load-balancing.md), // regardless of what LB policy is requested either here or via the - // client API. However, if the resolver returns at least one backend - // address in addition to the balancer address(es), the client may fall - // back to the requested policy if it is unable to reach any of the - // grpclb load balancers. + // client API. 'loadBalancingPolicy': string, // Per-method configuration. Optional. From fb0262b0fda6b3f605ef8f6a65ed628f22180d54 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 15:19:19 -0700 Subject: [PATCH 126/227] Eliminate lock in self-scheduling path in executor --- src/core/lib/iomgr/executor.c | 57 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index 892385d7d78..edc391331ff 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -38,10 +38,10 @@ typedef struct { gpr_mu mu; gpr_cv cv; grpc_closure_list elems; - size_t depth; bool shutdown; bool queued_long_job; gpr_thd_id id; + grpc_closure_list local_elems; } thread_state; static thread_state *g_thread_state; @@ -56,32 +56,30 @@ static grpc_tracer_flag executor_trace = static void executor_thread(void *arg); -static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) { - size_t n = 0; - - grpc_closure *c = list.head; - while (c != NULL) { - grpc_closure *next = c->next_data.next; - grpc_error *error = c->error_data.error; - if (GRPC_TRACER_ON(executor_trace)) { +static void run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { + while (!grpc_closure_list_empty(*list)) { + grpc_closure *c = list->head; + grpc_closure_list_init(list); + while (c != NULL) { + grpc_closure *next = c->next_data.next; + grpc_error *error = c->error_data.error; + if (GRPC_TRACER_ON(executor_trace)) { #ifndef NDEBUG - gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c, - c->file_created, c->line_created); + gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c, + c->file_created, c->line_created); #else - gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c); + gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c); #endif - } + } #ifndef NDEBUG - c->scheduled = false; + c->scheduled = false; #endif - c->cb(exec_ctx, c->cb_arg, error); - GRPC_ERROR_UNREF(error); - c = next; - n++; - grpc_exec_ctx_flush(exec_ctx); + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + c = next; + grpc_exec_ctx_flush(exec_ctx); + } } - - return n; } bool grpc_executor_is_threaded() { @@ -126,7 +124,7 @@ void grpc_executor_set_threading(grpc_exec_ctx *exec_ctx, bool threading) { for (size_t i = 0; i < g_max_threads; i++) { gpr_mu_destroy(&g_thread_state[i].mu); gpr_cv_destroy(&g_thread_state[i].cv); - run_closures(exec_ctx, g_thread_state[i].elems); + run_closures(exec_ctx, &g_thread_state[i].elems); } gpr_free(g_thread_state); gpr_tls_destroy(&g_this_thread_state); @@ -150,14 +148,11 @@ static void executor_thread(void *arg) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); - size_t subtract_depth = 0; for (;;) { if (GRPC_TRACER_ON(executor_trace)) { - gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")", - (int)(ts - g_thread_state), subtract_depth); + gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step", (int)(ts - g_thread_state)); } gpr_mu_lock(&ts->mu); - ts->depth -= subtract_depth; while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) { ts->queued_long_job = false; gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_REALTIME)); @@ -171,14 +166,15 @@ static void executor_thread(void *arg) { break; } GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx); - grpc_closure_list exec = ts->elems; + GPR_ASSERT(grpc_closure_list_empty(ts->local_elems)); + ts->local_elems = ts->elems; ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; gpr_mu_unlock(&ts->mu); if (GRPC_TRACER_ON(executor_trace)) { gpr_log(GPR_DEBUG, "EXECUTOR[%d]: execute", (int)(ts - g_thread_state)); } - subtract_depth = run_closures(&exec_ctx, exec); + run_closures(&exec_ctx, &ts->local_elems); } grpc_exec_ctx_finish(&exec_ctx); } @@ -211,6 +207,8 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)]; } else { GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx); + grpc_closure_list_append(&ts->local_elems, closure, error); + return; } thread_state *orig_ts = ts; @@ -250,8 +248,7 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, gpr_cv_signal(&ts->cv); } grpc_closure_list_append(&ts->elems, closure, error); - ts->depth++; - try_new_thread = ts->depth > MAX_DEPTH && + try_new_thread = ts->elems.head != closure && cur_thread_count < g_max_threads && !ts->shutdown; if (!is_short) ts->queued_long_job = true; gpr_mu_unlock(&ts->mu); From 21b2c189d2b09b72c1afd9e5ff3cebb0b9b70a08 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 15:21:10 -0700 Subject: [PATCH 127/227] Remove old constant --- src/core/lib/iomgr/executor.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index edc391331ff..fa8c5bfaab8 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -32,8 +32,6 @@ #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/support/spinlock.h" -#define MAX_DEPTH 2 - typedef struct { gpr_mu mu; gpr_cv cv; From 213d2744b543cf8593838a5eb995bf9c7924c7d5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 15:22:14 -0700 Subject: [PATCH 128/227] Only use fast path for short items --- src/core/lib/iomgr/executor.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index fa8c5bfaab8..e5372c7b08c 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -205,8 +205,10 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)]; } else { GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx); - grpc_closure_list_append(&ts->local_elems, closure, error); - return; + if (!is_short) { + grpc_closure_list_append(&ts->local_elems, closure, error); + return; + } } thread_state *orig_ts = ts; From 7f10c299c7eb5d6688c20ae43c63b0548ca8acd1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 15:26:56 -0700 Subject: [PATCH 129/227] Windows --- src/core/lib/iomgr/pollset_windows.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index ea017a60546..eb295d3eebd 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -98,7 +98,7 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { pollset->shutting_down = 1; - grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + grpc_pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset->is_iocp_worker) { GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_ERROR_NONE); } else { @@ -181,7 +181,7 @@ done: return GRPC_ERROR_NONE; } -grpc_error *grpc_pollset_kick(grpc_pollset *p, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { if (specific_worker != NULL) { if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { @@ -209,7 +209,7 @@ grpc_error *grpc_pollset_kick(grpc_pollset *p, specific_worker = pop_front_worker(&p->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET); if (specific_worker != NULL) { - grpc_pollset_kick(p, specific_worker); + grpc_pollset_kick(exec_ctx, p, specific_worker); } else if (p->is_iocp_worker) { grpc_iocp_kick(); } else { From c7cebe8d8c17df2d0d2f97e402294b2717e2ff73 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 13 Sep 2017 17:51:26 -0700 Subject: [PATCH 130/227] Consistent acq/rel semantics for pending_events atomic --- src/core/lib/surface/completion_queue.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 47265039946..c4eff350da0 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -444,7 +444,7 @@ grpc_completion_queue *grpc_completion_queue_create_internal( static void cq_init_next(void *ptr) { cq_next_data *cqd = (cq_next_data *)ptr; /* Initial count is dropped by grpc_completion_queue_shutdown */ - gpr_atm_no_barrier_store(&cqd->pending_events, 1); + gpr_atm_rel_store(&cqd->pending_events, 1); cqd->shutdown_called = false; gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0); cq_event_queue_init(&cqd->queue); @@ -459,7 +459,7 @@ static void cq_destroy_next(void *ptr) { static void cq_init_pluck(void *ptr) { cq_pluck_data *cqd = (cq_pluck_data *)ptr; /* Initial count is dropped by grpc_completion_queue_shutdown */ - gpr_atm_no_barrier_store(&cqd->pending_events, 1); + gpr_atm_rel_store(&cqd->pending_events, 1); cqd->completed_tail = &cqd->completed_head; cqd->completed_head.next = (uintptr_t)cqd->completed_tail; gpr_atm_no_barrier_store(&cqd->shutdown, 0); @@ -560,13 +560,13 @@ static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) {} * true if the increment was successful; false if the counter is zero */ static bool atm_inc_if_nonzero(gpr_atm *counter) { while (true) { - gpr_atm count = gpr_atm_no_barrier_load(counter); + gpr_atm count = gpr_atm_acq_load(counter); /* If zero, we are done. If not, we must to a CAS (instead of an atomic * increment) to maintain the contract: do not increment the counter if it * is zero. */ if (count == 0) { return false; - } else if (gpr_atm_no_barrier_cas(counter, count, count + 1)) { + } else if (gpr_atm_full_cas(counter, count, count + 1)) { break; } } @@ -638,8 +638,7 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, /* Add the completion to the queue */ bool is_first = cq_event_queue_push(&cqd->queue, storage); gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1); - bool will_definitely_shutdown = - gpr_atm_no_barrier_load(&cqd->pending_events) == 1; + bool will_definitely_shutdown = gpr_atm_acq_load(&cqd->pending_events) == 1; if (!will_definitely_shutdown) { /* Only kick if this is the first item queued */ @@ -883,7 +882,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, } } - if (gpr_atm_no_barrier_load(&cqd->pending_events) == 0) { + if (gpr_atm_acq_load(&cqd->pending_events) == 0) { /* Before returning, check if the queue has any items left over (since gpr_mpscq_pop() can sometimes return NULL even if the queue is not empty. If so, keep retrying but do not return GRPC_QUEUE_SHUTDOWN */ @@ -929,7 +928,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, } if (cq_event_queue_num_items(&cqd->queue) > 0 && - gpr_atm_no_barrier_load(&cqd->pending_events) > 0) { + gpr_atm_acq_load(&cqd->pending_events) > 0) { gpr_mu_lock(cq->mu); cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); @@ -956,7 +955,7 @@ static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx, cq_next_data *cqd = DATA_FROM_CQ(cq); GPR_ASSERT(cqd->shutdown_called); - GPR_ASSERT(gpr_atm_no_barrier_load(&cqd->pending_events) == 0); + GPR_ASSERT(gpr_atm_acq_load(&cqd->pending_events) == 0); cq->poller_vtable->shutdown(exec_ctx, POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done); From 319652fa85c01543d6695ba721d8fc82b9860726 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 18:13:48 -0700 Subject: [PATCH 131/227] Track pings in bmdiff --- tools/profiling/microbenchmarks/bm_diff/bm_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py index ad79a0a1972..f8989b17fc8 100644 --- a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py @@ -28,4 +28,4 @@ _INTERESTING = ('cpu_time', 'real_time', 'locks_per_iteration', 'atm_cas_per_iteration', 'atm_add_per_iteration', 'nows_per_iteration', 'cli_transport_stalls_per_iteration', 'cli_stream_stalls_per_iteration', 'svr_transport_stalls_per_iteration', - 'svr_stream_stalls_per_iteration',) + 'svr_stream_stalls_per_iteration', 'http2_pings_sent_per_iteration') From 438304960f95169a7a0272ff684b80805ced9cba Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 13 Sep 2017 18:27:55 -0700 Subject: [PATCH 132/227] Add allow-ssh tags to newly created GCE VMs --- tools/gce/create_interop_worker.sh | 3 ++- tools/gce/create_linux_performance_worker.sh | 3 ++- tools/gce/create_linux_worker.sh | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/gce/create_interop_worker.sh b/tools/gce/create_interop_worker.sh index 76e4905272a..3e59dc501ad 100755 --- a/tools/gce/create_interop_worker.sh +++ b/tools/gce/create_interop_worker.sh @@ -33,7 +33,8 @@ gcloud compute instances create $INSTANCE_NAME \ --machine-type n1-standard-16 \ --image ubuntu-15-10 \ --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/xapi.zoo + --scopes https://www.googleapis.com/auth/xapi.zoo \ + --tags=allow-ssh echo 'Created GCE instance, waiting 60 seconds for it to come online.' sleep 60 diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh index 7f53732c05d..4270f5cbfb6 100755 --- a/tools/gce/create_linux_performance_worker.sh +++ b/tools/gce/create_linux_performance_worker.sh @@ -36,7 +36,8 @@ gcloud compute instances create $INSTANCE_NAME \ --image-project ubuntu-os-cloud \ --image-family ubuntu-1704 \ --boot-disk-size 300 \ - --scopes https://www.googleapis.com/auth/bigquery + --scopes https://www.googleapis.com/auth/bigquery \ + --tags=allow-ssh echo 'Created GCE instance, waiting 60 seconds for it to come online.' sleep 60 diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh index c96f3281fc7..8bf44aa7294 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_worker.sh @@ -31,7 +31,8 @@ gcloud compute instances create $INSTANCE_NAME \ --image=ubuntu-1510 \ --image-project=grpc-testing \ --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/bigquery + --scopes https://www.googleapis.com/auth/bigquery \ + --tags=allow-ssh echo 'Created GCE instance, waiting 60 seconds for it to come online.' sleep 60 From 01e83b55e4700459ae82efeac088b512d5dfabd7 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Wed, 13 Sep 2017 19:45:07 -0700 Subject: [PATCH 133/227] Revert "Add fallback (use backends from resolver if can't reach balancer) to grpclb." This reverts commit aba0a0a54412fe59bc2090334ae26c99bb148097. --- include/grpc++/support/channel_arguments.h | 6 - include/grpc/impl/codegen/grpc_types.h | 6 +- .../client_channel/lb_policy/grpclb/grpclb.c | 197 ++++-------------- .../client_channel/lb_policy_factory.c | 2 +- .../client_channel/lb_policy_factory.h | 2 +- src/cpp/common/channel_arguments.cc | 4 - test/cpp/end2end/grpclb_end2end_test.cc | 74 +------ 7 files changed, 47 insertions(+), 244 deletions(-) diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h index 9dc505f0082..7b6befeaf1e 100644 --- a/include/grpc++/support/channel_arguments.h +++ b/include/grpc++/support/channel_arguments.h @@ -64,12 +64,6 @@ class ChannelArguments { /// Set the compression algorithm for the channel. void SetCompressionAlgorithm(grpc_compression_algorithm algorithm); - /// Set the grpclb fallback timeout (in ms) for the channel. If this amount - /// of time has passed but we have not gotten any non-empty \a serverlist from - /// the balancer, we will fall back to use the backend address(es) returned by - /// the resolver. - void SetGrpclbFallbackTimeout(int fallback_timeout); - /// Set the socket mutator for the channel. void SetSocketMutator(grpc_socket_mutator* mutator); diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index beb6080c944..748dc717a33 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -287,11 +287,7 @@ typedef struct { "grpc.experimental.tcp_max_read_chunk_size" /* Timeout in milliseconds to use for calls to the grpclb load balancer. If 0 or unset, the balancer calls will have no deadline. */ -#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms" -/* Timeout in milliseconds to wait for the serverlist from the grpclb load - balancer before using fallback backend addresses from the resolver. - If 0, fallback will never be used. */ -#define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms" +#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_timeout_ms" /** If non-zero, grpc server's cronet compression workaround will be enabled */ #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \ "grpc.workaround.cronet_compression" diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 18979829bd2..5aafed1374d 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -122,7 +122,6 @@ #define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6 #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 -#define GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS 10000 grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false, "glb"); @@ -299,10 +298,6 @@ typedef struct glb_lb_policy { /** timeout in milliseconds for the LB call. 0 means no deadline. */ int lb_call_timeout_ms; - /** timeout in milliseconds for before using fallback backend addresses. - * 0 means not using fallback. */ - int lb_fallback_timeout_ms; - /** for communicating with the LB server */ grpc_channel *lb_channel; @@ -329,9 +324,6 @@ typedef struct glb_lb_policy { * Otherwise, we delegate to the RR policy. */ size_t serverlist_index; - /** stores the backend addresses from the resolver */ - grpc_lb_addresses *fallback_backend_addresses; - /** list of picks that are waiting on RR's policy connectivity */ pending_pick *pending_picks; @@ -352,9 +344,6 @@ typedef struct glb_lb_policy { /** is \a lb_call_retry_timer active? */ bool retry_timer_active; - /** is \a lb_fallback_timer active? */ - bool fallback_timer_active; - /** called upon changes to the LB channel's connectivity. */ grpc_closure lb_channel_on_connectivity_changed; @@ -377,9 +366,6 @@ typedef struct glb_lb_policy { /* LB call retry timer callback. */ grpc_closure lb_on_call_retry; - /* LB fallback timer callback. */ - grpc_closure lb_on_fallback; - grpc_call *lb_call; /* streaming call to the LB server, */ grpc_metadata_array lb_initial_metadata_recv; /* initial MD from LB server */ @@ -403,9 +389,6 @@ typedef struct glb_lb_policy { /** LB call retry timer */ grpc_timer lb_call_retry_timer; - /** LB fallback timer */ - grpc_timer lb_fallback_timer; - bool initial_request_sent; bool seen_initial_response; @@ -552,32 +535,6 @@ static grpc_lb_addresses *process_serverlist_locked( return lb_addresses; } -/* Returns the backend addresses extracted from the given addresses */ -static grpc_lb_addresses *extract_backend_addresses_locked( - grpc_exec_ctx *exec_ctx, const grpc_lb_addresses *addresses) { - /* first pass: count the number of backend addresses */ - size_t num_backends = 0; - for (size_t i = 0; i < addresses->num_addresses; ++i) { - if (!addresses->addresses[i].is_balancer) { - ++num_backends; - } - } - /* second pass: actually populate the addresses and (empty) LB tokens */ - grpc_lb_addresses *backend_addresses = - grpc_lb_addresses_create(num_backends, &lb_token_vtable); - size_t num_copied = 0; - for (size_t i = 0; i < addresses->num_addresses; ++i) { - if (addresses->addresses[i].is_balancer) continue; - const grpc_resolved_address *addr = &addresses->addresses[i].address; - grpc_lb_addresses_set_address(backend_addresses, num_copied, &addr->addr, - addr->len, false /* is_balancer */, - NULL /* balancer_name */, - (void *)GRPC_MDELEM_LB_TOKEN_EMPTY.payload); - ++num_copied; - } - return backend_addresses; -} - static void update_lb_connectivity_status_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, grpc_connectivity_state rr_state, grpc_error *rr_state_error) { @@ -645,38 +602,35 @@ static bool pick_from_internal_rr_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, const grpc_lb_policy_pick_args *pick_args, bool force_async, grpc_connected_subchannel **target, wrapped_rr_closure_arg *wc_arg) { - // Check for drops if we are not using fallback backend addresses. - if (glb_policy->serverlist != NULL) { - // Look at the index into the serverlist to see if we should drop this call. - grpc_grpclb_server *server = - glb_policy->serverlist->servers[glb_policy->serverlist_index++]; - if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { - glb_policy->serverlist_index = 0; // Wrap-around. + // Look at the index into the serverlist to see if we should drop this call. + grpc_grpclb_server *server = + glb_policy->serverlist->servers[glb_policy->serverlist_index++]; + if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { + glb_policy->serverlist_index = 0; // Wrap-around. + } + if (server->drop) { + // Not using the RR policy, so unref it. + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", + (intptr_t)wc_arg->rr_policy); } - if (server->drop) { - // Not using the RR policy, so unref it. - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", - (intptr_t)wc_arg->rr_policy); - } - GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); - // Update client load reporting stats to indicate the number of - // dropped calls. Note that we have to do this here instead of in - // the client_load_reporting filter, because we do not create a - // subchannel call (and therefore no client_load_reporting filter) - // for dropped calls. - grpc_grpclb_client_stats_add_call_dropped_locked( - server->load_balance_token, wc_arg->client_stats); - grpc_grpclb_client_stats_unref(wc_arg->client_stats); - if (force_async) { - GPR_ASSERT(wc_arg->wrapped_closure != NULL); - GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); - gpr_free(wc_arg->free_when_done); - return false; - } + GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); + // Update client load reporting stats to indicate the number of + // dropped calls. Note that we have to do this here instead of in + // the client_load_reporting filter, because we do not create a + // subchannel call (and therefore no client_load_reporting filter) + // for dropped calls. + grpc_grpclb_client_stats_add_call_dropped_locked(server->load_balance_token, + wc_arg->client_stats); + grpc_grpclb_client_stats_unref(wc_arg->client_stats); + if (force_async) { + GPR_ASSERT(wc_arg->wrapped_closure != NULL); + GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); gpr_free(wc_arg->free_when_done); - return true; + return false; } + gpr_free(wc_arg->free_when_done); + return true; } // Pick via the RR policy. const bool pick_done = grpc_lb_policy_pick_locked( @@ -714,18 +668,8 @@ static bool pick_from_internal_rr_locked( static grpc_lb_policy_args *lb_policy_args_create(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - grpc_lb_addresses *addresses; - if (glb_policy->serverlist != NULL) { - GPR_ASSERT(glb_policy->serverlist->num_servers > 0); - addresses = process_serverlist_locked(exec_ctx, glb_policy->serverlist); - } else { - // If rr_handover_locked() is invoked when we haven't received any - // serverlist from the balancer, we use the fallback backends returned by - // the resolver. Note that the fallback backend list may be empty, in which - // case the new round_robin policy will keep the requested picks pending. - GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); - addresses = grpc_lb_addresses_copy(glb_policy->fallback_backend_addresses); - } + grpc_lb_addresses *addresses = + process_serverlist_locked(exec_ctx, glb_policy->serverlist); GPR_ASSERT(addresses != NULL); grpc_lb_policy_args *args = (grpc_lb_policy_args *)gpr_zalloc(sizeof(*args)); args->client_channel_factory = glb_policy->cc_factory; @@ -831,6 +775,8 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, /* glb_policy->rr_policy may be NULL (initial handover) */ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { + GPR_ASSERT(glb_policy->serverlist != NULL && + glb_policy->serverlist->num_servers > 0); if (glb_policy->shutting_down) return; grpc_lb_policy_args *args = lb_policy_args_create(exec_ctx, glb_policy); GPR_ASSERT(args != NULL); @@ -971,7 +917,13 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. */ + /* Count the number of gRPC-LB addresses. There must be at least one. + * TODO(roth): For now, we ignore non-balancer addresses, but in the + * future, we may change the behavior such that we fall back to using + * the non-balancer addresses if we cannot reach any balancers. In the + * fallback case, we should use the LB policy indicated by + * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is + * unset, we should default to pick_first). */ const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); if (arg == NULL || arg->type != GRPC_ARG_POINTER) { @@ -1007,11 +959,6 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->lb_call_timeout_ms = grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); - arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); - glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( - arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, - INT_MAX}); - // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. grpc_arg new_arg = @@ -1020,11 +967,6 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); - /* Extract the backend addresses (may be empty) from the resolver for - * fallback. */ - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - /* Create a client channel over them to communicate with a LB service */ glb_policy->response_generator = grpc_fake_resolver_response_generator_create(); @@ -1068,9 +1010,6 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { if (glb_policy->serverlist != NULL) { grpc_grpclb_destroy_serverlist(glb_policy->serverlist); } - if (glb_policy->fallback_backend_addresses != NULL) { - grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); - } grpc_fake_resolver_response_generator_unref(glb_policy->response_generator); if (glb_policy->pending_update_args != NULL) { grpc_channel_args_destroy(exec_ctx, glb_policy->pending_update_args->args); @@ -1211,28 +1150,10 @@ static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } -static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error); static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy); static void start_picking_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - /* start a timer to fall back */ - if (glb_policy->lb_fallback_timeout_ms > 0 && - glb_policy->serverlist == NULL) { - gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_timespec deadline = gpr_time_add( - now, - gpr_time_from_millis(glb_policy->lb_fallback_timeout_ms, GPR_TIMESPAN)); - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer"); - GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked, - glb_policy, - grpc_combiner_scheduler(glb_policy->base.combiner)); - glb_policy->fallback_timer_active = true; - grpc_timer_init(exec_ctx, &glb_policy->lb_fallback_timer, deadline, - &glb_policy->lb_on_fallback, now); - } - glb_policy->started_picking = true; gpr_backoff_reset(&glb_policy->lb_call_backoff_state); query_for_backends_locked(exec_ctx, glb_policy); @@ -1686,15 +1607,6 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, if (glb_policy->serverlist != NULL) { /* dispose of the old serverlist */ grpc_grpclb_destroy_serverlist(glb_policy->serverlist); - } else { - /* or dispose of the fallback */ - grpc_lb_addresses_destroy(exec_ctx, - glb_policy->fallback_backend_addresses); - glb_policy->fallback_backend_addresses = NULL; - if (glb_policy->fallback_timer_active) { - grpc_timer_cancel(exec_ctx, &glb_policy->lb_fallback_timer); - glb_policy->fallback_timer_active = false; - } } /* and update the copy in the glb_lb_policy instance. This * serverlist instance will be destroyed either upon the next @@ -1705,7 +1617,9 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, } } else { if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Received empty server list, ignoring."); + gpr_log(GPR_INFO, + "Received empty server list. Picks will stay pending until " + "a response with > 0 servers is received"); } grpc_grpclb_destroy_serverlist(serverlist); } @@ -1752,26 +1666,6 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer"); } -static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - glb_lb_policy *glb_policy = arg; - /* If we receive a serverlist after the timer fires but before this callback - * actually runs, don't do anything. */ - if (glb_policy->serverlist != NULL) return; - glb_policy->fallback_timer_active = false; - if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) { - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, - "Falling back to use backends from resolver (grpclb %p)", - (void *)glb_policy); - } - GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); - rr_handover_locked(exec_ctx, glb_policy); - } - GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, - "grpclb_fallback_timer"); -} - static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { glb_lb_policy *glb_policy = (glb_lb_policy *)arg; @@ -1892,17 +1786,6 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, &glb_policy->lb_channel_connectivity, &glb_policy->lb_channel_on_connectivity_changed, NULL); } - - // Propagate update to fallback_backend_addresses if a non-empty serverlist - // hasn't been received from the balancer. - if (glb_policy->serverlist == NULL) { - grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - if (glb_policy->rr_policy != NULL) { - rr_handover_locked(exec_ctx, glb_policy); - } - } } // Invoked as part of the update process. It continues watching the LB channel diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index 918bab745c6..cdcaf17544b 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -56,7 +56,7 @@ grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses) { } void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index, - const void* address, size_t address_len, + void* address, size_t address_len, bool is_balancer, const char* balancer_name, void* user_data) { GPR_ASSERT(index < addresses->num_addresses); diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.h b/src/core/ext/filters/client_channel/lb_policy_factory.h index cf0f8cb6157..9d9fb143df5 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.h +++ b/src/core/ext/filters/client_channel/lb_policy_factory.h @@ -73,7 +73,7 @@ grpc_lb_addresses *grpc_lb_addresses_copy(const grpc_lb_addresses *addresses); * \a address is a socket address of length \a address_len. * Takes ownership of \a balancer_name. */ void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index, - const void *address, size_t address_len, + void *address, size_t address_len, bool is_balancer, const char *balancer_name, void *user_data); diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index f89f5f1f03d..f130aecd4b5 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -86,10 +86,6 @@ void ChannelArguments::SetCompressionAlgorithm( SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, algorithm); } -void ChannelArguments::SetGrpclbFallbackTimeout(int fallback_timeout) { - SetInt(GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS, fallback_timeout); -} - void ChannelArguments::SetSocketMutator(grpc_socket_mutator* mutator) { if (!mutator) { return; diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 17a094f7a21..570a3d10677 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -368,9 +368,8 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_fake_resolver_response_generator_unref(response_generator_); } - void ResetStub(int fallback_timeout = 0) { + void ResetStub() { ChannelArguments args; - args.SetGrpclbFallbackTimeout(fallback_timeout); args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR, response_generator_); std::ostringstream uri; @@ -442,10 +441,10 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_exec_ctx_finish(&exec_ctx); } - const std::vector GetBackendPorts(const size_t start_index = 0) const { + const std::vector GetBackendPorts() const { std::vector backend_ports; - for (size_t i = start_index; i < backend_servers_.size(); ++i) { - backend_ports.push_back(backend_servers_[i].port_); + for (const auto& bs : backend_servers_) { + backend_ports.push_back(bs.port_); } return backend_ports; } @@ -616,71 +615,6 @@ TEST_F(SingleBalancerTest, InitiallyEmptyServerlist) { EXPECT_EQ("grpclb", channel_->GetLoadBalancingPolicyName()); } -TEST_F(SingleBalancerTest, Fallback) { - const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); - const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor(); - const size_t kNumBackendInResolution = backends_.size() / 2; - - ResetStub(kFallbackTimeoutMs); - std::vector addresses; - addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""}); - for (size_t i = 0; i < kNumBackendInResolution; ++i) { - addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""}); - } - SetNextResolution(addresses); - - // Send non-empty serverlist only after kServerlistDelayMs - ScheduleResponseForBalancer( - 0, BalancerServiceImpl::BuildResponseForBackends( - GetBackendPorts(kNumBackendInResolution /* start_index */), {}), - kServerlistDelayMs); - - // The first request. The client will block while it's still trying to - // contact the balancer. - gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); - CheckRpcSendOk(kNumBackendInResolution); - gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); - - // Fallback is used: each backend returned by the resolver should have - // gotten one request. - for (size_t i = 0; i < kNumBackendInResolution; ++i) { - EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); - } - for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { - EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); - } - - // Wait until update has been processed, as signaled by the backend returned - // by the balancer receiving a request. - do { - CheckRpcSendOk(1); - } while ( - backend_servers_[kNumBackendInResolution].service_->request_count() == 0); - for (size_t i = 0; i < backends_.size(); ++i) { - backend_servers_[i].service_->ResetCounters(); - } - - // Send out the second request. - gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); - CheckRpcSendOk(backends_.size() - kNumBackendInResolution); - gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); - - // Serverlist is used: each backend returned by the balancer should - // have gotten one request. - for (size_t i = 0; i < kNumBackendInResolution; ++i) { - EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); - } - for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { - EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); - } - - balancers_[0]->NotifyDoneWithServerlists(); - // The balancer got a single request. - EXPECT_EQ(1U, balancer_servers_[0].service_->request_count()); - // and sent a single response. - EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); -} - TEST_F(SingleBalancerTest, BackendsRestart) { const size_t kNumRpcsPerAddress = 100; ScheduleResponseForBalancer( From 759c7029b18f8bbcf6a6399d512abed8b6e12436 Mon Sep 17 00:00:00 2001 From: Vladimir Moskva Date: Thu, 14 Sep 2017 15:40:27 +0200 Subject: [PATCH 134/227] Rename set to depset --- bazel/generate_cc.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl index 7fffb582188..f88ee2f56f1 100644 --- a/bazel/generate_cc.bzl +++ b/bazel/generate_cc.bzl @@ -55,7 +55,7 @@ def generate_cc_impl(ctx): arguments = arguments, ) - return struct(files=set(out_files)) + return struct(files=depset(out_files)) _generate_cc = rule( attrs = { From 5432dd887430984713f9fc3ad595c85e995d6451 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 14 Sep 2017 17:55:42 +0200 Subject: [PATCH 135/227] Revert "Allow SerializationTraits to use grpc::ByteBuffer rather than only grpc_byte_buffer" --- BUILD | 2 - CMakeLists.txt | 15 -- Makefile | 15 -- build.yaml | 2 - gRPC-Core.podspec | 1 - grpc.def | 20 +-- grpc.gemspec | 1 - include/grpc++/impl/codegen/byte_buffer.h | 141 ------------------ include/grpc++/impl/codegen/call.h | 114 +++----------- .../grpc++/impl/codegen/method_handler_impl.h | 9 +- .../grpc++/impl/codegen/rpc_service_method.h | 14 +- .../impl/codegen/serialization_traits.h | 25 +--- include/grpc++/impl/codegen/slice.h | 78 ---------- include/grpc++/support/byte_buffer.h | 68 ++++++++- include/grpc++/support/slice.h | 80 +++++++++- include/grpc/byte_buffer.h | 64 +++++++- include/grpc/impl/codegen/byte_buffer.h | 86 ----------- package.xml | 1 - src/core/lib/iomgr/resource_quota.c | 1 - .../health/default_health_check_service.cc | 1 - src/cpp/util/byte_buffer_cc.cc | 37 ++--- src/cpp/util/slice_cc.cc | 2 - src/ruby/ext/grpc/rb_grpc_imports.generated.c | 40 ++--- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 62 ++++---- .../core/surface/public_headers_must_be_c89.c | 1 - test/cpp/util/byte_buffer_test.cc | 5 +- tools/doxygen/Doxyfile.c++ | 2 - tools/doxygen/Doxyfile.c++.internal | 2 - tools/doxygen/Doxyfile.core | 1 - tools/doxygen/Doxyfile.core.internal | 1 - .../generated/sources_and_headers.json | 4 - 31 files changed, 330 insertions(+), 565 deletions(-) delete mode 100644 include/grpc++/impl/codegen/byte_buffer.h delete mode 100644 include/grpc/impl/codegen/byte_buffer.h diff --git a/BUILD b/BUILD index d76b821be92..281375fc1db 100644 --- a/BUILD +++ b/BUILD @@ -989,7 +989,6 @@ grpc_cc_library( name = "grpc_codegen", language = "c", public_hdrs = [ - "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -1488,7 +1487,6 @@ grpc_cc_library( public_hdrs = [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index eac63a9fe3c..fa1d7d87dd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1238,7 +1238,6 @@ target_link_libraries(grpc ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1545,7 +1544,6 @@ target_link_libraries(grpc_cronet ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1822,7 +1820,6 @@ target_link_libraries(grpc_test_util ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2083,7 +2080,6 @@ target_link_libraries(grpc_test_util_unsecure ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2379,7 +2375,6 @@ target_link_libraries(grpc_unsecure ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2689,7 +2684,6 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2700,7 +2694,6 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3183,7 +3176,6 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3194,7 +3186,6 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3555,7 +3546,6 @@ target_link_libraries(grpc++_test_util foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3583,7 +3573,6 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3695,7 +3684,6 @@ target_link_libraries(grpc++_test_util_unsecure foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3723,7 +3711,6 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3926,7 +3913,6 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3937,7 +3923,6 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h diff --git a/Makefile b/Makefile index 1ed1877f77c..0c3648566a9 100644 --- a/Makefile +++ b/Makefile @@ -3190,7 +3190,6 @@ LIBGRPC_SRC = \ src/core/plugin_registry/grpc_plugin_registry.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3497,7 +3496,6 @@ LIBGRPC_CRONET_SRC = \ src/core/plugin_registry/grpc_cronet_plugin_registry.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3775,7 +3773,6 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4027,7 +4024,6 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4300,7 +4296,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/plugin_registry/grpc_unsecure_plugin_registry.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4589,7 +4584,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4600,7 +4594,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5084,7 +5077,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5095,7 +5087,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5449,7 +5440,6 @@ LIBGRPC++_TEST_UTIL_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5477,7 +5467,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5566,7 +5555,6 @@ LIBGRPC++_TEST_UTIL_UNSECURE_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5594,7 +5582,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5802,7 +5789,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5813,7 +5799,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ diff --git a/build.yaml b/build.yaml index 92ca2b6237b..d15cf0339ca 100644 --- a/build.yaml +++ b/build.yaml @@ -502,7 +502,6 @@ filegroups: - grpc_deadline_filter - name: grpc_codegen public_headers: - - include/grpc/impl/codegen/byte_buffer.h - include/grpc/impl/codegen/byte_buffer_reader.h - include/grpc/impl/codegen/compression_types.h - include/grpc/impl/codegen/connectivity_state.h @@ -970,7 +969,6 @@ filegroups: public_headers: - include/grpc++/impl/codegen/async_stream.h - include/grpc++/impl/codegen/async_unary_call.h - - include/grpc++/impl/codegen/byte_buffer.h - include/grpc++/impl/codegen/call.h - include/grpc++/impl/codegen/call_hook.h - include/grpc++/impl/codegen/channel_interface.h diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 6f84ef235db..2f1f4158662 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -141,7 +141,6 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_generic.h', 'include/grpc/impl/codegen/sync_posix.h', 'include/grpc/impl/codegen/sync_windows.h', - 'include/grpc/impl/codegen/byte_buffer.h', 'include/grpc/impl/codegen/byte_buffer_reader.h', 'include/grpc/impl/codegen/compression_types.h', 'include/grpc/impl/codegen/connectivity_state.h', diff --git a/grpc.def b/grpc.def index 558be60c3c1..a7a11601ed9 100644 --- a/grpc.def +++ b/grpc.def @@ -1,4 +1,14 @@ EXPORTS + grpc_raw_byte_buffer_create + grpc_raw_compressed_byte_buffer_create + grpc_byte_buffer_copy + grpc_byte_buffer_length + grpc_byte_buffer_destroy + grpc_byte_buffer_reader_init + grpc_byte_buffer_reader_destroy + grpc_byte_buffer_reader_next + grpc_byte_buffer_reader_readall + grpc_raw_byte_buffer_from_reader census_initialize census_shutdown census_supported @@ -135,16 +145,6 @@ EXPORTS grpc_server_add_secure_http2_port grpc_call_set_credentials grpc_server_credentials_set_auth_metadata_processor - grpc_raw_byte_buffer_create - grpc_raw_compressed_byte_buffer_create - grpc_byte_buffer_copy - grpc_byte_buffer_length - grpc_byte_buffer_destroy - grpc_byte_buffer_reader_init - grpc_byte_buffer_reader_destroy - grpc_byte_buffer_reader_next - grpc_byte_buffer_reader_readall - grpc_raw_byte_buffer_from_reader grpc_slice_ref grpc_slice_unref grpc_slice_copy diff --git a/grpc.gemspec b/grpc.gemspec index a226e9bc31a..2d0f9fd4504 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -145,7 +145,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/support/tmpfile_posix.c ) s.files += %w( src/core/lib/support/tmpfile_windows.c ) s.files += %w( src/core/lib/support/wrap_memcpy.c ) - s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h ) s.files += %w( include/grpc/impl/codegen/compression_types.h ) s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h deleted file mode 100644 index 87d390c688a..00000000000 --- a/include/grpc++/impl/codegen/byte_buffer.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H -#define GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H - -#include - -#include -#include -#include -#include -#include - -#include - -namespace grpc { - -template -class CallOpRecvMessage; -class MethodHandler; -namespace internal { -template -class MessageDeserializer; -} - -/// A sequence of bytes. -class ByteBuffer final { - public: - /// Constuct an empty buffer. - ByteBuffer() : buffer_(nullptr) {} - - /// Construct buffer from \a slices, of which there are \a nslices. - ByteBuffer(const Slice* slices, size_t nslices); - - /// Constuct a byte buffer by referencing elements of existing buffer - /// \a buf. Wrapper of core function grpc_byte_buffer_copy - ByteBuffer(const ByteBuffer& buf); - - ~ByteBuffer() { - if (buffer_) { - g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); - } - } - - ByteBuffer& operator=(const ByteBuffer&); - - /// Dump (read) the buffer contents into \a slices. - Status Dump(std::vector* slices) const; - - /// Remove all data. - void Clear() { - if (buffer_) { - g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); - buffer_ = nullptr; - } - } - - /// Make a duplicate copy of the internals of this byte - /// buffer so that we have our own owned version of it. - /// bbuf.Duplicate(); is equivalent to bbuf=bbuf; but is actually readable - void Duplicate() { - buffer_ = g_core_codegen_interface->grpc_byte_buffer_copy(buffer_); - } - - /// Forget underlying byte buffer without destroying - /// Use this only for un-owned byte buffers - void Release() { buffer_ = nullptr; } - - /// Buffer size in bytes. - size_t Length() const; - - /// Swap the state of *this and *other. - void Swap(ByteBuffer* other); - - /// Is this ByteBuffer valid? - bool Valid() const { return (buffer_ != nullptr); } - - private: - friend class SerializationTraits; - friend class CallOpSendMessage; - template - friend class CallOpRecvMessage; - friend class CallOpGenericRecvMessage; - friend class MethodHandler; - template - friend class internal::MessageDeserializer; - - // takes ownership - void set_buffer(grpc_byte_buffer* buf) { - if (buffer_) { - Clear(); - } - buffer_ = buf; - } - - grpc_byte_buffer* c_buffer() { return buffer_; } - grpc_byte_buffer** c_buffer_ptr() { return &buffer_; } - - // DEPRECATED: Implicit conversion to transparently - // support deprecated SerializationTraits API - // No need to inline since deprecated - operator grpc_byte_buffer*(); - operator const grpc_byte_buffer*() const; - - grpc_byte_buffer* buffer_; -}; - -template <> -class SerializationTraits { - public: - static Status Deserialize(const ByteBuffer& byte_buffer, ByteBuffer* dest) { - dest->set_buffer(byte_buffer.buffer_); - return Status::OK; - } - static Status Serialize(const ByteBuffer& source, ByteBuffer* buffer, - bool* own_buffer) { - *buffer = source; - *own_buffer = true; - return Status::OK; - } -}; - -} // namespace grpc - -#endif // GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 3c418c7ae2b..74ed5cbfb9e 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -25,7 +25,6 @@ #include #include -#include #include #include #include @@ -40,6 +39,8 @@ #include #include +struct grpc_byte_buffer; + namespace grpc { class ByteBuffer; @@ -280,7 +281,7 @@ class CallOpSendInitialMetadata { class CallOpSendMessage { public: - CallOpSendMessage() : send_buf_() {} + CallOpSendMessage() : send_buf_(nullptr) {} /// Send \a message using \a options for the write. The \a options are cleared /// after use. @@ -293,67 +294,33 @@ class CallOpSendMessage { protected: void AddOp(grpc_op* ops, size_t* nops) { - if (!send_buf_.Valid()) return; + if (send_buf_ == nullptr) return; grpc_op* op = &ops[(*nops)++]; op->op = GRPC_OP_SEND_MESSAGE; op->flags = write_options_.flags(); op->reserved = NULL; - op->data.send_message.send_message = send_buf_.c_buffer(); + op->data.send_message.send_message = send_buf_; // Flags are per-message: clear them after use. write_options_.Clear(); } - void FinishOp(bool* status) { send_buf_.Clear(); } + void FinishOp(bool* status) { + g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_); + send_buf_ = nullptr; + } private: - template - class MessageSerializer; - - ByteBuffer send_buf_; + grpc_byte_buffer* send_buf_; WriteOptions write_options_; }; -namespace internal { -template -T Example(); -} // namespace internal - -template -class CallOpSendMessage::MessageSerializer< - M, typename std::enable_if::Serialize( - internal::Example(), - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, - bool* own_buf) { - return SerializationTraits::Serialize(message, bbuf->c_buffer_ptr(), - own_buf); - } -}; - -template -class CallOpSendMessage::MessageSerializer< - M, typename std::enable_if::Serialize( - internal::Example(), - internal::Example<::grpc::ByteBuffer*>(), - internal::Example()))>::value>::type> { - public: - static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, - bool* own_buf) { - return SerializationTraits::Serialize(message, bbuf, own_buf); - } -}; - template Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) { write_options_ = options; bool own_buf; Status result = - MessageSerializer::SendMessageInternal(message, &send_buf_, &own_buf); + SerializationTraits::Serialize(message, &send_buf_, &own_buf); if (!own_buf) { - send_buf_.Duplicate(); + send_buf_ = g_core_codegen_interface->grpc_byte_buffer_copy(send_buf_); } return result; } @@ -363,36 +330,6 @@ Status CallOpSendMessage::SendMessage(const M& message) { return SendMessage(message, WriteOptions()); } -namespace internal { -template -class MessageDeserializer; - -template -class MessageDeserializer< - M, typename std::enable_if::Deserialize( - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status Deserialize(const ByteBuffer& bbuf, M* message) { - return SerializationTraits::Deserialize(bbuf, message); - } -}; - -template -class MessageDeserializer< - M, typename std::enable_if::Deserialize( - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status Deserialize(const ByteBuffer& bbuf, M* message) { - return SerializationTraits::Deserialize( - const_cast(bbuf).c_buffer(), message); - } -}; -} // namespace internal - template class CallOpRecvMessage { public: @@ -415,20 +352,18 @@ class CallOpRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); + op->data.recv_message.recv_message = &recv_buf_; } void FinishOp(bool* status) { if (message_ == nullptr) return; - if (recv_buf_.Valid()) { + if (recv_buf_) { if (*status) { got_message = *status = - internal::MessageDeserializer::Deserialize(recv_buf_, message_) - .ok(); - recv_buf_.Release(); + SerializationTraits::Deserialize(recv_buf_, message_).ok(); } else { got_message = false; - recv_buf_.Clear(); + g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); } } else { got_message = false; @@ -441,14 +376,14 @@ class CallOpRecvMessage { private: R* message_; - ByteBuffer recv_buf_; + grpc_byte_buffer* recv_buf_; bool allow_not_getting_message_; }; namespace CallOpGenericRecvMessageHelper { class DeserializeFunc { public: - virtual Status Deserialize(const ByteBuffer& buf) = 0; + virtual Status Deserialize(grpc_byte_buffer* buf) = 0; virtual ~DeserializeFunc() {} }; @@ -456,8 +391,8 @@ template class DeserializeFuncType final : public DeserializeFunc { public: DeserializeFuncType(R* message) : message_(message) {} - Status Deserialize(const ByteBuffer& buf) override { - return grpc::internal::MessageDeserializer::Deserialize(buf, message_); + Status Deserialize(grpc_byte_buffer* buf) override { + return SerializationTraits::Deserialize(buf, message_); } ~DeserializeFuncType() override {} @@ -493,19 +428,18 @@ class CallOpGenericRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); + op->data.recv_message.recv_message = &recv_buf_; } void FinishOp(bool* status) { if (!deserialize_) return; - if (recv_buf_.Valid()) { + if (recv_buf_) { if (*status) { got_message = true; *status = deserialize_->Deserialize(recv_buf_).ok(); - recv_buf_.Release(); } else { got_message = false; - recv_buf_.Clear(); + g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); } } else { got_message = false; @@ -518,7 +452,7 @@ class CallOpGenericRecvMessage { private: std::unique_ptr deserialize_; - ByteBuffer recv_buf_; + grpc_byte_buffer* recv_buf_; bool allow_not_getting_message_; }; diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h index 8125e0a6513..15e24bdcdcb 100644 --- a/include/grpc++/impl/codegen/method_handler_impl.h +++ b/include/grpc++/impl/codegen/method_handler_impl.h @@ -19,7 +19,6 @@ #ifndef GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H #define GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H -#include #include #include #include @@ -38,8 +37,8 @@ class RpcMethodHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = internal::MessageDeserializer::Deserialize( - param.request, &req); + Status status = + SerializationTraits::Deserialize(param.request, &req); ResponseType rsp; if (status.ok()) { status = func_(service_, param.server_context, &req, &rsp); @@ -124,8 +123,8 @@ class ServerStreamingHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = internal::MessageDeserializer::Deserialize( - param.request, &req); + Status status = + SerializationTraits::Deserialize(param.request, &req); if (status.ok()) { ServerWriter writer(param.call, param.server_context); diff --git a/include/grpc++/impl/codegen/rpc_service_method.h b/include/grpc++/impl/codegen/rpc_service_method.h index d356012ad60..7165774172c 100644 --- a/include/grpc++/impl/codegen/rpc_service_method.h +++ b/include/grpc++/impl/codegen/rpc_service_method.h @@ -25,11 +25,14 @@ #include #include -#include #include #include #include +extern "C" { +struct grpc_byte_buffer; +} + namespace grpc { class ServerContext; class StreamContextInterface; @@ -40,14 +43,11 @@ class MethodHandler { virtual ~MethodHandler() {} struct HandlerParameter { HandlerParameter(Call* c, ServerContext* context, grpc_byte_buffer* req) - : call(c), server_context(context) { - request.set_buffer(req); - } - ~HandlerParameter() { request.Release(); } + : call(c), server_context(context), request(req) {} Call* call; ServerContext* server_context; - // Handler required to destroy these contents - ByteBuffer request; + // Handler required to grpc_byte_buffer_destroy this + grpc_byte_buffer* request; }; virtual void RunHandler(const HandlerParameter& param) = 0; }; diff --git a/include/grpc++/impl/codegen/serialization_traits.h b/include/grpc++/impl/codegen/serialization_traits.h index 7fc4e24172f..b72d4741267 100644 --- a/include/grpc++/impl/codegen/serialization_traits.h +++ b/include/grpc++/impl/codegen/serialization_traits.h @@ -26,24 +26,15 @@ namespace grpc { /// Used for hooking different message serialization API's into GRPC. /// Each SerializationTraits implementation must provide the following /// functions: -/// 1. static Status Serialize(const Message& msg, -/// ByteBuffer* buffer, -/// bool* own_buffer); -/// AND/OR -/// static Status Serialize(const Message& msg, -/// grpc_byte_buffer** buffer, -/// bool* own_buffer); -/// The former is preferred; the latter is deprecated +/// static Status Serialize(const Message& msg, +/// grpc_byte_buffer** buffer, +/// bool* own_buffer); +/// static Status Deserialize(grpc_byte_buffer* buffer, +/// Message* msg, +/// int max_receive_message_size); /// -/// 2. static Status Deserialize(const ByteBuffer& buffer, -/// Message* msg); -/// AND/OR -/// static Status Deserialize(grpc_byte_buffer* buffer, -/// Message* msg); -/// The former is preferred; the latter is deprecated -/// -/// Serialize is required to convert message to a ByteBuffer, and -/// return that byte buffer through *buffer. *own_buffer should +/// Serialize is required to convert message to a grpc_byte_buffer, and +/// to store a pointer to that byte buffer at *buffer. *own_buffer should /// be set to true if the caller owns said byte buffer, or false if /// ownership is retained elsewhere. /// diff --git a/include/grpc++/impl/codegen/slice.h b/include/grpc++/impl/codegen/slice.h index c185bf4fd01..e682bdef6af 100644 --- a/include/grpc++/impl/codegen/slice.h +++ b/include/grpc++/impl/codegen/slice.h @@ -19,89 +19,11 @@ #ifndef GRPCXX_IMPL_CODEGEN_SLICE_H #define GRPCXX_IMPL_CODEGEN_SLICE_H -#include #include #include -#include - namespace grpc { -/// A wrapper around \a grpc_slice. -/// -/// A slice represents a contiguous reference counted array of bytes. -/// It is cheap to take references to a slice, and it is cheap to create a -/// slice pointing to a subset of another slice. -class Slice final { - public: - /// Construct an empty slice. - Slice(); - /// Destructor - drops one reference. - ~Slice(); - - enum AddRef { ADD_REF }; - /// Construct a slice from \a slice, adding a reference. - Slice(grpc_slice slice, AddRef); - - enum StealRef { STEAL_REF }; - /// Construct a slice from \a slice, stealing a reference. - Slice(grpc_slice slice, StealRef); - - /// Allocate a slice of specified size - Slice(size_t len); - - /// Construct a slice from a copied buffer - Slice(const void* buf, size_t len); - - /// Construct a slice from a copied string - Slice(const grpc::string& str); - - enum StaticSlice { STATIC_SLICE }; - - /// Construct a slice from a static buffer - Slice(const void* buf, size_t len, StaticSlice); - - /// Copy constructor, adds a reference. - Slice(const Slice& other); - - /// Assignment, reference count is unchanged. - Slice& operator=(Slice other) { - std::swap(slice_, other.slice_); - return *this; - } - - /// Create a slice pointing at some data. Calls malloc to allocate a refcount - /// for the object, and arranges that destroy will be called with the - /// user data pointer passed in at destruction. Can be the same as buf or - /// different (e.g., if data is part of a larger structure that must be - /// destroyed when the data is no longer needed) - Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); - - /// Specialization of above for common case where buf == user_data - Slice(void* buf, size_t len, void (*destroy)(void*)) - : Slice(buf, len, destroy, buf) {} - - /// Similar to the above but has a destroy that also takes slice length - Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); - - /// Byte size. - size_t size() const { return GRPC_SLICE_LENGTH(slice_); } - - /// Raw pointer to the beginning (first element) of the slice. - const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } - - /// Raw pointer to the end (one byte \em past the last element) of the slice. - const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } - - /// Raw C slice. Caller needs to call grpc_slice_unref when done. - grpc_slice c_slice() const; - - private: - friend class ByteBuffer; - - grpc_slice slice_; -}; - inline grpc::string_ref StringRefFromSlice(const grpc_slice* slice) { return grpc::string_ref( reinterpret_cast(GRPC_SLICE_START_PTR(*slice)), diff --git a/include/grpc++/support/byte_buffer.h b/include/grpc++/support/byte_buffer.h index 81fa3b0a184..df16c6a75e4 100644 --- a/include/grpc++/support/byte_buffer.h +++ b/include/grpc++/support/byte_buffer.h @@ -19,7 +19,6 @@ #ifndef GRPCXX_SUPPORT_BYTE_BUFFER_H #define GRPCXX_SUPPORT_BYTE_BUFFER_H -#include #include #include #include @@ -28,4 +27,71 @@ #include #include +#include + +namespace grpc { + +/// A sequence of bytes. +class ByteBuffer final { + public: + /// Constuct an empty buffer. + ByteBuffer() : buffer_(nullptr) {} + + /// Construct buffer from \a slices, of which there are \a nslices. + ByteBuffer(const Slice* slices, size_t nslices); + + /// Constuct a byte buffer by referencing elements of existing buffer + /// \a buf. Wrapper of core function grpc_byte_buffer_copy + ByteBuffer(const ByteBuffer& buf); + + ~ByteBuffer(); + + ByteBuffer& operator=(const ByteBuffer&); + + /// Dump (read) the buffer contents into \a slices. + Status Dump(std::vector* slices) const; + + /// Remove all data. + void Clear(); + + /// Buffer size in bytes. + size_t Length() const; + + /// Swap the state of *this and *other. + void Swap(ByteBuffer* other); + + private: + friend class SerializationTraits; + + // takes ownership + void set_buffer(grpc_byte_buffer* buf) { + if (buffer_) { + Clear(); + } + buffer_ = buf; + } + + // For \a SerializationTraits's usage. + grpc_byte_buffer* buffer() const { return buffer_; } + + grpc_byte_buffer* buffer_; +}; + +template <> +class SerializationTraits { + public: + static Status Deserialize(grpc_byte_buffer* byte_buffer, ByteBuffer* dest) { + dest->set_buffer(byte_buffer); + return Status::OK; + } + static Status Serialize(const ByteBuffer& source, grpc_byte_buffer** buffer, + bool* own_buffer) { + *buffer = grpc_byte_buffer_copy(source.buffer()); + *own_buffer = true; + return Status::OK; + } +}; + +} // namespace grpc + #endif // GRPCXX_SUPPORT_BYTE_BUFFER_H diff --git a/include/grpc++/support/slice.h b/include/grpc++/support/slice.h index 10db10d79c3..bbf97f280ef 100644 --- a/include/grpc++/support/slice.h +++ b/include/grpc++/support/slice.h @@ -19,8 +19,86 @@ #ifndef GRPCXX_SUPPORT_SLICE_H #define GRPCXX_SUPPORT_SLICE_H -#include #include #include +namespace grpc { + +/// A wrapper around \a grpc_slice. +/// +/// A slice represents a contiguous reference counted array of bytes. +/// It is cheap to take references to a slice, and it is cheap to create a +/// slice pointing to a subset of another slice. +class Slice final { + public: + /// Construct an empty slice. + Slice(); + /// Destructor - drops one reference. + ~Slice(); + + enum AddRef { ADD_REF }; + /// Construct a slice from \a slice, adding a reference. + Slice(grpc_slice slice, AddRef); + + enum StealRef { STEAL_REF }; + /// Construct a slice from \a slice, stealing a reference. + Slice(grpc_slice slice, StealRef); + + /// Allocate a slice of specified size + Slice(size_t len); + + /// Construct a slice from a copied buffer + Slice(const void* buf, size_t len); + + /// Construct a slice from a copied string + Slice(const grpc::string& str); + + enum StaticSlice { STATIC_SLICE }; + + /// Construct a slice from a static buffer + Slice(const void* buf, size_t len, StaticSlice); + + /// Copy constructor, adds a reference. + Slice(const Slice& other); + + /// Assignment, reference count is unchanged. + Slice& operator=(Slice other) { + std::swap(slice_, other.slice_); + return *this; + } + + /// Create a slice pointing at some data. Calls malloc to allocate a refcount + /// for the object, and arranges that destroy will be called with the + /// user data pointer passed in at destruction. Can be the same as buf or + /// different (e.g., if data is part of a larger structure that must be + /// destroyed when the data is no longer needed) + Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); + + /// Specialization of above for common case where buf == user_data + Slice(void* buf, size_t len, void (*destroy)(void*)) + : Slice(buf, len, destroy, buf) {} + + /// Similar to the above but has a destroy that also takes slice length + Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); + + /// Byte size. + size_t size() const { return GRPC_SLICE_LENGTH(slice_); } + + /// Raw pointer to the beginning (first element) of the slice. + const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } + + /// Raw pointer to the end (one byte \em past the last element) of the slice. + const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } + + /// Raw C slice. Caller needs to call grpc_slice_unref when done. + grpc_slice c_slice() const { return grpc_slice_ref(slice_); } + + private: + friend class ByteBuffer; + + grpc_slice slice_; +}; + +} // namespace grpc + #endif // GRPCXX_SUPPORT_SLICE_H diff --git a/include/grpc/byte_buffer.h b/include/grpc/byte_buffer.h index 7669582af27..55e191da315 100644 --- a/include/grpc/byte_buffer.h +++ b/include/grpc/byte_buffer.h @@ -19,7 +19,69 @@ #ifndef GRPC_BYTE_BUFFER_H #define GRPC_BYTE_BUFFER_H -#include +#include #include +#ifdef __cplusplus +extern "C" { +#endif + +/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, + size_t nslices); + +/** Returns a *compressed* RAW byte buffer instance over the given slices (up to + * \a nslices). The \a compression argument defines the compression algorithm + * used to generate the data in \a slices. + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( + grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); + +/** Copies input byte buffer \a bb. + * + * Increases the reference count of all the source slices. The user is + * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ +GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); + +/** Returns the size of the given byte buffer, in bytes. */ +GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); + +/** Destroys \a byte_buffer deallocating all its memory. */ +GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); + +/** Reader for byte buffers. Iterates over slices in the byte buffer */ +struct grpc_byte_buffer_reader; +typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; + +/** Initialize \a reader to read over \a buffer. + * Returns 1 upon success, 0 otherwise. */ +GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, + grpc_byte_buffer *buffer); + +/** Cleanup and destroy \a reader */ +GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); + +/** Updates \a slice with the next piece of data from from \a reader and returns + * 1. Returns 0 at the end of the stream. Caller is responsible for calling + * grpc_slice_unref on the result. */ +GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, + grpc_slice *slice); + +/** Merge all data from \a reader into single slice */ +GRPCAPI grpc_slice +grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); + +/** Returns a RAW byte buffer instance from the output of \a reader. */ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( + grpc_byte_buffer_reader *reader); + +#ifdef __cplusplus +} +#endif + #endif /* GRPC_BYTE_BUFFER_H */ diff --git a/include/grpc/impl/codegen/byte_buffer.h b/include/grpc/impl/codegen/byte_buffer.h deleted file mode 100644 index fc333057134..00000000000 --- a/include/grpc/impl/codegen/byte_buffer.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_IMPL_CODEGEN_BYTE_BUFFER_H -#define GRPC_IMPL_CODEGEN_BYTE_BUFFER_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, - size_t nslices); - -/** Returns a *compressed* RAW byte buffer instance over the given slices (up to - * \a nslices). The \a compression argument defines the compression algorithm - * used to generate the data in \a slices. - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( - grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); - -/** Copies input byte buffer \a bb. - * - * Increases the reference count of all the source slices. The user is - * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ -GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); - -/** Returns the size of the given byte buffer, in bytes. */ -GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); - -/** Destroys \a byte_buffer deallocating all its memory. */ -GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); - -/** Reader for byte buffers. Iterates over slices in the byte buffer */ -struct grpc_byte_buffer_reader; -typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; - -/** Initialize \a reader to read over \a buffer. - * Returns 1 upon success, 0 otherwise. */ -GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, - grpc_byte_buffer *buffer); - -/** Cleanup and destroy \a reader */ -GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); - -/** Updates \a slice with the next piece of data from from \a reader and returns - * 1. Returns 0 at the end of the stream. Caller is responsible for calling - * grpc_slice_unref on the result. */ -GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, - grpc_slice *slice); - -/** Merge all data from \a reader into single slice */ -GRPCAPI grpc_slice -grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); - -/** Returns a RAW byte buffer instance from the output of \a reader. */ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( - grpc_byte_buffer_reader *reader); - -#ifdef __cplusplus -} -#endif - -#endif /* GRPC_IMPL_CODEGEN_BYTE_BUFFER_H */ diff --git a/package.xml b/package.xml index 4f67e46673b..b7c0e679e5d 100644 --- a/package.xml +++ b/package.xml @@ -155,7 +155,6 @@ - diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 4d69986fbc2..4895e0d1c92 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -22,7 +22,6 @@ #include #include -#include #include #include #include diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index d2cba6d6627..815b6070320 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -20,7 +20,6 @@ #include #include -#include #include #include diff --git a/src/cpp/util/byte_buffer_cc.cc b/src/cpp/util/byte_buffer_cc.cc index 1ebe22b6a49..b1ff25252a1 100644 --- a/src/cpp/util/byte_buffer_cc.cc +++ b/src/cpp/util/byte_buffer_cc.cc @@ -16,15 +16,11 @@ * */ -#include #include -#include #include namespace grpc { -static internal::GrpcLibraryInitializer g_gli_initializer; - ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { // The following assertions check that the representation of a grpc::Slice is // identical to that of a grpc_slice: it has a grpc_slice field, and nothing @@ -33,7 +29,6 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { "Slice must have same representation as grpc_slice"); static_assert(sizeof(Slice) == sizeof(grpc_slice), "Slice must have same representation as grpc_slice"); - g_gli_initializer.summon(); // Make sure that initializer linked in // The const_cast is legal if grpc_raw_byte_buffer_create() does no more // than its advertised side effect of increasing the reference count of the // slices it processes, and such an increase does not affect the semantics @@ -42,6 +37,19 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { reinterpret_cast(const_cast(slices)), nslices); } +ByteBuffer::~ByteBuffer() { + if (buffer_) { + grpc_byte_buffer_destroy(buffer_); + } +} + +void ByteBuffer::Clear() { + if (buffer_) { + grpc_byte_buffer_destroy(buffer_); + buffer_ = nullptr; + } +} + Status ByteBuffer::Dump(std::vector* slices) const { slices->clear(); if (!buffer_) { @@ -72,9 +80,7 @@ ByteBuffer::ByteBuffer(const ByteBuffer& buf) : buffer_(grpc_byte_buffer_copy(buf.buffer_)) {} ByteBuffer& ByteBuffer::operator=(const ByteBuffer& buf) { - if (this != &buf) { - Clear(); // first remove existing data - } + Clear(); // first remove existing data if (buf.buffer_) { buffer_ = grpc_byte_buffer_copy(buf.buffer_); // then copy } @@ -87,19 +93,4 @@ void ByteBuffer::Swap(ByteBuffer* other) { buffer_ = tmp; } -ByteBuffer::operator grpc_byte_buffer*() { - // The following assertions check that the representation of a ByteBuffer is - // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, - // and nothing else. - static_assert(std::is_same::value, - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - return buffer_; -} - -ByteBuffer::operator const grpc_byte_buffer*() const { return buffer_; } - } // namespace grpc diff --git a/src/cpp/util/slice_cc.cc b/src/cpp/util/slice_cc.cc index 3ae17e80520..486d0cdf0ec 100644 --- a/src/cpp/util/slice_cc.cc +++ b/src/cpp/util/slice_cc.cc @@ -50,6 +50,4 @@ Slice::Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data) Slice::Slice(void* buf, size_t len, void (*destroy)(void*, size_t)) : slice_(grpc_slice_new_with_len(buf, len, destroy)) {} -grpc_slice Slice::c_slice() const { return grpc_slice_ref(slice_); } - } // namespace grpc diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 70831494fad..57b543967eb 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -22,6 +22,16 @@ #include "rb_grpc_imports.generated.h" +grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; census_initialize_type census_initialize_import; census_shutdown_type census_shutdown_import; census_supported_type census_supported_import; @@ -158,16 +168,6 @@ grpc_ssl_server_credentials_create_ex_type grpc_ssl_server_credentials_create_ex grpc_server_add_secure_http2_port_type grpc_server_add_secure_http2_port_import; grpc_call_set_credentials_type grpc_call_set_credentials_import; grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; -grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; grpc_slice_ref_type grpc_slice_ref_import; grpc_slice_unref_type grpc_slice_unref_import; grpc_slice_copy_type grpc_slice_copy_import; @@ -330,6 +330,16 @@ gpr_sleep_until_type gpr_sleep_until_import; gpr_timespec_to_micros_type gpr_timespec_to_micros_import; void grpc_rb_load_imports(HMODULE library) { + grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); + grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); + grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); + grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); + grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); + grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); + grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); + grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); + grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); + grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize"); census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown"); census_supported_import = (census_supported_type) GetProcAddress(library, "census_supported"); @@ -466,16 +476,6 @@ void grpc_rb_load_imports(HMODULE library) { grpc_server_add_secure_http2_port_import = (grpc_server_add_secure_http2_port_type) GetProcAddress(library, "grpc_server_add_secure_http2_port"); grpc_call_set_credentials_import = (grpc_call_set_credentials_type) GetProcAddress(library, "grpc_call_set_credentials"); grpc_server_credentials_set_auth_metadata_processor_import = (grpc_server_credentials_set_auth_metadata_processor_type) GetProcAddress(library, "grpc_server_credentials_set_auth_metadata_processor"); - grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); - grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); - grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); - grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); - grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); - grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); - grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); - grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); - grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); - grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); grpc_slice_ref_import = (grpc_slice_ref_type) GetProcAddress(library, "grpc_slice_ref"); grpc_slice_unref_import = (grpc_slice_unref_type) GetProcAddress(library, "grpc_slice_unref"); grpc_slice_copy_import = (grpc_slice_copy_type) GetProcAddress(library, "grpc_slice_copy"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 263057f309f..c5c848ae447 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -25,12 +25,12 @@ #include +#include #include #include #include #include #include -#include #include #include #include @@ -47,6 +47,36 @@ #include #include +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); +extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); +extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import +typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +#define grpc_byte_buffer_length grpc_byte_buffer_length_import +typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); +extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import +typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); +extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import +typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import +typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); +extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import +typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); +extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; +#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef int(*census_initialize_type)(int features); extern census_initialize_type census_initialize_import; #define census_initialize census_initialize_import @@ -455,36 +485,6 @@ extern grpc_call_set_credentials_type grpc_call_set_credentials_import; typedef void(*grpc_server_credentials_set_auth_metadata_processor_type)(grpc_server_credentials *creds, grpc_auth_metadata_processor processor); extern grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; #define grpc_server_credentials_set_auth_metadata_processor grpc_server_credentials_set_auth_metadata_processor_import -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); -extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); -extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import -typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -#define grpc_byte_buffer_length grpc_byte_buffer_length_import -typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); -extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import -typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); -extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import -typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import -typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); -extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import -typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); -extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; -#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef grpc_slice(*grpc_slice_ref_type)(grpc_slice s); extern grpc_slice_ref_type grpc_slice_ref_import; #define grpc_slice_ref grpc_slice_ref_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index d36d116afb8..0d7f68c0add 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include diff --git a/test/cpp/util/byte_buffer_test.cc b/test/cpp/util/byte_buffer_test.cc index 8fb51bc6635..cac01a73078 100644 --- a/test/cpp/util/byte_buffer_test.cc +++ b/test/cpp/util/byte_buffer_test.cc @@ -93,7 +93,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { std::vector slices; slices.push_back(Slice(hello, Slice::STEAL_REF)); slices.push_back(Slice(world, Slice::STEAL_REF)); - ByteBuffer send_buffer; + grpc_byte_buffer* send_buffer = nullptr; bool owned = false; ByteBuffer buffer(&slices[0], 2); slices.clear(); @@ -101,7 +101,8 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { buffer, &send_buffer, &owned); EXPECT_TRUE(status.ok()); EXPECT_TRUE(owned); - EXPECT_TRUE(send_buffer.Valid()); + EXPECT_TRUE(send_buffer != nullptr); + grpc_byte_buffer_destroy(send_buffer); } } // namespace diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index d81b7b4d115..62f113907db 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -811,7 +811,6 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ -include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -882,7 +881,6 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ -include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 263c6c1afbc..49919415bd5 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -811,7 +811,6 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ -include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -883,7 +882,6 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ -include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index b8514fe3114..632735342b3 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -811,7 +811,6 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ -include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 77cd28e9dfd..e352cb78f10 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -811,7 +811,6 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ -include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 5581240a9a1..2f6e34bfb38 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8395,7 +8395,6 @@ "gpr_codegen" ], "headers": [ - "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -8409,7 +8408,6 @@ "language": "c", "name": "grpc_codegen", "src": [ - "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -9288,7 +9286,6 @@ "headers": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", @@ -9323,7 +9320,6 @@ "src": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", From f68978c41f13197171325d944f971db7d9d85a54 Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Thu, 14 Sep 2017 09:18:24 -0700 Subject: [PATCH 136/227] Pass exec_ctx in TSI zero_copy_protector create --- src/core/lib/security/transport/security_handshaker.c | 2 +- src/core/tsi/fake_transport_security.c | 3 ++- src/core/tsi/transport_security.h | 10 ++++++++-- src/core/tsi/transport_security_grpc.c | 8 +++++--- src/core/tsi/transport_security_grpc.h | 3 ++- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c index 975d5995232..3d196056177 100644 --- a/src/core/lib/security/transport/security_handshaker.c +++ b/src/core/lib/security/transport/security_handshaker.c @@ -137,7 +137,7 @@ static void on_peer_checked_inner(grpc_exec_ctx *exec_ctx, // Create zero-copy frame protector, if implemented. tsi_zero_copy_grpc_protector *zero_copy_protector = NULL; tsi_result result = tsi_handshaker_result_create_zero_copy_grpc_protector( - h->handshaker_result, NULL, &zero_copy_protector); + exec_ctx, h->handshaker_result, NULL, &zero_copy_protector); if (result != TSI_OK && result != TSI_UNIMPLEMENTED) { error = grpc_set_tsi_error_result( GRPC_ERROR_CREATE_FROM_STATIC_STRING( diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index e7b3be3d86f..64043fea088 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -493,7 +493,8 @@ static tsi_result fake_handshaker_result_extract_peer( } static tsi_result fake_handshaker_result_create_zero_copy_grpc_protector( - const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + void *exec_ctx, const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector) { *protector = tsi_create_fake_zero_copy_grpc_protector(max_output_protected_frame_size); diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index b0d7039850d..3bba38149c8 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -84,11 +84,17 @@ struct tsi_handshaker { }; /* Base for tsi_handshaker_result implementations. - See transport_security_interface.h for documentation. */ + See transport_security_interface.h for documentation. + The exec_ctx parameter in create_zero_copy_grpc_protector is supposed to be + of type grpc_exec_ctx*, but we're using void* instead to avoid making the TSI + API depend on grpc. The create_zero_copy_grpc_protector() method is only used + in grpc, where we do need the exec_ctx passed through, but the API still + needs to compile in other applications, where grpc_exec_ctx is not defined. +*/ typedef struct { tsi_result (*extract_peer)(const tsi_handshaker_result *self, tsi_peer *peer); tsi_result (*create_zero_copy_grpc_protector)( - const tsi_handshaker_result *self, + void *exec_ctx, const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector); tsi_result (*create_frame_protector)(const tsi_handshaker_result *self, diff --git a/src/core/tsi/transport_security_grpc.c b/src/core/tsi/transport_security_grpc.c index 773b35e7179..affd995230e 100644 --- a/src/core/tsi/transport_security_grpc.c +++ b/src/core/tsi/transport_security_grpc.c @@ -20,16 +20,18 @@ /* This method creates a tsi_zero_copy_grpc_protector object. */ tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector( - const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + grpc_exec_ctx *exec_ctx, const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector) { - if (self == NULL || self->vtable == NULL || protector == NULL) { + if (exec_ctx == NULL || self == NULL || self->vtable == NULL || + protector == NULL) { return TSI_INVALID_ARGUMENT; } if (self->vtable->create_zero_copy_grpc_protector == NULL) { return TSI_UNIMPLEMENTED; } return self->vtable->create_zero_copy_grpc_protector( - self, max_output_protected_frame_size, protector); + exec_ctx, self, max_output_protected_frame_size, protector); } /* --- tsi_zero_copy_grpc_protector common implementation. --- diff --git a/src/core/tsi/transport_security_grpc.h b/src/core/tsi/transport_security_grpc.h index 375a758888b..ca6755c12fa 100644 --- a/src/core/tsi/transport_security_grpc.h +++ b/src/core/tsi/transport_security_grpc.h @@ -30,7 +30,8 @@ extern "C" { assuming there is no fatal error. The caller is responsible for destroying the protector. */ tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector( - const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + grpc_exec_ctx *exec_ctx, const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector); /* -- tsi_zero_copy_grpc_protector object -- */ From 3218e2b975a71cc32aedad71d00267e07244c109 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 14 Sep 2017 10:29:42 -0700 Subject: [PATCH 137/227] minor build fix --- src/core/lib/compression/stream_compression_gzip.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/compression/stream_compression_gzip.c b/src/core/lib/compression/stream_compression_gzip.c index 587bf8379eb..abcbdb3a91f 100644 --- a/src/core/lib/compression/stream_compression_gzip.c +++ b/src/core/lib/compression/stream_compression_gzip.c @@ -183,7 +183,7 @@ grpc_stream_compression_context_create_gzip( GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_GZIP_COMPRESS || method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_stream_compression_context_gzip *gzip_ctx = - (grpc_stream_compression_context *)gpr_zalloc( + (grpc_stream_compression_context_gzip *)gpr_zalloc( sizeof(grpc_stream_compression_context_gzip)); int r; if (gzip_ctx == NULL) { From 3ff1fa6e05726fb40cc60271a8e6a189bf901eb8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 14 Sep 2017 11:07:48 -0700 Subject: [PATCH 138/227] Collect stats on why writes are initiated --- .../chttp2/transport/chttp2_transport.c | 239 +++++++++++++---- .../transport/chttp2/transport/frame_ping.c | 3 +- .../chttp2/transport/frame_window_update.c | 10 +- .../transport/chttp2/transport/hpack_parser.c | 3 +- .../ext/transport/chttp2/transport/internal.h | 37 ++- .../ext/transport/chttp2/transport/writing.c | 5 +- src/core/lib/debug/stats_data.c | 45 ++++ src/core/lib/debug/stats_data.h | 110 ++++++++ src/core/lib/debug/stats_data.yaml | 42 +++ src/core/lib/debug/stats_data_bq_schema.sql | 25 ++ .../performance/massage_qps_stats.py | 25 ++ .../performance/scenario_result_schema.json | 250 ++++++++++++++++++ 12 files changed, 733 insertions(+), 61 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 930fa805a09..7e392b6d630 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -144,10 +144,11 @@ static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, static void cancel_pings(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_error *error); -static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_chttp2_ping_type ping_type, - grpc_closure *on_initiate, - grpc_closure *on_complete); +static void send_ping_locked( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_ping_type ping_type, grpc_closure *on_initiate, + grpc_closure *on_complete, + grpc_chttp2_initiate_write_reason initiate_write_reason); static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error); @@ -346,7 +347,6 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (is_client) { grpc_slice_buffer_add(&t->outbuf, grpc_slice_from_copied_string( GRPC_CHTTP2_CLIENT_CONNECT_STRING)); - grpc_chttp2_initiate_write(exec_ctx, t, "initial_write"); } /* configure http2 the way we like it */ @@ -578,7 +578,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_DISABLED; } - grpc_chttp2_initiate_write(exec_ctx, t, "init"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE); post_benign_reclaimer(exec_ctx, t); } @@ -846,13 +847,91 @@ static void set_write_state(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } } +static void inc_initiate_write_reason( + grpc_exec_ctx *exec_ctx, grpc_chttp2_initiate_write_reason reason) { + switch (reason) { + case GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM(exec_ctx); + break; + } +} + void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, const char *reason) { + grpc_chttp2_transport *t, + grpc_chttp2_initiate_write_reason reason) { GPR_TIMER_BEGIN("grpc_chttp2_initiate_write", 0); switch (t->write_state) { case GRPC_CHTTP2_WRITE_STATE_IDLE: - set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, reason); + inc_initiate_write_reason(exec_ctx, reason); + set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, + grpc_chttp2_initiate_write_reason_string(reason)); t->is_first_write_in_batch = true; GRPC_CHTTP2_REF_TRANSPORT(t, "writing"); GRPC_CLOSURE_SCHED( @@ -864,7 +943,7 @@ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, break; case GRPC_CHTTP2_WRITE_STATE_WRITING: set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE, - reason); + grpc_chttp2_initiate_write_reason_string(reason)); break; case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE: break; @@ -872,16 +951,12 @@ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("grpc_chttp2_initiate_write", 0); } -void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, - grpc_chttp2_stream *s, - bool also_initiate_write, const char *reason) { +void grpc_chttp2_mark_stream_writable(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s)) { GRPC_CHTTP2_STREAM_REF(s, "chttp2_writing:become"); } - if (also_initiate_write) { - grpc_chttp2_initiate_write(exec_ctx, t, reason); - } } static grpc_closure_scheduler *write_scheduler(grpc_chttp2_transport *t, @@ -1105,7 +1180,9 @@ static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream_map_add(&t->stream_map, s->id, s); post_destructive_reclaimer(exec_ctx, t); - grpc_chttp2_become_writable(exec_ctx, t, s, true, "new_stream"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM); } /* cancel out streams that will never be started */ while (t->next_stream_id >= MAX_CLIENT_STREAM_ID && @@ -1202,7 +1279,9 @@ static void maybe_become_writable_due_to_send_msg(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s) { if (s->id != 0 && (!s->write_buffering || s->flow_controlled_buffer.length > t->write_buffer_size)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, "op.send_message"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE); } } @@ -1404,14 +1483,13 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } } else { GPR_ASSERT(s->id != 0); - bool initiate_write = true; - if (op->send_message && - (op->payload->send_message.send_message->flags & - GRPC_WRITE_BUFFER_HINT)) { - initiate_write = false; + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + if (!(op->send_message && + (op->payload->send_message.send_message->flags & + GRPC_WRITE_BUFFER_HINT))) { + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA); } - grpc_chttp2_become_writable(exec_ctx, t, s, initiate_write, - "op.send_initial_metadata"); } } else { s->send_initial_metadata = NULL; @@ -1519,8 +1597,9 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } else if (s->id != 0) { /* TODO(ctiller): check if there's flow control for any outstanding bytes before going writable */ - grpc_chttp2_become_writable(exec_ctx, t, s, true, - "op.send_trailing_metadata"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA); } } } @@ -1632,15 +1711,17 @@ static void cancel_pings(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ERROR_UNREF(error); } -static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_chttp2_ping_type ping_type, - grpc_closure *on_initiate, grpc_closure *on_ack) { +static void send_ping_locked( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_ping_type ping_type, grpc_closure *on_initiate, + grpc_closure *on_ack, + grpc_chttp2_initiate_write_reason initiate_write_reason) { grpc_chttp2_ping_queue *pq = &t->ping_queues[ping_type]; grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_INITIATE], on_initiate, GRPC_ERROR_NONE); if (grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_NEXT], on_ack, GRPC_ERROR_NONE)) { - grpc_chttp2_initiate_write(exec_ctx, t, "send_ping"); + grpc_chttp2_initiate_write(exec_ctx, t, initiate_write_reason); } } @@ -1648,7 +1729,8 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)tp; t->ping_state.is_delayed_ping_timer_set = false; - grpc_chttp2_initiate_write(exec_ctx, t, "retry_send_ping"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING); } void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, @@ -1663,7 +1745,8 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } GRPC_CLOSURE_LIST_SCHED(exec_ctx, &pq->lists[GRPC_CHTTP2_PCL_INFLIGHT]); if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_NEXT])) { - grpc_chttp2_initiate_write(exec_ctx, t, "continue_pings"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS); } } @@ -1676,7 +1759,8 @@ static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &slice, &http_error); grpc_chttp2_goaway_append(t->last_new_stream_id, (uint32_t)http_error, grpc_slice_ref_internal(slice), &t->qbuf); - grpc_chttp2_initiate_write(exec_ctx, t, "goaway_sent"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT); GRPC_ERROR_UNREF(error); } @@ -1723,7 +1807,8 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, if (op->send_ping) { send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_ON_NEXT_WRITE, NULL, - op->send_ping); + op->send_ping, + GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING); } if (op->on_connectivity_state_change != NULL) { @@ -1968,7 +2053,8 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, grpc_slice_buffer_add( &t->qbuf, grpc_chttp2_rst_stream_create(s->id, (uint32_t)http_error, &s->stats.outgoing)); - grpc_chttp2_initiate_write(exec_ctx, t, "rst_stream"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM); } } if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { @@ -2289,7 +2375,8 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &s->stats.outgoing)); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); - grpc_chttp2_initiate_write(exec_ctx, t, "close_from_api"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API); } typedef struct { @@ -2324,19 +2411,20 @@ void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx, case GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED: break; case GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY: - grpc_chttp2_become_writable(exec_ctx, t, s, true, - "immediate stream flowctl"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL); break; case GRPC_CHTTP2_FLOWCTL_QUEUE_UPDATE: - grpc_chttp2_become_writable(exec_ctx, t, s, false, - "queue stream flowctl"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); break; } switch (action.send_transport_update) { case GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED: break; case GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY: - grpc_chttp2_initiate_write(exec_ctx, t, "immediate transport flowctl"); + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL); break; // this is the same as no action b/c every time the transport enters the // writing path it will maybe do an update @@ -2354,7 +2442,8 @@ void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx, (uint32_t)action.max_frame_size); } if (action.send_setting_update == GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY) { - grpc_chttp2_initiate_write(exec_ctx, t, "immediate setting update"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS); } } if (action.need_ping) { @@ -2362,7 +2451,8 @@ void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx, grpc_bdp_estimator_schedule_ping(&t->flow_control.bdp_estimator); send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE, - &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked); + &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked, + GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING); } } @@ -2441,7 +2531,10 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, if (t->flow_control.initial_window_update > 0) { grpc_chttp2_stream *s; while (grpc_chttp2_list_pop_stalled_by_stream(t, &s)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, "unstalled"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING); } } t->flow_control.initial_window_update = 0; @@ -2556,7 +2649,8 @@ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_CHTTP2_REF_TRANSPORT(t, "keepalive ping end"); send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_ON_NEXT_WRITE, &t->start_keepalive_ping_locked, - &t->finish_keepalive_ping_locked); + &t->finish_keepalive_ping_locked, + GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING); } else { GRPC_CHTTP2_REF_TRANSPORT(t, "init keepalive ping"); grpc_timer_init( @@ -2912,8 +3006,7 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, uint32_t frame_size, uint32_t flags) { grpc_chttp2_incoming_byte_stream *incoming_byte_stream = - (grpc_chttp2_incoming_byte_stream *)gpr_malloc( - sizeof(*incoming_byte_stream)); + gpr_malloc(sizeof(*incoming_byte_stream)); incoming_byte_stream->base.length = frame_size; incoming_byte_stream->remaining_bytes = frame_size; incoming_byte_stream->base.flags = flags; @@ -3017,6 +3110,56 @@ static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, /******************************************************************************* * MONITORING */ + +const char *grpc_chttp2_initiate_write_reason_string( + grpc_chttp2_initiate_write_reason reason) { + switch (reason) { + case GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE: + return "INITIAL_WRITE"; + case GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM: + return "START_NEW_STREAM"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE: + return "SEND_MESSAGE"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA: + return "SEND_INITIAL_METADATA"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA: + return "SEND_TRAILING_METADATA"; + case GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING: + return "RETRY_SEND_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS: + return "CONTINUE_PINGS"; + case GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT: + return "GOAWAY_SENT"; + case GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM: + return "RST_STREAM"; + case GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API: + return "CLOSE_FROM_API"; + case GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL: + return "STREAM_FLOW_CONTROL"; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL: + return "TRANSPORT_FLOW_CONTROL"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS: + return "SEND_SETTINGS"; + case GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING: + return "BDP_ESTIMATOR_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING: + return "FLOW_CONTROL_UNSTALLED_BY_SETTING"; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE: + return "FLOW_CONTROL_UNSTALLED_BY_UPDATE"; + case GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING: + return "APPLICATION_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING: + return "KEEPALIVE_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED: + return "TRANSPORT_FLOW_CONTROL_UNSTALLED"; + case GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE: + return "PING_RESPONSE"; + case GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM: + return "FORCE_RST_STREAM"; + } + GPR_UNREACHABLE_CODE(return "unknown"); +} + static grpc_endpoint *chttp2_get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) { return ((grpc_chttp2_transport *)t)->ep; diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 582fd7bfaac..81bd02ae702 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -117,7 +117,8 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, t->ping_acks, t->ping_ack_capacity * sizeof(*t->ping_acks)); } t->ping_acks[t->ping_ack_count++] = p->opaque_8bytes; - grpc_chttp2_initiate_write(exec_ctx, t, "ping response"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE); } } } diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.c b/src/core/ext/transport/chttp2/transport/frame_window_update.c index c94f7725bf6..c9ab8d1b50f 100644 --- a/src/core/ext/transport/chttp2/transport/frame_window_update.c +++ b/src/core/ext/transport/chttp2/transport/frame_window_update.c @@ -99,8 +99,10 @@ grpc_error *grpc_chttp2_window_update_parser_parse( grpc_chttp2_flowctl_recv_stream_update( &t->flow_control, &s->flow_control, received_update); if (grpc_chttp2_list_remove_stalled_by_stream(t, s)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, - "stream.read_flow_control"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE); } } } else { @@ -109,7 +111,9 @@ grpc_error *grpc_chttp2_window_update_parser_parse( received_update); bool is_zero = t->flow_control.remote_window <= 0; if (was_zero && !is_zero) { - grpc_chttp2_initiate_write(exec_ctx, t, "new_global_flow_control"); + grpc_chttp2_initiate_write( + exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED); } } } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 82ff2c8e2cf..901e7413c0a 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1649,7 +1649,8 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, grpc_slice_buffer_add( &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); - grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 0fbedd1e567..c2dfce7c9c7 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -79,6 +79,33 @@ typedef enum { GRPC_CHTTP2_PCL_COUNT /* must be last */ } grpc_chttp2_ping_closure_list; +typedef enum { + GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE, + GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM, + GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE, + GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA, + GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA, + GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING, + GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS, + GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT, + GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM, + GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API, + GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL, + GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL, + GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS, + GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE, + GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING, + GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING, + GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED, + GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE, + GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM, +} grpc_chttp2_initiate_write_reason; + +const char *grpc_chttp2_initiate_write_reason_string( + grpc_chttp2_initiate_write_reason reason); + typedef struct { grpc_closure_list lists[GRPC_CHTTP2_PCL_COUNT]; uint64_t inflight_id; @@ -599,7 +626,8 @@ struct grpc_chttp2_stream { The actual call chain is documented in the implementation of this function. */ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, const char *reason); + grpc_chttp2_transport *t, + grpc_chttp2_initiate_write_reason reason); typedef struct { /** are we writing? */ @@ -851,10 +879,9 @@ void grpc_chttp2_add_ping_strike(grpc_exec_ctx *exec_ctx, /** add a ref to the stream and add it to the writable list; ref will be dropped in writing.c */ -void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, - grpc_chttp2_stream *s, - bool also_initiate_write, const char *reason); +void grpc_chttp2_mark_stream_writable(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s); void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 3fa38db4db6..3ded801985d 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -201,9 +201,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (t->flow_control.remote_window > 0) { while (grpc_chttp2_list_pop_stalled_by_transport(t, &s)) { - if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s) && - stream_ref_if_not_destroyed(&s->refcount->refs)) { - grpc_chttp2_initiate_write(exec_ctx, t, "transport.read_flow_control"); + if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s)) { + stream_ref_if_not_destroyed(&s->refcount->refs); } } } diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 2c567653432..b3e1ee9b4ec 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -50,6 +50,27 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "http2_writes_offloaded", "http2_writes_continued", "http2_partial_writes", + "http2_initiate_write_due_to_initial_write", + "http2_initiate_write_due_to_start_new_stream", + "http2_initiate_write_due_to_send_message", + "http2_initiate_write_due_to_send_initial_metadata", + "http2_initiate_write_due_to_send_trailing_metadata", + "http2_initiate_write_due_to_retry_send_ping", + "http2_initiate_write_due_to_continue_pings", + "http2_initiate_write_due_to_goaway_sent", + "http2_initiate_write_due_to_rst_stream", + "http2_initiate_write_due_to_close_from_api", + "http2_initiate_write_due_to_stream_flow_control", + "http2_initiate_write_due_to_transport_flow_control", + "http2_initiate_write_due_to_send_settings", + "http2_initiate_write_due_to_bdp_estimator_ping", + "http2_initiate_write_due_to_flow_control_unstalled_by_setting", + "http2_initiate_write_due_to_flow_control_unstalled_by_update", + "http2_initiate_write_due_to_application_ping", + "http2_initiate_write_due_to_keepalive_ping", + "http2_initiate_write_due_to_transport_flow_control_unstalled", + "http2_initiate_write_due_to_ping_response", + "http2_initiate_write_due_to_force_rst_stream", "combiner_locks_initiated", "combiner_locks_scheduled_items", "combiner_locks_scheduled_final_items", @@ -92,6 +113,30 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "written", "Number of HTTP2 writes that were made knowing there was still more data " "to be written (we cap maximum write size to syscall_write)", + "Number of HTTP2 writes initiated due to 'initial_write'", + "Number of HTTP2 writes initiated due to 'start_new_stream'", + "Number of HTTP2 writes initiated due to 'send_message'", + "Number of HTTP2 writes initiated due to 'send_initial_metadata'", + "Number of HTTP2 writes initiated due to 'send_trailing_metadata'", + "Number of HTTP2 writes initiated due to 'retry_send_ping'", + "Number of HTTP2 writes initiated due to 'continue_pings'", + "Number of HTTP2 writes initiated due to 'goaway_sent'", + "Number of HTTP2 writes initiated due to 'rst_stream'", + "Number of HTTP2 writes initiated due to 'close_from_api'", + "Number of HTTP2 writes initiated due to 'stream_flow_control'", + "Number of HTTP2 writes initiated due to 'transport_flow_control'", + "Number of HTTP2 writes initiated due to 'send_settings'", + "Number of HTTP2 writes initiated due to 'bdp_estimator_ping'", + "Number of HTTP2 writes initiated due to " + "'flow_control_unstalled_by_setting'", + "Number of HTTP2 writes initiated due to " + "'flow_control_unstalled_by_update'", + "Number of HTTP2 writes initiated due to 'application_ping'", + "Number of HTTP2 writes initiated due to 'keepalive_ping'", + "Number of HTTP2 writes initiated due to " + "'transport_flow_control_unstalled'", + "Number of HTTP2 writes initiated due to 'ping_response'", + "Number of HTTP2 writes initiated due to 'force_rst_stream'", "Number of combiner lock entries by process (first items queued to a " "combiner)", "Number of items scheduled against combiner locks", diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index e93db32c6a0..c9871c4a56c 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -52,6 +52,27 @@ typedef enum { GRPC_STATS_COUNTER_HTTP2_WRITES_OFFLOADED, GRPC_STATS_COUNTER_HTTP2_WRITES_CONTINUED, GRPC_STATS_COUNTER_HTTP2_PARTIAL_WRITES, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM, GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_ITEMS, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS, @@ -169,6 +190,95 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_WRITES_CONTINUED) #define GRPC_STATS_INC_HTTP2_PARTIAL_WRITES(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_PARTIAL_WRITES) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM) #define GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), \ GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED) diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 37dff9c5f48..84727fe6c47 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -117,6 +117,48 @@ - counter: http2_partial_writes doc: Number of HTTP2 writes that were made knowing there was still more data to be written (we cap maximum write size to syscall_write) +- counter: http2_initiate_write_due_to_initial_write + doc: Number of HTTP2 writes initiated due to 'initial_write' +- counter: http2_initiate_write_due_to_start_new_stream + doc: Number of HTTP2 writes initiated due to 'start_new_stream' +- counter: http2_initiate_write_due_to_send_message + doc: Number of HTTP2 writes initiated due to 'send_message' +- counter: http2_initiate_write_due_to_send_initial_metadata + doc: Number of HTTP2 writes initiated due to 'send_initial_metadata' +- counter: http2_initiate_write_due_to_send_trailing_metadata + doc: Number of HTTP2 writes initiated due to 'send_trailing_metadata' +- counter: http2_initiate_write_due_to_retry_send_ping + doc: Number of HTTP2 writes initiated due to 'retry_send_ping' +- counter: http2_initiate_write_due_to_continue_pings + doc: Number of HTTP2 writes initiated due to 'continue_pings' +- counter: http2_initiate_write_due_to_goaway_sent + doc: Number of HTTP2 writes initiated due to 'goaway_sent' +- counter: http2_initiate_write_due_to_rst_stream + doc: Number of HTTP2 writes initiated due to 'rst_stream' +- counter: http2_initiate_write_due_to_close_from_api + doc: Number of HTTP2 writes initiated due to 'close_from_api' +- counter: http2_initiate_write_due_to_stream_flow_control + doc: Number of HTTP2 writes initiated due to 'stream_flow_control' +- counter: http2_initiate_write_due_to_transport_flow_control + doc: Number of HTTP2 writes initiated due to 'transport_flow_control' +- counter: http2_initiate_write_due_to_send_settings + doc: Number of HTTP2 writes initiated due to 'send_settings' +- counter: http2_initiate_write_due_to_bdp_estimator_ping + doc: Number of HTTP2 writes initiated due to 'bdp_estimator_ping' +- counter: http2_initiate_write_due_to_flow_control_unstalled_by_setting + doc: Number of HTTP2 writes initiated due to 'flow_control_unstalled_by_setting' +- counter: http2_initiate_write_due_to_flow_control_unstalled_by_update + doc: Number of HTTP2 writes initiated due to 'flow_control_unstalled_by_update' +- counter: http2_initiate_write_due_to_application_ping + doc: Number of HTTP2 writes initiated due to 'application_ping' +- counter: http2_initiate_write_due_to_keepalive_ping + doc: Number of HTTP2 writes initiated due to 'keepalive_ping' +- counter: http2_initiate_write_due_to_transport_flow_control_unstalled + doc: Number of HTTP2 writes initiated due to 'transport_flow_control_unstalled' +- counter: http2_initiate_write_due_to_ping_response + doc: Number of HTTP2 writes initiated due to 'ping_response' +- counter: http2_initiate_write_due_to_force_rst_stream + doc: Number of HTTP2 writes initiated due to 'force_rst_stream' # combiner locks - counter: combiner_locks_initiated doc: Number of combiner lock entries by process diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index 53547692918..d21afbbfe45 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -1,5 +1,9 @@ client_calls_created_per_iteration:FLOAT, server_calls_created_per_iteration:FLOAT, +cqs_created_per_iteration:FLOAT, +client_channels_created_per_iteration:FLOAT, +client_subchannels_created_per_iteration:FLOAT, +server_channels_created_per_iteration:FLOAT, syscall_poll_per_iteration:FLOAT, syscall_wait_per_iteration:FLOAT, histogram_slow_lookups_per_iteration:FLOAT, @@ -21,6 +25,27 @@ http2_writes_begun_per_iteration:FLOAT, http2_writes_offloaded_per_iteration:FLOAT, http2_writes_continued_per_iteration:FLOAT, http2_partial_writes_per_iteration:FLOAT, +http2_initiate_write_due_to_initial_write_per_iteration:FLOAT, +http2_initiate_write_due_to_start_new_stream_per_iteration:FLOAT, +http2_initiate_write_due_to_send_message_per_iteration:FLOAT, +http2_initiate_write_due_to_send_initial_metadata_per_iteration:FLOAT, +http2_initiate_write_due_to_send_trailing_metadata_per_iteration:FLOAT, +http2_initiate_write_due_to_retry_send_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_continue_pings_per_iteration:FLOAT, +http2_initiate_write_due_to_goaway_sent_per_iteration:FLOAT, +http2_initiate_write_due_to_rst_stream_per_iteration:FLOAT, +http2_initiate_write_due_to_close_from_api_per_iteration:FLOAT, +http2_initiate_write_due_to_stream_flow_control_per_iteration:FLOAT, +http2_initiate_write_due_to_transport_flow_control_per_iteration:FLOAT, +http2_initiate_write_due_to_send_settings_per_iteration:FLOAT, +http2_initiate_write_due_to_bdp_estimator_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_flow_control_unstalled_by_setting_per_iteration:FLOAT, +http2_initiate_write_due_to_flow_control_unstalled_by_update_per_iteration:FLOAT, +http2_initiate_write_due_to_application_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_keepalive_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_transport_flow_control_unstalled_per_iteration:FLOAT, +http2_initiate_write_due_to_ping_response_per_iteration:FLOAT, +http2_initiate_write_due_to_force_rst_stream_per_iteration:FLOAT, combiner_locks_initiated_per_iteration:FLOAT, combiner_locks_scheduled_items_per_iteration:FLOAT, combiner_locks_scheduled_final_items_per_iteration:FLOAT, diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 30a94ca7cdf..7994b544ac2 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -22,6 +22,10 @@ def massage_qps_stats(scenario_result): del stats["coreStats"] stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created") stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") + stats["core_cqs_created"] = massage_qps_stats_helpers.counter(core_stats, "cqs_created") + stats["core_client_channels_created"] = massage_qps_stats_helpers.counter(core_stats, "client_channels_created") + stats["core_client_subchannels_created"] = massage_qps_stats_helpers.counter(core_stats, "client_subchannels_created") + stats["core_server_channels_created"] = massage_qps_stats_helpers.counter(core_stats, "server_channels_created") stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") @@ -43,6 +47,27 @@ def massage_qps_stats(scenario_result): stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded") stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued") stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes") + stats["core_http2_initiate_write_due_to_initial_write"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_initial_write") + stats["core_http2_initiate_write_due_to_start_new_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_start_new_stream") + stats["core_http2_initiate_write_due_to_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_message") + stats["core_http2_initiate_write_due_to_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_initial_metadata") + stats["core_http2_initiate_write_due_to_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_trailing_metadata") + stats["core_http2_initiate_write_due_to_retry_send_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_retry_send_ping") + stats["core_http2_initiate_write_due_to_continue_pings"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_continue_pings") + stats["core_http2_initiate_write_due_to_goaway_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_goaway_sent") + stats["core_http2_initiate_write_due_to_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_rst_stream") + stats["core_http2_initiate_write_due_to_close_from_api"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_close_from_api") + stats["core_http2_initiate_write_due_to_stream_flow_control"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_stream_flow_control") + stats["core_http2_initiate_write_due_to_transport_flow_control"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control") + stats["core_http2_initiate_write_due_to_send_settings"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_settings") + stats["core_http2_initiate_write_due_to_bdp_estimator_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_bdp_estimator_ping") + stats["core_http2_initiate_write_due_to_flow_control_unstalled_by_setting"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_flow_control_unstalled_by_setting") + stats["core_http2_initiate_write_due_to_flow_control_unstalled_by_update"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_flow_control_unstalled_by_update") + stats["core_http2_initiate_write_due_to_application_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_application_ping") + stats["core_http2_initiate_write_due_to_keepalive_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_keepalive_ping") + stats["core_http2_initiate_write_due_to_transport_flow_control_unstalled"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control_unstalled") + stats["core_http2_initiate_write_due_to_ping_response"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_ping_response") + stats["core_http2_initiate_write_due_to_force_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_force_rst_stream") stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index 84b3cb7c0a3..12a56dbec0e 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -120,6 +120,26 @@ "name": "core_server_calls_created", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_cqs_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_channels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_subchannels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_channels_created", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_syscall_poll", @@ -225,6 +245,111 @@ "name": "core_http2_partial_writes", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_initial_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_start_new_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_retry_send_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_continue_pings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_goaway_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_rst_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_close_from_api", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_stream_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_settings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_bdp_estimator_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_setting", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_update", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_application_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_keepalive_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control_unstalled", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_ping_response", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_force_rst_stream", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_combiner_locks_initiated", @@ -597,6 +722,26 @@ "name": "core_server_calls_created", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_cqs_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_channels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_subchannels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_channels_created", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_syscall_poll", @@ -702,6 +847,111 @@ "name": "core_http2_partial_writes", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_initial_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_start_new_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_retry_send_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_continue_pings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_goaway_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_rst_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_close_from_api", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_stream_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_settings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_bdp_estimator_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_setting", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_update", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_application_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_keepalive_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control_unstalled", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_ping_response", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_force_rst_stream", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_combiner_locks_initiated", From 1c87515f8da87dd96182dc3070029599b0a7165f Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Mon, 11 Sep 2017 16:11:46 -0700 Subject: [PATCH 139/227] add php extension --- src/ruby/qps/proxy-worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/qps/proxy-worker.rb b/src/ruby/qps/proxy-worker.rb index d7a9f114bdb..488610ae74d 100755 --- a/src/ruby/qps/proxy-worker.rb +++ b/src/ruby/qps/proxy-worker.rb @@ -41,7 +41,7 @@ class ProxyBenchmarkClientServiceImpl < Grpc::Testing::ProxyClientService::Servi @histogram = Histogram.new(@histres, @histmax) @start_time = Time.now # TODO(vjpai): Support multiple client channels by spawning off a PHP client per channel - command = "php " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget + command = "php -d extension=" + File.expand_path(File.dirname(__FILE__)) + "/../../php/ext/grpc/modules/grpc.so " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget puts "Starting command: " + command @php_pid = spawn(command) end From 3ad5c74311b6f603ed590c5330ac71d3387c7710 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 14 Sep 2017 11:29:30 -0700 Subject: [PATCH 140/227] Fix errant change --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 7e392b6d630..79a9ed827f7 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -3006,7 +3006,8 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, uint32_t frame_size, uint32_t flags) { grpc_chttp2_incoming_byte_stream *incoming_byte_stream = - gpr_malloc(sizeof(*incoming_byte_stream)); + (grpc_chttp2_incoming_byte_stream *)gpr_malloc( + sizeof(*incoming_byte_stream)); incoming_byte_stream->base.length = frame_size; incoming_byte_stream->remaining_bytes = frame_size; incoming_byte_stream->base.flags = flags; From 9fa10cce9ffe2689174af7b69940c8c2f2b51d0f Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 14 Sep 2017 11:49:52 -0700 Subject: [PATCH 141/227] Fix merge issue --- src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 707fc293adb..a776a07d99a 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -1857,6 +1857,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, gpr_free(glb_policy); return NULL; } + grpc_subchannel_index_ref(); GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed, glb_lb_channel_on_connectivity_changed_cb, glb_policy, grpc_combiner_scheduler(args->combiner)); From f66ed288538a7e28cdbdc694bef80bb86b696b90 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 14 Sep 2017 12:30:27 -0700 Subject: [PATCH 142/227] Instrument executor a little better --- src/core/lib/debug/stats_data.c | 48 +++++++++++++++++++++++++----- src/core/lib/debug/stats_data.h | 24 +++++++++++---- src/core/lib/debug/stats_data.yaml | 8 +++++ src/core/lib/iomgr/executor.c | 9 ++++++ 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 3fd8ee38ef8..b52852d0000 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -56,6 +56,8 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "executor_wakeup_initiated", "executor_queue_drained", "executor_push_retries", + "executor_threads_created", + "executor_threads_used", }; const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of client side calls created by this process", @@ -98,6 +100,8 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of times an executor queue was drained", "Number of times we raced and were forced to retry pushing a closure to " "the executor", + "Size of the backing thread pool for overflow gRPC Core work", + "How many executor threads actually got used", }; const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { "tcp_write_size", @@ -110,6 +114,7 @@ const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { "http2_send_message_per_write", "http2_send_trailing_metadata_per_write", "http2_send_flowctl_per_write", + "executor_closures_per_wakeup", }; const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = { "Number of bytes offered to each syscall_write", @@ -122,6 +127,7 @@ const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = { "Number of streams whose payload was written per TCP write", "Number of streams terminated per TCP write", "Number of flow control updates written per TCP write", + "Number of closures executed each time an executor wakes up", }; const int grpc_stats_table_0[65] = { 0, 1, 2, 3, 4, 6, 8, 11, @@ -418,16 +424,43 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, grpc_stats_histo_find_bucket_slow( (exec_ctx), value, grpc_stats_table_2, 64)); } -const int grpc_stats_histo_buckets[10] = {64, 64, 64, 64, 64, +void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, + int value) { + value = GPR_CLAMP(value, 0, 1024); + if (value < 13) { + GRPC_STATS_INC_HISTOGRAM( + (exec_ctx), GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, value); + return; + } + union { + double dbl; + uint64_t uint; + } _val, _bkt; + _val.dbl = value; + if (_val.uint < 4637863191261478912ull) { + int bucket = + grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13; + _bkt.dbl = grpc_stats_table_2[bucket]; + bucket -= (_val.uint < _bkt.uint); + GRPC_STATS_INC_HISTOGRAM( + (exec_ctx), GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, bucket); + return; + } + GRPC_STATS_INC_HISTOGRAM((exec_ctx), + GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, + grpc_stats_histo_find_bucket_slow( + (exec_ctx), value, grpc_stats_table_2, 64)); +} +const int grpc_stats_histo_buckets[11] = {64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64}; -const int grpc_stats_histo_start[10] = {0, 64, 128, 192, 256, - 320, 384, 448, 512, 576}; -const int *const grpc_stats_histo_bucket_boundaries[10] = { +const int grpc_stats_histo_start[11] = {0, 64, 128, 192, 256, 320, + 384, 448, 512, 576, 640}; +const int *const grpc_stats_histo_bucket_boundaries[11] = { grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_2, grpc_stats_table_2, - grpc_stats_table_2}; -void (*const grpc_stats_inc_histogram[10])(grpc_exec_ctx *exec_ctx, int x) = { + grpc_stats_table_2, grpc_stats_table_2}; +void (*const grpc_stats_inc_histogram[11])(grpc_exec_ctx *exec_ctx, int x) = { grpc_stats_inc_tcp_write_size, grpc_stats_inc_tcp_write_iov_size, grpc_stats_inc_tcp_read_size, @@ -437,4 +470,5 @@ void (*const grpc_stats_inc_histogram[10])(grpc_exec_ctx *exec_ctx, int x) = { grpc_stats_inc_http2_send_initial_metadata_per_write, grpc_stats_inc_http2_send_message_per_write, grpc_stats_inc_http2_send_trailing_metadata_per_write, - grpc_stats_inc_http2_send_flowctl_per_write}; + grpc_stats_inc_http2_send_flowctl_per_write, + grpc_stats_inc_executor_closures_per_wakeup}; diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index b7c15c08a5e..cf7d3b60e29 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -58,6 +58,8 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED, GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED, GRPC_STATS_COUNTER_EXECUTOR_PUSH_RETRIES, + GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED, + GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED, GRPC_STATS_COUNTER_COUNT } grpc_stats_counters; extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT]; @@ -73,6 +75,7 @@ typedef enum { GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_PER_WRITE, GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE, + GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, GRPC_STATS_HISTOGRAM_COUNT } grpc_stats_histograms; extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT]; @@ -98,7 +101,9 @@ typedef enum { GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE_BUCKETS = 64, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_FIRST_SLOT = 576, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_BUCKETS = 640 + GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_FIRST_SLOT = 640, + GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_BUCKETS = 64, + GRPC_STATS_HISTOGRAM_BUCKETS = 704 } grpc_stats_histogram_constants; #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) @@ -179,6 +184,11 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED) #define GRPC_STATS_INC_EXECUTOR_PUSH_RETRIES(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_PUSH_RETRIES) +#define GRPC_STATS_INC_EXECUTOR_THREADS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED) +#define GRPC_STATS_INC_EXECUTOR_THREADS_USED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED) #define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \ grpc_stats_inc_tcp_write_size((exec_ctx), (int)(value)) void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int x); @@ -214,10 +224,14 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write( grpc_stats_inc_http2_send_flowctl_per_write((exec_ctx), (int)(value)) void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, int x); -extern const int grpc_stats_histo_buckets[10]; -extern const int grpc_stats_histo_start[10]; -extern const int *const grpc_stats_histo_bucket_boundaries[10]; -extern void (*const grpc_stats_inc_histogram[10])(grpc_exec_ctx *exec_ctx, +#define GRPC_STATS_INC_EXECUTOR_CLOSURES_PER_WAKEUP(exec_ctx, value) \ + grpc_stats_inc_executor_closures_per_wakeup((exec_ctx), (int)(value)) +void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, + int x); +extern const int grpc_stats_histo_buckets[11]; +extern const int grpc_stats_histo_start[11]; +extern const int *const grpc_stats_histo_bucket_boundaries[11]; +extern void (*const grpc_stats_inc_histogram[11])(grpc_exec_ctx *exec_ctx, int x); #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index a9d71f4fcb3..08f49d11509 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -135,3 +135,11 @@ - counter: executor_push_retries doc: Number of times we raced and were forced to retry pushing a closure to the executor +- counter: executor_threads_created + doc: Size of the backing thread pool for overflow gRPC Core work +- counter: executor_threads_used + doc: How many executor threads actually got used +- histogram: executor_closures_per_wakeup + max: 1024 + buckets: 64 + doc: Number of closures executed each time an executor wakes up diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index 892385d7d78..41c7ff959eb 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -81,6 +81,8 @@ static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) { grpc_exec_ctx_flush(exec_ctx); } + GRPC_STATS_INC_EXECUTOR_CLOSURES_PER_WAKEUP(exec_ctx, n); + return n; } @@ -150,7 +152,10 @@ static void executor_thread(void *arg) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); + GRPC_STATS_INC_EXECUTOR_THREADS_CREATED(&exec_ctx); + size_t subtract_depth = 0; + bool used = false; for (;;) { if (GRPC_TRACER_ON(executor_trace)) { gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")", @@ -170,6 +175,10 @@ static void executor_thread(void *arg) { gpr_mu_unlock(&ts->mu); break; } + if (!used) { + GRPC_STATS_INC_EXECUTOR_THREADS_USED(&exec_ctx); + used = true; + } GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx); grpc_closure_list exec = ts->elems; ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; From b2b4a85c86b6e1e8f9b4e278355e77b4a4b5c682 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 14 Sep 2017 12:38:38 -0700 Subject: [PATCH 143/227] Correct comparison --- src/core/lib/iomgr/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index cf2c051c923..2439f15a8a9 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -217,7 +217,7 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)]; } else { GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx); - if (!is_short) { + if (is_short) { grpc_closure_list_append(&ts->local_elems, closure, error); return; } From 27bf05d003f48e6f95b836a5e5450e4bb787f9ef Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Mon, 7 Aug 2017 18:09:11 -0700 Subject: [PATCH 144/227] Add tests for c-ares wrapper using a local DNS server. --- CMakeLists.txt | 184 ++++++++++ Makefile | 188 ++++++++++ bazel/grpc_build_system.bzl | 16 + .../resolver/dns/c_ares/dns_resolver_ares.c | 2 +- .../resolver_component_tests_defs.include | 92 +++++ ...esolver_component_tests_runner.sh.template | 4 + .../tools/dockerfile/apt_get_basic.include | 1 + .../tools/dockerfile/python_deps.include | 2 +- test/cpp/naming/BUILD | 49 +++ test/cpp/naming/gen_build_yaml.py | 99 ++++++ .../generate_resolver_component_tests.bzl | 64 ++++ test/cpp/naming/resolver_component_test.cc | 323 ++++++++++++++++++ .../naming/resolver_component_tests_runner.sh | 173 ++++++++++ ...resolver_component_tests_runner_invoker.cc | 189 ++++++++++ .../naming/resolver_test_record_groups.yaml | 155 +++++++++ test/cpp/naming/test_dns_server.py | 134 ++++++++ tools/buildgen/generate_build_additions.sh | 1 + .../grpc_interop_csharp/Dockerfile | 3 +- .../interoptest/grpc_interop_cxx/Dockerfile | 3 +- .../interoptest/grpc_interop_go/Dockerfile | 2 +- .../interoptest/grpc_interop_go1.7/Dockerfile | 2 +- .../interoptest/grpc_interop_go1.8/Dockerfile | 2 +- .../interoptest/grpc_interop_http2/Dockerfile | 2 +- .../interoptest/grpc_interop_java/Dockerfile | 2 +- .../grpc_interop_java_oracle8/Dockerfile | 2 +- .../interoptest/grpc_interop_node/Dockerfile | 3 +- .../interoptest/grpc_interop_php/Dockerfile | 1 + .../grpc_interop_python/Dockerfile | 3 +- .../interoptest/grpc_interop_ruby/Dockerfile | 3 +- .../test/csharp_jessie_x64/Dockerfile | 3 +- .../dockerfile/test/cxx_jessie_x64/Dockerfile | 3 +- .../dockerfile/test/cxx_jessie_x86/Dockerfile | 3 +- .../test/cxx_ubuntu1404_x64/Dockerfile | 3 +- .../test/cxx_ubuntu1604_x64/Dockerfile | 3 +- tools/dockerfile/test/fuzzer/Dockerfile | 3 +- .../test/multilang_jessie_x64/Dockerfile | 3 +- .../test/node_jessie_x64/Dockerfile | 3 +- .../test/php7_jessie_x64/Dockerfile | 2 +- .../dockerfile/test/php_jessie_x64/Dockerfile | 3 +- .../test/python_jessie_x64/Dockerfile | 3 +- .../test/python_pyenv_x64/Dockerfile | 3 +- .../test/ruby_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/sanity/Dockerfile | 3 +- .../helper_scripts/prepare_build_macos_rc | 2 +- .../generated/sources_and_headers.json | 80 +++++ tools/run_tests/generated/tests.json | 46 +++ 46 files changed, 1845 insertions(+), 28 deletions(-) create mode 100644 templates/test/cpp/naming/resolver_component_tests_defs.include create mode 100644 templates/test/cpp/naming/resolver_component_tests_runner.sh.template create mode 100644 test/cpp/naming/BUILD create mode 100755 test/cpp/naming/gen_build_yaml.py create mode 100755 test/cpp/naming/generate_resolver_component_tests.bzl create mode 100644 test/cpp/naming/resolver_component_test.cc create mode 100755 test/cpp/naming/resolver_component_tests_runner.sh create mode 100644 test/cpp/naming/resolver_component_tests_runner_invoker.cc create mode 100644 test/cpp/naming/resolver_test_record_groups.yaml create mode 100755 test/cpp/naming/test_dns_server.py diff --git a/CMakeLists.txt b/CMakeLists.txt index dd68016be86..1186e2b5038 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -760,6 +760,18 @@ add_dependencies(buildtests_cxx thread_stress_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx writes_per_rpc_test) endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_test_unsecure) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_test) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker_unsecure) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker) +endif() add_custom_target(buildtests DEPENDS buildtests_c buildtests_cxx) @@ -14113,6 +14125,178 @@ target_link_libraries(inproc_nosec_test gpr ) +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_test_unsecure + test/cpp/naming/resolver_component_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_test_unsecure + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_test_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util_unsecure + grpc_test_util_unsecure + gpr_test_util + grpc++_unsecure + grpc_unsecure + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_test + test/cpp/naming/resolver_component_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_tests_runner_invoker_unsecure + test/cpp/naming/resolver_component_tests_runner_invoker.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_tests_runner_invoker_unsecure + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_tests_runner_invoker_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_tests_runner_invoker + test/cpp/naming/resolver_component_tests_runner_invoker.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_tests_runner_invoker + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_tests_runner_invoker + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/Makefile b/Makefile index 0c3648566a9..61d6cae6ab5 100644 --- a/Makefile +++ b/Makefile @@ -1266,6 +1266,10 @@ h2_sockpair+trace_nosec_test: $(BINDIR)/$(CONFIG)/h2_sockpair+trace_nosec_test h2_sockpair_1byte_nosec_test: $(BINDIR)/$(CONFIG)/h2_sockpair_1byte_nosec_test h2_uds_nosec_test: $(BINDIR)/$(CONFIG)/h2_uds_nosec_test inproc_nosec_test: $(BINDIR)/$(CONFIG)/inproc_nosec_test +resolver_component_test_unsecure: $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure +resolver_component_test: $(BINDIR)/$(CONFIG)/resolver_component_test +resolver_component_tests_runner_invoker_unsecure: $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure +resolver_component_tests_runner_invoker: $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker api_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/api_fuzzer_one_entry client_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/client_fuzzer_one_entry hpack_parser_fuzzer_test_one_entry: $(BINDIR)/$(CONFIG)/hpack_parser_fuzzer_test_one_entry @@ -1652,6 +1656,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/boringssl_x509_test \ $(BINDIR)/$(CONFIG)/boringssl_tab_test \ $(BINDIR)/$(CONFIG)/boringssl_v3name_test \ + $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_test \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker \ else buildtests_cxx: privatelibs_cxx \ @@ -1730,6 +1738,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/thread_manager_test \ $(BINDIR)/$(CONFIG)/thread_stress_test \ $(BINDIR)/$(CONFIG)/writes_per_rpc_test \ + $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_test \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker \ endif @@ -2141,6 +2153,10 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/thread_stress_test || ( echo test thread_stress_test failed ; exit 1 ) $(E) "[RUN] Testing writes_per_rpc_test" $(Q) $(BINDIR)/$(CONFIG)/writes_per_rpc_test || ( echo test writes_per_rpc_test failed ; exit 1 ) + $(E) "[RUN] Testing resolver_component_tests_runner_invoker_unsecure" + $(Q) $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure || ( echo test resolver_component_tests_runner_invoker_unsecure failed ; exit 1 ) + $(E) "[RUN] Testing resolver_component_tests_runner_invoker" + $(Q) $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker || ( echo test resolver_component_tests_runner_invoker failed ; exit 1 ) flaky_test_cxx: buildtests_cxx @@ -19480,6 +19496,178 @@ ifneq ($(NO_DEPS),true) endif +RESOLVER_COMPONENT_TEST_UNSECURE_SRC = \ + test/cpp/naming/resolver_component_test.cc \ + +RESOLVER_COMPONENT_TEST_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TEST_UNSECURE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_test_unsecure: $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TEST_SRC = \ + test/cpp/naming/resolver_component_test.cc \ + +RESOLVER_COMPONENT_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_test: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_test: $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_SRC = \ + test/cpp/naming/resolver_component_tests_runner_invoker.cc \ + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_tests_runner_invoker.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_tests_runner_invoker_unsecure: $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_SRC = \ + test/cpp/naming/resolver_component_tests_runner_invoker.cc \ + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_tests_runner_invoker.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_tests_runner_invoker: $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS:.o=.dep) +endif +endif + + API_FUZZER_ONE_ENTRY_SRC = \ test/core/end2end/fuzzers/api_fuzzer.c \ test/core/util/one_corpus_entry_fuzzer.c \ diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index ba7ffcc9beb..6cfed6b0e72 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -106,6 +106,22 @@ def grpc_sh_test(name, srcs, args = [], data = []): args = args, data = data) +def grpc_sh_binary(name, srcs, data = []): + native.sh_test( + name = name, + srcs = srcs, + data = data) + +def grpc_py_binary(name, srcs, data = [], deps = []): + if name == "test_dns_server": + # TODO: allow running test_dns_server in oss bazel test suite + deps = [] + native.py_binary( + name = name, + srcs = srcs, + data = data, + deps = deps) + def grpc_package(name, visibility = "private", features = []): if visibility == "tests": visibility = ["//test:__subpackages__"] diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index b87a3b70820..371c59b8cf3 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -204,7 +204,7 @@ static char *choose_service_config(char *service_config_choice_json) { int random_pct = rand() % 100; int percentage; if (sscanf(field->value, "%d", &percentage) != 1 || - random_pct > percentage) { + random_pct > percentage || percentage == 0) { service_config_json = NULL; break; } diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include new file mode 100644 index 00000000000..6fa91c741ad --- /dev/null +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -0,0 +1,92 @@ +<%def name="resolver_component_tests(tests)">#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +# all command args required in this set order +FLAGS_test_bin_path=`echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2` +FLAGS_dns_server_bin_path=`echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2` +FLAGS_records_config_path=`echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2` +FLAGS_test_dns_server_port=`echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2` + +for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do + if [[ "$cmd_arg" == "" ]]; then + echo "Missing a CMD arg" && exit 1 + fi +done + +if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "This test only works under GRPC_DNS_RESOLVER=ares. Have GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER" && exit 1 +fi +export GRPC_DNS_RESOLVER=ares + +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" 2>&1 > /dev/null & +DNS_SERVER_PID=$! +echo "Local DNS server started. PID: $DNS_SERVER_PID" + +# Health check local DNS server TCP and UDP ports +for ((i=0;i<30;i++)); +do + echo "Retry health-check DNS query to local DNS server over tcp and udp" + RETRY=0 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1 + if [[ "$RETRY" == 0 ]]; then + break + fi; + sleep 0.1 +done + +if [[ $RETRY == 1 ]]; then + echo "FAILED TO START LOCAL DNS SERVER" + kill -SIGTERM $DNS_SERVER_PID + wait + exit 1 +fi + +function terminate_all { + echo "Received signal. Terminating $! and $DNS_SERVER_PID" + kill -SIGTERM $! || true + kill -SIGTERM $DNS_SERVER_PID || true + wait + exit 1 +} + +trap terminate_all SIGTERM SIGINT + +EXIT_CODE=0 +# TODO: this test should check for GCE residency and skip tests using _grpclb._tcp.* SRV records once GCE residency checks are made +# in the resolver. + +% for test in tests: +$FLAGS_test_bin_path \\ + + --target_name='${test['target_name']}' \\ + + --expected_addrs='${test['expected_addrs']}' \\ + + --expected_chosen_service_config='${test['expected_chosen_service_config']}' \\ + + --expected_lb_policy='${test['expected_lb_policy']}' \\ + + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +% endfor +kill -SIGTERM $DNS_SERVER_PID || true +wait +exit $EXIT_CODE diff --git a/templates/test/cpp/naming/resolver_component_tests_runner.sh.template b/templates/test/cpp/naming/resolver_component_tests_runner.sh.template new file mode 100644 index 00000000000..86772dd1416 --- /dev/null +++ b/templates/test/cpp/naming/resolver_component_tests_runner.sh.template @@ -0,0 +1,4 @@ +%YAML 1.2 +--- | + <%namespace file="resolver_component_tests_defs.include" import="*"/>\ + ${resolver_component_tests(resolver_component_test_cases)} diff --git a/templates/tools/dockerfile/apt_get_basic.include b/templates/tools/dockerfile/apt_get_basic.include index 9237e7dacef..6e19e65157a 100644 --- a/templates/tools/dockerfile/apt_get_basic.include +++ b/templates/tools/dockerfile/apt_get_basic.include @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y ${'\\'} bzip2 ${'\\'} ccache ${'\\'} curl ${'\\'} + dnsutils ${'\\'} gcc ${'\\'} gcc-multilib ${'\\'} git ${'\\'} diff --git a/templates/tools/dockerfile/python_deps.include b/templates/tools/dockerfile/python_deps.include index 2c129814184..94b854ad212 100644 --- a/templates/tools/dockerfile/python_deps.include +++ b/templates/tools/dockerfile/python_deps.include @@ -11,4 +11,4 @@ RUN apt-get update && apt-get install -y ${'\\'} # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/test/cpp/naming/BUILD b/test/cpp/naming/BUILD new file mode 100644 index 00000000000..24c3d1a4439 --- /dev/null +++ b/test/cpp/naming/BUILD @@ -0,0 +1,49 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package( + default_visibility = ["//visibility:public"], + features = [ + "-layering_check", + "-parse_headers", + ], +) + +licenses(["notice"]) # Apache v2 + +load("//bazel:grpc_build_system.bzl", "grpc_sh_binary", "grpc_py_binary") + +load(":generate_resolver_component_tests.bzl", "generate_resolver_component_tests") + +# Meant to be invoked only through the top-level shell script driver. +grpc_sh_binary( + name = "resolver_component_tests_runner", + srcs = [ + "resolver_component_tests_runner.sh", + ], +) + +grpc_py_binary( + name = "test_dns_server", + srcs = ["test_dns_server.py"], + data = [ + "resolver_test_record_groups.yaml", + ], + deps = [ + "twisted", + "yaml", + ] +) + +generate_resolver_component_tests() diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py new file mode 100755 index 00000000000..3a51fef7a0e --- /dev/null +++ b/test/cpp/naming/gen_build_yaml.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Generates the appropriate build.json data for all the naming tests.""" + + +import yaml +import collections +import hashlib +import json + +_LOCAL_DNS_SERVER_ADDRESS = '127.0.0.1:15353' + +def _append_zone_name(name, zone_name): + return '%s.%s' % (name, zone_name) + +def _build_expected_addrs_cmd_arg(expected_addrs): + out = [] + for addr in expected_addrs: + out.append('%s,%s' % (addr['address'], str(addr['is_balancer']))) + return ';'.join(out) + +def main(): + resolver_component_data = '' + with open('test/cpp/naming/resolver_test_record_groups.yaml') as f: + resolver_component_data = yaml.load(f) + + json = { + 'resolver_component_test_cases': [ + { + 'target_name': _append_zone_name(test_case['record_to_resolve'], + resolver_component_data['resolver_component_tests_common_zone_name']), + 'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']), + 'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''), + 'expected_lb_policy': (test_case['expected_lb_policy'] or ''), + } for test_case in resolver_component_data['resolver_component_tests'] + ], + 'targets': [ + { + 'name': 'resolver_component_test' + unsecure_build_config_suffix, + 'build': 'test', + 'language': 'c++', + 'gtest': False, + 'run': False, + 'src': ['test/cpp/naming/resolver_component_test.cc'], + 'platforms': ['linux', 'posix', 'mac'], + 'deps': [ + 'grpc++_test_util' + unsecure_build_config_suffix, + 'grpc_test_util' + unsecure_build_config_suffix, + 'gpr_test_util', + 'grpc++' + unsecure_build_config_suffix, + 'grpc' + unsecure_build_config_suffix, + 'gpr', + 'grpc++_test_config', + ], + } for unsecure_build_config_suffix in ['_unsecure', ''] + ] + [ + { + 'name': 'resolver_component_tests_runner_invoker' + unsecure_build_config_suffix, + 'build': 'test', + 'language': 'c++', + 'gtest': False, + 'run': True, + 'src': ['test/cpp/naming/resolver_component_tests_runner_invoker.cc'], + 'platforms': ['linux', 'posix', 'mac'], + 'deps': [ + 'grpc++_test_util', + 'grpc_test_util', + 'gpr_test_util', + 'grpc++', + 'grpc', + 'gpr', + 'grpc++_test_config', + ], + 'args': [ + '--test_bin_name=resolver_component_test%s' % unsecure_build_config_suffix, + '--running_under_bazel=false', + ], + } for unsecure_build_config_suffix in ['_unsecure', ''] + ] + } + + print(yaml.dump(json)) + +if __name__ == '__main__': + main() diff --git a/test/cpp/naming/generate_resolver_component_tests.bzl b/test/cpp/naming/generate_resolver_component_tests.bzl new file mode 100755 index 00000000000..118d9452d9f --- /dev/null +++ b/test/cpp/naming/generate_resolver_component_tests.bzl @@ -0,0 +1,64 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:grpc_build_system.bzl", "grpc_sh_binary", "grpc_cc_test", "grpc_cc_binary") + +def generate_resolver_component_tests(): + for unsecure_build_config_suffix in ['_unsecure', '']: + # meant to be invoked only through the top-level shell script driver + grpc_cc_binary( + name = "resolver_component_test%s" % unsecure_build_config_suffix, + testonly = 1, + srcs = [ + "resolver_component_test.cc", + ], + external_deps = [ + "gmock", + ], + deps = [ + "//test/cpp/util:test_util%s" % unsecure_build_config_suffix, + "//test/core/util:grpc_test_util%s" % unsecure_build_config_suffix, + "//test/core/util:gpr_test_util", + "//:grpc++%s" % unsecure_build_config_suffix, + "//:grpc%s" % unsecure_build_config_suffix, + "//:gpr", + "//test/cpp/util:test_config", + ], + ) + grpc_cc_test( + name = "resolver_component_tests_runner_invoker%s" % unsecure_build_config_suffix, + srcs = [ + "resolver_component_tests_runner_invoker.cc", + ], + deps = [ + "//test/cpp/util:test_util", + "//test/core/util:grpc_test_util", + "//test/core/util:gpr_test_util", + "//:grpc++", + "//:grpc", + "//:gpr", + "//test/cpp/util:test_config", + ], + data = [ + ":resolver_component_tests_runner", + ":resolver_component_test%s" % unsecure_build_config_suffix, + ":test_dns_server", + "resolver_test_record_groups.yaml", # include the transitive dependency so that the dns sever py binary can locate this + ], + args = [ + "--test_bin_name=resolver_component_test%s" % unsecure_build_config_suffix, + "--running_under_bazel=true", + ] + ) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc new file mode 100644 index 00000000000..0857fb6a32c --- /dev/null +++ b/test/cpp/naming/resolver_component_test.cc @@ -0,0 +1,323 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "test/cpp/util/subprocess.h" +#include "test/cpp/util/test_config.h" + +extern "C" { +#include "src/core/ext/filters/client_channel/client_channel.h" +#include "src/core/ext/filters/client_channel/resolver.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" +#include "src/core/ext/filters/client_channel/resolver_registry.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/executor.h" +#include "src/core/lib/iomgr/iomgr.h" +#include "src/core/lib/iomgr/resolve_address.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/support/env.h" +#include "src/core/lib/support/string.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" +} + +using std::vector; +using grpc::SubProcess; +using testing::UnorderedElementsAreArray; + +// Hack copied from "test/cpp/end2end/server_crash_test_client.cc"! +// In some distros, gflags is in the namespace google, and in some others, +// in gflags. This hack is enabling us to find both. +namespace google {} +namespace gflags {} +using namespace google; +using namespace gflags; + +DEFINE_string(target_name, "", "Target name to resolve."); +DEFINE_string(expected_addrs, "", + "Comma-separated list of expected " + "',;,;...' " + "addresses of " + "backend and/or balancers. 'is_balancer' should be bool, i.e. " + "true or false."); +DEFINE_string(expected_chosen_service_config, "", + "Expected service config json string that gets chosen (no " + "whitespace). Empty for none."); +DEFINE_string( + local_dns_server_address, "", + "Optional. This address is placed as the uri authority if present."); +DEFINE_string(expected_lb_policy, "", + "Expected lb policy name that appears in resolver result channel " + "arg. Empty for none."); + +namespace { + +class GrpcLBAddress final { + public: + GrpcLBAddress(std::string address, bool is_balancer) + : is_balancer(is_balancer), address(address) {} + + bool operator==(const GrpcLBAddress &other) const { + return this->is_balancer == other.is_balancer && + this->address == other.address; + } + + bool operator!=(const GrpcLBAddress &other) const { + return !(*this == other); + } + + bool is_balancer; + std::string address; +}; + +vector ParseExpectedAddrs(std::string expected_addrs) { + std::vector out; + while (expected_addrs.size() != 0) { + // get the next , (v4 or v6) + size_t next_comma = expected_addrs.find(","); + if (next_comma == std::string::npos) { + gpr_log( + GPR_ERROR, + "Missing ','. Expected_addrs arg should be a semi-colon-separated " + "list of " + ", pairs. Left-to-be-parsed arg is |%s|", + expected_addrs.c_str()); + abort(); + } + std::string next_addr = expected_addrs.substr(0, next_comma); + expected_addrs = expected_addrs.substr(next_comma + 1, std::string::npos); + // get the next is_balancer 'bool' associated with this address + size_t next_semicolon = expected_addrs.find(";"); + bool is_balancer = + gpr_is_true(expected_addrs.substr(0, next_semicolon).c_str()); + out.emplace_back(GrpcLBAddress(next_addr, is_balancer)); + if (next_semicolon == std::string::npos) { + break; + } + expected_addrs = + expected_addrs.substr(next_semicolon + 1, std::string::npos); + } + if (out.size() == 0) { + gpr_log(GPR_ERROR, + "expected_addrs arg should be a comma-separated list of " + ", pairs"); + abort(); + } + return out; +} + +gpr_timespec TestDeadline(void) { + return grpc_timeout_seconds_to_deadline(100); +} + +struct ArgsStruct { + gpr_event ev; + gpr_atm done_atm; + gpr_mu *mu; + grpc_pollset *pollset; + grpc_pollset_set *pollset_set; + grpc_combiner *lock; + grpc_channel_args *channel_args; + vector expected_addrs; + std::string expected_service_config_string; + std::string expected_lb_policy; +}; + +void ArgsInit(grpc_exec_ctx *exec_ctx, ArgsStruct *args) { + gpr_event_init(&args->ev); + args->pollset = (grpc_pollset *)gpr_zalloc(grpc_pollset_size()); + grpc_pollset_init(args->pollset, &args->mu); + args->pollset_set = grpc_pollset_set_create(); + grpc_pollset_set_add_pollset(exec_ctx, args->pollset_set, args->pollset); + args->lock = grpc_combiner_create(); + gpr_atm_rel_store(&args->done_atm, 0); + args->channel_args = NULL; +} + +void DoNothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} + +void ArgsFinish(grpc_exec_ctx *exec_ctx, ArgsStruct *args) { + GPR_ASSERT(gpr_event_wait(&args->ev, TestDeadline())); + grpc_pollset_set_del_pollset(exec_ctx, args->pollset_set, args->pollset); + grpc_pollset_set_destroy(exec_ctx, args->pollset_set); + grpc_closure DoNothing_cb; + GRPC_CLOSURE_INIT(&DoNothing_cb, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_pollset_shutdown(exec_ctx, args->pollset, &DoNothing_cb); + // exec_ctx needs to be flushed before calling grpc_pollset_destroy() + grpc_channel_args_destroy(exec_ctx, args->channel_args); + grpc_exec_ctx_flush(exec_ctx); + grpc_pollset_destroy(exec_ctx, args->pollset); + gpr_free(args->pollset); + GRPC_COMBINER_UNREF(exec_ctx, args->lock, NULL); +} + +gpr_timespec NSecondDeadline(int seconds) { + return gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_seconds(seconds, GPR_TIMESPAN)); +} + +void PollPollsetUntilRequestDone(ArgsStruct *args) { + gpr_timespec deadline = NSecondDeadline(10); + while (true) { + bool done = gpr_atm_acq_load(&args->done_atm) != 0; + if (done) { + break; + } + gpr_timespec time_left = + gpr_time_sub(deadline, gpr_now(GPR_CLOCK_REALTIME)); + gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRId64 ".%09d", done, + time_left.tv_sec, time_left.tv_nsec); + GPR_ASSERT(gpr_time_cmp(time_left, gpr_time_0(GPR_TIMESPAN)) >= 0); + grpc_pollset_worker *worker = NULL; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_mu_lock(args->mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(&exec_ctx, args->pollset, &worker, + gpr_now(GPR_CLOCK_REALTIME), NSecondDeadline(1))); + gpr_mu_unlock(args->mu); + grpc_exec_ctx_finish(&exec_ctx); + } + gpr_event_set(&args->ev, (void *)1); +} + +void CheckServiceConfigResultLocked(grpc_channel_args *channel_args, + ArgsStruct *args) { + const grpc_arg *service_config_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_SERVICE_CONFIG); + if (args->expected_service_config_string != "") { + GPR_ASSERT(service_config_arg != NULL); + GPR_ASSERT(service_config_arg->type == GRPC_ARG_STRING); + EXPECT_EQ(service_config_arg->value.string, + args->expected_service_config_string); + } else { + GPR_ASSERT(service_config_arg == NULL); + } +} + +void CheckLBPolicyResultLocked(grpc_channel_args *channel_args, + ArgsStruct *args) { + const grpc_arg *lb_policy_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_LB_POLICY_NAME); + if (args->expected_lb_policy != "") { + GPR_ASSERT(lb_policy_arg != NULL); + GPR_ASSERT(lb_policy_arg->type == GRPC_ARG_STRING); + EXPECT_EQ(lb_policy_arg->value.string, args->expected_lb_policy); + } else { + GPR_ASSERT(lb_policy_arg == NULL); + } +} + +void CheckResolverResultLocked(grpc_exec_ctx *exec_ctx, void *argsp, + grpc_error *err) { + ArgsStruct *args = (ArgsStruct *)argsp; + grpc_channel_args *channel_args = args->channel_args; + const grpc_arg *channel_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_LB_ADDRESSES); + GPR_ASSERT(channel_arg != NULL); + GPR_ASSERT(channel_arg->type == GRPC_ARG_POINTER); + grpc_lb_addresses *addresses = + (grpc_lb_addresses *)channel_arg->value.pointer.p; + gpr_log(GPR_INFO, "num addrs found: %" PRIdPTR ". expected %" PRIdPTR, + addresses->num_addresses, args->expected_addrs.size()); + GPR_ASSERT(addresses->num_addresses == args->expected_addrs.size()); + std::vector found_lb_addrs; + for (size_t i = 0; i < addresses->num_addresses; i++) { + grpc_lb_address addr = addresses->addresses[i]; + char *str; + grpc_sockaddr_to_string(&str, &addr.address, 1 /* normalize */); + gpr_log(GPR_INFO, "%s", str); + found_lb_addrs.emplace_back( + GrpcLBAddress(std::string(str), addr.is_balancer)); + gpr_free(str); + } + if (args->expected_addrs.size() != found_lb_addrs.size()) { + gpr_log(GPR_DEBUG, "found lb addrs size is: %" PRIdPTR + ". expected addrs size is %" PRIdPTR, + found_lb_addrs.size(), args->expected_addrs.size()); + abort(); + } + EXPECT_THAT(args->expected_addrs, UnorderedElementsAreArray(found_lb_addrs)); + CheckServiceConfigResultLocked(channel_args, args); + CheckLBPolicyResultLocked(channel_args, args); + gpr_atm_rel_store(&args->done_atm, 1); + gpr_mu_lock(args->mu); + GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + gpr_mu_unlock(args->mu); +} + +TEST(ResolverComponentTest, TestResolvesRelevantRecords) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + ArgsStruct args; + ArgsInit(&exec_ctx, &args); + args.expected_addrs = ParseExpectedAddrs(FLAGS_expected_addrs); + args.expected_service_config_string = FLAGS_expected_chosen_service_config; + args.expected_lb_policy = FLAGS_expected_lb_policy; + // maybe build the address with an authority + char *whole_uri = NULL; + GPR_ASSERT(asprintf(&whole_uri, "dns://%s/%s", + FLAGS_local_dns_server_address.c_str(), + FLAGS_target_name.c_str())); + // create resolver and resolve + grpc_resolver *resolver = grpc_resolver_create(&exec_ctx, whole_uri, NULL, + args.pollset_set, args.lock); + gpr_free(whole_uri); + grpc_closure on_resolver_result_changed; + GRPC_CLOSURE_INIT(&on_resolver_result_changed, CheckResolverResultLocked, + (void *)&args, grpc_combiner_scheduler(args.lock)); + grpc_resolver_next_locked(&exec_ctx, resolver, &args.channel_args, + &on_resolver_result_changed); + grpc_exec_ctx_flush(&exec_ctx); + PollPollsetUntilRequestDone(&args); + GRPC_RESOLVER_UNREF(&exec_ctx, resolver, NULL); + ArgsFinish(&exec_ctx, &args); + grpc_exec_ctx_finish(&exec_ctx); +} + +} // namespace + +int main(int argc, char **argv) { + grpc_init(); + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_target_name == "") { + gpr_log(GPR_ERROR, "Missing target_name param."); + abort(); + } + if (FLAGS_local_dns_server_address != "") { + gpr_log(GPR_INFO, "Specifying authority in uris to: %s", + FLAGS_local_dns_server_address.c_str()); + } + auto result = RUN_ALL_TESTS(); + grpc_shutdown(); + return result; +} diff --git a/test/cpp/naming/resolver_component_tests_runner.sh b/test/cpp/naming/resolver_component_tests_runner.sh new file mode 100755 index 00000000000..83b03b67a30 --- /dev/null +++ b/test/cpp/naming/resolver_component_tests_runner.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +# all command args required in this set order +FLAGS_test_bin_path=`echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2` +FLAGS_dns_server_bin_path=`echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2` +FLAGS_records_config_path=`echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2` +FLAGS_test_dns_server_port=`echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2` + +for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do + if [[ "$cmd_arg" == "" ]]; then + echo "Missing a CMD arg" && exit 1 + fi +done + +if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "This test only works under GRPC_DNS_RESOLVER=ares. Have GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER" && exit 1 +fi +export GRPC_DNS_RESOLVER=ares + +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" 2>&1 > /dev/null & +DNS_SERVER_PID=$! +echo "Local DNS server started. PID: $DNS_SERVER_PID" + +# Health check local DNS server TCP and UDP ports +for ((i=0;i<30;i++)); +do + echo "Retry health-check DNS query to local DNS server over tcp and udp" + RETRY=0 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1 + if [[ "$RETRY" == 0 ]]; then + break + fi; + sleep 0.1 +done + +if [[ $RETRY == 1 ]]; then + echo "FAILED TO START LOCAL DNS SERVER" + kill -SIGTERM $DNS_SERVER_PID + wait + exit 1 +fi + +function terminate_all { + echo "Received signal. Terminating $! and $DNS_SERVER_PID" + kill -SIGTERM $! || true + kill -SIGTERM $DNS_SERVER_PID || true + wait + exit 1 +} + +trap terminate_all SIGTERM SIGINT + +EXIT_CODE=0 +# TODO: this test should check for GCE residency and skip tests using _grpclb._tcp.* SRV records once GCE residency checks are made +# in the resolver. + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-single-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-multi-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-single-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-multi-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-simple-service-config.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-no-srv-simple-service-config.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-no-config-for-cpp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-second-language-is-cpp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-config-with-percentages.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +kill -SIGTERM $DNS_SERVER_PID || true +wait +exit $EXIT_CODE diff --git a/test/cpp/naming/resolver_component_tests_runner_invoker.cc b/test/cpp/naming/resolver_component_tests_runner_invoker.cc new file mode 100644 index 00000000000..b14391284d5 --- /dev/null +++ b/test/cpp/naming/resolver_component_tests_runner_invoker.cc @@ -0,0 +1,189 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "test/cpp/util/subprocess.h" +#include "test/cpp/util/test_config.h" + +extern "C" { +#include "src/core/lib/support/env.h" +#include "test/core/util/port.h" +} + +DEFINE_bool( + running_under_bazel, false, + "True if this test is running under bazel. " + "False indicates that this test is running under run_tests.py. " + "Child process test binaries are located differently based on this flag. "); + +DEFINE_string(test_bin_name, "", + "Name, without the preceding path, of the test binary"); + +DEFINE_string(grpc_test_directory_relative_to_test_srcdir, "/__main__", + "This flag only applies if runner_under_bazel is true. This " + "flag is ignored if runner_under_bazel is false. " + "Directory of the /test directory relative to bazel's " + "TEST_SRCDIR environment variable"); + +using grpc::SubProcess; + +static volatile sig_atomic_t abort_wait_for_child = 0; + +static void sighandler(int sig) { abort_wait_for_child = 1; } + +static void register_sighandler() { + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = sighandler; + sigaction(SIGINT, &act, NULL); + sigaction(SIGTERM, &act, NULL); +} + +namespace { + +const int kTestTimeoutSeconds = 60 * 2; + +void RunSigHandlingThread(SubProcess *test_driver, gpr_mu *test_driver_mu, + gpr_cv *test_driver_cv, int *test_driver_done) { + gpr_timespec overall_deadline = + gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_seconds(kTestTimeoutSeconds, GPR_TIMESPAN)); + while (true) { + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + if (gpr_time_cmp(now, overall_deadline) > 0 || abort_wait_for_child) break; + gpr_mu_lock(test_driver_mu); + if (*test_driver_done) { + gpr_mu_unlock(test_driver_mu); + return; + } + gpr_timespec wait_deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(1, GPR_TIMESPAN)); + gpr_cv_wait(test_driver_cv, test_driver_mu, wait_deadline); + gpr_mu_unlock(test_driver_mu); + } + gpr_log(GPR_DEBUG, + "Test timeout reached or received signal. Interrupting test driver " + "child process."); + test_driver->Interrupt(); + return; +} +} + +namespace grpc { + +namespace testing { + +void InvokeResolverComponentTestsRunner(std::string test_runner_bin_path, + std::string test_bin_path, + std::string dns_server_bin_path, + std::string records_config_path) { + int test_dns_server_port = grpc_pick_unused_port_or_die(); + + SubProcess *test_driver = new SubProcess( + {test_runner_bin_path, "--test_bin_path=" + test_bin_path, + "--dns_server_bin_path=" + dns_server_bin_path, + "--records_config_path=" + records_config_path, + "--test_dns_server_port=" + std::to_string(test_dns_server_port)}); + gpr_mu test_driver_mu; + gpr_mu_init(&test_driver_mu); + gpr_cv test_driver_cv; + gpr_cv_init(&test_driver_cv); + int test_driver_done = 0; + register_sighandler(); + std::thread sig_handling_thread(RunSigHandlingThread, test_driver, + &test_driver_mu, &test_driver_cv, + &test_driver_done); + int status = test_driver->Join(); + if (WIFEXITED(status)) { + if (WEXITSTATUS(status)) { + gpr_log(GPR_INFO, + "Resolver component test test-runner exited with code %d", + WEXITSTATUS(status)); + abort(); + } + } else if (WIFSIGNALED(status)) { + gpr_log(GPR_INFO, + "Resolver component test test-runner ended from signal %d", + WTERMSIG(status)); + abort(); + } else { + gpr_log(GPR_INFO, + "Resolver component test test-runner ended with unknown status %d", + status); + abort(); + } + gpr_mu_lock(&test_driver_mu); + test_driver_done = 1; + gpr_cv_signal(&test_driver_cv); + gpr_mu_unlock(&test_driver_mu); + sig_handling_thread.join(); + delete test_driver; + gpr_mu_destroy(&test_driver_mu); + gpr_cv_destroy(&test_driver_cv); +} + +} // namespace testing + +} // namespace grpc + +int main(int argc, char **argv) { + grpc::testing::InitTest(&argc, &argv, true); + grpc_init(); + GPR_ASSERT(FLAGS_test_bin_name != ""); + std::string my_bin = argv[0]; + if (FLAGS_running_under_bazel) { + GPR_ASSERT(FLAGS_grpc_test_directory_relative_to_test_srcdir != ""); + // Use bazel's TEST_SRCDIR environment variable to locate the "test data" + // binaries. + std::string const bin_dir = + gpr_getenv("TEST_SRCDIR") + + FLAGS_grpc_test_directory_relative_to_test_srcdir + + std::string("/test/cpp/naming"); + // Invoke bazel's executeable links to the .sh and .py scripts (don't use + // the .sh and .py suffixes) to make + // sure that we're using bazel's test environment. + grpc::testing::InvokeResolverComponentTestsRunner( + bin_dir + "/resolver_component_tests_runner", + bin_dir + "/" + FLAGS_test_bin_name, bin_dir + "/test_dns_server", + bin_dir + "/resolver_test_record_groups.yaml"); + } else { + // Get the current binary's directory relative to repo root to invoke the + // correct build config (asan/tsan/dbg, etc.). + std::string const bin_dir = my_bin.substr(0, my_bin.rfind('/')); + // Invoke the .sh and .py scripts directly where they are in source code. + grpc::testing::InvokeResolverComponentTestsRunner( + "test/cpp/naming/resolver_component_tests_runner.sh", + bin_dir + "/" + FLAGS_test_bin_name, + "test/cpp/naming/test_dns_server.py", + "test/cpp/naming/resolver_test_record_groups.yaml"); + } + grpc_shutdown(); + return 0; +} diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml new file mode 100644 index 00000000000..67c611d831f --- /dev/null +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -0,0 +1,155 @@ +resolver_component_tests_common_zone_name: resolver-tests.grpctestingexp. +resolver_component_tests: +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-single-target + records: + _grpclb._tcp.srv-ipv4-single-target: + - {TTL: '2100', data: 0 0 1234 ipv4-single-target, type: SRV} + ipv4-single-target: + - {TTL: '2100', data: 1.2.3.4, type: A} +- expected_addrs: + - {address: '1.2.3.5:1234', is_balancer: true} + - {address: '1.2.3.6:1234', is_balancer: true} + - {address: '1.2.3.7:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-multi-target + records: + _grpclb._tcp.srv-ipv4-multi-target: + - {TTL: '2100', data: 0 0 1234 ipv4-multi-target, type: SRV} + ipv4-multi-target: + - {TTL: '2100', data: 1.2.3.5, type: A} + - {TTL: '2100', data: 1.2.3.6, type: A} + - {TTL: '2100', data: 1.2.3.7, type: A} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1001]:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-single-target + records: + _grpclb._tcp.srv-ipv6-single-target: + - {TTL: '2100', data: 0 0 1234 ipv6-single-target, type: SRV} + ipv6-single-target: + - {TTL: '2100', data: '2607:f8b0:400a:801::1001', type: AAAA} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1002]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1003]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1004]:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-multi-target + records: + _grpclb._tcp.srv-ipv6-multi-target: + - {TTL: '2100', data: 0 0 1234 ipv6-multi-target, type: SRV} + ipv6-multi-target: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + - {TTL: '2100', data: '2607:f8b0:400a:801::1003', type: AAAA} + - {TTL: '2100', data: '2607:f8b0:400a:801::1004', type: AAAA} +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: srv-ipv4-simple-service-config + records: + _grpclb._tcp.srv-ipv4-simple-service-config: + - {TTL: '2100', data: 0 0 1234 ipv4-simple-service-config, type: SRV} + ipv4-simple-service-config: + - {TTL: '2100', data: 1.2.3.4, type: A} + srv-ipv4-simple-service-config: + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-no-srv-simple-service-config + records: + ipv4-no-srv-simple-service-config: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: ipv4-no-config-for-cpp + records: + ipv4-no-config-for-cpp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"clientLanguage":["python"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"PythonService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: ipv4-cpp-config-has-zero-percentage + records: + ipv4-cpp-config-has-zero-percentage: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"percentage":0,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-second-language-is-cpp + records: + ipv4-second-language-is-cpp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"clientLanguage":["go"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"GoService","waitForReady":true}]}]}},{"clientLanguage":["c++"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-config-with-percentages + records: + ipv4-config-with-percentages: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"percentage":0,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NeverPickedService","waitForReady":true}]}]}},{"percentage":100,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-target-has-backend-and-balancer + records: + _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer: + - {TTL: '2100', data: 0 0 1234 balancer-for-ipv4-has-backend-and-balancer, type: SRV} + balancer-for-ipv4-has-backend-and-balancer: + - {TTL: '2100', data: 1.2.3.4, type: A} + srv-ipv4-target-has-backend-and-balancer: + - {TTL: '2100', data: 1.2.3.4, type: A} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1002]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1002]:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-target-has-backend-and-balancer + records: + _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer: + - {TTL: '2100', data: 0 0 1234 balancer-for-ipv6-has-backend-and-balancer, type: SRV} + balancer-for-ipv6-has-backend-and-balancer: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + srv-ipv6-target-has-backend-and-balancer: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + +resolver_component_tests_TODO: +- 'TODO: enable this large-txt-record test once working. (it is much longer than 512 + bytes, likely to cause use of TCP even if max payload for UDP is changed somehow, + e.g. via notes in RFC 2671)' +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' + expected_lb_policy: null + record_to_resolve: srv-ipv6-target-has-backend-and-balancer + record_to_resolve: ipv4-config-causing-fallback-to-tcp + records: + ipv4-config-causing-fallback-to-tcp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}}]', + type: TXT} diff --git a/test/cpp/naming/test_dns_server.py b/test/cpp/naming/test_dns_server.py new file mode 100755 index 00000000000..9d4b89cffb5 --- /dev/null +++ b/test/cpp/naming/test_dns_server.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starts a local DNS server for use in tests""" + +import argparse +import sys +import yaml +import signal +import os + +import twisted +import twisted.internet +import twisted.internet.reactor +import twisted.internet.threads +import twisted.internet.defer +import twisted.internet.protocol +import twisted.names +import twisted.names.client +import twisted.names.dns +import twisted.names.server +from twisted.names import client, server, common, authority, dns +import argparse + +_SERVER_HEALTH_CHECK_RECORD_NAME = 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp' # missing end '.' for twisted syntax +_SERVER_HEALTH_CHECK_RECORD_DATA = '123.123.123.123' + +class NoFileAuthority(authority.FileAuthority): + def __init__(self, soa, records): + # skip FileAuthority + common.ResolverBase.__init__(self) + self.soa = soa + self.records = records + +def start_local_dns_server(args): + all_records = {} + def _push_record(name, r): + print('pushing record: |%s|' % name) + if all_records.get(name) is not None: + all_records[name].append(r) + return + all_records[name] = [r] + + def _maybe_split_up_txt_data(name, txt_data, r_ttl): + start = 0 + txt_data_list = [] + while len(txt_data[start:]) > 0: + next_read = len(txt_data[start:]) + if next_read > 255: + next_read = 255 + txt_data_list.append(txt_data[start:start+next_read]) + start += next_read + _push_record(name, dns.Record_TXT(*txt_data_list, ttl=r_ttl)) + + with open(args.records_config_path) as config: + test_records_config = yaml.load(config) + common_zone_name = test_records_config['resolver_component_tests_common_zone_name'] + for group in test_records_config['resolver_component_tests']: + for name in group['records'].keys(): + for record in group['records'][name]: + r_type = record['type'] + r_data = record['data'] + r_ttl = int(record['TTL']) + record_full_name = '%s.%s' % (name, common_zone_name) + assert record_full_name[-1] == '.' + record_full_name = record_full_name[:-1] + if r_type == 'A': + _push_record(record_full_name, dns.Record_A(r_data, ttl=r_ttl)) + if r_type == 'AAAA': + _push_record(record_full_name, dns.Record_AAAA(r_data, ttl=r_ttl)) + if r_type == 'SRV': + p, w, port, target = r_data.split(' ') + p = int(p) + w = int(w) + port = int(port) + target_full_name = '%s.%s' % (target, common_zone_name) + r_data = '%s %s %s %s' % (p, w, port, target_full_name) + _push_record(record_full_name, dns.Record_SRV(p, w, port, target_full_name, ttl=r_ttl)) + if r_type == 'TXT': + _maybe_split_up_txt_data(record_full_name, r_data, r_ttl) + # Server health check record + _push_record(_SERVER_HEALTH_CHECK_RECORD_NAME, dns.Record_A(_SERVER_HEALTH_CHECK_RECORD_DATA, ttl=0)) + soa_record = dns.Record_SOA(mname = common_zone_name) + test_domain_com = NoFileAuthority( + soa = (common_zone_name, soa_record), + records = all_records, + ) + server = twisted.names.server.DNSServerFactory( + authorities=[test_domain_com], verbose=2) + server.noisy = 2 + twisted.internet.reactor.listenTCP(args.port, server) + dns_proto = twisted.names.dns.DNSDatagramProtocol(server) + dns_proto.noisy = 2 + twisted.internet.reactor.listenUDP(args.port, dns_proto) + print('starting local dns server on 127.0.0.1:%s' % args.port) + print('starting twisted.internet.reactor') + twisted.internet.reactor.suggestThreadPoolSize(1) + twisted.internet.reactor.run() + +def _quit_on_signal(signum, _frame): + print('Received SIGNAL %d. Quitting with exit code 0' % signum) + twisted.internet.reactor.stop() + sys.stdout.flush() + sys.exit(0) + +def main(): + argp = argparse.ArgumentParser(description='Local DNS Server for resolver tests') + argp.add_argument('-p', '--port', default=None, type=int, + help='Port for DNS server to listen on for TCP and UDP.') + argp.add_argument('-r', '--records_config_path', default=None, type=str, + help=('Directory of resolver_test_record_groups.yaml file. ' + 'Defauls to path needed when the test is invoked as part of run_tests.py.')) + args = argp.parse_args() + signal.signal(signal.SIGALRM, _quit_on_signal) + signal.signal(signal.SIGTERM, _quit_on_signal) + signal.signal(signal.SIGINT, _quit_on_signal) + # Prevent zombies. Tests that use this server are short-lived. + signal.alarm(2 * 60) + start_local_dns_server(args) + +if __name__ == '__main__': + main() diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index 7649145f860..693c02fdb2c 100644 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -24,6 +24,7 @@ gen_build_yaml_dirs=" \ test/core/bad_client \ test/core/bad_ssl \ test/core/end2end \ + test/cpp/naming \ test/cpp/qps" gen_build_files="" for gen_build_yaml in $gen_build_yaml_dirs diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index 4ccfbc43c30..dbf58023c53 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================ # C# dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile index 824ecf4604a..7cfe98cbc84 100644 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index 5f7bedde735..febe2fa251b 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile index 2bfa87c4ea2..3a516cbb62a 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile index 96391372cab..acb640a81dc 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 27a924343ff..354b7bfdb59 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 RUN pip install twisted h2==2.6.1 hyper diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile index 264a0f92713..92a542ff76b 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Trigger download of as many Gradle artifacts as possible. diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile index 264a0f92713..92a542ff76b 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Trigger download of as many Gradle artifacts as possible. diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile index c59b12d1556..4343d56cb49 100644 --- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Node dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile index f859a465149..d6e229f84a2 100644 --- a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index 42b1107c119..271c6e75e9d 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index 3c744b434e6..7bcada68e0a 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 60dc7eb21aa..40d46fcf58b 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================ # C# dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index f9622dd7577..888a37baca7 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index 2d396ce2ff0..319f1e18893 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index 349a6eff837..61f005d9da3 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index bccfdba7cd3..f35247eccb5 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index 94c37b3f631..ce1badfeb1b 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index b7373b5d9c3..59fe4d8f93b 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -122,7 +123,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Install coverage for Python test coverage reporting RUN pip install coverage diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 31602e594ec..103be8412b4 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -76,7 +77,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Node dependencies diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index e79305bb27b..f6d426bcd6a 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -77,7 +77,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index 245a6423678..ae82a8d99f1 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # PHP dependencies diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index 5093b793b06..d5d781cd1ac 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index ff8f0231cb8..3b4ad12b6d9 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Install dependencies for pyenv RUN apt-get update && apt-get install -y \ diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index d77917c87fa..3d879bb0c90 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 0f2f9ede58c..44732a5ae42 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #======================== # Sanity test dependencies diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index dd2741e595b..3319aa4394f 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -46,7 +46,7 @@ pod repo update # needed by python # python brew install coreutils # we need grealpath pip install virtualenv --user python -pip install -U six tox setuptools --user python +pip install -U six tox setuptools twisted pyyaml --user python export PYTHONPATH=/Library/Python/3.4/site-packages # set xcode version for Obj-C tests diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 2f6e34bfb38..8d42c3e4328 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5627,6 +5627,86 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc++_test_config", + "grpc++_test_util_unsecure", + "grpc++_unsecure", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_test_unsecure", + "src": [ + "test/cpp/naming/resolver_component_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_test", + "src": [ + "test/cpp/naming/resolver_component_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker_unsecure", + "src": [ + "test/cpp/naming/resolver_component_tests_runner_invoker.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker", + "src": [ + "test/cpp/naming/resolver_component_tests_runner_invoker.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 45d7145e8e3..72dfbc0b982 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4404,6 +4404,52 @@ "posix" ] }, + { + "args": [ + "--test_bin_name=resolver_component_test_unsecure", + "--running_under_bazel=false" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker_unsecure", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "--test_bin_name=resolver_component_test", + "--running_under_bazel=false" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "third_party/boringssl/crypto/aes/aes_tests.txt" From b2a54ac4cb95742a11325c923f63a8aa1dd145e0 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 10:18:07 -0700 Subject: [PATCH 145/227] Reverting suffixes and changing field names instead --- src/core/lib/iomgr/ev_epoll1_linux.c | 54 ++++---- src/core/lib/iomgr/ev_epollex_linux.c | 186 +++++++++++++------------- src/core/lib/surface/call.c | 66 ++++----- src/core/lib/surface/call.h | 2 +- src/core/lib/surface/channel.c | 2 +- 5 files changed, 157 insertions(+), 153 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 6946a2cbf52..0d57833d859 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -130,9 +130,9 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state_t; +typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state; -static const char *kick_state_string(kick_state_t st) { +static const char *kick_state_string(kick_state st) { switch (st) { case UNKICKED: return "UNKICKED"; @@ -145,7 +145,7 @@ static const char *kick_state_string(kick_state_t st) { } struct grpc_pollset_worker { - kick_state_t kick_state; + kick_state state; int kick_state_mutator; // which line of code last changed kick state bool initialized_cv; grpc_pollset_worker *next; @@ -154,9 +154,9 @@ struct grpc_pollset_worker { grpc_closure_list schedule_on_end_work; }; -#define SET_KICK_STATE(worker, state) \ +#define SET_KICK_STATE(worker, kick_state) \ do { \ - (worker)->kick_state = (state); \ + (worker)->state = (kick_state); \ (worker)->kick_state_mutator = __LINE__; \ } while (false) @@ -508,7 +508,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { - switch (worker->kick_state) { + switch (worker->state) { case KICKED: break; case UNKICKED: @@ -688,7 +688,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_mu_lock(&pollset->mu); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d", - pollset, worker, kick_state_string(worker->kick_state), + pollset, worker, kick_state_string(worker->state), is_reassigning); } if (pollset->seen_inactive) { @@ -708,12 +708,12 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, at this point is if it were "kicked specifically". Since the worker has not added itself to the pollset yet (by calling worker_insert()), it is not visible in the "kick any" path yet */ - if (worker->kick_state == UNKICKED) { + if (worker->state == UNKICKED) { pollset->seen_inactive = false; if (neighborhood->active_root == NULL) { neighborhood->active_root = pollset->next = pollset->prev = pollset; /* Make this the designated poller if there isn't one already */ - if (worker->kick_state == UNKICKED && + if (worker->state == UNKICKED && gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { SET_KICK_STATE(worker, DESIGNATED_POLLER); } @@ -733,19 +733,19 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker_insert(pollset, worker); pollset->begin_refs--; - if (worker->kick_state == UNKICKED && !pollset->kicked_without_poller) { + if (worker->state == UNKICKED && !pollset->kicked_without_poller) { GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); worker->initialized_cv = true; gpr_cv_init(&worker->cv); - while (worker->kick_state == UNKICKED && !pollset->shutting_down) { + while (worker->state == UNKICKED && !pollset->shutting_down) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, "PS:%p BEGIN_WAIT:%p kick_state=%s shutdown=%d", - pollset, worker, kick_state_string(worker->kick_state), + pollset, worker, kick_state_string(worker->state), pollset->shutting_down); } if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) && - worker->kick_state == UNKICKED) { + worker->state == UNKICKED) { /* If gpr_cv_wait returns true (i.e a timeout), pretend that the worker received a kick */ SET_KICK_STATE(worker, KICKED); @@ -758,7 +758,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_log(GPR_ERROR, "PS:%p BEGIN_DONE:%p kick_state=%s shutdown=%d " "kicked_without_poller: %d", - pollset, worker, kick_state_string(worker->kick_state), + pollset, worker, kick_state_string(worker->state), pollset->shutting_down, pollset->kicked_without_poller); } @@ -778,7 +778,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } GPR_TIMER_END("begin_worker", 0); - return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down; + return worker->state == DESIGNATED_POLLER && !pollset->shutting_down; } static bool check_neighborhood_for_available_poller( @@ -795,7 +795,7 @@ static bool check_neighborhood_for_available_poller( grpc_pollset_worker *inspect_worker = inspect->root_worker; if (inspect_worker != NULL) { do { - switch (inspect_worker->kick_state) { + switch (inspect_worker->state) { case UNKICKED: if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)inspect_worker)) { @@ -858,7 +858,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure_list_move(&worker->schedule_on_end_work, &exec_ctx->closure_list); if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { - if (worker->next != worker && worker->next->kick_state == UNKICKED) { + if (worker->next != worker && worker->next->state == UNKICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, " .. choose next poller to be peer %p", worker); } @@ -993,14 +993,14 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, gpr_strvec_add(&log, tmp); if (pollset->root_worker != NULL) { gpr_asprintf(&tmp, " {kick_state=%s next=%p {kick_state=%s}}", - kick_state_string(pollset->root_worker->kick_state), + kick_state_string(pollset->root_worker->state), pollset->root_worker->next, - kick_state_string(pollset->root_worker->next->kick_state)); + kick_state_string(pollset->root_worker->next->state)); gpr_strvec_add(&log, tmp); } if (specific_worker != NULL) { gpr_asprintf(&tmp, " worker_kick_state=%s", - kick_state_string(specific_worker->kick_state)); + kick_state_string(specific_worker->state)); gpr_strvec_add(&log, tmp); } tmp = gpr_strvec_flatten(&log, NULL); @@ -1020,13 +1020,13 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } grpc_pollset_worker *next_worker = root_worker->next; - if (root_worker->kick_state == KICKED) { + if (root_worker->state == KICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", root_worker); } SET_KICK_STATE(root_worker, KICKED); goto done; - } else if (next_worker->kick_state == KICKED) { + } else if (next_worker->state == KICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", next_worker); } @@ -1043,7 +1043,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, SET_KICK_STATE(root_worker, KICKED); ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; - } else if (next_worker->kick_state == UNKICKED) { + } else if (next_worker->state == UNKICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", next_worker); } @@ -1051,8 +1051,8 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, SET_KICK_STATE(next_worker, KICKED); gpr_cv_signal(&next_worker->cv); goto done; - } else if (next_worker->kick_state == DESIGNATED_POLLER) { - if (root_worker->kick_state != DESIGNATED_POLLER) { + } else if (next_worker->state == DESIGNATED_POLLER) { + if (root_worker->state != DESIGNATED_POLLER) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log( GPR_ERROR, @@ -1074,7 +1074,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } } else { - GPR_ASSERT(next_worker->kick_state == KICKED); + GPR_ASSERT(next_worker->state == KICKED); SET_KICK_STATE(next_worker, KICKED); goto done; } @@ -1088,7 +1088,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, GPR_UNREACHABLE_CODE(goto done); } - if (specific_worker->kick_state == KICKED) { + if (specific_worker->state == KICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. specific worker already kicked"); } diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index df69025f1a3..9934f8d3299 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -97,12 +97,12 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, * pollable Declarations */ -typedef struct pollable_t { +typedef struct pollable { polling_obj po; int epfd; grpc_wakeup_fd wakeup; grpc_pollset_worker *root_worker; -} pollable_t; +} pollable; static const char *polling_obj_type_string(polling_obj_type t) { switch (t) { @@ -122,7 +122,7 @@ static const char *polling_obj_type_string(polling_obj_type t) { return ""; } -static char *pollable_desc(pollable_t *p) { +static char *pollable_desc(pollable *p) { char *out; gpr_asprintf(&out, "type=%s group=%p epfd=%d wakeup=%d", polling_obj_type_string(p->po.type), p->po.group, p->epfd, @@ -130,19 +130,19 @@ static char *pollable_desc(pollable_t *p) { return out; } -static pollable_t g_empty_pollable; +static pollable g_empty_pollable; -static void pollable_init(pollable_t *p, polling_obj_type type); -static void pollable_destroy(pollable_t *p); +static void pollable_init(pollable *p, polling_obj_type type); +static void pollable_destroy(pollable *p); /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable_t *p); +static grpc_error *pollable_materialize(pollable *p); /******************************************************************************* * Fd Declarations */ struct grpc_fd { - pollable_t pollable; + pollable pollable_obj; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -193,15 +193,15 @@ struct grpc_pollset_worker { pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; gpr_cv cv; grpc_pollset *pollset; - pollable_t *pollable; + pollable *pollable_obj; }; #define MAX_EPOLL_EVENTS 100 #define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 5 struct grpc_pollset { - pollable_t pollable; - pollable_t *current_pollable; + pollable pollable_obj; + pollable *current_pollable; int kick_alls_pending; bool kicked_without_poller; grpc_closure *shutdown_closure; @@ -282,7 +282,7 @@ static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_fd *fd = (grpc_fd *)arg; /* Add the fd to the freelist */ grpc_iomgr_unregister_object(&fd->iomgr_object); - pollable_destroy(&fd->pollable); + pollable_destroy(&fd->pollable_obj); gpr_mu_destroy(&fd->orphaned_mu); gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; @@ -343,7 +343,7 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd = (grpc_fd *)gpr_malloc(sizeof(grpc_fd)); } - pollable_init(&new_fd->pollable, PO_FD); + pollable_init(&new_fd->pollable_obj, PO_FD); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -385,7 +385,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, bool is_fd_closed = already_closed; grpc_error *error = GRPC_ERROR_NONE; - gpr_mu_lock(&fd->pollable.po.mu); + gpr_mu_lock(&fd->pollable_obj.po.mu); gpr_mu_lock(&fd->orphaned_mu); fd->on_done_closure = on_done; @@ -411,7 +411,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, GRPC_CLOSURE_SCHED(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); gpr_mu_unlock(&fd->orphaned_mu); - gpr_mu_unlock(&fd->pollable.po.mu); + gpr_mu_unlock(&fd->pollable_obj.po.mu); UNREF_BY(exec_ctx, fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); @@ -451,13 +451,13 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, * Pollable Definitions */ -static void pollable_init(pollable_t *p, polling_obj_type type) { +static void pollable_init(pollable *p, polling_obj_type type) { po_init(&p->po, type); p->root_worker = NULL; p->epfd = -1; } -static void pollable_destroy(pollable_t *p) { +static void pollable_destroy(pollable *p) { po_destroy(&p->po); if (p->epfd != -1) { close(p->epfd); @@ -466,7 +466,7 @@ static void pollable_destroy(pollable_t *p) { } /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable_t *p) { +static grpc_error *pollable_materialize(pollable *p) { if (p->epfd == -1) { int new_epfd = epoll_create1(EPOLL_CLOEXEC); if (new_epfd < 0) { @@ -492,7 +492,7 @@ static grpc_error *pollable_materialize(pollable_t *p) { } /* pollable must be materialized */ -static grpc_error *pollable_add_fd(pollable_t *p, grpc_fd *fd) { +static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "pollable_add_fd"; const int epfd = p->epfd; @@ -557,30 +557,33 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error_unused) { grpc_error *error = GRPC_ERROR_NONE; grpc_pollset *pollset = (grpc_pollset *)arg; - gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { - if (worker->pollable != &pollset->pollable) { - gpr_mu_lock(&worker->pollable->po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_lock(&worker->pollable_obj->po.mu); } if (worker->initialized_cv && worker != pollset->root_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kickall_via_cv %p (pollable %p vs %p)", - pollset, worker, &pollset->pollable, worker->pollable); + pollset, worker, &pollset->pollable_obj, + worker->pollable_obj); } worker->kicked = true; gpr_cv_signal(&worker->cv); } else { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kickall_via_wakeup %p (pollable %p vs %p)", - pollset, worker, &pollset->pollable, worker->pollable); + pollset, worker, &pollset->pollable_obj, + worker->pollable_obj); } - append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), + append_error(&error, + grpc_wakeup_fd_wakeup(&worker->pollable_obj->wakeup), "pollset_shutdown"); } - if (worker->pollable != &pollset->pollable) { - gpr_mu_unlock(&worker->pollable->po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker->pollable_obj->po.mu); } worker = worker->links[PWL_POLLSET].next; @@ -588,7 +591,7 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, } pollset->kick_alls_pending--; pollset_maybe_finish_shutdown(exec_ctx, pollset); - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); GRPC_LOG_IF_ERROR("kick_all", error); } @@ -599,7 +602,7 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -664,24 +667,24 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - pollable_t *p = pollset->current_pollable; - if (p != &pollset->pollable) { + pollable *p = pollset->current_pollable; + if (p != &pollset->pollable_obj) { gpr_mu_lock(&p->po.mu); } grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); - if (p != &pollset->pollable) { + if (p != &pollset->pollable_obj) { gpr_mu_unlock(&p->po.mu); } return error; } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - pollable_init(&pollset->pollable, PO_POLLSET); + pollable_init(&pollset->pollable_obj, PO_POLLSET); pollset->current_pollable = &g_empty_pollable; pollset->kicked_without_poller = false; pollset->shutdown_closure = NULL; pollset->root_worker = NULL; - *mu = &pollset->pollable.po.mu; + *mu = &pollset->pollable_obj.po.mu; } /* Convert a timespec to milliseconds: @@ -729,8 +732,8 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { static grpc_error *fd_become_pollable_locked(grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "fd_become_pollable"; - if (append_error(&error, pollable_materialize(&fd->pollable), err_desc)) { - append_error(&error, pollable_add_fd(&fd->pollable, fd), err_desc); + if (append_error(&error, pollable_materialize(&fd->pollable_obj), err_desc)) { + append_error(&error, pollable_add_fd(&fd->pollable_obj, fd), err_desc); } return error; } @@ -744,8 +747,8 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable_t *p) { - return p != &g_empty_pollable && p != &pollset->pollable; +static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { + return p != &g_empty_pollable && p != &pollset->pollable_obj; } static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, @@ -791,7 +794,7 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - pollable_destroy(&pollset->pollable); + pollable_destroy(&pollset->pollable_obj); if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); @@ -801,7 +804,7 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { } static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - pollable_t *p, gpr_timespec now, + pollable *p, gpr_timespec now, gpr_timespec deadline) { int timeout = poll_deadline_to_millis_timeout(deadline, now); @@ -883,68 +886,69 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker->initialized_cv = false; worker->kicked = false; worker->pollset = pollset; - worker->pollable = pollset->current_pollable; + worker->pollable_obj = pollset->current_pollable; - if (pollset_is_pollable_fd(pollset, worker->pollable)) { - REF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); + if (pollset_is_pollable_fd(pollset, worker->pollable_obj)) { + REF_BY((grpc_fd *)worker->pollable_obj, 2, "one_poll"); } worker_insert(&pollset->root_worker, PWL_POLLSET, worker); - if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { + if (!worker_insert(&worker->pollable_obj->root_worker, PWL_POLLABLE, + worker)) { worker->initialized_cv = true; gpr_cv_init(&worker->cv); - if (worker->pollable != &pollset->pollable) { - gpr_mu_unlock(&pollset->pollable.po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&pollset->pollable_obj.po.mu); } if (GRPC_TRACER_ON(grpc_polling_trace) && - worker->pollable->root_worker != worker) { + worker->pollable_obj->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, - worker->pollable, worker, + worker->pollable_obj, worker, poll_deadline_to_millis_timeout(deadline, *now)); } - while (do_poll && worker->pollable->root_worker != worker) { - if (gpr_cv_wait(&worker->cv, &worker->pollable->po.mu, deadline)) { + while (do_poll && worker->pollable_obj->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &worker->pollable_obj->po.mu, deadline)) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p timeout_wait %p w=%p", pollset, - worker->pollable, worker); + worker->pollable_obj, worker); } do_poll = false; } else if (worker->kicked) { if (GRPC_TRACER_ON(grpc_polling_trace)) { - gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, worker->pollable, - worker); + gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, + worker->pollable_obj, worker); } do_poll = false; } else if (GRPC_TRACER_ON(grpc_polling_trace) && - worker->pollable->root_worker != worker) { + worker->pollable_obj->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p spurious_wakeup %p w=%p", pollset, - worker->pollable, worker); + worker->pollable_obj, worker); } } - if (worker->pollable != &pollset->pollable) { - gpr_mu_unlock(&worker->pollable->po.mu); - gpr_mu_lock(&pollset->pollable.po.mu); - gpr_mu_lock(&worker->pollable->po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker->pollable_obj->po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); + gpr_mu_lock(&worker->pollable_obj->po.mu); } *now = gpr_now(now->clock_type); } return do_poll && pollset->shutdown_closure == NULL && - pollset->current_pollable == worker->pollable; + pollset->current_pollable == worker->pollable_obj; } static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { if (NEW_ROOT == - worker_remove(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { - gpr_cv_signal(&worker->pollable->root_worker->cv); + worker_remove(&worker->pollable_obj->root_worker, PWL_POLLABLE, worker)) { + gpr_cv_signal(&worker->pollable_obj->root_worker->cv); } if (worker->initialized_cv) { gpr_cv_destroy(&worker->cv); } - if (pollset_is_pollable_fd(pollset, worker->pollable)) { - UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable, 2, "one_poll"); + if (pollset_is_pollable_fd(pollset, worker->pollable_obj)) { + UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable_obj, 2, "one_poll"); } if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { pollset_maybe_finish_shutdown(exec_ctx, pollset); @@ -972,41 +976,41 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->kicked_without_poller = false; return GRPC_ERROR_NONE; } - if (pollset->current_pollable != &pollset->pollable) { + if (pollset->current_pollable != &pollset->pollable_obj) { gpr_mu_lock(&pollset->current_pollable->po.mu); } if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); - append_error(&error, pollable_materialize(worker.pollable), err_desc); - if (worker.pollable != &pollset->pollable) { - gpr_mu_unlock(&worker.pollable->po.mu); + append_error(&error, pollable_materialize(worker.pollable_obj), err_desc); + if (worker.pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker.pollable_obj->po.mu); } - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); if (pollset->event_cursor == pollset->event_count) { - append_error(&error, pollset_epoll(exec_ctx, pollset, worker.pollable, + append_error(&error, pollset_epoll(exec_ctx, pollset, worker.pollable_obj, now, deadline), err_desc); } append_error(&error, pollset_process_events(exec_ctx, pollset, false), err_desc); - gpr_mu_lock(&pollset->pollable.po.mu); - if (worker.pollable != &pollset->pollable) { - gpr_mu_lock(&worker.pollable->po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); + if (worker.pollable_obj != &pollset->pollable_obj) { + gpr_mu_lock(&worker.pollable_obj->po.mu); } gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); pollset_maybe_finish_shutdown(exec_ctx, pollset); } end_worker(exec_ctx, pollset, &worker, worker_hdl); - if (worker.pollable != &pollset->pollable) { - gpr_mu_unlock(&worker.pollable->po.mu); + if (worker.pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker.pollable_obj->po.mu); } if (grpc_exec_ctx_has_work(exec_ctx)) { - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); } return error; } @@ -1029,20 +1033,20 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, "PS:%p add fd %p; transition pollable from empty to fd", pollset, fd); } - /* empty pollable --> single fd pollable_t */ + /* empty pollable --> single fd pollable */ pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &fd->pollable; - if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu); + pollset->current_pollable = &fd->pollable_obj; + if (!fd_locked) gpr_mu_lock(&fd->pollable_obj.po.mu); append_error(&error, fd_become_pollable_locked(fd), err_desc); - if (!fd_locked) gpr_mu_unlock(&fd->pollable.po.mu); + if (!fd_locked) gpr_mu_unlock(&fd->pollable_obj.po.mu); REF_BY(fd, 2, "pollset_pollable"); - } else if (pollset->current_pollable == &pollset->pollable) { + } else if (pollset->current_pollable == &pollset->pollable_obj) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; already multipolling", pollset, fd); } append_error(&error, pollable_add_fd(pollset->current_pollable, fd), err_desc); - } else if (pollset->current_pollable != &fd->pollable) { + } else if (pollset->current_pollable != &fd->pollable_obj) { grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -1055,11 +1059,11 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_lfev_set_ready(exec_ctx, &had_fd->read_closure, "read"); grpc_lfev_set_ready(exec_ctx, &had_fd->write_closure, "write"); pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &pollset->pollable; - if (append_error(&error, pollable_materialize(&pollset->pollable), + pollset->current_pollable = &pollset->pollable_obj; + if (append_error(&error, pollable_materialize(&pollset->pollable_obj), err_desc)) { - pollable_add_fd(&pollset->pollable, had_fd); - pollable_add_fd(&pollset->pollable, fd); + pollable_add_fd(&pollset->pollable_obj, had_fd); + pollable_add_fd(&pollset->pollable_obj, fd); } GRPC_CLOSURE_SCHED(exec_ctx, GRPC_CLOSURE_CREATE(unref_fd_no_longer_poller, had_fd, @@ -1071,9 +1075,9 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { - gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); grpc_error *error = pollset_add_fd_locked(exec_ctx, pollset, fd, false); - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); GRPC_LOG_IF_ERROR("pollset_add_fd", error); } @@ -1095,7 +1099,7 @@ static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_fd *fd) { - po_join(exec_ctx, &pss->po, &fd->pollable.po); + po_join(exec_ctx, &pss->po, &fd->pollable_obj.po); } static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, @@ -1103,7 +1107,7 @@ static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_pollset *ps) { - po_join(exec_ctx, &pss->po, &ps->pollable.po); + po_join(exec_ctx, &pss->po, &ps->pollable_obj.po); } static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index de5aa7b6bec..e258719c4da 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -135,7 +135,7 @@ typedef struct batch_control { typedef struct { gpr_mu child_list_mu; grpc_call *first_child; -} parent_call_t; +} parent_call; typedef struct { grpc_call *parent; @@ -144,7 +144,7 @@ typedef struct { parent->mu */ grpc_call *sibling_next; grpc_call *sibling_prev; -} child_call_t; +} child_call; #define RECV_NONE ((gpr_atm)0) #define RECV_INITIAL_METADATA_FIRST ((gpr_atm)1) @@ -157,8 +157,8 @@ struct grpc_call { grpc_polling_entity pollent; grpc_channel *channel; gpr_timespec start_time; - /* parent_call_t* */ gpr_atm parent_call_atm; - child_call_t *child_call; + /* parent_call* */ gpr_atm parent_call_atm; + child_call *child; /* client or server call */ bool is_client; @@ -304,21 +304,21 @@ void *grpc_call_arena_alloc(grpc_call *call, size_t size) { return gpr_arena_alloc(call->arena, size); } -static parent_call_t *get_or_create_parent_call(grpc_call *call) { - parent_call_t *p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call *get_or_create_parent_call(grpc_call *call) { + parent_call *p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); if (p == NULL) { - p = (parent_call_t *)gpr_arena_alloc(call->arena, sizeof(*p)); + p = (parent_call *)gpr_arena_alloc(call->arena, sizeof(*p)); gpr_mu_init(&p->child_list_mu); if (!gpr_atm_rel_cas(&call->parent_call_atm, (gpr_atm)NULL, (gpr_atm)p)) { gpr_mu_destroy(&p->child_list_mu); - p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); + p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); } } return p; } -static parent_call_t *get_parent_call(grpc_call *call) { - return (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call *get_parent_call(grpc_call *call) { + return (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); } grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, @@ -377,24 +377,24 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, bool immediately_cancel = false; - if (args->parent_call != NULL) { - child_call_t *cc = call->child_call = - (child_call_t *)gpr_arena_alloc(arena, sizeof(child_call_t)); - call->child_call->parent = args->parent_call; + if (args->parent != NULL) { + child_call *cc = call->child = + (child_call *)gpr_arena_alloc(arena, sizeof(child_call)); + call->child->parent = args->parent; - GRPC_CALL_INTERNAL_REF(args->parent_call, "child"); + GRPC_CALL_INTERNAL_REF(args->parent, "child"); GPR_ASSERT(call->is_client); - GPR_ASSERT(!args->parent_call->is_client); + GPR_ASSERT(!args->parent->is_client); - parent_call_t *pc = get_or_create_parent_call(args->parent_call); + parent_call *pc = get_or_create_parent_call(args->parent); gpr_mu_lock(&pc->child_list_mu); if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) { send_deadline = gpr_time_min( gpr_convert_clock_type(send_deadline, - args->parent_call->send_deadline.clock_type), - args->parent_call->send_deadline); + args->parent->send_deadline.clock_type), + args->parent->send_deadline); } /* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with * GRPC_PROPAGATE_STATS_CONTEXT */ @@ -406,9 +406,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, "Census tracing propagation requested " "without Census context propagation")); } - grpc_call_context_set( - call, GRPC_CONTEXT_TRACING, - args->parent_call->context[GRPC_CONTEXT_TRACING].value, NULL); + grpc_call_context_set(call, GRPC_CONTEXT_TRACING, + args->parent->context[GRPC_CONTEXT_TRACING].value, + NULL); } else if (args->propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) { add_init_error(&error, GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Census context propagation requested " @@ -416,7 +416,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, } if (args->propagation_mask & GRPC_PROPAGATE_CANCELLATION) { call->cancellation_is_inherited = 1; - if (gpr_atm_acq_load(&args->parent_call->received_final_op_atm)) { + if (gpr_atm_acq_load(&args->parent->received_final_op_atm)) { immediately_cancel = true; } } @@ -426,9 +426,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, cc->sibling_next = cc->sibling_prev = call; } else { cc->sibling_next = pc->first_child; - cc->sibling_prev = pc->first_child->child_call->sibling_prev; - cc->sibling_next->child_call->sibling_prev = - cc->sibling_prev->child_call->sibling_next = call; + cc->sibling_prev = pc->first_child->child->sibling_prev; + cc->sibling_next->child->sibling_prev = + cc->sibling_prev->child->sibling_next = call; } gpr_mu_unlock(&pc->child_list_mu); @@ -533,7 +533,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - parent_call_t *pc = get_parent_call(c); + parent_call *pc = get_parent_call(c); if (pc != NULL) { gpr_mu_destroy(&pc->child_list_mu); } @@ -570,14 +570,14 @@ void grpc_call_ref(grpc_call *c) { gpr_ref(&c->ext_ref); } void grpc_call_unref(grpc_call *c) { if (!gpr_unref(&c->ext_ref)) return; - child_call_t *cc = c->child_call; + child_call *cc = c->child; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_call_unref", 0); GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (c)); if (cc) { - parent_call_t *pc = get_parent_call(cc->parent); + parent_call *pc = get_parent_call(cc->parent); gpr_mu_lock(&pc->child_list_mu); if (c == pc->first_child) { pc->first_child = cc->sibling_next; @@ -585,8 +585,8 @@ void grpc_call_unref(grpc_call *c) { pc->first_child = NULL; } } - cc->sibling_prev->child_call->sibling_next = cc->sibling_next; - cc->sibling_next->child_call->sibling_prev = cc->sibling_prev; + cc->sibling_prev->child->sibling_next = cc->sibling_next; + cc->sibling_next->child->sibling_prev = cc->sibling_prev; gpr_mu_unlock(&pc->child_list_mu); GRPC_CALL_INTERNAL_UNREF(&exec_ctx, cc->parent, "child"); } @@ -1309,14 +1309,14 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, /* propagate cancellation to any interested children */ gpr_atm_rel_store(&call->received_final_op_atm, 1); - parent_call_t *pc = get_parent_call(call); + parent_call *pc = get_parent_call(call); if (pc != NULL) { grpc_call *child; gpr_mu_lock(&pc->child_list_mu); child = pc->first_child; if (child != NULL) { do { - next_child_call = child->child_call->sibling_next; + next_child_call = child->child->sibling_next; if (child->cancellation_is_inherited) { GRPC_CALL_INTERNAL_REF(child, "propagate_cancel"); cancel_with_error(exec_ctx, child, STATUS_FROM_API_OVERRIDE, diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index d537637cbb8..c680139cf68 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -37,7 +37,7 @@ typedef void (*grpc_ioreq_completion_func)(grpc_exec_ctx *exec_ctx, typedef struct grpc_call_create_args { grpc_channel *channel; - grpc_call *parent_call; + grpc_call *parent; uint32_t propagation_mask; grpc_completion_queue *cq; diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 2f9b677c85e..48962e5e455 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -282,7 +282,7 @@ static grpc_call *grpc_channel_create_call_internal( grpc_call_create_args args; memset(&args, 0, sizeof(args)); args.channel = channel; - args.parent_call = parent_call; + args.parent = parent_call; args.propagation_mask = propagation_mask; args.cq = cq; args.pollset_set_alternative = pollset_set_alternative; From 8f798e9ca153f844033ce49f261eeb345bd09c4f Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 15:45:47 -0700 Subject: [PATCH 146/227] %s/current_pollable/current_pollable_obj --- src/core/lib/iomgr/ev_epollex_linux.c | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 9934f8d3299..51200fc0643 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -201,7 +201,7 @@ struct grpc_pollset_worker { struct grpc_pollset { pollable pollable_obj; - pollable *current_pollable; + pollable *current_pollable_obj; int kick_alls_pending; bool kicked_without_poller; grpc_closure *shutdown_closure; @@ -667,7 +667,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - pollable *p = pollset->current_pollable; + pollable *p = pollset->current_pollable_obj; if (p != &pollset->pollable_obj) { gpr_mu_lock(&p->po.mu); } @@ -680,7 +680,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollable_init(&pollset->pollable_obj, PO_POLLSET); - pollset->current_pollable = &g_empty_pollable; + pollset->current_pollable_obj = &g_empty_pollable; pollset->kicked_without_poller = false; pollset->shutdown_closure = NULL; pollset->root_worker = NULL; @@ -795,8 +795,8 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { pollable_destroy(&pollset->pollable_obj); - if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { - UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, + if (pollset_is_pollable_fd(pollset, pollset->current_pollable_obj)) { + UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable_obj, 2, "pollset_pollable"); } GRPC_LOG_IF_ERROR("pollset_process_events", @@ -886,7 +886,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker->initialized_cv = false; worker->kicked = false; worker->pollset = pollset; - worker->pollable_obj = pollset->current_pollable; + worker->pollable_obj = pollset->current_pollable_obj; if (pollset_is_pollable_fd(pollset, worker->pollable_obj)) { REF_BY((grpc_fd *)worker->pollable_obj, 2, "one_poll"); @@ -934,7 +934,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } return do_poll && pollset->shutdown_closure == NULL && - pollset->current_pollable == worker->pollable_obj; + pollset->current_pollable_obj == worker->pollable_obj; } static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -976,8 +976,8 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->kicked_without_poller = false; return GRPC_ERROR_NONE; } - if (pollset->current_pollable != &pollset->pollable_obj) { - gpr_mu_lock(&pollset->current_pollable->po.mu); + if (pollset->current_pollable_obj != &pollset->pollable_obj) { + gpr_mu_lock(&pollset->current_pollable_obj->po.mu); } if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); @@ -1027,7 +1027,7 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, bool fd_locked) { static const char *err_desc = "pollset_add_fd"; grpc_error *error = GRPC_ERROR_NONE; - if (pollset->current_pollable == &g_empty_pollable) { + if (pollset->current_pollable_obj == &g_empty_pollable) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; transition pollable from empty to fd", pollset, @@ -1035,19 +1035,19 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, } /* empty pollable --> single fd pollable */ pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &fd->pollable_obj; + pollset->current_pollable_obj = &fd->pollable_obj; if (!fd_locked) gpr_mu_lock(&fd->pollable_obj.po.mu); append_error(&error, fd_become_pollable_locked(fd), err_desc); if (!fd_locked) gpr_mu_unlock(&fd->pollable_obj.po.mu); REF_BY(fd, 2, "pollset_pollable"); - } else if (pollset->current_pollable == &pollset->pollable_obj) { + } else if (pollset->current_pollable_obj == &pollset->pollable_obj) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; already multipolling", pollset, fd); } - append_error(&error, pollable_add_fd(pollset->current_pollable, fd), + append_error(&error, pollable_add_fd(pollset->current_pollable_obj, fd), err_desc); - } else if (pollset->current_pollable != &fd->pollable_obj) { - grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; + } else if (pollset->current_pollable_obj != &fd->pollable_obj) { + grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable_obj; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; transition pollable from fd %p to multipoller", @@ -1059,7 +1059,7 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_lfev_set_ready(exec_ctx, &had_fd->read_closure, "read"); grpc_lfev_set_ready(exec_ctx, &had_fd->write_closure, "write"); pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &pollset->pollable_obj; + pollset->current_pollable_obj = &pollset->pollable_obj; if (append_error(&error, pollable_materialize(&pollset->pollable_obj), err_desc)) { pollable_add_fd(&pollset->pollable_obj, had_fd); From 989466171296e296867a3009f615a90d46acf28d Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 14 Sep 2017 15:21:16 -0700 Subject: [PATCH 147/227] Polish comment --- src/objective-c/GRPCClient/GRPCCall.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 3b0ef56c3af..df563ca36cf 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -170,9 +170,9 @@ extern id const kGRPCTrailersKey; @property (atomic, copy, readwrite) NSString *serverName; /** - * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. If set to - * positive, the gRPC call returns with status GRPCErrorCodeDeadlineExceeded. A negative value is - * not allowed. + * The timeout for the RPC call in seconds. If set to 0, the call will not timeout. If set to + * positive, the gRPC call returns with status GRPCErrorCodeDeadlineExceeded if it is not completed + * within \a timeout seconds. A negative value is not allowed. */ @property NSTimeInterval timeout; From 4b047a3bff92ff377205be38d9044b9d99e06fbb Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 21 Aug 2017 22:34:37 -0700 Subject: [PATCH 148/227] Construction of streams shouldn't require triggering async ops --- .../cpp/helloworld/greeter_async_client.cc | 12 +- .../cpp/helloworld/greeter_async_client2.cc | 14 +- include/grpc++/generic/generic_stub.h | 7 + include/grpc++/impl/codegen/async_stream.h | 136 +++-- .../grpc++/impl/codegen/async_unary_call.h | 41 +- src/compiler/cpp_generator.cc | 578 +++++++++++------- src/cpp/client/generic_stub.cc | 23 +- test/cpp/codegen/compiler_test_golden | 40 ++ test/cpp/codegen/compiler_test_mock_golden | 5 + test/cpp/qps/client_async.cc | 153 ++--- 10 files changed, 651 insertions(+), 358 deletions(-) diff --git a/examples/cpp/helloworld/greeter_async_client.cc b/examples/cpp/helloworld/greeter_async_client.cc index 64da044ea6f..ddf6c1aaf3d 100644 --- a/examples/cpp/helloworld/greeter_async_client.cc +++ b/examples/cpp/helloworld/greeter_async_client.cc @@ -60,11 +60,15 @@ class GreeterClient { // Storage for the status of the RPC upon completion. Status status; - // stub_->AsyncSayHello() performs the RPC call, returning an instance we - // store in "rpc". Because we are using the asynchronous API, we need to - // hold on to the "rpc" instance in order to get updates on the ongoing RPC. + // stub_->PrepareAsyncSayHello() creates an RPC object, returning + // an instance to store in "call" but does not actually start the RPC + // Because we are using the asynchronous API, we need to hold on to + // the "call" instance in order to get updates on the ongoing RPC. std::unique_ptr > rpc( - stub_->AsyncSayHello(&context, request, &cq)); + stub_->PrepareAsyncSayHello(&context, request, &cq)); + + // StartCall initiates the RPC call + rpc->StartCall(); // Request that, upon completion of the RPC, "reply" be updated with the // server's response; "status" with the indication of whether the operation diff --git a/examples/cpp/helloworld/greeter_async_client2.cc b/examples/cpp/helloworld/greeter_async_client2.cc index 4192546beff..3154e84d854 100644 --- a/examples/cpp/helloworld/greeter_async_client2.cc +++ b/examples/cpp/helloworld/greeter_async_client2.cc @@ -49,11 +49,15 @@ class GreeterClient { // Call object to store rpc data AsyncClientCall* call = new AsyncClientCall; - // stub_->AsyncSayHello() performs the RPC call, returning an instance to - // store in "call". Because we are using the asynchronous API, we need to - // hold on to the "call" instance in order to get updates on the ongoing RPC. - call->response_reader = stub_->AsyncSayHello(&call->context, request, &cq_); - + // stub_->PrepareAsyncSayHello() creates an RPC object, returning + // an instance to store in "call" but does not actually start the RPC + // Because we are using the asynchronous API, we need to hold on to + // the "call" instance in order to get updates on the ongoing RPC. + call->response_reader = + stub_->PrepareAsyncSayHello(&call->context, request, &cq_); + + // StartCall initiates the RPC call + call->response_reader->StartCall(); // Request that, upon completion of the RPC, "reply" be updated with the // server's response; "status" with the indication of whether the operation diff --git a/include/grpc++/generic/generic_stub.h b/include/grpc++/generic/generic_stub.h index 7b269d0887a..2b3ff59ea24 100644 --- a/include/grpc++/generic/generic_stub.h +++ b/include/grpc++/generic/generic_stub.h @@ -44,6 +44,13 @@ class GenericStub final { ClientContext* context, const grpc::string& method, CompletionQueue* cq, void* tag); + /// Setup a call to a named method \a method using \a context, but don't + /// start it. Let it be started explicitly with StartCall and a tag. + /// The return value only indicates whether or not registration of the call + /// succeeded (i.e. the call won't proceed if the return value is nullptr). + std::unique_ptr PrepareCall( + ClientContext* context, const grpc::string& method, CompletionQueue* cq); + private: std::shared_ptr channel_; }; diff --git a/include/grpc++/impl/codegen/async_stream.h b/include/grpc++/impl/codegen/async_stream.h index 9cf7ac30ddf..e60572fc939 100644 --- a/include/grpc++/impl/codegen/async_stream.h +++ b/include/grpc++/impl/codegen/async_stream.h @@ -35,6 +35,11 @@ class ClientAsyncStreamingInterface { public: virtual ~ClientAsyncStreamingInterface() {} + /// Start the call that was set up by the constructor, but only if the + /// constructor was invoked through the "Prepare" API which doesn't actually + /// start the call + virtual void StartCall(void* tag) = 0; + /// Request notification of the reading of the initial metadata. Completion /// will be notified by \a tag on the associated completion queue. /// This call is optional, but if it is used, it cannot be used concurrently @@ -156,20 +161,22 @@ class ClientAsyncReaderInterface : public ClientAsyncStreamingInterface, template class ClientAsyncReader final : public ClientAsyncReaderInterface { public: - /// Create a stream and write the first request out. + /// Create a stream object. + /// Write the first request out if \a start is set. /// \a tag will be notified on \a cq when the call has been started and - /// \a request has been written out. + /// \a request has been written out. If \a start is not set, \a tag must be + /// nullptr and the actual call must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. template static ClientAsyncReader* Create(ChannelInterface* channel, CompletionQueue* cq, const RpcMethod& method, ClientContext* context, const W& request, - void* tag) { + bool start, void* tag) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncReader))) - ClientAsyncReader(call, context, request, tag); + ClientAsyncReader(call, context, request, start, tag); } // always allocated against a call arena, no memory free required @@ -177,6 +184,12 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { assert(size == sizeof(ClientAsyncReader)); } + void StartCall(void* tag) override { + assert(!started_); + started_ = true; + StartCallInternal(tag); + } + /// See the \a ClientAsyncStreamingInterface.ReadInitialMetadata /// method for semantics. /// @@ -186,6 +199,7 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { /// calling code can access the received metadata through the /// \a ClientContext. void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_ops_.set_output_tag(tag); @@ -194,6 +208,7 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { } void Read(R* msg, void* tag) override { + assert(started_); read_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { read_ops_.RecvInitialMetadata(context_); @@ -208,6 +223,7 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata received from the server. void Finish(Status* status, void* tag) override { + assert(started_); finish_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_ops_.RecvInitialMetadata(context_); @@ -219,19 +235,28 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { private: template ClientAsyncReader(Call call, ClientContext* context, const W& request, - void* tag) - : context_(context), call_(call) { - init_ops_.set_output_tag(tag); - init_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + bool start, void* tag) + : context_(context), call_(call), started_(start) { // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(init_ops_.SendMessage(request).ok()); init_ops_.ClientSendClose(); + if (start) { + StartCallInternal(tag); + } else { + assert(tag == nullptr); + } + } + + void StartCallInternal(void* tag) { + init_ops_.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); + init_ops_.set_output_tag(tag); call_.PerformOps(&init_ops_); } ClientContext* context_; Call call_; + bool started_; CallOpSet init_ops_; CallOpSet meta_ops_; @@ -257,9 +282,12 @@ class ClientAsyncWriterInterface : public ClientAsyncStreamingInterface, template class ClientAsyncWriter final : public ClientAsyncWriterInterface { public: - /// Create a stream and write the first request out. + /// Create a stream object. + /// Start the RPC if \a start is set /// \a tag will be notified on \a cq when the call has been started (i.e. /// intitial metadata sent) and \a request has been written out. + /// If \a start is not set, \a tag must be nullptr and the actual call + /// must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. /// \a response will be filled in with the single expected response @@ -269,11 +297,11 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { static ClientAsyncWriter* Create(ChannelInterface* channel, CompletionQueue* cq, const RpcMethod& method, ClientContext* context, R* response, - void* tag) { + bool start, void* tag) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncWriter))) - ClientAsyncWriter(call, context, response, tag); + ClientAsyncWriter(call, context, response, start, tag); } // always allocated against a call arena, no memory free required @@ -281,6 +309,12 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { assert(size == sizeof(ClientAsyncWriter)); } + void StartCall(void* tag) override { + assert(!started_); + started_ = true; + StartCallInternal(tag); + } + /// See the \a ClientAsyncStreamingInterface.ReadInitialMetadata method for /// semantics. /// @@ -289,6 +323,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { /// associated with this call is updated, and the calling code can access /// the received metadata through the \a ClientContext. void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_ops_.set_output_tag(tag); @@ -297,6 +332,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { } void Write(const W& msg, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok()); @@ -304,6 +340,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { } void Write(const W& msg, WriteOptions options, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); if (options.is_last_message()) { options.set_buffer_hint(); @@ -315,6 +352,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { } void WritesDone(void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); write_ops_.ClientSendClose(); call_.PerformOps(&write_ops_); @@ -328,6 +366,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { /// - attempts to fill in the \a response parameter passed to this class's /// constructor with the server's response message. void Finish(Status* status, void* tag) override { + assert(started_); finish_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_ops_.RecvInitialMetadata(context_); @@ -338,25 +377,32 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { private: template - ClientAsyncWriter(Call call, ClientContext* context, R* response, void* tag) - : context_(context), call_(call) { + ClientAsyncWriter(Call call, ClientContext* context, R* response, bool start, + void* tag) + : context_(context), call_(call), started_(start) { finish_ops_.RecvMessage(response); finish_ops_.AllowNoMessage(); - // if corked bit is set in context, we buffer up the initial metadata to - // coalesce with later message to be sent. No op is performed. - if (context_->initial_metadata_corked_) { - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + if (start) { + StartCallInternal(tag); } else { + assert(tag == nullptr); + } + } + + void StartCallInternal(void* tag) { + write_ops_.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); + // if corked bit is set in context, we just keep the initial metadata + // buffered up to coalesce with later message send. No op is performed. + if (!context_->initial_metadata_corked_) { write_ops_.set_output_tag(tag); - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); call_.PerformOps(&write_ops_); } } ClientContext* context_; Call call_; + bool started_; CallOpSet meta_ops_; CallOpSet write_ops_; @@ -388,20 +434,23 @@ template class ClientAsyncReaderWriter final : public ClientAsyncReaderWriterInterface { public: - /// Create a stream and write the first request out. + /// Create a stream object. + /// Start the RPC request if \a start is set. /// \a tag will be notified on \a cq when the call has been started (i.e. - /// intitial metadata sent). + /// intitial metadata sent). If \a start is not set, \a tag must be + /// nullptr and the actual call must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. static ClientAsyncReaderWriter* Create(ChannelInterface* channel, CompletionQueue* cq, const RpcMethod& method, - ClientContext* context, void* tag) { + ClientContext* context, bool start, + void* tag) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncReaderWriter))) - ClientAsyncReaderWriter(call, context, tag); + ClientAsyncReaderWriter(call, context, start, tag); } // always allocated against a call arena, no memory free required @@ -409,6 +458,12 @@ class ClientAsyncReaderWriter final assert(size == sizeof(ClientAsyncReaderWriter)); } + void StartCall(void* tag) override { + assert(!started_); + started_ = true; + StartCallInternal(tag); + } + /// See the \a ClientAsyncStreamingInterface.ReadInitialMetadata method /// for semantics of this method. /// @@ -417,6 +472,7 @@ class ClientAsyncReaderWriter final /// is updated with it, and then the receiving initial metadata can /// be accessed through this \a ClientContext. void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_ops_.set_output_tag(tag); @@ -425,6 +481,7 @@ class ClientAsyncReaderWriter final } void Read(R* msg, void* tag) override { + assert(started_); read_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { read_ops_.RecvInitialMetadata(context_); @@ -434,6 +491,7 @@ class ClientAsyncReaderWriter final } void Write(const W& msg, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok()); @@ -441,6 +499,7 @@ class ClientAsyncReaderWriter final } void Write(const W& msg, WriteOptions options, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); if (options.is_last_message()) { options.set_buffer_hint(); @@ -452,6 +511,7 @@ class ClientAsyncReaderWriter final } void WritesDone(void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); write_ops_.ClientSendClose(); call_.PerformOps(&write_ops_); @@ -462,6 +522,7 @@ class ClientAsyncReaderWriter final /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata sent from the server. void Finish(Status* status, void* tag) override { + assert(started_); finish_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_ops_.RecvInitialMetadata(context_); @@ -471,23 +532,30 @@ class ClientAsyncReaderWriter final } private: - ClientAsyncReaderWriter(Call call, ClientContext* context, void* tag) - : context_(context), call_(call) { - if (context_->initial_metadata_corked_) { - // if corked bit is set in context, we buffer up the initial metadata to - // coalesce with later message to be sent. No op is performed. - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + ClientAsyncReaderWriter(Call call, ClientContext* context, bool start, + void* tag) + : context_(context), call_(call), started_(start) { + if (start) { + StartCallInternal(tag); } else { + assert(tag == nullptr); + } + } + + void StartCallInternal(void* tag) { + write_ops_.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); + // if corked bit is set in context, we just keep the initial metadata + // buffered up to coalesce with later message send. No op is performed. + if (!context_->initial_metadata_corked_) { write_ops_.set_output_tag(tag); - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); call_.PerformOps(&write_ops_); } } ClientContext* context_; Call call_; + bool started_; CallOpSet meta_ops_; CallOpSet> read_ops_; CallOpSet diff --git a/include/grpc++/impl/codegen/async_unary_call.h b/include/grpc++/impl/codegen/async_unary_call.h index f0f909686bc..e472f04f56e 100644 --- a/include/grpc++/impl/codegen/async_unary_call.h +++ b/include/grpc++/impl/codegen/async_unary_call.h @@ -32,13 +32,18 @@ namespace grpc { class CompletionQueue; extern CoreCodegenInterface* g_core_codegen_interface; -/// An interface relevant for async client side unary RPCS (which send +/// An interface relevant for async client side unary RPCs (which send /// one request message to a server and receive one response message). template class ClientAsyncResponseReaderInterface { public: virtual ~ClientAsyncResponseReaderInterface() {} + /// Start the call that was set up by the constructor, but only if the + /// constructor was invoked through the "Prepare" API which doesn't actually + /// start the call + virtual void StartCall() = 0; + /// Request notification of the reading of initial metadata. Completion /// will be notified by \a tag on the associated completion queue. /// This call is optional, but if it is used, it cannot be used concurrently @@ -70,9 +75,10 @@ template class ClientAsyncResponseReader final : public ClientAsyncResponseReaderInterface { public: - /// Start a call and write the request out. + /// Start a call and write the request out if \a start is set. /// \a tag will be notified on \a cq when the call has been started (i.e. /// intitial metadata sent) and \a request has been written out. + /// If \a start is not set, the actual call must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. template @@ -80,11 +86,11 @@ class ClientAsyncResponseReader final CompletionQueue* cq, const RpcMethod& method, ClientContext* context, - const W& request) { + const W& request, bool start) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncResponseReader))) - ClientAsyncResponseReader(call, context, request); + ClientAsyncResponseReader(call, context, request, start); } // always allocated against a call arena, no memory free required @@ -92,13 +98,20 @@ class ClientAsyncResponseReader final assert(size == sizeof(ClientAsyncResponseReader)); } + void StartCall() override { + assert(!started_); + started_ = true; + StartCallInternal(); + } + /// See \a ClientAsyncResponseReaderInterface::ReadInitialMetadata for /// semantics. /// /// Side effect: /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata sent from the server. - void ReadInitialMetadata(void* tag) { + void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_buf.set_output_tag(tag); @@ -111,7 +124,8 @@ class ClientAsyncResponseReader final /// Side effect: /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata sent from the server. - void Finish(R* msg, Status* status, void* tag) { + void Finish(R* msg, Status* status, void* tag) override { + assert(started_); finish_buf.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_buf.RecvInitialMetadata(context_); @@ -125,15 +139,22 @@ class ClientAsyncResponseReader final private: ClientContext* const context_; Call call_; + bool started_; template - ClientAsyncResponseReader(Call call, ClientContext* context, const W& request) - : context_(context), call_(call) { - init_buf.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + ClientAsyncResponseReader(Call call, ClientContext* context, const W& request, + bool start) + : context_(context), call_(call), started_(start) { + // Bind the metadata at time of StartCallInternal but set up the rest here // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(init_buf.SendMessage(request).ok()); init_buf.ClientSendClose(); + if (start) StartCallInternal(); + } + + void StartCallInternal() { + init_buf.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); call_.PerformOps(&init_buf); } diff --git a/src/compiler/cpp_generator.cc b/src/compiler/cpp_generator.cc index b09bf996774..c2db8eff714 100644 --- a/src/compiler/cpp_generator.cc +++ b/src/compiler/cpp_generator.cc @@ -165,25 +165,37 @@ void PrintHeaderClientMethodInterfaces( (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string method_params; // extra arguments to method + grpc::string raw_args; // extra arguments to raw version of method + } async_prefixes[] = {{"Async", ", void* tag", ", tag"}, + {"PrepareAsync", "", ""}}; + if (is_public) { if (method->NoStreaming()) { printer->Print( *vars, "virtual ::grpc::Status $Method$(::grpc::ClientContext* context, " "const $Request$& request, $Response$* response) = 0;\n"); - printer->Print(*vars, - "std::unique_ptr< " - "::grpc::ClientAsyncResponseReaderInterface< $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncResponseReaderInterface< $Response$>>(" - "Async$Method$Raw(context, request, cq));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "std::unique_ptr< " + "::grpc::ClientAsyncResponseReaderInterface< $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncResponseReaderInterface< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, @@ -197,19 +209,26 @@ void PrintHeaderClientMethodInterfaces( "($Method$Raw(context, response));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncWriterInterface< $Request$>>" - " Async$Method$(::grpc::ClientContext* context, $Response$* " - "response, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncWriterInterface< $Request$>>(" - "Async$Method$Raw(context, response, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncWriterInterface< $Request$>>" + " $AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "$Response$* " + "response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print(*vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncWriterInterface< $Request$>>(" + "$AsyncPrefix$$Method$Raw(context, response, " + "cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, @@ -223,19 +242,25 @@ void PrintHeaderClientMethodInterfaces( "($Method$Raw(context, request));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncReaderInterface< $Response$>> " - "Async$Method$(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncReaderInterface< $Response$>>(" - "Async$Method$Raw(context, request, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncReaderInterface< $Response$>> " + "$AsyncPrefix$$Method$(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncReaderInterface< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (method->BidiStreaming()) { printer->Print(*vars, "std::unique_ptr< ::grpc::ClientReaderWriterInterface< " @@ -249,61 +274,83 @@ void PrintHeaderClientMethodInterfaces( "$Method$Raw(context));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< " - "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print( - *vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>>(" - "Async$Method$Raw(context, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< " + "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } } else { if (method->NoStreaming()) { - printer->Print( - *vars, - "virtual ::grpc::ClientAsyncResponseReaderInterface< $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncResponseReaderInterface< $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) = 0;\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, "virtual ::grpc::ClientWriterInterface< $Request$>*" " $Method$Raw(" "::grpc::ClientContext* context, $Response$* response) = 0;\n"); - printer->Print(*vars, - "virtual ::grpc::ClientAsyncWriterInterface< $Request$>*" - " Async$Method$Raw(::grpc::ClientContext* context, " - "$Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncWriterInterface< $Request$>*" + " $AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "$Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) = 0;\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, "virtual ::grpc::ClientReaderInterface< $Response$>* $Method$Raw(" "::grpc::ClientContext* context, const $Request$& request) = 0;\n"); - printer->Print( - *vars, - "virtual ::grpc::ClientAsyncReaderInterface< $Response$>* " - "Async$Method$Raw(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncReaderInterface< $Response$>* " + "$AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) = 0;\n"); + } } else if (method->BidiStreaming()) { printer->Print(*vars, "virtual ::grpc::ClientReaderWriterInterface< $Request$, " "$Response$>* " "$Method$Raw(::grpc::ClientContext* context) = 0;\n"); - printer->Print(*vars, - "virtual ::grpc::ClientAsyncReaderWriterInterface< " - "$Request$, $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncReaderWriterInterface< " + "$Request$, $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) = 0;\n"); + } } } } @@ -315,25 +362,35 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, (*vars)["Method"] = method->name(); (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string method_params; // extra arguments to method + grpc::string raw_args; // extra arguments to raw version of method + } async_prefixes[] = {{"Async", ", void* tag", ", tag"}, + {"PrepareAsync", "", ""}}; + if (is_public) { if (method->NoStreaming()) { printer->Print( *vars, "::grpc::Status $Method$(::grpc::ClientContext* context, " "const $Request$& request, $Response$* response) override;\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncResponseReader< $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncResponseReader< $Response$>>(" - "Async$Method$Raw(context, request, cq));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncResponseReader< $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) {\n"); + printer->Indent(); + printer->Print(*vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncResponseReader< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, @@ -346,18 +403,24 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, "($Method$Raw(context, response));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print(*vars, - "std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>" - " Async$Method$(::grpc::ClientContext* context, " - "$Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print( - *vars, - "return std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>(" - "Async$Method$Raw(context, response, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print(*vars, + "std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>" + " $AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "$Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>(" + "$AsyncPrefix$$Method$Raw(context, response, " + "cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, @@ -371,19 +434,24 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, "($Method$Raw(context, request));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>> " - "Async$Method$(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print( - *vars, - "return std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>>(" - "Async$Method$Raw(context, request, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>> " + "$AsyncPrefix$$Method$(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (method->BidiStreaming()) { printer->Print( *vars, @@ -396,53 +464,80 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, "$Method$Raw(context));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print(*vars, - "std::unique_ptr< ::grpc::ClientAsyncReaderWriter< " - "$Request$, $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>>(" - "Async$Method$Raw(context, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print(*vars, + "std::unique_ptr< ::grpc::ClientAsyncReaderWriter< " + "$Request$, $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } } else { if (method->NoStreaming()) { - printer->Print(*vars, - "::grpc::ClientAsyncResponseReader< $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "::grpc::ClientAsyncResponseReader< $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) override;\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print(*vars, "::grpc::ClientWriter< $Request$>* $Method$Raw(" "::grpc::ClientContext* context, $Response$* response) " "override;\n"); - printer->Print(*vars, - "::grpc::ClientAsyncWriter< $Request$>* Async$Method$Raw(" - "::grpc::ClientContext* context, $Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "::grpc::ClientAsyncWriter< $Request$>* $AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, $Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) override;\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print(*vars, "::grpc::ClientReader< $Response$>* $Method$Raw(" "::grpc::ClientContext* context, const $Request$& request)" " override;\n"); - printer->Print( - *vars, - "::grpc::ClientAsyncReader< $Response$>* Async$Method$Raw(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "::grpc::ClientAsyncReader< $Response$>* $AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) override;\n"); + } } else if (method->BidiStreaming()) { printer->Print(*vars, "::grpc::ClientReaderWriter< $Request$, $Response$>* " "$Method$Raw(::grpc::ClientContext* context) override;\n"); - printer->Print(*vars, - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) override;\n"); + } } } } @@ -1077,6 +1172,13 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, (*vars)["Method"] = method->name(); (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string start; // bool literal expressed as string + grpc::string method_params; // extra arguments to method + grpc::string create_args; // extra arguments to creator + } async_prefixes[] = {{"Async", "true", ", void* tag", ", tag"}, + {"PrepareAsync", "false", "", ", nullptr"}}; if (method->NoStreaming()) { printer->Print(*vars, "::grpc::Status $ns$$Service$::Stub::$Method$(" @@ -1087,19 +1189,23 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context, request, response);\n" "}\n\n"); - printer->Print( - *vars, - "::grpc::ClientAsyncResponseReader< $Response$>* " - "$ns$$Service$::Stub::Async$Method$Raw(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) {\n"); - printer->Print(*vars, - " return " - "::grpc::ClientAsyncResponseReader< $Response$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, request);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + printer->Print(*vars, + "::grpc::ClientAsyncResponseReader< $Response$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(::grpc::" + "ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) {\n"); + printer->Print(*vars, + " return " + "::grpc::ClientAsyncResponseReader< $Response$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, request, $AsyncStart$);\n" + "}\n\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print(*vars, "::grpc::ClientWriter< $Request$>* " @@ -1111,17 +1217,23 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context, response);\n" "}\n\n"); - printer->Print(*vars, - "::grpc::ClientAsyncWriter< $Request$>* " - "$ns$$Service$::Stub::Async$Method$Raw(" - "::grpc::ClientContext* context, $Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Print(*vars, - " return ::grpc::ClientAsyncWriter< $Request$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, response, tag);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncCreateArgs"] = async_prefix.create_args; + printer->Print(*vars, + "::grpc::ClientAsyncWriter< $Request$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, $Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Print(*vars, + " return ::grpc::ClientAsyncWriter< $Request$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, response, $AsyncStart$$AsyncCreateArgs$);\n" + "}\n\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, @@ -1134,17 +1246,24 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context, request);\n" "}\n\n"); - printer->Print(*vars, - "::grpc::ClientAsyncReader< $Response$>* " - "$ns$$Service$::Stub::Async$Method$Raw(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Print(*vars, - " return ::grpc::ClientAsyncReader< $Response$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, request, tag);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncCreateArgs"] = async_prefix.create_args; + printer->Print( + *vars, + "::grpc::ClientAsyncReader< $Response$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Print(*vars, + " return ::grpc::ClientAsyncReader< $Response$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, request, $AsyncStart$$AsyncCreateArgs$);\n" + "}\n\n"); + } } else if (method->BidiStreaming()) { printer->Print( *vars, @@ -1157,19 +1276,25 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context);\n" "}\n\n"); - printer->Print( - *vars, - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " - "$ns$$Service$::Stub::Async$Method$Raw(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Print( - *vars, - " return " - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, tag);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncCreateArgs"] = async_prefix.create_args; + printer->Print(*vars, + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(::grpc::" + "ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Print( + *vars, + " return " + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, $AsyncStart$$AsyncCreateArgs$);\n" + "}\n\n"); + } } } @@ -1460,50 +1585,79 @@ void PrintMockClientMethods(grpc_generator::Printer *printer, (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string method_params; // extra arguments to method + int extra_method_param_count; + } async_prefixes[] = {{"Async", ", void* tag", 1}, {"PrepareAsync", "", 0}}; + if (method->NoStreaming()) { printer->Print( *vars, "MOCK_METHOD3($Method$, ::grpc::Status(::grpc::ClientContext* context, " "const $Request$& request, $Response$* response));\n"); - printer->Print(*vars, - "MOCK_METHOD3(Async$Method$Raw, " - "::grpc::ClientAsyncResponseReaderInterface< $Response$>*" - "(::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "MOCK_METHOD3($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncResponseReaderInterface< $Response$>*" + "(::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq));\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, "MOCK_METHOD2($Method$Raw, " "::grpc::ClientWriterInterface< $Request$>*" "(::grpc::ClientContext* context, $Response$* response));\n"); - printer->Print(*vars, - "MOCK_METHOD4(Async$Method$Raw, " - "::grpc::ClientAsyncWriterInterface< $Request$>*" - "(::grpc::ClientContext* context, $Response$* response, " - "::grpc::CompletionQueue* cq, void* tag));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["MockArgs"] = + std::to_string(3 + async_prefix.extra_method_param_count); + printer->Print(*vars, + "MOCK_METHOD$MockArgs$($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncWriterInterface< $Request$>*" + "(::grpc::ClientContext* context, $Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$));\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, "MOCK_METHOD2($Method$Raw, " "::grpc::ClientReaderInterface< $Response$>*" "(::grpc::ClientContext* context, const $Request$& request));\n"); - printer->Print(*vars, - "MOCK_METHOD4(Async$Method$Raw, " - "::grpc::ClientAsyncReaderInterface< $Response$>*" - "(::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["MockArgs"] = + std::to_string(3 + async_prefix.extra_method_param_count); + printer->Print( + *vars, + "MOCK_METHOD$MockArgs$($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncReaderInterface< $Response$>*" + "(::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$));\n"); + } } else if (method->BidiStreaming()) { printer->Print( *vars, "MOCK_METHOD1($Method$Raw, " "::grpc::ClientReaderWriterInterface< $Request$, $Response$>*" "(::grpc::ClientContext* context));\n"); - printer->Print( - *vars, - "MOCK_METHOD3(Async$Method$Raw, " - "::grpc::ClientAsyncReaderWriterInterface<$Request$, $Response$>*" - "(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, " - "void* tag));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["MockArgs"] = + std::to_string(2 + async_prefix.extra_method_param_count); + printer->Print( + *vars, + "MOCK_METHOD$MockArgs$($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncReaderWriterInterface<$Request$, $Response$>*" + "(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq" + "$AsyncMethodParams$));\n"); + } } } diff --git a/src/cpp/client/generic_stub.cc b/src/cpp/client/generic_stub.cc index 66b1ef0e39c..de2e449fe8f 100644 --- a/src/cpp/client/generic_stub.cc +++ b/src/cpp/client/generic_stub.cc @@ -22,14 +22,29 @@ namespace grpc { +namespace { +std::unique_ptr CallInternal( + ChannelInterface* channel, ClientContext* context, + const grpc::string& method, CompletionQueue* cq, bool start, void* tag) { + return std::unique_ptr( + GenericClientAsyncReaderWriter::Create( + channel, cq, RpcMethod(method.c_str(), RpcMethod::BIDI_STREAMING), + context, start, tag)); +} + +} // namespace + // begin a call to a named method std::unique_ptr GenericStub::Call( ClientContext* context, const grpc::string& method, CompletionQueue* cq, void* tag) { - return std::unique_ptr( - GenericClientAsyncReaderWriter::Create( - channel_.get(), cq, - RpcMethod(method.c_str(), RpcMethod::BIDI_STREAMING), context, tag)); + return CallInternal(channel_.get(), context, method, cq, true, tag); +} + +// setup a call to a named method +std::unique_ptr GenericStub::PrepareCall( + ClientContext* context, const grpc::string& method, CompletionQueue* cq) { + return CallInternal(channel_.get(), context, method, cq, false, nullptr); } } // namespace grpc diff --git a/test/cpp/codegen/compiler_test_golden b/test/cpp/codegen/compiler_test_golden index b43c27f3f77..3d664e88254 100644 --- a/test/cpp/codegen/compiler_test_golden +++ b/test/cpp/codegen/compiler_test_golden @@ -65,6 +65,9 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> AsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(AsyncMethodA1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> PrepareAsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodA1Raw(context, request, cq)); + } // MethodA1 trailing comment 1 // MethodA2 detached leading comment 1 // @@ -76,6 +79,9 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>> AsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>>(AsyncMethodA2Raw(context, response, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>> PrepareAsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>>(PrepareAsyncMethodA2Raw(context, response, cq)); + } // MethodA2 trailing comment 1 // Method A3 leading comment 1 std::unique_ptr< ::grpc::ClientReaderInterface< ::grpc::testing::Response>> MethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request) { @@ -84,6 +90,9 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>> AsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>>(AsyncMethodA3Raw(context, request, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>> PrepareAsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodA3Raw(context, request, cq)); + } // Method A3 trailing comment 1 // Method A4 leading comment 1 std::unique_ptr< ::grpc::ClientReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>> MethodA4(::grpc::ClientContext* context) { @@ -92,15 +101,22 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>> AsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>>(AsyncMethodA4Raw(context, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>> PrepareAsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>>(PrepareAsyncMethodA4Raw(context, cq)); + } // Method A4 trailing comment 1 private: virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* AsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientWriterInterface< ::grpc::testing::Request>* MethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response) = 0; virtual ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>* AsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) = 0; + virtual ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>* PrepareAsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientReaderInterface< ::grpc::testing::Response>* MethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request) = 0; virtual ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>* AsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) = 0; + virtual ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>* MethodA4Raw(::grpc::ClientContext* context) = 0; virtual ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>* AsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) = 0; + virtual ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>* PrepareAsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) = 0; }; class Stub final : public StubInterface { public: @@ -109,34 +125,50 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> AsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(AsyncMethodA1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> PrepareAsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(PrepareAsyncMethodA1Raw(context, request, cq)); + } std::unique_ptr< ::grpc::ClientWriter< ::grpc::testing::Request>> MethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response) { return std::unique_ptr< ::grpc::ClientWriter< ::grpc::testing::Request>>(MethodA2Raw(context, response)); } std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>> AsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>>(AsyncMethodA2Raw(context, response, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>> PrepareAsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>>(PrepareAsyncMethodA2Raw(context, response, cq)); + } std::unique_ptr< ::grpc::ClientReader< ::grpc::testing::Response>> MethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request) { return std::unique_ptr< ::grpc::ClientReader< ::grpc::testing::Response>>(MethodA3Raw(context, request)); } std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>> AsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>>(AsyncMethodA3Raw(context, request, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>> PrepareAsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>>(PrepareAsyncMethodA3Raw(context, request, cq)); + } std::unique_ptr< ::grpc::ClientReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> MethodA4(::grpc::ClientContext* context) { return std::unique_ptr< ::grpc::ClientReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(MethodA4Raw(context)); } std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> AsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(AsyncMethodA4Raw(context, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> PrepareAsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(PrepareAsyncMethodA4Raw(context, cq)); + } private: std::shared_ptr< ::grpc::ChannelInterface> channel_; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* AsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* PrepareAsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientWriter< ::grpc::testing::Request>* MethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response) override; ::grpc::ClientAsyncWriter< ::grpc::testing::Request>* AsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) override; + ::grpc::ClientAsyncWriter< ::grpc::testing::Request>* PrepareAsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) override; ::grpc::ClientReader< ::grpc::testing::Response>* MethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request) override; ::grpc::ClientAsyncReader< ::grpc::testing::Response>* AsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) override; + ::grpc::ClientAsyncReader< ::grpc::testing::Response>* PrepareAsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>* MethodA4Raw(::grpc::ClientContext* context) override; ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>* AsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) override; + ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>* PrepareAsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) override; const ::grpc::RpcMethod rpcmethod_MethodA1_; const ::grpc::RpcMethod rpcmethod_MethodA2_; const ::grpc::RpcMethod rpcmethod_MethodA3_; @@ -372,9 +404,13 @@ class ServiceB final { std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> AsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(AsyncMethodB1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> PrepareAsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodB1Raw(context, request, cq)); + } // MethodB1 trailing comment 1 private: virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* AsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; }; class Stub final : public StubInterface { public: @@ -383,10 +419,14 @@ class ServiceB final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> AsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(AsyncMethodB1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> PrepareAsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(PrepareAsyncMethodB1Raw(context, request, cq)); + } private: std::shared_ptr< ::grpc::ChannelInterface> channel_; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* AsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* PrepareAsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; const ::grpc::RpcMethod rpcmethod_MethodB1_; }; static std::unique_ptr NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options = ::grpc::StubOptions()); diff --git a/test/cpp/codegen/compiler_test_mock_golden b/test/cpp/codegen/compiler_test_mock_golden index 8e4b4d59112..f97c2dd83a4 100644 --- a/test/cpp/codegen/compiler_test_mock_golden +++ b/test/cpp/codegen/compiler_test_mock_golden @@ -15,18 +15,23 @@ class MockServiceAStub : public ServiceA::StubInterface { public: MOCK_METHOD3(MethodA1, ::grpc::Status(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::testing::Response* response)); MOCK_METHOD3(AsyncMethodA1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); + MOCK_METHOD3(PrepareAsyncMethodA1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); MOCK_METHOD2(MethodA2Raw, ::grpc::ClientWriterInterface< ::grpc::testing::Request>*(::grpc::ClientContext* context, ::grpc::testing::Response* response)); MOCK_METHOD4(AsyncMethodA2Raw, ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>*(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag)); + MOCK_METHOD3(PrepareAsyncMethodA2Raw, ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>*(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq)); MOCK_METHOD2(MethodA3Raw, ::grpc::ClientReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request)); MOCK_METHOD4(AsyncMethodA3Raw, ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag)); + MOCK_METHOD3(PrepareAsyncMethodA3Raw, ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); MOCK_METHOD1(MethodA4Raw, ::grpc::ClientReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>*(::grpc::ClientContext* context)); MOCK_METHOD3(AsyncMethodA4Raw, ::grpc::ClientAsyncReaderWriterInterface<::grpc::testing::Request, ::grpc::testing::Response>*(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag)); + MOCK_METHOD2(PrepareAsyncMethodA4Raw, ::grpc::ClientAsyncReaderWriterInterface<::grpc::testing::Request, ::grpc::testing::Response>*(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq)); }; class MockServiceBStub : public ServiceB::StubInterface { public: MOCK_METHOD3(MethodB1, ::grpc::Status(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::testing::Response* response)); MOCK_METHOD3(AsyncMethodB1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); + MOCK_METHOD3(PrepareAsyncMethodB1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); }; } // namespace grpc diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 912c871482a..1a4438047dd 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -56,11 +56,6 @@ class ClientRpcContext { } virtual void Start(CompletionQueue* cq, const ClientConfig& config) = 0; - void lock() { mu_.lock(); } - void unlock() { mu_.unlock(); } - - private: - std::mutex mu_; }; template @@ -73,7 +68,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::unique_ptr>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, CompletionQueue*)> - start_req, + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -83,7 +78,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { next_state_(State::READY), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextUnaryImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq); @@ -92,7 +87,8 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { switch (next_state_) { case State::READY: start_ = UsageTimer::Now(); - response_reader_ = start_req_(stub_, &context_, req_, cq_); + response_reader_ = prepare_req_(stub_, &context_, req_, cq_); + response_reader_->StartCall(); next_state_ = State::RESP_DONE; response_reader_->Finish(&response_, &status_, ClientRpcContext::tag(this)); @@ -111,8 +107,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextUnaryImpl(stub_, req_, next_issue_, - start_req_, callback_); - std::lock_guard lclone(*clone); + prepare_req_, callback_); clone->StartInternal(cq); } @@ -130,7 +125,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, CompletionQueue*)> - start_req_; + prepare_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -252,29 +247,13 @@ class AsyncClient : public ClientImpl { // this thread isn't supposed to shut down std::lock_guard l(shutdown_state_[thread_idx]->mutex); if (shutdown_state_[thread_idx]->shutdown) { - // We want to delete the context. However, it is possible that - // another thread that just initiated an action on this - // context still has its lock even though the action on the - // context has completed. To delay for that, just grab the - // lock for serialization. Take a new scope. - { std::lock_guard lctx(*ctx); } delete ctx; return true; } - bool del = false; - - // Create a new scope for a lock_guard'ed region - { - std::lock_guard lctx(*ctx); - if (!ctx->RunNextState(ok, entry)) { - // The RPC and callback are done, so clone the ctx - // and kickstart the new one - ctx->StartNewClone(cli_cqs_[cq_[thread_idx]].get()); - // set the old version to delete - del = true; - } - } - if (del) { + if (!ctx->RunNextState(ok, entry)) { + // The RPC and callback are done, so clone the ctx + // and kickstart the new one + ctx->StartNewClone(cli_cqs_[cq_[thread_idx]].get()); delete ctx; } return true; @@ -311,15 +290,15 @@ class AsyncUnaryClient final entry->set_status(s.error_code()); } static std::unique_ptr> - StartReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - const SimpleRequest& request, CompletionQueue* cq) { - return stub->AsyncUnaryCall(ctx, request, cq); + PrepareReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, + const SimpleRequest& request, CompletionQueue* cq) { + return stub->PrepareAsyncUnaryCall(ctx, request, cq); }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, std::function next_issue, const SimpleRequest& req) { return new ClientRpcContextUnaryImpl( - stub, req, next_issue, AsyncUnaryClient::StartReq, + stub, req, next_issue, AsyncUnaryClient::PrepareReq, AsyncUnaryClient::CheckDone); } }; @@ -332,9 +311,8 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { std::function next_issue, std::function>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, - void*)> - start_req, + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*)> + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -344,7 +322,7 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextStreamingPingPongImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq, config.messages_per_stream()); @@ -407,8 +385,7 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextStreamingPingPongImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } @@ -432,10 +409,10 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { State next_state_; std::function callback_; std::function next_issue_; - std::function>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, void*)> - start_req_; + std::function< + std::unique_ptr>( + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -449,8 +426,9 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { cq_ = cq; messages_per_stream_ = messages_per_stream; messages_issued_ = 0; + stream_ = prepare_req_(stub_, &context_, cq); next_state_ = State::STREAM_IDLE; - stream_ = start_req_(stub_, &context_, cq, ClientRpcContext::tag(this)); + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -469,9 +447,9 @@ class AsyncStreamingPingPongClient final static void CheckDone(grpc::Status s, SimpleResponse* response) {} static std::unique_ptr< grpc::ClientAsyncReaderWriter> - StartReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - CompletionQueue* cq, void* tag) { - auto stream = stub->AsyncStreamingCall(ctx, cq, tag); + PrepareReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, + CompletionQueue* cq) { + auto stream = stub->PrepareAsyncStreamingCall(ctx, cq); return stream; }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, @@ -479,7 +457,7 @@ class AsyncStreamingPingPongClient final const SimpleRequest& req) { return new ClientRpcContextStreamingPingPongImpl( - stub, req, next_issue, AsyncStreamingPingPongClient::StartReq, + stub, req, next_issue, AsyncStreamingPingPongClient::PrepareReq, AsyncStreamingPingPongClient::CheckDone); } }; @@ -492,8 +470,8 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { std::function next_issue, std::function>( BenchmarkService::Stub*, grpc::ClientContext*, ResponseType*, - CompletionQueue*, void*)> - start_req, + CompletionQueue*)> + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -503,7 +481,7 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextStreamingFromClientImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq); @@ -546,8 +524,7 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextStreamingFromClientImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } @@ -570,17 +547,17 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { std::function next_issue_; std::function>( BenchmarkService::Stub*, grpc::ClientContext*, ResponseType*, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr> stream_; void StartInternal(CompletionQueue* cq) { cq_ = cq; - stream_ = start_req_(stub_, &context_, &response_, cq, - ClientRpcContext::tag(this)); + stream_ = prepare_req_(stub_, &context_, &response_, cq); next_state_ = State::STREAM_IDLE; + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -597,10 +574,10 @@ class AsyncStreamingFromClientClient final private: static void CheckDone(grpc::Status s, SimpleResponse* response) {} - static std::unique_ptr> StartReq( + static std::unique_ptr> PrepareReq( BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - SimpleResponse* resp, CompletionQueue* cq, void* tag) { - auto stream = stub->AsyncStreamingFromClient(ctx, resp, cq, tag); + SimpleResponse* resp, CompletionQueue* cq) { + auto stream = stub->PrepareAsyncStreamingFromClient(ctx, resp, cq); return stream; }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, @@ -608,7 +585,7 @@ class AsyncStreamingFromClientClient final const SimpleRequest& req) { return new ClientRpcContextStreamingFromClientImpl( - stub, req, next_issue, AsyncStreamingFromClientClient::StartReq, + stub, req, next_issue, AsyncStreamingFromClientClient::PrepareReq, AsyncStreamingFromClientClient::CheckDone); } }; @@ -621,8 +598,8 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { std::function next_issue, std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*, void*)> - start_req, + CompletionQueue*)> + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -632,7 +609,7 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextStreamingFromServerImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq); @@ -664,8 +641,7 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextStreamingFromServerImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } @@ -682,8 +658,8 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { std::function next_issue_; std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr> stream_; @@ -691,9 +667,9 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { void StartInternal(CompletionQueue* cq) { // TODO(vjpai): Add support to rate-pace this cq_ = cq; + stream_ = prepare_req_(stub_, &context_, req_, cq); next_state_ = State::STREAM_IDLE; - stream_ = - start_req_(stub_, &context_, req_, cq, ClientRpcContext::tag(this)); + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -710,10 +686,10 @@ class AsyncStreamingFromServerClient final private: static void CheckDone(grpc::Status s, SimpleResponse* response) {} - static std::unique_ptr> StartReq( + static std::unique_ptr> PrepareReq( BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - const SimpleRequest& req, CompletionQueue* cq, void* tag) { - auto stream = stub->AsyncStreamingFromServer(ctx, req, cq, tag); + const SimpleRequest& req, CompletionQueue* cq) { + auto stream = stub->PrepareAsyncStreamingFromServer(ctx, req, cq); return stream; }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, @@ -721,7 +697,7 @@ class AsyncStreamingFromServerClient final const SimpleRequest& req) { return new ClientRpcContextStreamingFromServerImpl( - stub, req, next_issue, AsyncStreamingFromServerClient::StartReq, + stub, req, next_issue, AsyncStreamingFromServerClient::PrepareReq, AsyncStreamingFromServerClient::CheckDone); } }; @@ -733,8 +709,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function next_issue, std::function( grpc::GenericStub*, grpc::ClientContext*, - const grpc::string& method_name, CompletionQueue*, void*)> - start_req, + const grpc::string& method_name, CompletionQueue*)> + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -744,7 +720,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextGenericStreamingImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq, config.messages_per_stream()); @@ -807,8 +783,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextGenericStreamingImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } @@ -834,8 +809,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function next_issue_; std::function( grpc::GenericStub*, grpc::ClientContext*, const grpc::string&, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr stream_; @@ -850,9 +825,9 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { "/grpc.testing.BenchmarkService/StreamingCall"); messages_per_stream_ = messages_per_stream; messages_issued_ = 0; + stream_ = prepare_req_(stub_, &context_, kMethodName, cq); next_state_ = State::STREAM_IDLE; - stream_ = start_req_(stub_, &context_, kMethodName, cq, - ClientRpcContext::tag(this)); + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -874,17 +849,17 @@ class GenericAsyncStreamingClient final private: static void CheckDone(grpc::Status s, ByteBuffer* response) {} - static std::unique_ptr StartReq( + static std::unique_ptr PrepareReq( grpc::GenericStub* stub, grpc::ClientContext* ctx, - const grpc::string& method_name, CompletionQueue* cq, void* tag) { - auto stream = stub->Call(ctx, method_name, cq, tag); + const grpc::string& method_name, CompletionQueue* cq) { + auto stream = stub->PrepareCall(ctx, method_name, cq); return stream; }; static ClientRpcContext* SetupCtx(grpc::GenericStub* stub, std::function next_issue, const ByteBuffer& req) { return new ClientRpcContextGenericStreamingImpl( - stub, req, next_issue, GenericAsyncStreamingClient::StartReq, + stub, req, next_issue, GenericAsyncStreamingClient::PrepareReq, GenericAsyncStreamingClient::CheckDone); } }; From 67fa513fbaaaffc9fc8b9bd1328c20cbcbc7746d Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Sun, 30 Jul 2017 00:12:23 -0700 Subject: [PATCH 149/227] make sure calls created in ruby unit tests are completed timely --- src/ruby/spec/client_server_spec.rb | 270 +++++++++++++++------- src/ruby/spec/generic/active_call_spec.rb | 57 ++++- 2 files changed, 237 insertions(+), 90 deletions(-) diff --git a/src/ruby/spec/client_server_spec.rb b/src/ruby/spec/client_server_spec.rb index 1a9b47e2c3b..adab8c9d14d 100644 --- a/src/ruby/spec/client_server_spec.rb +++ b/src/ruby/spec/client_server_spec.rb @@ -29,14 +29,18 @@ shared_context 'setup: tags' do expect(recvd_rpc).to_not eq nil server_call = recvd_rpc.call ops = { CallOps::SEND_INITIAL_METADATA => metadata } - svr_batch = server_call.run_batch(ops) - expect(svr_batch.send_metadata).to be true + server_batch = server_call.run_batch(ops) + expect(server_batch.send_metadata).to be true server_call end def new_client_call @ch.create_call(nil, nil, '/method', nil, deadline) end + + def ok_status + Struct::Status.new(StatusCodes::OK, 'OK') + end end shared_examples 'basic GRPC message delivery is OK' do @@ -70,19 +74,32 @@ shared_examples 'basic GRPC message delivery is OK' do client_ops = { CallOps::SEND_INITIAL_METADATA => {}, - CallOps::SEND_MESSAGE => sent_message + CallOps::SEND_MESSAGE => sent_message, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true + expect(client_batch.send_close).to be true # confirm the server can read the inbound message server_thread.join server_ops = { - CallOps::RECV_MESSAGE => nil + CallOps::RECV_MESSAGE => nil, + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq(sent_message) + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq(sent_message) + expect(server_batch.send_close).to be true + expect(server_batch.send_status).to be true + + # finish the call + final_client_batch = call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end it 'responses written by servers are received by the client' do @@ -95,21 +112,36 @@ shared_examples 'basic GRPC message delivery is OK' do client_ops = { CallOps::SEND_INITIAL_METADATA => {}, - CallOps::SEND_MESSAGE => sent_message + CallOps::SEND_MESSAGE => sent_message, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true + expect(client_batch.send_close).to be true # confirm the server can read the inbound message server_thread.join server_ops = { CallOps::RECV_MESSAGE => nil, - CallOps::SEND_MESSAGE => reply_text + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_MESSAGE => reply_text, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq(sent_message) - expect(svr_batch.send_message).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq(sent_message) + expect(server_batch.send_close).to be true + expect(server_batch.send_message).to be true + expect(server_batch.send_status).to be true + + # finish the call + final_client_batch = call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_MESSAGE => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.message).to eq(reply_text) + expect(final_client_batch.status.code).to eq(0) end it 'compressed messages can be sent and received' do @@ -125,30 +157,37 @@ shared_examples 'basic GRPC message delivery is OK' do client_ops = { CallOps::SEND_INITIAL_METADATA => md, - CallOps::SEND_MESSAGE => long_request_str + CallOps::SEND_MESSAGE => long_request_str, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true + expect(client_batch.send_close).to be true # confirm the server can read the inbound message server_thread.join server_ops = { CallOps::RECV_MESSAGE => nil, - CallOps::SEND_MESSAGE => long_response_str + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_MESSAGE => long_response_str, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq(long_request_str) - expect(svr_batch.send_message).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq(long_request_str) + expect(server_batch.send_close).to be true + expect(server_batch.send_message).to be true + expect(server_batch.send_status).to be true client_ops = { - CallOps::SEND_CLOSE_FROM_CLIENT => nil, CallOps::RECV_INITIAL_METADATA => nil, - CallOps::RECV_MESSAGE => nil + CallOps::RECV_MESSAGE => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_close).to be true - expect(batch_result.message).to eq long_response_str + final_client_batch = call.run_batch(client_ops) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.message).to eq long_response_str + expect(final_client_batch.status.code).to eq(0) end it 'servers can ignore a client write and send a status' do @@ -161,11 +200,13 @@ shared_examples 'basic GRPC message delivery is OK' do client_ops = { CallOps::SEND_INITIAL_METADATA => {}, - CallOps::SEND_MESSAGE => sent_message + CallOps::SEND_MESSAGE => sent_message, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true + expect(client_batch.send_close).to be true # confirm the server can read the inbound message the_status = Struct::Status.new(StatusCodes::OK, 'OK') @@ -173,9 +214,15 @@ shared_examples 'basic GRPC message delivery is OK' do server_ops = { CallOps::SEND_STATUS_FROM_SERVER => the_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq nil - expect(svr_batch.send_status).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq nil + expect(server_batch.send_status).to be true + + final_client_batch = call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end it 'completes calls by sending status to client and server' do @@ -190,9 +237,9 @@ shared_examples 'basic GRPC message delivery is OK' do CallOps::SEND_INITIAL_METADATA => {}, CallOps::SEND_MESSAGE => sent_message } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true # confirm the server can read the inbound message and respond the_status = Struct::Status.new(StatusCodes::OK, 'OK', {}) @@ -202,10 +249,10 @@ shared_examples 'basic GRPC message delivery is OK' do CallOps::SEND_MESSAGE => reply_text, CallOps::SEND_STATUS_FROM_SERVER => the_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq sent_message - expect(svr_batch.send_status).to be true - expect(svr_batch.send_message).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq sent_message + expect(server_batch.send_status).to be true + expect(server_batch.send_message).to be true # confirm the client can receive the server response and status. client_ops = { @@ -214,17 +261,17 @@ shared_examples 'basic GRPC message delivery is OK' do CallOps::RECV_MESSAGE => nil, CallOps::RECV_STATUS_ON_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_close).to be true - expect(batch_result.message).to eq reply_text - expect(batch_result.status).to eq the_status + final_client_batch = call.run_batch(client_ops) + expect(final_client_batch.send_close).to be true + expect(final_client_batch.message).to eq reply_text + expect(final_client_batch.status).to eq the_status # confirm the server can receive the client close. server_ops = { CallOps::RECV_CLOSE_ON_SERVER => nil } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.send_close).to be true + final_server_batch = server_call.run_batch(server_ops) + expect(final_server_batch.send_close).to be true end def client_cancel_test(cancel_proc, expected_code, @@ -240,9 +287,9 @@ shared_examples 'basic GRPC message delivery is OK' do CallOps::SEND_INITIAL_METADATA => {}, CallOps::RECV_INITIAL_METADATA => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.metadata).to eq({}) + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.metadata).to eq({}) cancel_proc.call(call) @@ -250,16 +297,16 @@ shared_examples 'basic GRPC message delivery is OK' do server_ops = { CallOps::RECV_CLOSE_ON_SERVER => nil } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.send_close).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.send_close).to be true client_ops = { CallOps::RECV_STATUS_ON_CLIENT => {} } - batch_result = call.run_batch(client_ops) + client_batch = call.run_batch(client_ops) - expect(batch_result.status.code).to be expected_code - expect(batch_result.status.details).to eq expected_details + expect(client_batch.status.code).to be expected_code + expect(client_batch.status.details).to eq expected_details end it 'clients can cancel a call on the server' do @@ -325,10 +372,11 @@ shared_examples 'GRPC metadata delivery works OK' do call = new_client_call client_ops = { - CallOps::SEND_INITIAL_METADATA => md + CallOps::SEND_INITIAL_METADATA => md, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true # confirm the server can receive the client metadata rcv_thread.join @@ -336,6 +384,21 @@ shared_examples 'GRPC metadata delivery works OK' do recvd_md = recvd_rpc.metadata replace_symbols = Hash[md.each_pair.collect { |x, y| [x.to_s, y] }] expect(recvd_md).to eq(recvd_md.merge(replace_symbols)) + + # finish the call + final_server_batch = recvd_rpc.call.run_batch( + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status) + expect(final_server_batch.send_close).to be(true) + expect(final_server_batch.send_metadata).to be(true) + expect(final_server_batch.send_status).to be(true) + + final_client_batch = call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end end end @@ -381,6 +444,9 @@ shared_examples 'GRPC metadata delivery works OK' do recvd_rpc.call.run_batch(server_ops) end expect(&blk).to raise_error + + # cancel the call so the server can shut down immediately + call.cancel end end @@ -394,25 +460,37 @@ shared_examples 'GRPC metadata delivery works OK' do # client signals that it's done sending metadata to allow server to # respond client_ops = { - CallOps::SEND_INITIAL_METADATA => nil + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - call.run_batch(client_ops) + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_close).to be true # server gets the invocation but sends no metadata back rcv_thread.join expect(recvd_rpc).to_not eq nil server_call = recvd_rpc.call server_ops = { - CallOps::SEND_INITIAL_METADATA => nil + # receive close and send status to finish the call + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - server_call.run_batch(server_ops) + srv_batch = server_call.run_batch(server_ops) + expect(srv_batch.send_close).to be true + expect(srv_batch.send_metadata).to be true + expect(srv_batch.send_status).to be true # client receives nothing as expected client_ops = { - CallOps::RECV_INITIAL_METADATA => nil + CallOps::RECV_INITIAL_METADATA => nil, + # receive status to finish the call + CallOps::RECV_STATUS_ON_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.metadata).to eq({}) + final_client_batch = call.run_batch(client_ops) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end it 'sends all the pairs when keys and values are valid' do @@ -426,26 +504,36 @@ shared_examples 'GRPC metadata delivery works OK' do # client signals that it's done sending metadata to allow server to # respond client_ops = { - CallOps::SEND_INITIAL_METADATA => nil + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - call.run_batch(client_ops) + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_close).to be true # server gets the invocation but sends no metadata back rcv_thread.join expect(recvd_rpc).to_not eq nil server_call = recvd_rpc.call server_ops = { - CallOps::SEND_INITIAL_METADATA => md + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_INITIAL_METADATA => md, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - server_call.run_batch(server_ops) + srv_batch = server_call.run_batch(server_ops) + expect(srv_batch.send_close).to be true + expect(srv_batch.send_metadata).to be true + expect(srv_batch.send_status).to be true # client receives nothing as expected client_ops = { - CallOps::RECV_INITIAL_METADATA => nil + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil } - batch_result = call.run_batch(client_ops) + final_client_batch = call.run_batch(client_ops) replace_symbols = Hash[md.each_pair.collect { |x, y| [x.to_s, y] }] - expect(batch_result.metadata).to eq(replace_symbols) + expect(final_client_batch.metadata).to eq(replace_symbols) + expect(final_client_batch.status.code).to eq(0) end end end @@ -510,8 +598,7 @@ describe 'the secure http client/server' do initial_md_key = 'k2' initial_md_val = 'v2' - initial_md = {} - initial_md[initial_md_key] = initial_md_val + initial_md = { initial_md_key => initial_md_val } expected_md = creds_update_md.clone fail 'bad test param' unless expected_md[initial_md_key].nil? expected_md[initial_md_key] = initial_md_val @@ -523,11 +610,12 @@ describe 'the secure http client/server' do call = new_client_call call.set_credentials! call_creds - client_ops = { - CallOps::SEND_INITIAL_METADATA => initial_md - } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true + + client_batch = call.run_batch( + CallOps::SEND_INITIAL_METADATA => initial_md, + CallOps::SEND_CLOSE_FROM_CLIENT => nil) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_close).to be true # confirm the server can receive the client metadata rcv_thread.join @@ -535,6 +623,24 @@ describe 'the secure http client/server' do recvd_md = recvd_rpc.metadata replace_symbols = Hash[expected_md.each_pair.collect { |x, y| [x.to_s, y] }] expect(recvd_md).to eq(recvd_md.merge(replace_symbols)) + + credentials_update_test_finish_call(call, recvd_rpc.call) + end + + def credentials_update_test_finish_call(client_call, server_call) + final_server_batch = server_call.run_batch( + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status) + expect(final_server_batch.send_close).to be(true) + expect(final_server_batch.send_metadata).to be(true) + expect(final_server_batch.send_status).to be(true) + + final_client_batch = client_call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end it 'modifies metadata with CallCredentials' do diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb index ec0c2941741..a00df9236d2 100644 --- a/src/ruby/spec/generic/active_call_spec.rb +++ b/src/ruby/spec/generic/active_call_spec.rb @@ -22,6 +22,21 @@ describe GRPC::ActiveCall do CallOps = GRPC::Core::CallOps WriteFlags = GRPC::Core::WriteFlags + def ok_status + Struct::Status.new(OK, 'OK') + end + + def send_and_receive_close_and_status(client_call, server_call) + client_call.run_batch(CallOps::SEND_CLOSE_FROM_CLIENT => nil) + server_call.run_batch(CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status) + client_call.run_batch(CallOps::RECV_STATUS_ON_CLIENT => nil) + end + + def inner_call_of_active_call(active_call) + active_call.instance_variable_get(:@call) + end + before(:each) do @pass_through = proc { |x| x } host = '0.0.0.0:0' @@ -70,13 +85,13 @@ describe GRPC::ActiveCall do end describe '#remote_send' do - it 'allows a client to send a payload to the server' do + it 'allows a client to send a payload to the server', test: true do call = make_test_call ActiveCall.client_invoke(call) - @client_call = ActiveCall.new(call, @pass_through, - @pass_through, deadline) + client_call = ActiveCall.new(call, @pass_through, + @pass_through, deadline) msg = 'message is a string' - @client_call.remote_send(msg) + client_call.remote_send(msg) # check that server rpc new was received recvd_rpc = @server.request_call @@ -86,8 +101,13 @@ describe GRPC::ActiveCall do # Accept the call, and verify that the server reads the response ok. server_call = ActiveCall.new(recvd_call, @pass_through, @pass_through, deadline, - metadata_received: true) + metadata_received: true, + started: false) expect(server_call.remote_read).to eq(msg) + # finish the call + server_call.send_initial_metadata + call.run_batch(CallOps::RECV_INITIAL_METADATA => nil) + send_and_receive_close_and_status(call, recvd_call) end it 'marshals the payload using the marshal func' do @@ -109,6 +129,9 @@ describe GRPC::ActiveCall do @pass_through, deadline, metadata_received: true) expect(server_call.remote_read).to eq('marshalled:' + msg) + # finish the call + call.run_batch(CallOps::RECV_INITIAL_METADATA => nil) + send_and_receive_close_and_status(call, recvd_call) end TEST_WRITE_FLAGS = [WriteFlags::BUFFER_HINT, WriteFlags::NO_COMPRESS] @@ -136,6 +159,9 @@ describe GRPC::ActiveCall do @pass_through, deadline, metadata_received: true) expect(server_call.remote_read).to eq('marshalled:' + msg) + # finish the call + server_call.send_status(OK, '', true) + client_call.receive_and_check_status end end end @@ -177,7 +203,6 @@ describe GRPC::ActiveCall do @pass_through, @pass_through, deadline) - expect(@client_call.metadata_sent).to eql(true) expect(call).to( receive(:run_batch).with(hash_including( @@ -291,6 +316,10 @@ describe GRPC::ActiveCall do expect(recvd_rpc.metadata).to_not be_nil expect(recvd_rpc.metadata['k1']).to eq('v1') expect(recvd_rpc.metadata['k2']).to eq('v2') + # finish the call + recvd_call.run_batch(CallOps::SEND_INITIAL_METADATA => {}) + call.run_batch(CallOps::RECV_INITIAL_METADATA => nil) + send_and_receive_close_and_status(call, recvd_call) end end @@ -324,6 +353,8 @@ describe GRPC::ActiveCall do server_call = expect_server_to_receive(msg) server_call.remote_send('server_response') expect(client_call.remote_read).to eq('server_response') + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end it 'saves no metadata when the server adds no metadata' do @@ -338,6 +369,8 @@ describe GRPC::ActiveCall do expect(client_call.metadata).to be_nil client_call.remote_read expect(client_call.metadata).to eq({}) + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end it 'saves metadata add by the server' do @@ -353,6 +386,8 @@ describe GRPC::ActiveCall do client_call.remote_read expected = { 'k1' => 'v1', 'k2' => 'v2' } expect(client_call.metadata).to eq(expected) + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end it 'get a status from server when nothing else sent from server' do @@ -409,6 +444,8 @@ describe GRPC::ActiveCall do server_call = expect_server_to_receive(msg) server_call.remote_send('server_response') expect(client_call.remote_read).to eq('unmarshalled:server_response') + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end end @@ -418,9 +455,11 @@ describe GRPC::ActiveCall do client_call = ActiveCall.new(call, @pass_through, @pass_through, deadline) expect(client_call.each_remote_read).to be_a(Enumerator) + # finish the call + client_call.cancel end - it 'the returns an enumerator that can read n responses' do + it 'the returned enumerator can read n responses' do call = make_test_call ActiveCall.client_invoke(call) client_call = ActiveCall.new(call, @pass_through, @@ -435,6 +474,8 @@ describe GRPC::ActiveCall do server_call.remote_send(reply) expect(e.next).to eq(reply) end + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end it 'the returns an enumerator that stops after an OK Status' do @@ -453,7 +494,7 @@ describe GRPC::ActiveCall do server_call.remote_send(reply) expect(e.next).to eq(reply) end - server_call.send_status(OK, 'OK') + server_call.send_status(OK, 'OK', true) expect { e.next }.to raise_error(StopIteration) end end From 05eadf21e05d521d73a05babf7c8f45218728355 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 11 Sep 2017 11:46:59 +0200 Subject: [PATCH 150/227] reveal leftover processes --- tools/internal_ci/linux/grpc_run_tests_matrix.sh | 12 +++++++++++- tools/internal_ci/macos/grpc_run_tests_matrix.sh | 5 +++++ tools/internal_ci/windows/grpc_run_tests_matrix.bat | 10 ++++++---- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tools/internal_ci/linux/grpc_run_tests_matrix.sh b/tools/internal_ci/linux/grpc_run_tests_matrix.sh index 028704b3ffe..bd1430b7415 100755 --- a/tools/internal_ci/linux/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/linux/grpc_run_tests_matrix.sh @@ -20,4 +20,14 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS +tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" + +# Reveal leftover processes that might be left behind by the build +ps aux | grep -i kbuilder + +echo 'Exiting gRPC main test script.' + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/grpc_run_tests_matrix.sh b/tools/internal_ci/macos/grpc_run_tests_matrix.sh index 9a43e4869b0..8e7fd54a629 100755 --- a/tools/internal_ci/macos/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/macos/grpc_run_tests_matrix.sh @@ -25,6 +25,11 @@ tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" # kill port_server.py to prevent the build from hanging ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 +# Reveal leftover processes that might be left behind by the build +ps aux | grep -i kbuilder + +echo 'Exiting gRPC main test script.' + if [ "$FAILED" != "" ] then exit 1 diff --git a/tools/internal_ci/windows/grpc_run_tests_matrix.bat b/tools/internal_ci/windows/grpc_run_tests_matrix.bat index 08d834f8b06..10627add25c 100644 --- a/tools/internal_ci/windows/grpc_run_tests_matrix.bat +++ b/tools/internal_ci/windows/grpc_run_tests_matrix.bat @@ -17,8 +17,10 @@ cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat -python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% || goto :error -goto :EOF +python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +@rem Reveal leftover processes that might be left behind by the build +tasklist /V + +exit /b %RUNTESTS_EXITCODE% From 8d3cc5ff0c2b86bd6766944891c1706153135a47 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Fri, 15 Sep 2017 16:43:33 +0000 Subject: [PATCH 151/227] Clean up a few more uses of gRPC in _pb2.py files Were this not done these would break when the default behavior of gRPC Python Protoc Plug-In is changed to be the put-gRPC-code-elements-only- in-_pb2_grpc.py-files behavior that currently happens only when the grpc_2_0 flag is passed. --- .../protoc_plugin/_python_plugin_test.py | 23 +++++++++++-------- .../tests/qps/benchmark_server.py | 6 ++--- .../tests/stress/metrics_server.py | 3 ++- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py index 71493bfec63..5b84001aab3 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py @@ -33,7 +33,7 @@ from tests.unit.framework.common import test_constants import tests.protoc_plugin.protos.payload.test_payload_pb2 as payload_pb2 import tests.protoc_plugin.protos.requests.r.test_requests_pb2 as request_pb2 import tests.protoc_plugin.protos.responses.test_responses_pb2 as response_pb2 -import tests.protoc_plugin.protos.service.test_service_pb2 as service_pb2 +import tests.protoc_plugin.protos.service.test_service_pb2_grpc as service_pb2_grpc # Identifiers of entities we expect to find in the generated module. STUB_IDENTIFIER = 'TestServiceStub' @@ -138,7 +138,7 @@ def _CreateService(): """ servicer_methods = _ServicerMethods() - class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)): + class Servicer(getattr(service_pb2_grpc, SERVICER_IDENTIFIER)): def UnaryCall(self, request, context): return servicer_methods.UnaryCall(request, context) @@ -157,11 +157,12 @@ def _CreateService(): server = grpc.server( futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), server) + getattr(service_pb2_grpc, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), + server) port = server.add_insecure_port('[::]:0') server.start() channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = getattr(service_pb2, STUB_IDENTIFIER)(channel) + stub = getattr(service_pb2_grpc, STUB_IDENTIFIER)(channel) return _Service(servicer_methods, server, stub) @@ -173,16 +174,17 @@ def _CreateIncompleteService(): servicer_methods implements none of the methods required of it. """ - class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)): + class Servicer(getattr(service_pb2_grpc, SERVICER_IDENTIFIER)): pass server = grpc.server( futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), server) + getattr(service_pb2_grpc, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), + server) port = server.add_insecure_port('[::]:0') server.start() channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = getattr(service_pb2, STUB_IDENTIFIER)(channel) + stub = getattr(service_pb2_grpc, STUB_IDENTIFIER)(channel) return _Service(None, server, stub) @@ -223,10 +225,11 @@ class PythonPluginTest(unittest.TestCase): def testImportAttributes(self): # check that we can access the generated module and its members. - self.assertIsNotNone(getattr(service_pb2, STUB_IDENTIFIER, None)) - self.assertIsNotNone(getattr(service_pb2, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone(getattr(service_pb2_grpc, STUB_IDENTIFIER, None)) self.assertIsNotNone( - getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER, None)) + getattr(service_pb2_grpc, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone( + getattr(service_pb2_grpc, ADD_SERVICER_TO_SERVER_IDENTIFIER, None)) def testUpDown(self): service = _CreateService() diff --git a/src/python/grpcio_tests/tests/qps/benchmark_server.py b/src/python/grpcio_tests/tests/qps/benchmark_server.py index 05101fdc6db..bb078444912 100644 --- a/src/python/grpcio_tests/tests/qps/benchmark_server.py +++ b/src/python/grpcio_tests/tests/qps/benchmark_server.py @@ -13,10 +13,10 @@ # limitations under the License. from src.proto.grpc.testing import messages_pb2 -from src.proto.grpc.testing import services_pb2 +from src.proto.grpc.testing import services_pb2_grpc -class BenchmarkServer(services_pb2.BenchmarkServiceServicer): +class BenchmarkServer(services_pb2_grpc.BenchmarkServiceServicer): """Synchronous Server implementation for the Benchmark service.""" def UnaryCall(self, request, context): @@ -29,7 +29,7 @@ class BenchmarkServer(services_pb2.BenchmarkServiceServicer): yield messages_pb2.SimpleResponse(payload=payload) -class GenericBenchmarkServer(services_pb2.BenchmarkServiceServicer): +class GenericBenchmarkServer(services_pb2_grpc.BenchmarkServiceServicer): """Generic Server implementation for the Benchmark service.""" def __init__(self, resp_size): diff --git a/src/python/grpcio_tests/tests/stress/metrics_server.py b/src/python/grpcio_tests/tests/stress/metrics_server.py index 11ab6c3f4e9..33a74b4a388 100644 --- a/src/python/grpcio_tests/tests/stress/metrics_server.py +++ b/src/python/grpcio_tests/tests/stress/metrics_server.py @@ -16,11 +16,12 @@ import time from src.proto.grpc.testing import metrics_pb2 +from src.proto.grpc.testing import metrics_pb2_grpc GAUGE_NAME = 'python_overall_qps' -class MetricsServer(metrics_pb2.MetricsServiceServicer): +class MetricsServer(metrics_pb2_grpc.MetricsServiceServicer): def __init__(self, histogram): self._start_time = time.time() From 4984135a78594215b3fbf04e33262b36a99bdb14 Mon Sep 17 00:00:00 2001 From: Justin Burke Date: Thu, 31 Aug 2017 17:42:54 -0700 Subject: [PATCH 152/227] Add reference counting to tsi_ssl_handshaker_factories --- .../lib/http/httpcli_security_connector.c | 3 +- .../security/transport/security_connector.c | 22 ++-- src/core/tsi/ssl_transport_security.c | 113 ++++++++++++++-- src/core/tsi/ssl_transport_security.h | 37 +++++- test/core/tsi/ssl_transport_security_test.c | 121 +++++++++++++++++- 5 files changed, 265 insertions(+), 31 deletions(-) diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c index 97c28865258..c553fa3981e 100644 --- a/src/core/lib/http/httpcli_security_connector.c +++ b/src/core/lib/http/httpcli_security_connector.c @@ -43,7 +43,8 @@ static void httpcli_ssl_destroy(grpc_exec_ctx *exec_ctx, grpc_httpcli_ssl_channel_security_connector *c = (grpc_httpcli_ssl_channel_security_connector *)sc; if (c->handshaker_factory != NULL) { - tsi_ssl_client_handshaker_factory_destroy(c->handshaker_factory); + tsi_ssl_client_handshaker_factory_unref(c->handshaker_factory); + c->handshaker_factory = NULL; } if (c->secure_peer_name != NULL) gpr_free(c->secure_peer_name); gpr_free(sc); diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index a7568b995f5..2a9e939d40a 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -455,14 +455,14 @@ grpc_server_security_connector *grpc_fake_server_security_connector_create( typedef struct { grpc_channel_security_connector base; - tsi_ssl_client_handshaker_factory *handshaker_factory; + tsi_ssl_client_handshaker_factory *client_handshaker_factory; char *target_name; char *overridden_target_name; } grpc_ssl_channel_security_connector; typedef struct { grpc_server_security_connector base; - tsi_ssl_server_handshaker_factory *handshaker_factory; + tsi_ssl_server_handshaker_factory *server_handshaker_factory; } grpc_ssl_server_security_connector; static void ssl_channel_destroy(grpc_exec_ctx *exec_ctx, @@ -470,9 +470,8 @@ static void ssl_channel_destroy(grpc_exec_ctx *exec_ctx, grpc_ssl_channel_security_connector *c = (grpc_ssl_channel_security_connector *)sc; grpc_call_credentials_unref(exec_ctx, c->base.request_metadata_creds); - if (c->handshaker_factory != NULL) { - tsi_ssl_client_handshaker_factory_destroy(c->handshaker_factory); - } + tsi_ssl_client_handshaker_factory_unref(c->client_handshaker_factory); + c->client_handshaker_factory = NULL; if (c->target_name != NULL) gpr_free(c->target_name); if (c->overridden_target_name != NULL) gpr_free(c->overridden_target_name); gpr_free(sc); @@ -482,9 +481,8 @@ static void ssl_server_destroy(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc) { grpc_ssl_server_security_connector *c = (grpc_ssl_server_security_connector *)sc; - if (c->handshaker_factory != NULL) { - tsi_ssl_server_handshaker_factory_destroy(c->handshaker_factory); - } + tsi_ssl_server_handshaker_factory_unref(c->server_handshaker_factory); + c->server_handshaker_factory = NULL; gpr_free(sc); } @@ -496,7 +494,7 @@ static void ssl_channel_add_handshakers(grpc_exec_ctx *exec_ctx, // Instantiate TSI handshaker. tsi_handshaker *tsi_hs = NULL; tsi_result result = tsi_ssl_client_handshaker_factory_create_handshaker( - c->handshaker_factory, + c->client_handshaker_factory, c->overridden_target_name != NULL ? c->overridden_target_name : c->target_name, &tsi_hs); @@ -521,7 +519,7 @@ static void ssl_server_add_handshakers(grpc_exec_ctx *exec_ctx, // Instantiate TSI handshaker. tsi_handshaker *tsi_hs = NULL; tsi_result result = tsi_ssl_server_handshaker_factory_create_handshaker( - c->handshaker_factory, &tsi_hs); + c->server_handshaker_factory, &tsi_hs); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.", tsi_result_to_string(result)); @@ -852,7 +850,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create( result = tsi_create_ssl_client_handshaker_factory( has_key_cert_pair ? &config->pem_key_cert_pair : NULL, pem_root_certs, ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, - &c->handshaker_factory); + &c->client_handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); @@ -897,7 +895,7 @@ grpc_security_status grpc_ssl_server_security_connector_create( config->pem_root_certs, get_tsi_client_certificate_request_type( config->client_certificate_request), ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, - &c->handshaker_factory); + &c->server_handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 1fd65928f9c..7ebf9dd96f0 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -67,7 +67,13 @@ /* --- Structure definitions. ---*/ +struct tsi_ssl_handshaker_factory { + const tsi_ssl_handshaker_factory_vtable *vtable; + gpr_refcount refcount; +}; + struct tsi_ssl_client_handshaker_factory { + tsi_ssl_handshaker_factory base; SSL_CTX *ssl_context; unsigned char *alpn_protocol_list; size_t alpn_protocol_list_length; @@ -77,6 +83,7 @@ struct tsi_ssl_server_handshaker_factory { /* Several contexts to support SNI. The tsi_peer array contains the subject names of the server certificates associated with the contexts at the same index. */ + tsi_ssl_handshaker_factory base; SSL_CTX **ssl_contexts; tsi_peer *ssl_context_x509_subject_names; size_t ssl_context_count; @@ -90,6 +97,7 @@ typedef struct { BIO *into_ssl; BIO *from_ssl; tsi_result result; + tsi_ssl_handshaker_factory *factory_ref; } tsi_ssl_handshaker; typedef struct { @@ -846,6 +854,47 @@ static const tsi_frame_protector_vtable frame_protector_vtable = { ssl_protector_destroy, }; +/* --- tsi_server_handshaker_factory methods implementation. --- */ + +static void tsi_ssl_handshaker_factory_destroy( + tsi_ssl_handshaker_factory *self) { + if (self == NULL) return; + + if (self->vtable != NULL && self->vtable->destroy != NULL) { + self->vtable->destroy(self); + } + /* Note, we don't free(self) here because this object is always directly + * embedded in another object. If tsi_ssl_handshaker_factory_init allocates + * any memory, it should be free'd here. */ +} + +static tsi_ssl_handshaker_factory *tsi_ssl_handshaker_factory_ref( + tsi_ssl_handshaker_factory *self) { + if (self == NULL) return NULL; + gpr_refn(&self->refcount, 1); + return self; +} + +static void tsi_ssl_handshaker_factory_unref(tsi_ssl_handshaker_factory *self) { + if (self == NULL) return; + + if (gpr_unref(&self->refcount)) { + tsi_ssl_handshaker_factory_destroy(self); + } +} + +static tsi_ssl_handshaker_factory_vtable handshaker_factory_vtable = {NULL}; + +/* Initializes a tsi_ssl_handshaker_factory object. Caller is responsible for + * allocating memory for the factory. */ +static void tsi_ssl_handshaker_factory_init( + tsi_ssl_handshaker_factory *factory) { + GPR_ASSERT(factory != NULL); + + factory->vtable = &handshaker_factory_vtable; + gpr_ref_init(&factory->refcount, 1); +} + /* --- tsi_handshaker methods implementation. ---*/ static tsi_result ssl_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self, @@ -1013,6 +1062,7 @@ static tsi_result ssl_handshaker_create_frame_protector( static void ssl_handshaker_destroy(tsi_handshaker *self) { tsi_ssl_handshaker *impl = (tsi_ssl_handshaker *)self; SSL_free(impl->ssl); /* The BIO objects are owned by ssl */ + tsi_ssl_handshaker_factory_unref(impl->factory_ref); gpr_free(impl); } @@ -1030,6 +1080,7 @@ static const tsi_handshaker_vtable handshaker_vtable = { static tsi_result create_tsi_ssl_handshaker(SSL_CTX *ctx, int is_client, const char *server_name_indication, + tsi_ssl_handshaker_factory *factory, tsi_handshaker **handshaker) { SSL *ssl = SSL_new(ctx); BIO *into_ssl = NULL; @@ -1085,6 +1136,8 @@ static tsi_result create_tsi_ssl_handshaker(SSL_CTX *ctx, int is_client, impl->from_ssl = from_ssl; impl->result = TSI_HANDSHAKE_IN_PROGRESS; impl->base.vtable = &handshaker_vtable; + impl->factory_ref = tsi_ssl_handshaker_factory_ref(factory); + *handshaker = &impl->base; return TSI_OK; } @@ -1121,11 +1174,20 @@ tsi_result tsi_ssl_client_handshaker_factory_create_handshaker( tsi_ssl_client_handshaker_factory *self, const char *server_name_indication, tsi_handshaker **handshaker) { return create_tsi_ssl_handshaker(self->ssl_context, 1, server_name_indication, - handshaker); + &self->base, handshaker); } -void tsi_ssl_client_handshaker_factory_destroy( +void tsi_ssl_client_handshaker_factory_unref( tsi_ssl_client_handshaker_factory *self) { + if (self == NULL) return; + tsi_ssl_handshaker_factory_unref(&self->base); +} + +static void tsi_ssl_client_handshaker_factory_destroy( + tsi_ssl_handshaker_factory *factory) { + if (factory == NULL) return; + tsi_ssl_client_handshaker_factory *self = + (tsi_ssl_client_handshaker_factory *)factory; if (self->ssl_context != NULL) SSL_CTX_free(self->ssl_context); if (self->alpn_protocol_list != NULL) gpr_free(self->alpn_protocol_list); gpr_free(self); @@ -1150,11 +1212,21 @@ tsi_result tsi_ssl_server_handshaker_factory_create_handshaker( if (self->ssl_context_count == 0) return TSI_INVALID_ARGUMENT; /* Create the handshaker with the first context. We will switch if needed because of SNI in ssl_server_handshaker_factory_servername_callback. */ - return create_tsi_ssl_handshaker(self->ssl_contexts[0], 0, NULL, handshaker); + return create_tsi_ssl_handshaker(self->ssl_contexts[0], 0, NULL, &self->base, + handshaker); } -void tsi_ssl_server_handshaker_factory_destroy( +void tsi_ssl_server_handshaker_factory_unref( tsi_ssl_server_handshaker_factory *self) { + if (self == NULL) return; + tsi_ssl_handshaker_factory_unref(&self->base); +} + +static void tsi_ssl_server_handshaker_factory_destroy( + tsi_ssl_handshaker_factory *factory) { + if (factory == NULL) return; + tsi_ssl_server_handshaker_factory *self = + (tsi_ssl_server_handshaker_factory *)factory; size_t i; for (i = 0; i < self->ssl_context_count; i++) { if (self->ssl_contexts[i] != NULL) { @@ -1263,6 +1335,9 @@ static int server_handshaker_factory_npn_advertised_callback( /* --- tsi_ssl_handshaker_factory constructors. --- */ +static tsi_ssl_handshaker_factory_vtable client_handshaker_factory_vtable = { + tsi_ssl_client_handshaker_factory_destroy}; + tsi_result tsi_create_ssl_client_handshaker_factory( const tsi_ssl_pem_key_cert_pair *pem_key_cert_pair, const char *pem_root_certs, const char *cipher_suites, @@ -1285,6 +1360,9 @@ tsi_result tsi_create_ssl_client_handshaker_factory( } impl = gpr_zalloc(sizeof(*impl)); + tsi_ssl_handshaker_factory_init(&impl->base); + impl->base.vtable = &client_handshaker_factory_vtable; + impl->ssl_context = ssl_context; do { @@ -1322,7 +1400,7 @@ tsi_result tsi_create_ssl_client_handshaker_factory( } } while (0); if (result != TSI_OK) { - tsi_ssl_client_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return result; } SSL_CTX_set_verify(ssl_context, SSL_VERIFY_PEER, NULL); @@ -1332,6 +1410,9 @@ tsi_result tsi_create_ssl_client_handshaker_factory( return TSI_OK; } +static tsi_ssl_handshaker_factory_vtable server_handshaker_factory_vtable = { + tsi_ssl_server_handshaker_factory_destroy}; + tsi_result tsi_create_ssl_server_handshaker_factory( const tsi_ssl_pem_key_cert_pair *pem_key_cert_pairs, size_t num_key_cert_pairs, const char *pem_client_root_certs, @@ -1364,12 +1445,15 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( } impl = gpr_zalloc(sizeof(*impl)); + tsi_ssl_handshaker_factory_init(&impl->base); + impl->base.vtable = &server_handshaker_factory_vtable; + impl->ssl_contexts = gpr_zalloc(num_key_cert_pairs * sizeof(SSL_CTX *)); impl->ssl_context_x509_subject_names = gpr_zalloc(num_key_cert_pairs * sizeof(tsi_peer)); if (impl->ssl_contexts == NULL || impl->ssl_context_x509_subject_names == NULL) { - tsi_ssl_server_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return TSI_OUT_OF_RESOURCES; } impl->ssl_context_count = num_key_cert_pairs; @@ -1379,7 +1463,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( &impl->alpn_protocol_list, &impl->alpn_protocol_list_length); if (result != TSI_OK) { - tsi_ssl_server_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return result; } } @@ -1451,10 +1535,11 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( } while (0); if (result != TSI_OK) { - tsi_ssl_server_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return result; } } + *factory = impl; return TSI_OK; } @@ -1501,3 +1586,15 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) { return 0; /* Not found. */ } + +/* --- Testing support. --- */ +const tsi_ssl_handshaker_factory_vtable *tsi_ssl_handshaker_factory_swap_vtable( + tsi_ssl_handshaker_factory *factory, + tsi_ssl_handshaker_factory_vtable *new_vtable) { + GPR_ASSERT(factory != NULL); + GPR_ASSERT(factory->vtable != NULL); + + const tsi_ssl_handshaker_factory_vtable *orig_vtable = factory->vtable; + factory->vtable = new_vtable; + return orig_vtable; +} diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index 177599930bc..3abfdf5ed87 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -96,10 +96,10 @@ tsi_result tsi_ssl_client_handshaker_factory_create_handshaker( tsi_ssl_client_handshaker_factory *self, const char *server_name_indication, tsi_handshaker **handshaker); -/* Destroys the handshaker factory. WARNING: it is unsafe to destroy a factory - while handshakers created with this factory are still in use. */ -void tsi_ssl_client_handshaker_factory_destroy( - tsi_ssl_client_handshaker_factory *self); +/* Decrements reference count of the handshaker factory. Handshaker factory will + * be destroyed once no references exist. */ +void tsi_ssl_client_handshaker_factory_unref( + tsi_ssl_client_handshaker_factory *factory); /* --- tsi_ssl_server_handshaker_factory object --- @@ -158,9 +158,9 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( tsi_result tsi_ssl_server_handshaker_factory_create_handshaker( tsi_ssl_server_handshaker_factory *self, tsi_handshaker **handshaker); -/* Destroys the handshaker factory. WARNING: it is unsafe to destroy a factory - while handshakers created with this factory are still in use. */ -void tsi_ssl_server_handshaker_factory_destroy( +/* Decrements reference count of the handshaker factory. Handshaker factory will + * be destroyed once no references exist. */ +void tsi_ssl_server_handshaker_factory_unref( tsi_ssl_server_handshaker_factory *self); /* Util that checks that an ssl peer matches a specific name. @@ -170,6 +170,29 @@ void tsi_ssl_server_handshaker_factory_destroy( - handle public suffix wildchar more strictly (e.g. *.co.uk) */ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name); +/* --- Testing support. --- + + These functions and typedefs are not intended to be used outside of testing. + */ + +/* Base type of client and server handshaker factories. */ +typedef struct tsi_ssl_handshaker_factory tsi_ssl_handshaker_factory; + +/* Function pointer to handshaker_factory destructor. */ +typedef void (*tsi_ssl_handshaker_factory_destructor)( + tsi_ssl_handshaker_factory *factory); + +/* Virtual table for tsi_ssl_handshaker_factory. */ +typedef struct { + tsi_ssl_handshaker_factory_destructor destroy; +} tsi_ssl_handshaker_factory_vtable; + +/* Set destructor of handshaker_factory to new_destructor, returns previous + destructor. */ +const tsi_ssl_handshaker_factory_vtable *tsi_ssl_handshaker_factory_swap_vtable( + tsi_ssl_handshaker_factory *factory, + tsi_ssl_handshaker_factory_vtable *new_vtable); + #ifdef __cplusplus } #endif diff --git a/test/core/tsi/ssl_transport_security_test.c b/test/core/tsi/ssl_transport_security_test.c index 364dfa1b73f..2399b054b1b 100644 --- a/test/core/tsi/ssl_transport_security_test.c +++ b/test/core/tsi/ssl_transport_security_test.c @@ -23,7 +23,9 @@ #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/transport/security_connector.h" #include "src/core/tsi/ssl_transport_security.h" +#include "src/core/tsi/transport_security.h" #include "src/core/tsi/transport_security_adapter.h" +#include "src/core/tsi/transport_security_interface.h" #include "test/core/tsi/transport_security_test_lib.h" #include "test/core/util/test_config.h" @@ -312,10 +314,10 @@ static void ssl_test_destruct(tsi_test_fixture *fixture) { key_cert_lib->bad_client_pem_key_cert_pair); gpr_free(key_cert_lib->root_cert); gpr_free(key_cert_lib); - /* Destroy others. */ - tsi_ssl_server_handshaker_factory_destroy( + /* Unreference others. */ + tsi_ssl_server_handshaker_factory_unref( ssl_fixture->server_handshaker_factory); - tsi_ssl_client_handshaker_factory_destroy( + tsi_ssl_client_handshaker_factory_unref( ssl_fixture->client_handshaker_factory); } @@ -536,6 +538,118 @@ void ssl_tsi_test_do_round_trip_odd_buffer_size() { } } +static const tsi_ssl_handshaker_factory_vtable *original_vtable; +static bool handshaker_factory_destructor_called; + +static void ssl_tsi_test_handshaker_factory_destructor( + tsi_ssl_handshaker_factory *factory) { + GPR_ASSERT(factory != NULL); + handshaker_factory_destructor_called = true; + if (original_vtable != NULL && original_vtable->destroy != NULL) { + original_vtable->destroy(factory); + } +} + +static tsi_ssl_handshaker_factory_vtable test_handshaker_factory_vtable = { + ssl_tsi_test_handshaker_factory_destructor}; + +void test_tsi_ssl_client_handshaker_factory_refcounting() { + int i; + const char *cert_chain = + load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "client.pem"); + + tsi_ssl_client_handshaker_factory *client_handshaker_factory; + GPR_ASSERT(tsi_create_ssl_client_handshaker_factory( + NULL, cert_chain, NULL, NULL, 0, &client_handshaker_factory) == + TSI_OK); + + handshaker_factory_destructor_called = false; + original_vtable = tsi_ssl_handshaker_factory_swap_vtable( + (tsi_ssl_handshaker_factory *)client_handshaker_factory, + &test_handshaker_factory_vtable); + + tsi_handshaker *handshaker[3]; + + for (i = 0; i < 3; ++i) { + GPR_ASSERT(tsi_ssl_client_handshaker_factory_create_handshaker( + client_handshaker_factory, "google.com", &handshaker[i]) == + TSI_OK); + } + + tsi_handshaker_destroy(handshaker[1]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[0]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_ssl_client_handshaker_factory_unref(client_handshaker_factory); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[2]); + GPR_ASSERT(handshaker_factory_destructor_called); + + gpr_free((void *)cert_chain); +} + +void test_tsi_ssl_server_handshaker_factory_refcounting() { + int i; + tsi_ssl_server_handshaker_factory *server_handshaker_factory; + tsi_handshaker *handshaker[3]; + const char *cert_chain = + load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "server0.pem"); + tsi_ssl_pem_key_cert_pair cert_pair; + + cert_pair.cert_chain = cert_chain; + cert_pair.private_key = + load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "server0.key"); + + GPR_ASSERT(tsi_create_ssl_server_handshaker_factory( + &cert_pair, 1, cert_chain, 0, NULL, NULL, 0, + &server_handshaker_factory) == TSI_OK); + + handshaker_factory_destructor_called = false; + original_vtable = tsi_ssl_handshaker_factory_swap_vtable( + (tsi_ssl_handshaker_factory *)server_handshaker_factory, + &test_handshaker_factory_vtable); + + for (i = 0; i < 3; ++i) { + GPR_ASSERT(tsi_ssl_server_handshaker_factory_create_handshaker( + server_handshaker_factory, &handshaker[i]) == TSI_OK); + } + + tsi_handshaker_destroy(handshaker[1]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[0]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_ssl_server_handshaker_factory_unref(server_handshaker_factory); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[2]); + GPR_ASSERT(handshaker_factory_destructor_called); + + ssl_test_pem_key_cert_pair_destroy(cert_pair); +} + +/* Attempting to create a handshaker factory with invalid parameters should fail + * but not crash. */ +void test_tsi_ssl_client_handshaker_factory_bad_params() { + const char *cert_chain = "This is not a valid PEM file."; + + tsi_ssl_client_handshaker_factory *client_handshaker_factory; + GPR_ASSERT(tsi_create_ssl_client_handshaker_factory( + NULL, cert_chain, NULL, NULL, 0, &client_handshaker_factory) == + TSI_INVALID_ARGUMENT); + tsi_ssl_client_handshaker_factory_unref(client_handshaker_factory); +} + +void ssl_tsi_test_handshaker_factory_internals() { + test_tsi_ssl_client_handshaker_factory_refcounting(); + test_tsi_ssl_server_handshaker_factory_refcounting(); + test_tsi_ssl_client_handshaker_factory_bad_params(); +} + int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); @@ -553,6 +667,7 @@ int main(int argc, char **argv) { ssl_tsi_test_do_handshake_alpn_client_server_ok(); ssl_tsi_test_do_round_trip_for_all_configs(); ssl_tsi_test_do_round_trip_odd_buffer_size(); + ssl_tsi_test_handshaker_factory_internals(); grpc_shutdown(); return 0; } From 3fb250bdb16dda206596f28442502523e19bd2c4 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 15 Sep 2017 11:47:50 -0700 Subject: [PATCH 153/227] Polish output more --- src/objective-c/tests/run_tests.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/objective-c/tests/run_tests.sh b/src/objective-c/tests/run_tests.sh index 5b7a2d104aa..608ae6884b6 100755 --- a/src/objective-c/tests/run_tests.sh +++ b/src/objective-c/tests/run_tests.sh @@ -49,7 +49,8 @@ xcodebuild \ HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \ test \ | egrep -v "$XCODEBUILD_FILTER" \ - | egrep -v '^$' - + | egrep -v '^$' \ + | egrep -v "(GPBDictionary|GPBArray)" - echo "TIME: $(date)" xcodebuild \ @@ -57,7 +58,8 @@ xcodebuild \ -scheme CoreCronetEnd2EndTests \ -destination name="iPhone 6" \ test \ - | egrep "$XCODEBUILD_FILTER" \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ | egrep -v "(GPBDictionary|GPBArray)" - echo "TIME: $(date)" @@ -65,7 +67,10 @@ xcodebuild \ -workspace Tests.xcworkspace \ -scheme CronetUnitTests \ -destination name="iPhone 6" \ - test | xcpretty + test \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ + | egrep -v "(GPBDictionary|GPBArray)" - echo "TIME: $(date)" xcodebuild \ @@ -74,5 +79,6 @@ xcodebuild \ -destination name="iPhone 6" \ HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \ test \ - | egrep "$XCODEBUILD_FILTER" \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ | egrep -v "(GPBDictionary|GPBArray)" - From 4478a88ad92562a315267cfea8e6f3a29d1da654 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 15 Sep 2017 14:25:35 -0700 Subject: [PATCH 154/227] More detailed error when double-scheduling a closure --- src/core/lib/iomgr/closure.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index 7236e23cf70..2253604342e 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -167,7 +167,15 @@ void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *c, GPR_TIMER_BEGIN("grpc_closure_sched", 0); if (c != NULL) { #ifndef NDEBUG - GPR_ASSERT(!c->scheduled); + if (c->scheduled) { + gpr_log(GPR_ERROR, + "Closure already scheduled. (closure: %p, created: [%s:%d], " + "previously scheduled at: [%s: %d] run?: %s", + closure, closure->file_created, closure->line_created, + closure->file_initiated, closure->line_initiated, + closure->run ? "true" : "false"); + abort(); + } c->scheduled = true; c->file_initiated = file; c->line_initiated = line; From f68858464afd15402e09f480938e5efc6c7ff02f Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 21 Aug 2017 08:09:06 -0700 Subject: [PATCH 155/227] Add Kokoro perf PR jobs --- .../prepare_build_linux_perf_rc | 15 +++++++ .../linux/grpc_microbenchmark_diff.sh | 38 +++++++++++++++++ tools/internal_ci/linux/grpc_trickle_diff.sh | 42 +++++++++++++++++++ .../pull_request/grpc_microbenchmark_diff.cfg | 25 +++++++++++ .../linux/pull_request/grpc_trickle_diff.cfg | 25 +++++++++++ 5 files changed, 145 insertions(+) create mode 100755 tools/internal_ci/linux/grpc_microbenchmark_diff.sh create mode 100755 tools/internal_ci/linux/grpc_trickle_diff.sh create mode 100644 tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc index 1b7779caa82..dc3c75a6a12 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc @@ -19,4 +19,19 @@ ulimit -n 32768 ulimit -c unlimited +# Performance PR testing needs GH API key and PR metadata to comment results +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then + set +x + sudo apt-get install -y jq + export ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) + + gsutil cp gs://grpc-testing-secrets/github_credentials/oauth_token.txt ~/ + # TODO(matt-kwong): rename this to GITHUB_OAUTH_TOKEN after Jenkins deprecation + export JENKINS_OAUTH_TOKEN=$(cat ~/oauth_token.txt) + export ghprbPullId=$KOKORO_GITHUB_PULL_REQUEST_NUMBER + set -x +fi + +sudo pip install tabulate + git submodule update --init diff --git a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh new file mode 100755 index 00000000000..58ffcf336b8 --- /dev/null +++ b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/run_tests/start_port_server.py +tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ + -d origin/$ghprbTargetBranch \ + -b $BENCHMARKS_TO_RUN || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_trickle_diff.sh b/tools/internal_ci/linux/grpc_trickle_diff.sh new file mode 100755 index 00000000000..624031afa78 --- /dev/null +++ b/tools/internal_ci/linux/grpc_trickle_diff.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="cli_transport_stalls_per_iteration cli_stream_stalls_per_iteration svr_transport_stalls_per_iteration svr_stream_stalls_per_iteration" + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/run_tests/start_port_server.py +tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ + -d origin/$ghprbTargetBranch \ + -b bm_fullstack_trickle \ + -l 4 \ + -t $BENCHMARKS_TO_RUN \ + --no-counters \ + --pr_comment_name trickle || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg new file mode 100644 index 00000000000..9269c345f0a --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_microbenchmark_diff.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg new file mode 100644 index 00000000000..e86b3ab4758 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_trickle_diff.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} From 49d75e3fc7022f7ff409fcb9025fe36e76d0f4ed Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 15 Sep 2017 15:25:50 -0700 Subject: [PATCH 156/227] Address some review comments --- src/core/lib/surface/completion_queue.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index b7109c41047..1329dec35d7 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -449,7 +449,7 @@ grpc_completion_queue *grpc_completion_queue_create_internal( static void cq_init_next(void *ptr) { cq_next_data *cqd = (cq_next_data *)ptr; /* Initial count is dropped by grpc_completion_queue_shutdown */ - gpr_atm_rel_store(&cqd->pending_events, 1); + gpr_atm_no_barrier_store(&cqd->pending_events, 1); cqd->shutdown_called = false; gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0); cq_event_queue_init(&cqd->queue); @@ -464,7 +464,7 @@ static void cq_destroy_next(void *ptr) { static void cq_init_pluck(void *ptr) { cq_pluck_data *cqd = (cq_pluck_data *)ptr; /* Initial count is dropped by grpc_completion_queue_shutdown */ - gpr_atm_rel_store(&cqd->pending_events, 1); + gpr_atm_no_barrier_store(&cqd->pending_events, 1); cqd->completed_tail = &cqd->completed_head; cqd->completed_head.next = (uintptr_t)cqd->completed_tail; gpr_atm_no_barrier_store(&cqd->shutdown, 0); @@ -643,6 +643,11 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, /* Add the completion to the queue */ bool is_first = cq_event_queue_push(&cqd->queue, storage); gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1); + + /* Since we do not hold the cq lock here, it is important to do an 'acquire' + load here (instead of a 'no_barrier' load) to match with the release store + (done via gpr_atm_full_fetch_add(pending_events, -1)) in cq_shutdown_next + */ bool will_definitely_shutdown = gpr_atm_acq_load(&cqd->pending_events) == 1; if (!will_definitely_shutdown) { From 2795eae06eca4f649682f06099520fdedc7e215a Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Sat, 16 Sep 2017 09:59:20 -0700 Subject: [PATCH 157/227] Fix timeout test --- src/objective-c/tests/GRPCClientTests.m | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 4be3d1c5e87..82ac2600fab 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -28,6 +28,7 @@ #import #import #import +#import #define TEST_TIMEOUT 16 @@ -39,6 +40,7 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; static GRPCProtoMethod *kInexistentMethod; static GRPCProtoMethod *kEmptyCallMethod; static GRPCProtoMethod *kUnaryCallMethod; +static GRPCProtoMethod *kFullDuplexCallMethod; /** Observer class for testing that responseMetadata is KVO-compliant */ @interface PassthroughObserver : NSObject @@ -106,6 +108,9 @@ static GRPCProtoMethod *kUnaryCallMethod; kUnaryCallMethod = [[GRPCProtoMethod alloc] initWithPackage:kPackage service:kService method:@"UnaryCall"]; + kFullDuplexCallMethod = [[GRPCProtoMethod alloc] initWithPackage:kPackage + service:kService + method:@"FullDuplexCall"]; } - (void)testConnectionToRemoteServer { @@ -423,11 +428,12 @@ static GRPCProtoMethod *kUnaryCallMethod; } - (void)testTimeout { - __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; + __weak XCTestExpectation *completion = [self expectationWithDescription:@"RPC completed."]; + GRXBufferedPipe *pipe = [GRXBufferedPipe pipe]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - path:kEmptyCallMethod.HTTPPath - requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; + path:kFullDuplexCallMethod.HTTPPath + requestsWriter:pipe]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { XCTAssert(0, @"Failure: response received; Expect: no response received."); From 44038eb9b630eafad57e30e09d021de8d68e94d5 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Sat, 16 Sep 2017 16:13:09 +0000 Subject: [PATCH 158/227] Tweak Python sanity test - Move it out of the "unit" package, as it's not itself a unit test. - Suffix the test class with "Test" as we do with every other subclass of unittest.TestCase. - Add a larger-than-we'll-need-any-time-soon maxDiff so that failures are fully described. - Relax the assertion from assertListEqual to assertSequenceEqual since we don't actually care whether or not the sequences being compared are list instances. - Change the order of the assertions arguments to match the ", " convention used in our assert*Equal calls elsewhere throughout the test corpus. - Internal implementation simplifications. --- .../tests/{unit => }/_sanity/__init__.py | 0 .../tests/{unit => }/_sanity/_sanity_test.py | 17 +++++++++-------- src/python/grpcio_tests/tests/tests.json | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) rename src/python/grpcio_tests/tests/{unit => }/_sanity/__init__.py (100%) rename src/python/grpcio_tests/tests/{unit => }/_sanity/_sanity_test.py (78%) diff --git a/src/python/grpcio_tests/tests/unit/_sanity/__init__.py b/src/python/grpcio_tests/tests/_sanity/__init__.py similarity index 100% rename from src/python/grpcio_tests/tests/unit/_sanity/__init__.py rename to src/python/grpcio_tests/tests/_sanity/__init__.py diff --git a/src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py b/src/python/grpcio_tests/tests/_sanity/_sanity_test.py similarity index 78% rename from src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py rename to src/python/grpcio_tests/tests/_sanity/_sanity_test.py index 19bc8801ebc..b4079850ff7 100644 --- a/src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py +++ b/src/python/grpcio_tests/tests/_sanity/_sanity_test.py @@ -21,24 +21,25 @@ import six import tests -class Sanity(unittest.TestCase): +class SanityTest(unittest.TestCase): + + maxDiff = 32768 def testTestsJsonUpToDate(self): """Autodiscovers all test suites and checks that tests.json is up to date""" loader = tests.Loader() loader.loadTestsFromNames(['tests']) - test_suite_names = [ + test_suite_names = sorted({ test_case_class.id().rsplit('.', 1)[0] for test_case_class in tests._loader.iterate_suite_cases( loader.suite) - ] - test_suite_names = sorted(set(test_suite_names)) + }) tests_json_string = pkg_resources.resource_string('tests', 'tests.json') - if six.PY3: - tests_json_string = tests_json_string.decode() - tests_json = json.loads(tests_json_string) - self.assertListEqual(test_suite_names, tests_json) + tests_json = json.loads(tests_json_string.decode() + if six.PY3 else tests_json_string) + + self.assertSequenceEqual(tests_json, test_suite_names) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 4c078e6c22b..a3f2a9d8206 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -1,4 +1,5 @@ [ + "_sanity._sanity_test.SanityTest", "health_check._health_servicer_test.HealthServicerTest", "interop._insecure_intraop_test.InsecureIntraopTest", "interop._secure_intraop_test.SecureIntraopTest", @@ -41,7 +42,6 @@ "unit._reconnect_test.ReconnectTest", "unit._resource_exhausted_test.ResourceExhaustedTest", "unit._rpc_test.RPCTest", - "unit._sanity._sanity_test.Sanity", "unit._thread_cleanup_test.CleanupThreadTest", "unit.beta._beta_features_test.BetaFeaturesTest", "unit.beta._beta_features_test.ContextManagementAndLifecycleTest", From 719ade3cdcb5e132087fa8a443f1735adfe05634 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Sat, 16 Sep 2017 15:28:22 +0000 Subject: [PATCH 159/227] Overhaul protoc_plugins._split_definitions_test We were mistaken before when we were testing _pb2.py files being generated in one directory and _pb2_grpc.py files being generated in another directory. Sure, that was a thing our code generator could do, but because of the way paths and packages work in Python it wasn't a realistic use case for actual users. This test now tests _pb2.py files and _pb2_grpc.py files being generated either together or independently of one another, and if independently, in either order. Looking forward to an eventual py_grpc_library Bazel rule, that's what actually matters. --- .../protoc_plugin/_split_definitions_test.py | 519 +++++++++--------- .../split_messages/__init__.py | 13 - .../split_messages/{ => sub}/messages.proto | 0 .../split_services/__init__.py | 13 - src/python/grpcio_tests/tests/tests.json | 12 +- 5 files changed, 263 insertions(+), 294 deletions(-) delete mode 100644 src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py rename src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/{ => sub}/messages.proto (100%) delete mode 100644 src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py index 1aeb62a7c5c..7868cdbfb31 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py @@ -12,22 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections +import abc from concurrent import futures import contextlib -import distutils.spawn -import errno import importlib import os -import os.path +from os import path import pkgutil +import platform import shutil -import subprocess import sys import tempfile -import threading import unittest -import platform + +import six import grpc from grpc_tools import protoc @@ -37,292 +35,285 @@ _MESSAGES_IMPORT = b'import "messages.proto";' _SPLIT_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing.split;' _COMMON_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing;' +_RELATIVE_PROTO_PATH = 'relative_proto_path' +_RELATIVE_PYTHON_OUT = 'relative_python_out' + @contextlib.contextmanager -def _system_path(path): +def _system_path(path_insertion): old_system_path = sys.path[:] - sys.path = sys.path[0:1] + path + sys.path[1:] + sys.path = sys.path[0:1] + path_insertion + sys.path[1:] yield sys.path = old_system_path -class DummySplitServicer(object): +# NOTE(nathaniel): https://twitter.com/exoplaneteer/status/677259364256747520 +# Life lesson "just always default to idempotence" reinforced. +def _create_directory_tree(root, path_components_sequence): + created = set() + for path_components in path_components_sequence: + thus_far = '' + for path_component in path_components: + relative_path = path.join(thus_far, path_component) + if relative_path not in created: + os.makedirs(path.join(root, relative_path)) + created.add(relative_path) + thus_far = path.join(thus_far, path_component) + + +def _massage_proto_content(proto_content, test_name_bytes, + messages_proto_relative_file_name_bytes): + package_substitution = (b'package grpc_protoc_plugin.invocation_testing.' + + test_name_bytes + b';') + common_namespace_substituted = proto_content.replace(_COMMON_NAMESPACE, + package_substitution) + split_namespace_substituted = common_namespace_substituted.replace( + _SPLIT_NAMESPACE, package_substitution) + message_import_replaced = split_namespace_substituted.replace( + _MESSAGES_IMPORT, + b'import "' + messages_proto_relative_file_name_bytes + b'";') + return message_import_replaced + + +def _packagify(directory): + for subdirectory, _, _ in os.walk(directory): + init_file_name = path.join(subdirectory, '__init__.py') + with open(init_file_name, 'wb') as init_file: + init_file.write(b'') - def __init__(self, request_class, response_class): - self.request_class = request_class - self.response_class = response_class + +class _Servicer(object): + + def __init__(self, response_class): + self._response_class = response_class def Call(self, request, context): - return self.response_class() + return self._response_class() -class SeparateTestMixin(object): +def _protoc(proto_path, python_out, grpc_python_out_flag, grpc_python_out, + absolute_proto_file_names): + args = [ + '', + '--proto_path={}'.format(proto_path), + ] + if python_out is not None: + args.append('--python_out={}'.format(python_out)) + if grpc_python_out is not None: + args.append('--grpc_python_out={}:{}'.format(grpc_python_out_flag, + grpc_python_out)) + args.extend(absolute_proto_file_names) + return protoc.main(args) - def testImportAttributes(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - pb2.Request - pb2.Response - if self.should_find_services_in_pb2: - pb2.TestServiceServicer - else: - with self.assertRaises(AttributeError): - pb2.TestServiceServicer - - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - pb2_grpc.TestServiceServicer - with self.assertRaises(AttributeError): - pb2_grpc.Request - with self.assertRaises(AttributeError): - pb2_grpc.Response - - def testCall(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - server = grpc.server( - futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - pb2_grpc.add_TestServiceServicer_to_server( - DummySplitServicer(pb2.Request, pb2.Response), server) - port = server.add_insecure_port('[::]:0') - server.start() - channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = pb2_grpc.TestServiceStub(channel) - request = pb2.Request() - expected_response = pb2.Response() - response = stub.Call(request) - self.assertEqual(expected_response, response) - - -class CommonTestMixin(object): - - def testImportAttributes(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - pb2.Request - pb2.Response - if self.should_find_services_in_pb2: - pb2.TestServiceServicer - else: - with self.assertRaises(AttributeError): - pb2.TestServiceServicer - - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - pb2_grpc.TestServiceServicer - with self.assertRaises(AttributeError): - pb2_grpc.Request - with self.assertRaises(AttributeError): - pb2_grpc.Response - - def testCall(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - server = grpc.server( - futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - pb2_grpc.add_TestServiceServicer_to_server( - DummySplitServicer(pb2.Request, pb2.Response), server) - port = server.add_insecure_port('[::]:0') - server.start() - channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = pb2_grpc.TestServiceStub(channel) - request = pb2.Request() - expected_response = pb2.Response() - response = stub.Call(request) - self.assertEqual(expected_response, response) - - -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SameSeparateTest(unittest.TestCase, SeparateTestMixin): - def setUp(self): - same_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing', 'same.proto') - self.directory = tempfile.mkdtemp(suffix='same_separate', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = os.path.join(self.directory, - 'grpc_python_out') - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - os.makedirs(self.grpc_python_out_directory) - same_proto_file = os.path.join(self.proto_directory, - 'same_separate.proto') - open(same_proto_file, 'wb').write( - same_proto_contents.replace( - _COMMON_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.same_separate;')) - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out=grpc_2_0:{}'.format( - self.grpc_python_out_directory), - same_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.grpc_python_out_directory, '__init__.py'), - 'w').write('') - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'same_separate_pb2' - self.pb2_grpc_import = 'same_separate_pb2_grpc' - self.should_find_services_in_pb2 = False +class _Mid2016ProtocStyle(object): - def tearDown(self): - shutil.rmtree(self.directory) + def name(self): + return 'Mid2016ProtocStyle' + def grpc_in_pb2_expected(self): + return True -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SameCommonTest(unittest.TestCase, CommonTestMixin): + def protoc(self, proto_path, python_out, absolute_proto_file_names): + return (_protoc(proto_path, python_out, 'grpc_1_0', python_out, + absolute_proto_file_names),) - def setUp(self): - same_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing', 'same.proto') - self.directory = tempfile.mkdtemp(suffix='same_common', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = self.python_out_directory - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - same_proto_file = os.path.join(self.proto_directory, - 'same_common.proto') - open(same_proto_file, 'wb').write( - same_proto_contents.replace( - _COMMON_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.same_common;')) - - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out={}'.format(self.grpc_python_out_directory), - same_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'same_common_pb2' - self.pb2_grpc_import = 'same_common_pb2_grpc' - self.should_find_services_in_pb2 = True - def tearDown(self): - shutil.rmtree(self.directory) +class _SingleProtocExecutionProtocStyle(object): + def name(self): + return 'SingleProtocExecutionProtocStyle' -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SplitCommonTest(unittest.TestCase, CommonTestMixin): + def grpc_in_pb2_expected(self): + return False - def setUp(self): - services_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_services', - 'services.proto') - messages_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_messages', - 'messages.proto') - self.directory = tempfile.mkdtemp(suffix='split_common', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = self.python_out_directory - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - services_proto_file = os.path.join(self.proto_directory, - 'split_common_services.proto') - messages_proto_file = os.path.join(self.proto_directory, - 'split_common_messages.proto') - open(services_proto_file, 'wb').write( - services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_common_messages.proto";') - .replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_common;')) - open(messages_proto_file, 'wb').write( - messages_proto_contents.replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_common;')) - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out={}'.format(self.grpc_python_out_directory), - services_proto_file, - messages_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'split_common_messages_pb2' - self.pb2_grpc_import = 'split_common_services_pb2_grpc' - self.should_find_services_in_pb2 = False + def protoc(self, proto_path, python_out, absolute_proto_file_names): + return (_protoc(proto_path, python_out, 'grpc_2_0', python_out, + absolute_proto_file_names),) + + +class _ProtoBeforeGrpcProtocStyle(object): + + def name(self): + return 'ProtoBeforeGrpcProtocStyle' + + def grpc_in_pb2_expected(self): + return False + + def protoc(self, proto_path, python_out, absolute_proto_file_names): + pb2_protoc_exit_code = _protoc(proto_path, python_out, None, None, + absolute_proto_file_names) + pb2_grpc_protoc_exit_code = _protoc( + proto_path, None, 'grpc_2_0', python_out, absolute_proto_file_names) + return pb2_protoc_exit_code, pb2_grpc_protoc_exit_code, - def tearDown(self): - shutil.rmtree(self.directory) +class _GrpcBeforeProtoProtocStyle(object): -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SplitSeparateTest(unittest.TestCase, SeparateTestMixin): + def name(self): + return 'GrpcBeforeProtoProtocStyle' + + def grpc_in_pb2_expected(self): + return False + + def protoc(self, proto_path, python_out, absolute_proto_file_names): + pb2_grpc_protoc_exit_code = _protoc( + proto_path, None, 'grpc_2_0', python_out, absolute_proto_file_names) + pb2_protoc_exit_code = _protoc(proto_path, python_out, None, None, + absolute_proto_file_names) + return pb2_grpc_protoc_exit_code, pb2_protoc_exit_code, + + +_PROTOC_STYLES = (_Mid2016ProtocStyle(), _SingleProtocExecutionProtocStyle(), + _ProtoBeforeGrpcProtocStyle(), _GrpcBeforeProtoProtocStyle(),) + + +@unittest.skipIf(platform.python_implementation() == 'PyPy', + 'Skip test if run with PyPy!') +class _Test(six.with_metaclass(abc.ABCMeta, unittest.TestCase)): def setUp(self): - services_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_services', - 'services.proto') - messages_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_messages', - 'messages.proto') - self.directory = tempfile.mkdtemp(suffix='split_separate', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = os.path.join(self.directory, - 'grpc_python_out') - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - os.makedirs(self.grpc_python_out_directory) - services_proto_file = os.path.join(self.proto_directory, - 'split_separate_services.proto') - messages_proto_file = os.path.join(self.proto_directory, - 'split_separate_messages.proto') - open(services_proto_file, 'wb').write( - services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_separate_messages.proto";') - .replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_separate;' - )) - open(messages_proto_file, 'wb').write( - messages_proto_contents.replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_separate;' - )) - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out=grpc_2_0:{}'.format( - self.grpc_python_out_directory), - services_proto_file, - messages_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'split_separate_messages_pb2' - self.pb2_grpc_import = 'split_separate_services_pb2_grpc' - self.should_find_services_in_pb2 = False + self._directory = tempfile.mkdtemp(suffix=self.NAME, dir='.') + self._proto_path = path.join(self._directory, _RELATIVE_PROTO_PATH) + self._python_out = path.join(self._directory, _RELATIVE_PYTHON_OUT) + + os.makedirs(self._proto_path) + os.makedirs(self._python_out) + + proto_directories_and_names = { + (self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES, + self.MESSAGES_PROTO_FILE_NAME,), + (self.SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES, + self.SERVICES_PROTO_FILE_NAME,), + } + messages_proto_relative_file_name_forward_slashes = '/'.join( + self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES + ( + self.MESSAGES_PROTO_FILE_NAME,)) + _create_directory_tree(self._proto_path, ( + relative_proto_directory_names + for relative_proto_directory_names, _ in proto_directories_and_names + )) + self._absolute_proto_file_names = set() + for relative_directory_names, file_name in proto_directories_and_names: + absolute_proto_file_name = path.join( + self._proto_path, *relative_directory_names + (file_name,)) + raw_proto_content = pkgutil.get_data( + 'tests.protoc_plugin.protos.invocation_testing', + path.join(*relative_directory_names + (file_name,))) + massaged_proto_content = _massage_proto_content( + raw_proto_content, + self.NAME.encode(), + messages_proto_relative_file_name_forward_slashes.encode()) + with open(absolute_proto_file_name, 'wb') as proto_file: + proto_file.write(massaged_proto_content) + self._absolute_proto_file_names.add(absolute_proto_file_name) def tearDown(self): - shutil.rmtree(self.directory) + shutil.rmtree(self._directory) + + def _protoc(self): + protoc_exit_codes = self.PROTOC_STYLE.protoc( + self._proto_path, self._python_out, self._absolute_proto_file_names) + for protoc_exit_code in protoc_exit_codes: + self.assertEqual(0, protoc_exit_code) + + _packagify(self._python_out) + + generated_modules = {} + expected_generated_full_module_names = { + self.EXPECTED_MESSAGES_PB2, + self.EXPECTED_SERVICES_PB2, + self.EXPECTED_SERVICES_PB2_GRPC, + } + with _system_path([self._python_out]): + for full_module_name in expected_generated_full_module_names: + module = importlib.import_module(full_module_name) + generated_modules[full_module_name] = module + + self._messages_pb2 = generated_modules[self.EXPECTED_MESSAGES_PB2] + self._services_pb2 = generated_modules[self.EXPECTED_SERVICES_PB2] + self._services_pb2_grpc = generated_modules[ + self.EXPECTED_SERVICES_PB2_GRPC] + + def _services_modules(self): + if self.PROTOC_STYLE.grpc_in_pb2_expected(): + return self._services_pb2, self._services_pb2_grpc, + else: + return self._services_pb2_grpc, + + def test_imported_attributes(self): + self._protoc() + + self._messages_pb2.Request + self._messages_pb2.Response + self._services_pb2.DESCRIPTOR.services_by_name['TestService'] + for services_module in self._services_modules(): + services_module.TestServiceStub + services_module.TestServiceServicer + services_module.add_TestServiceServicer_to_server + + def test_call(self): + self._protoc() + + for services_module in self._services_modules(): + server = grpc.server( + futures.ThreadPoolExecutor( + max_workers=test_constants.POOL_SIZE)) + services_module.add_TestServiceServicer_to_server( + _Servicer(self._messages_pb2.Response), server) + port = server.add_insecure_port('[::]:0') + server.start() + channel = grpc.insecure_channel('localhost:{}'.format(port)) + stub = services_module.TestServiceStub(channel) + response = stub.Call(self._messages_pb2.Request()) + self.assertEqual(self._messages_pb2.Response(), response) + + +def _create_test_case_class(split_proto, protoc_style): + attributes = {} + + name = '{}{}'.format('SplitProto' if split_proto else 'SameProto', + protoc_style.name()) + attributes['NAME'] = name + + if split_proto: + attributes['MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES'] = ( + 'split_messages', 'sub',) + attributes['MESSAGES_PROTO_FILE_NAME'] = 'messages.proto' + attributes['SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES'] = ( + 'split_services',) + attributes['SERVICES_PROTO_FILE_NAME'] = 'services.proto' + attributes['EXPECTED_MESSAGES_PB2'] = 'split_messages.sub.messages_pb2' + attributes['EXPECTED_SERVICES_PB2'] = 'split_services.services_pb2' + attributes['EXPECTED_SERVICES_PB2_GRPC'] = ( + 'split_services.services_pb2_grpc') + else: + attributes['MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES'] = () + attributes['MESSAGES_PROTO_FILE_NAME'] = 'same.proto' + attributes['SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES'] = () + attributes['SERVICES_PROTO_FILE_NAME'] = 'same.proto' + attributes['EXPECTED_MESSAGES_PB2'] = 'same_pb2' + attributes['EXPECTED_SERVICES_PB2'] = 'same_pb2' + attributes['EXPECTED_SERVICES_PB2_GRPC'] = 'same_pb2_grpc' + + attributes['PROTOC_STYLE'] = protoc_style + + attributes['__module__'] = _Test.__module__ + + return type('{}Test'.format(name), (_Test,), attributes) + + +def _create_test_case_classes(): + for split_proto in (False, True,): + for protoc_style in _PROTOC_STYLES: + yield _create_test_case_class(split_proto, protoc_style) + + +def load_tests(loader, tests, pattern): + tests = tuple( + loader.loadTestsFromTestCase(test_case_class) + for test_case_class in _create_test_case_classes()) + return unittest.TestSuite(tests=tests) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py deleted file mode 100644 index 5772620b602..00000000000 --- a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/messages.proto b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/sub/messages.proto similarity index 100% rename from src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/messages.proto rename to src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/sub/messages.proto diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py deleted file mode 100644 index 5772620b602..00000000000 --- a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index d61297b9187..bbd682b3353 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -3,10 +3,14 @@ "interop._insecure_intraop_test.InsecureIntraopTest", "interop._secure_intraop_test.SecureIntraopTest", "protoc_plugin._python_plugin_test.PythonPluginTest", - "protoc_plugin._split_definitions_test.SameCommonTest", - "protoc_plugin._split_definitions_test.SameSeparateTest", - "protoc_plugin._split_definitions_test.SplitCommonTest", - "protoc_plugin._split_definitions_test.SplitSeparateTest", + "protoc_plugin._split_definitions_test.SameProtoGrpcBeforeProtoProtocStyleTest", + "protoc_plugin._split_definitions_test.SameProtoMid2016ProtocStyleTest", + "protoc_plugin._split_definitions_test.SameProtoProtoBeforeGrpcProtocStyleTest", + "protoc_plugin._split_definitions_test.SameProtoSingleProtocExecutionProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoGrpcBeforeProtoProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoMid2016ProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoProtoBeforeGrpcProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoSingleProtocExecutionProtocStyleTest", "protoc_plugin.beta_python_plugin_test.PythonPluginTest", "reflection._reflection_servicer_test.ReflectionServicerTest", "testing._client_test.ClientTest", From 9eb8672bbe02dc47f2f667b99670b215d7431ef0 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Sun, 17 Sep 2017 23:43:30 -0700 Subject: [PATCH 160/227] Removing deprecated string to char * conversions --- include/grpc/compression.h | 4 +-- src/core/ext/census/base_resources.c | 28 +++++++++---------- .../filters/client_channel/client_channel.c | 13 +++++---- .../client_channel/client_channel_factory.c | 2 +- .../client_channel/client_channel_plugin.c | 4 +-- .../ext/filters/client_channel/http_proxy.c | 6 ++-- .../client_channel/lb_policy/grpclb/grpclb.c | 4 +-- .../lb_policy/round_robin/round_robin.c | 2 +- .../client_channel/lb_policy_factory.c | 2 +- .../resolver/dns/c_ares/dns_resolver_ares.c | 4 +-- .../resolver/fake/fake_resolver.c | 2 +- .../ext/filters/client_channel/subchannel.c | 2 +- .../message_compress_filter.c | 4 +-- .../server_load_reporting_plugin.c | 3 +- .../chttp2/client/insecure/channel_create.c | 2 +- .../client/insecure/channel_create_posix.c | 2 +- .../ext/transport/inproc/inproc_transport.c | 4 +-- src/core/lib/channel/channel_args.c | 12 ++++---- src/core/lib/channel/channel_stack.h | 2 +- src/core/lib/compression/compression.c | 4 +-- src/core/lib/debug/trace.h | 2 +- src/core/lib/iomgr/error.c | 2 +- src/core/lib/iomgr/resolve_address_posix.c | 2 +- src/core/lib/support/log_linux.c | 2 +- src/core/lib/support/string.c | 2 +- src/core/lib/surface/call.c | 8 +++--- src/core/lib/transport/transport_op_string.c | 2 +- src/cpp/client/client_context.cc | 2 +- src/cpp/server/server_context.cc | 2 +- test/core/compression/algorithm_test.c | 2 +- test/core/compression/compression_test.c | 2 +- test/core/compression/message_compress_test.c | 2 +- test/core/end2end/tests/compressed_payload.c | 2 +- .../stream_compression_compressed_payload.c | 2 +- 34 files changed, 71 insertions(+), 69 deletions(-) diff --git a/include/grpc/compression.h b/include/grpc/compression.h index 3a8de4b7b8a..13a8dd66ad9 100644 --- a/include/grpc/compression.h +++ b/include/grpc/compression.h @@ -44,13 +44,13 @@ int grpc_stream_compression_algorithm_parse( * algorithm. Note that \a name is statically allocated and must *not* be freed. * Returns 1 upon success, 0 otherwise. */ GRPCAPI int grpc_compression_algorithm_name( - grpc_compression_algorithm algorithm, char **name); + grpc_compression_algorithm algorithm, const char **name); /** Updates \a name with the encoding name corresponding to a valid \a * algorithm. Note that \a name is statically allocated and must *not* be freed. * Returns 1 upon success, 0 otherwise. */ GRPCAPI int grpc_stream_compression_algorithm_name( - grpc_stream_compression_algorithm algorithm, char **name); + grpc_stream_compression_algorithm algorithm, const char **name); /** Returns the compression algorithm corresponding to \a level for the * compression algorithms encoded in the \a accepted_encodings bitset. diff --git a/src/core/ext/census/base_resources.c b/src/core/ext/census/base_resources.c index 2114bf04cd0..1f2bb39fe0c 100644 --- a/src/core/ext/census/base_resources.c +++ b/src/core/ext/census/base_resources.c @@ -37,20 +37,20 @@ void define_base_resources() { google_census_Resource_BasicUnit numerator = google_census_Resource_BasicUnit_SECS; - resource r = {"client_rpc_latency", // name - "Client RPC latency in seconds", // description - 0, // prefix - 1, // n_numerators - &numerator, // numerators - 0, // n_denominators - NULL}; // denominators + resource r = {(char *)"client_rpc_latency", // name + (char *)"Client RPC latency in seconds", // description + 0, // prefix + 1, // n_numerators + &numerator, // numerators + 0, // n_denominators + NULL}; // denominators define_resource(&r); - r = (resource){"server_rpc_latency", // name - "Server RPC latency in seconds", // description - 0, // prefix - 1, // n_numerators - &numerator, // numerators - 0, // n_denominators - NULL}; // denominators + r = (resource){(char *)"server_rpc_latency", // name + (char *)"Server RPC latency in seconds", // description + 0, // prefix + 1, // n_numerators + &numerator, // numerators + 0, // n_denominators + NULL}; // denominators define_resource(&r); } diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index 129d0f368b7..016199b1f4f 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -375,7 +375,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, } // Extract the following fields from the resolver result, if non-NULL. bool lb_policy_updated = false; - char *lb_policy_name = NULL; + char *lb_policy_name_dup = NULL; bool lb_policy_name_changed = false; grpc_lb_policy *new_lb_policy = NULL; char *service_config_json = NULL; @@ -383,6 +383,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, grpc_slice_hash_table *method_params_table = NULL; if (chand->resolver_result != NULL) { // Find LB policy name. + const char *lb_policy_name = NULL; const grpc_arg *channel_arg = grpc_channel_args_find(chand->resolver_result, GRPC_ARG_LB_POLICY_NAME); if (channel_arg != NULL) { @@ -473,7 +474,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, // Before we clean up, save a copy of lb_policy_name, since it might // be pointing to data inside chand->resolver_result. // The copy will be saved in chand->lb_policy_name below. - lb_policy_name = gpr_strdup(lb_policy_name); + lb_policy_name_dup = gpr_strdup(lb_policy_name); grpc_channel_args_destroy(exec_ctx, chand->resolver_result); chand->resolver_result = NULL; } @@ -481,8 +482,8 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, gpr_log(GPR_DEBUG, "chand=%p: resolver result: lb_policy_name=\"%s\"%s, " "service_config=\"%s\"", - chand, lb_policy_name, lb_policy_name_changed ? " (changed)" : "", - service_config_json); + chand, lb_policy_name_dup, + lb_policy_name_changed ? " (changed)" : "", service_config_json); } // Now swap out fields in chand. Note that the new values may still // be NULL if (e.g.) the resolver failed to return results or the @@ -490,9 +491,9 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, // // First, swap out the data used by cc_get_channel_info(). gpr_mu_lock(&chand->info_mu); - if (lb_policy_name != NULL) { + if (lb_policy_name_dup != NULL) { gpr_free(chand->info_lb_policy_name); - chand->info_lb_policy_name = lb_policy_name; + chand->info_lb_policy_name = lb_policy_name_dup; } if (service_config_json != NULL) { gpr_free(chand->info_service_config_json); diff --git a/src/core/ext/filters/client_channel/client_channel_factory.c b/src/core/ext/filters/client_channel/client_channel_factory.c index e8aa4cda290..57eac8f8757 100644 --- a/src/core/ext/filters/client_channel/client_channel_factory.c +++ b/src/core/ext/filters/client_channel/client_channel_factory.c @@ -63,6 +63,6 @@ static const grpc_arg_pointer_vtable factory_arg_vtable = { grpc_arg grpc_client_channel_factory_create_channel_arg( grpc_client_channel_factory* factory) { - return grpc_channel_arg_pointer_create(GRPC_ARG_CLIENT_CHANNEL_FACTORY, + return grpc_channel_arg_pointer_create((char*)GRPC_ARG_CLIENT_CHANNEL_FACTORY, factory, &factory_arg_vtable); } diff --git a/src/core/ext/filters/client_channel/client_channel_plugin.c b/src/core/ext/filters/client_channel/client_channel_plugin.c index c32e83d0125..1f71c5a7f92 100644 --- a/src/core/ext/filters/client_channel/client_channel_plugin.c +++ b/src/core/ext/filters/client_channel/client_channel_plugin.c @@ -54,8 +54,8 @@ static bool set_default_host_if_unset(grpc_exec_ctx *exec_ctx, char *default_authority = grpc_get_default_authority( exec_ctx, grpc_channel_stack_builder_get_target(builder)); if (default_authority != NULL) { - grpc_arg arg = grpc_channel_arg_string_create(GRPC_ARG_DEFAULT_AUTHORITY, - default_authority); + grpc_arg arg = grpc_channel_arg_string_create( + (char *)GRPC_ARG_DEFAULT_AUTHORITY, default_authority); grpc_channel_args *new_args = grpc_channel_args_copy_and_add(args, &arg, 1); grpc_channel_stack_builder_set_channel_arguments(exec_ctx, builder, new_args); diff --git a/src/core/ext/filters/client_channel/http_proxy.c b/src/core/ext/filters/client_channel/http_proxy.c index ef3512ed833..0c3b009e440 100644 --- a/src/core/ext/filters/client_channel/http_proxy.c +++ b/src/core/ext/filters/client_channel/http_proxy.c @@ -157,7 +157,7 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, } grpc_arg args_to_add[2]; args_to_add[0] = grpc_channel_arg_string_create( - GRPC_ARG_HTTP_CONNECT_SERVER, + (char*)GRPC_ARG_HTTP_CONNECT_SERVER, uri->path[0] == '/' ? uri->path + 1 : uri->path); if (user_cred != NULL) { /* Use base64 encoding for user credentials as stated in RFC 7617 */ @@ -166,8 +166,8 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, char* header; gpr_asprintf(&header, "Proxy-Authorization:Basic %s", encoded_user_cred); gpr_free(encoded_user_cred); - args_to_add[1] = - grpc_channel_arg_string_create(GRPC_ARG_HTTP_CONNECT_HEADERS, header); + args_to_add[1] = grpc_channel_arg_string_create( + (char*)GRPC_ARG_HTTP_CONNECT_HEADERS, header); *new_args = grpc_channel_args_copy_and_add(args, args_to_add, 2); gpr_free(header); } else { diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index a776a07d99a..c4c4973b2df 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -1830,8 +1830,8 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. - grpc_arg new_arg = - grpc_channel_arg_string_create(GRPC_ARG_LB_POLICY_NAME, "grpclb"); + grpc_arg new_arg = grpc_channel_arg_string_create( + (char *)GRPC_ARG_LB_POLICY_NAME, (char *)"grpclb"); static const char *args_to_remove[] = {GRPC_ARG_LB_POLICY_NAME}; glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c index 8ac1a46abdf..a3a62e9f3ce 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c @@ -589,7 +589,7 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg, // Dispose of outdated subchannel lists. if (sd->subchannel_list != p->subchannel_list && sd->subchannel_list != p->latest_pending_subchannel_list) { - char *reason = NULL; + const char *reason = NULL; if (sd->subchannel_list->shutting_down) { reason = "sl_outdated_straggler"; rr_subchannel_list_unref(exec_ctx, sd->subchannel_list, reason); diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index acf5929746b..4d1405454c9 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -141,7 +141,7 @@ static const grpc_arg_pointer_vtable lb_addresses_arg_vtable = { grpc_arg grpc_lb_addresses_create_channel_arg( const grpc_lb_addresses* addresses) { return grpc_channel_arg_pointer_create( - GRPC_ARG_LB_ADDRESSES, (void*)addresses, &lb_addresses_arg_vtable); + (char*)GRPC_ARG_LB_ADDRESSES, (void*)addresses, &lb_addresses_arg_vtable); } grpc_lb_addresses* grpc_lb_addresses_find_channel_arg( diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index 371c59b8cf3..9bb229ad958 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -249,7 +249,7 @@ static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg, service_config_string); args_to_remove[num_args_to_remove++] = GRPC_ARG_SERVICE_CONFIG; new_args[num_args_to_add++] = grpc_channel_arg_string_create( - GRPC_ARG_SERVICE_CONFIG, service_config_string); + (char *)GRPC_ARG_SERVICE_CONFIG, service_config_string); service_config = grpc_service_config_create(service_config_string); if (service_config != NULL) { const char *lb_policy_name = @@ -257,7 +257,7 @@ static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg, if (lb_policy_name != NULL) { args_to_remove[num_args_to_remove++] = GRPC_ARG_LB_POLICY_NAME; new_args[num_args_to_add++] = grpc_channel_arg_string_create( - GRPC_ARG_LB_POLICY_NAME, (char *)lb_policy_name); + (char *)GRPC_ARG_LB_POLICY_NAME, (char *)lb_policy_name); } } } diff --git a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c index c4676e0299c..69ea440ae6a 100644 --- a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c +++ b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c @@ -210,7 +210,7 @@ grpc_arg grpc_fake_resolver_response_generator_arg( grpc_fake_resolver_response_generator* generator) { grpc_arg arg; arg.type = GRPC_ARG_POINTER; - arg.key = GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR; + arg.key = (char*)GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR; arg.value.pointer.p = generator; arg.value.pointer.vtable = &response_generator_arg_vtable; return arg; diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c index bc9c3cc782f..40a51c72d67 100644 --- a/src/core/ext/filters/client_channel/subchannel.c +++ b/src/core/ext/filters/client_channel/subchannel.c @@ -811,6 +811,6 @@ const char *grpc_get_subchannel_address_uri_arg(const grpc_channel_args *args) { grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address *addr) { return grpc_channel_arg_string_create( - GRPC_ARG_SUBCHANNEL_ADDRESS, + (char *)GRPC_ARG_SUBCHANNEL_ADDRESS, addr->len > 0 ? grpc_sockaddr_to_uri(addr) : gpr_strdup("")); } diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.c b/src/core/ext/filters/http/message_compress/message_compress_filter.c index 0145dffab01..f785e1355d3 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.c +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.c @@ -244,7 +244,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, &calld->slices, &tmp); if (did_compress) { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name; + const char *algo_name; const size_t before_size = calld->slices.length; const size_t after_size = tmp.length; const float savings_ratio = 1.0f - (float)after_size / (float)before_size; @@ -258,7 +258,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; } else { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name; + const char *algo_name; GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm, &algo_name)); gpr_log(GPR_DEBUG, diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c b/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c index f56afd27d2c..2486ead4272 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c +++ b/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c @@ -55,7 +55,8 @@ static bool maybe_add_server_load_reporting_filter( } grpc_arg grpc_load_reporting_enable_arg() { - return grpc_channel_arg_integer_create(GRPC_ARG_ENABLE_LOAD_REPORTING, 1); + return grpc_channel_arg_integer_create((char *)GRPC_ARG_ENABLE_LOAD_REPORTING, + 1); } /* Plugin registration */ diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c index cccb347bf10..6410a6043db 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c @@ -55,7 +55,7 @@ static grpc_channel *client_channel_factory_create_channel( } // Add channel arg containing the server URI. grpc_arg arg = grpc_channel_arg_string_create( - GRPC_ARG_SERVER_URI, + (char *)GRPC_ARG_SERVER_URI, grpc_resolver_factory_add_default_prefix_if_needed(exec_ctx, target)); const char *to_remove[] = {GRPC_ARG_SERVER_URI}; grpc_channel_args *new_args = diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c index 0346d50b6ce..dd88136f7b5 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c @@ -42,7 +42,7 @@ grpc_channel *grpc_insecure_channel_create_from_fd( (target, fd, args)); grpc_arg default_authority_arg = grpc_channel_arg_string_create( - GRPC_ARG_DEFAULT_AUTHORITY, "test.authority"); + (char *)GRPC_ARG_DEFAULT_AUTHORITY, (char *)"test.authority"); grpc_channel_args *final_args = grpc_channel_args_copy_and_add(args, &default_authority_arg, 1); diff --git a/src/core/ext/transport/inproc/inproc_transport.c b/src/core/ext/transport/inproc/inproc_transport.c index cd3e76a0b5b..31739d07dd6 100644 --- a/src/core/ext/transport/inproc/inproc_transport.c +++ b/src/core/ext/transport/inproc/inproc_transport.c @@ -1263,8 +1263,8 @@ grpc_channel *grpc_inproc_channel_create(grpc_server *server, grpc_arg default_authority_arg; default_authority_arg.type = GRPC_ARG_STRING; - default_authority_arg.key = GRPC_ARG_DEFAULT_AUTHORITY; - default_authority_arg.value.string = "inproc.authority"; + default_authority_arg.key = (char *)GRPC_ARG_DEFAULT_AUTHORITY; + default_authority_arg.value.string = (char *)"inproc.authority"; grpc_channel_args *client_args = grpc_channel_args_copy_and_add(args, &default_authority_arg, 1); diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 9215707f1c3..30248b3c609 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -243,7 +243,7 @@ grpc_channel_args *grpc_channel_args_set_compression_algorithm( GPR_ASSERT(algorithm < GRPC_COMPRESS_ALGORITHMS_COUNT); grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; + tmp.key = (char *)GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; tmp.value.integer = algorithm; return grpc_channel_args_copy_and_add(a, &tmp, 1); } @@ -253,7 +253,7 @@ grpc_channel_args *grpc_channel_args_set_stream_compression_algorithm( GPR_ASSERT(algorithm < GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT); grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; + tmp.key = (char *)GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; tmp.value.integer = algorithm; return grpc_channel_args_copy_and_add(a, &tmp, 1); } @@ -308,7 +308,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( if (grpc_channel_args_get_compression_algorithm(*a) == algorithm && state == 0) { - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT(grpc_compression_algorithm_name(algorithm, &algo_name) != 0); gpr_log(GPR_ERROR, "Tried to disable default compression algorithm '%s'. The " @@ -324,7 +324,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( /* create a new arg */ grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; + tmp.key = (char *)GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; /* all enabled by default */ tmp.value.integer = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; if (state != 0) { @@ -349,7 +349,7 @@ grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state( if (grpc_channel_args_get_stream_compression_algorithm(*a) == algorithm && state == 0) { - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT(grpc_stream_compression_algorithm_name(algorithm, &algo_name) != 0); gpr_log(GPR_ERROR, @@ -366,7 +366,7 @@ grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state( /* create a new arg */ grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; + tmp.key = (char *)GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; /* all enabled by default */ tmp.value.integer = (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1; if (state != 0) { diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index ae1cac31f74..f0de80f0c04 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -281,7 +281,7 @@ grpc_channel_stack *grpc_channel_stack_from_top_element( /* Given the top element of a call stack, get the call stack itself */ grpc_call_stack *grpc_call_stack_from_top_element(grpc_call_element *elem); -void grpc_call_log_op(char *file, int line, gpr_log_severity severity, +void grpc_call_log_op(const char *file, int line, gpr_log_severity severity, grpc_call_element *elem, grpc_transport_stream_op_batch *op); diff --git a/src/core/lib/compression/compression.c b/src/core/lib/compression/compression.c index ec84c018117..1cfac231295 100644 --- a/src/core/lib/compression/compression.c +++ b/src/core/lib/compression/compression.c @@ -60,7 +60,7 @@ int grpc_stream_compression_algorithm_parse( } int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, - char **name) { + const char **name) { GRPC_API_TRACE("grpc_compression_algorithm_parse(algorithm=%d, name=%p)", 2, ((int)algorithm, name)); switch (algorithm) { @@ -80,7 +80,7 @@ int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, } int grpc_stream_compression_algorithm_name( - grpc_stream_compression_algorithm algorithm, char **name) { + grpc_stream_compression_algorithm algorithm, const char **name) { GRPC_API_TRACE( "grpc_stream_compression_algorithm_parse(algorithm=%d, name=%p)", 2, ((int)algorithm, name)); diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h index dd9e6a30fe2..64f2e3fc331 100644 --- a/src/core/lib/debug/trace.h +++ b/src/core/lib/debug/trace.h @@ -35,7 +35,7 @@ typedef struct { #else bool value; #endif - char *name; + const char *name; } grpc_tracer_flag; #ifdef GRPC_THREADSAFE_TRACER diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index d5e00a8f643..aa05501537b 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -641,7 +641,7 @@ static char *key_time(grpc_error_times which) { static char *fmt_time(gpr_timespec tm) { char *out; - char *pfx = "!!"; + const char *pfx = "!!"; switch (tm.clock_type) { case GPR_CLOCK_MONOTONIC: pfx = "@monotonic:"; diff --git a/src/core/lib/iomgr/resolve_address_posix.c b/src/core/lib/iomgr/resolve_address_posix.c index 082e3b7947b..60cfeebd47d 100644 --- a/src/core/lib/iomgr/resolve_address_posix.c +++ b/src/core/lib/iomgr/resolve_address_posix.c @@ -85,7 +85,7 @@ static grpc_error *blocking_resolve_address_impl( if (s != 0) { /* Retry if well-known service name is recognized */ - char *svc[][2] = {{"http", "80"}, {"https", "443"}}; + const char *svc[][2] = {{"http", "80"}, {"https", "443"}}; for (i = 0; i < GPR_ARRAY_SIZE(svc); i++) { if (strcmp(port, svc[i][0]) == 0) { GRPC_SCHEDULING_START_BLOCKING_REGION; diff --git a/src/core/lib/support/log_linux.c b/src/core/lib/support/log_linux.c index 61d23464276..77550186932 100644 --- a/src/core/lib/support/log_linux.c +++ b/src/core/lib/support/log_linux.c @@ -57,7 +57,7 @@ void gpr_log(const char *file, int line, gpr_log_severity severity, } void gpr_default_log(gpr_log_func_args *args) { - char *final_slash; + const char *final_slash; char *prefix; const char *display_file; char time_buffer[64]; diff --git a/src/core/lib/support/string.c b/src/core/lib/support/string.c index 523e43445be..6b172df82f6 100644 --- a/src/core/lib/support/string.c +++ b/src/core/lib/support/string.c @@ -276,7 +276,7 @@ static void add_string_to_split(const char *beg, const char *end, char ***strs, void gpr_string_split(const char *input, const char *sep, char ***strs, size_t *nstrs) { - char *next; + const char *next; *strs = NULL; *nstrs = 0; size_t capstrs = 0; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index e258719c4da..a41eb336d48 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -1511,7 +1511,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, } else if (grpc_compression_options_is_stream_compression_algorithm_enabled( &compression_options, algo) == 0) { /* check if algorithm is supported by current channel config */ - char *algo_name = NULL; + const char *algo_name = NULL; grpc_stream_compression_algorithm_name(algo, &algo_name); gpr_asprintf(&error_msg, "Stream compression algorithm '%s' is disabled.", algo_name); @@ -1525,7 +1525,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, if (!GPR_BITGET(call->stream_encodings_accepted_by_peer, call->incoming_stream_compression_algorithm)) { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name = NULL; + const char *algo_name = NULL; grpc_stream_compression_algorithm_name( call->incoming_stream_compression_algorithm, &algo_name); gpr_log( @@ -1552,7 +1552,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, } else if (grpc_compression_options_is_algorithm_enabled( &compression_options, algo) == 0) { /* check if algorithm is supported by current channel config */ - char *algo_name = NULL; + const char *algo_name = NULL; grpc_compression_algorithm_name(algo, &algo_name); gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.", algo_name); @@ -1568,7 +1568,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, if (!GPR_BITGET(call->encodings_accepted_by_peer, call->incoming_compression_algorithm)) { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name = NULL; + const char *algo_name = NULL; grpc_compression_algorithm_name(call->incoming_compression_algorithm, &algo_name); gpr_log(GPR_ERROR, diff --git a/src/core/lib/transport/transport_op_string.c b/src/core/lib/transport/transport_op_string.c index 409a6c41039..858664715c6 100644 --- a/src/core/lib/transport/transport_op_string.c +++ b/src/core/lib/transport/transport_op_string.c @@ -197,7 +197,7 @@ char *grpc_transport_op_string(grpc_transport_op *op) { return out; } -void grpc_call_log_op(char *file, int line, gpr_log_severity severity, +void grpc_call_log_op(const char *file, int line, gpr_log_severity severity, grpc_call_element *elem, grpc_transport_stream_op_batch *op) { char *str = grpc_transport_stream_op_batch_string(op); diff --git a/src/cpp/client/client_context.cc b/src/cpp/client/client_context.cc index 3af8bdc11ac..40e95f3c05d 100644 --- a/src/cpp/client/client_context.cc +++ b/src/cpp/client/client_context.cc @@ -96,7 +96,7 @@ void ClientContext::set_call(grpc_call* call, void ClientContext::set_compression_algorithm( grpc_compression_algorithm algorithm) { - char* algorithm_name = nullptr; + const char* algorithm_name = nullptr; if (!grpc_compression_algorithm_name(algorithm, &algorithm_name)) { gpr_log(GPR_ERROR, "Name for compression algorithm '%d' unknown.", algorithm); diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index 4913682f1d1..d7876a000b4 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -190,7 +190,7 @@ bool ServerContext::IsCancelled() const { void ServerContext::set_compression_algorithm( grpc_compression_algorithm algorithm) { - char* algorithm_name = NULL; + const char* algorithm_name = NULL; if (!grpc_compression_algorithm_name(algorithm, &algorithm_name)) { gpr_log(GPR_ERROR, "Name for compression algorithm '%d' unknown.", algorithm); diff --git a/test/core/compression/algorithm_test.c b/test/core/compression/algorithm_test.c index e37c8c30100..a11e6e90ace 100644 --- a/test/core/compression/algorithm_test.c +++ b/test/core/compression/algorithm_test.c @@ -35,7 +35,7 @@ static void test_algorithm_mesh(void) { gpr_log(GPR_DEBUG, "test_algorithm_mesh"); for (i = 0; i < GRPC_COMPRESS_ALGORITHMS_COUNT; i++) { - char *name; + const char *name; grpc_compression_algorithm parsed; grpc_slice mdstr; grpc_mdelem mdelem; diff --git a/test/core/compression/compression_test.c b/test/core/compression/compression_test.c index 4b6026ec775..326a8003003 100644 --- a/test/core/compression/compression_test.c +++ b/test/core/compression/compression_test.c @@ -57,7 +57,7 @@ static void test_compression_algorithm_parse(void) { static void test_compression_algorithm_name(void) { int success; - char *name; + const char *name; size_t i; const char *valid_names[] = {"identity", "gzip", "deflate"}; const grpc_compression_algorithm valid_algorithms[] = { diff --git a/test/core/compression/message_compress_test.c b/test/core/compression/message_compress_test.c index fd692fe5a51..f7f4893deef 100644 --- a/test/core/compression/message_compress_test.c +++ b/test/core/compression/message_compress_test.c @@ -49,7 +49,7 @@ static void assert_passthrough(grpc_slice value, grpc_slice_buffer output; grpc_slice final; int was_compressed; - char *algorithm_name; + const char *algorithm_name; GPR_ASSERT(grpc_compression_algorithm_name(algorithm, &algorithm_name) != 0); gpr_log( diff --git a/test/core/end2end/tests/compressed_payload.c b/test/core/end2end/tests/compressed_payload.c index 429717a7be0..639af15454b 100644 --- a/test/core/end2end/tests/compressed_payload.c +++ b/test/core/end2end/tests/compressed_payload.c @@ -228,7 +228,7 @@ static void request_for_disabled_algorithm( /* with a certain error */ GPR_ASSERT(status == expected_error); - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT(grpc_compression_algorithm_name(algorithm_to_disable, &algo_name)); char *expected_details = NULL; gpr_asprintf(&expected_details, "Compression algorithm '%s' is disabled.", diff --git a/test/core/end2end/tests/stream_compression_compressed_payload.c b/test/core/end2end/tests/stream_compression_compressed_payload.c index 11e7999a1c3..5a69091a57f 100644 --- a/test/core/end2end/tests/stream_compression_compressed_payload.c +++ b/test/core/end2end/tests/stream_compression_compressed_payload.c @@ -228,7 +228,7 @@ static void request_for_disabled_algorithm( /* with a certain error */ GPR_ASSERT(status == expected_error); - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT( grpc_stream_compression_algorithm_name(algorithm_to_disable, &algo_name)); char *expected_details = NULL; From 571fa0c5967a4e7a504e5ad5c495bb4a68953166 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 11 Sep 2017 16:06:11 -0700 Subject: [PATCH 161/227] Add stream state tracer --- .../chttp2/transport/chttp2_plugin.c | 1 + .../chttp2/transport/chttp2_transport.h | 1 + .../transport/chttp2/transport/stream_lists.c | 52 +++++++++++++------ 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c index 78551df9c3c..6d09953830c 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c @@ -23,6 +23,7 @@ void grpc_chttp2_plugin_init(void) { grpc_register_tracer(&grpc_http_trace); grpc_register_tracer(&grpc_flowctl_trace); + grpc_register_tracer(&grpc_trace_http2_stream_state); #ifndef NDEBUG grpc_register_tracer(&grpc_trace_chttp2_refcount); #endif diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index 0c4e2a91c03..55fb1a8343e 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -25,6 +25,7 @@ extern grpc_tracer_flag grpc_http_trace; extern grpc_tracer_flag grpc_flowctl_trace; +extern grpc_tracer_flag grpc_trace_http2_stream_state; #ifndef NDEBUG extern grpc_tracer_flag grpc_trace_chttp2_refcount; diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.c b/src/core/ext/transport/chttp2/transport/stream_lists.c index 7cc85dea9c8..ee23b77efc5 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.c +++ b/src/core/ext/transport/chttp2/transport/stream_lists.c @@ -20,6 +20,27 @@ #include +static char *stream_list_id_string(grpc_chttp2_stream_list_id id) { + switch (id) { + case GRPC_CHTTP2_LIST_WRITABLE: + return "writable"; + case GRPC_CHTTP2_LIST_WRITING: + return "writing"; + case GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT: + return "stalled_by_transport"; + case GRPC_CHTTP2_LIST_STALLED_BY_STREAM: + return "stalled_by_stream"; + case GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY: + return "waiting_for_concurrency"; + case STREAM_LIST_COUNT: + GPR_UNREACHABLE_CODE(return "unknown"); + } + GPR_UNREACHABLE_CODE(return "unknown"); +} + +grpc_tracer_flag grpc_trace_http2_stream_state = + GRPC_TRACER_INITIALIZER(false, "http2_stream_state"); + /* core list management */ static bool stream_list_empty(grpc_chttp2_transport *t, @@ -44,6 +65,10 @@ static bool stream_list_pop(grpc_chttp2_transport *t, s->included[id] = 0; } *stream = s; + if (s && GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { + gpr_log(GPR_DEBUG, "t[%p]: pop stream %d from %s", t, s->id, + stream_list_id_string(id)); + } return s != 0; } @@ -62,6 +87,10 @@ static void stream_list_remove(grpc_chttp2_transport *t, grpc_chttp2_stream *s, } else { t->lists[id].tail = s->links[id].prev; } + if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { + gpr_log(GPR_DEBUG, "t[%p]: remove stream %d from %s", t, s->id, + stream_list_id_string(id)); + } } static bool stream_list_maybe_remove(grpc_chttp2_transport *t, @@ -90,6 +119,10 @@ static void stream_list_add_tail(grpc_chttp2_transport *t, } t->lists[id].tail = s; s->included[id] = 1; + if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { + gpr_log(GPR_DEBUG, "t[%p]: add stream %d from %s", t, s->id, + stream_list_id_string(id)); + } } static bool stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, @@ -150,17 +183,12 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport *t, void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - GRPC_FLOW_CONTROL_IF_TRACING( - gpr_log(GPR_DEBUG, "stream %u stalled by transport", s->id)); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } bool grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t, grpc_chttp2_stream **s) { - bool ret = stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); - GRPC_FLOW_CONTROL_IF_TRACING(if (ret) gpr_log( - GPR_DEBUG, "stream %u un-stalled by transport", (*s)->id)); - return ret; + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, @@ -170,23 +198,15 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - GRPC_FLOW_CONTROL_IF_TRACING( - gpr_log(GPR_DEBUG, "stream %u stalled by stream", s->id)); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport *t, grpc_chttp2_stream **s) { - bool ret = stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); - GRPC_FLOW_CONTROL_IF_TRACING( - if (ret) gpr_log(GPR_DEBUG, "stream %u un-stalled by stream", (*s)->id)); - return ret; + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - bool ret = stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); - GRPC_FLOW_CONTROL_IF_TRACING( - if (ret) gpr_log(GPR_DEBUG, "stream %u un-stalled by stream", s->id)); - return ret; + return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } From 0252252fdd5c468079b3d5e32cc4516b02dccbcc Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 11 Sep 2017 18:01:23 -0700 Subject: [PATCH 162/227] Update env var documentation --- doc/environment_variables.md | 1 + .../ext/transport/chttp2/transport/stream_lists.c | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/environment_variables.md b/doc/environment_variables.md index b79cd973633..f90f1d5b10e 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -50,6 +50,7 @@ some configuration as environment variables that can be set. - channel_stack_builder - traces information about channel stacks being built - executor - traces grpc's internal thread pool ('the executor') - http - traces state in the http2 transport engine + - http2_stream_state - traces all http2 stream state mutations. - http1 - traces HTTP/1.x operations performed by gRPC - inproc - traces the in-process transport - flowctl - traces http2 flow control diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.c b/src/core/ext/transport/chttp2/transport/stream_lists.c index ee23b77efc5..47cd22d1773 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.c +++ b/src/core/ext/transport/chttp2/transport/stream_lists.c @@ -66,8 +66,8 @@ static bool stream_list_pop(grpc_chttp2_transport *t, } *stream = s; if (s && GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { - gpr_log(GPR_DEBUG, "t[%p]: pop stream %d from %s", t, s->id, - stream_list_id_string(id)); + gpr_log(GPR_DEBUG, "%p[%d][%s]: pop from %s", t, s->id, + t->is_client ? "cli" : "svr", stream_list_id_string(id)); } return s != 0; } @@ -88,8 +88,8 @@ static void stream_list_remove(grpc_chttp2_transport *t, grpc_chttp2_stream *s, t->lists[id].tail = s->links[id].prev; } if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { - gpr_log(GPR_DEBUG, "t[%p]: remove stream %d from %s", t, s->id, - stream_list_id_string(id)); + gpr_log(GPR_DEBUG, "%p[%d][%s]: remove from %s", t, s->id, + t->is_client ? "cli" : "svr", stream_list_id_string(id)); } } @@ -120,8 +120,8 @@ static void stream_list_add_tail(grpc_chttp2_transport *t, t->lists[id].tail = s; s->included[id] = 1; if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { - gpr_log(GPR_DEBUG, "t[%p]: add stream %d from %s", t, s->id, - stream_list_id_string(id)); + gpr_log(GPR_DEBUG, "%p[%d][%s]: add to %s", t, s->id, + t->is_client ? "cli" : "svr", stream_list_id_string(id)); } } From 86eb4ec0ef68482faa960801899be964e545a8b2 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 18 Sep 2017 14:22:16 +0000 Subject: [PATCH 163/227] Fix a six.with_metaclass usage mistake I made this mistake in 2010985ab269c8df0443e4f3782cbdffb083e9d4 but only with yesterday's release of six 1.11.0 has it started failing ("TypeError: metaclass conflict: the metaclassof a derived class must be a (non-strict) subclass of the metaclasses of all its bases"). --- src/python/grpcio_testing/grpc_testing/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio_testing/grpc_testing/__init__.py b/src/python/grpcio_testing/grpc_testing/__init__.py index 917e11808e0..994274500c7 100644 --- a/src/python/grpcio_testing/grpc_testing/__init__.py +++ b/src/python/grpcio_testing/grpc_testing/__init__.py @@ -213,7 +213,7 @@ class StreamStreamChannelRpc(six.with_metaclass(abc.ABCMeta)): raise NotImplementedError() -class Channel(six.with_metaclass(abc.ABCMeta), grpc.Channel): +class Channel(six.with_metaclass(abc.ABCMeta, grpc.Channel)): """A grpc.Channel double with which to test a system that invokes RPCs.""" @abc.abstractmethod From a29032fd32d1e275bd05afc011e7a7c8db61f644 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 18 Sep 2017 16:33:37 +0200 Subject: [PATCH 164/227] test everything if internal_ci scripts change --- tools/run_tests/python_utils/filter_pull_request_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 4ad981237b7..f99143fdd78 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -86,7 +86,6 @@ _WHITELIST_DICT = { '^test/distrib/php/': [_PHP_TEST_SUITE], '^test/distrib/python/': [_PYTHON_TEST_SUITE], '^test/distrib/ruby/': [_RUBY_TEST_SUITE], - '^tools/internal_ci/': [], '^vsprojects/': [_WINDOWS_TEST_SUITE], 'binding\.gyp$': [_NODE_TEST_SUITE], 'composer\.json$': [_PHP_TEST_SUITE], From e4260e5b445ba01d141e2df2b642a846c5675ffa Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Mon, 11 Sep 2017 16:11:02 -0700 Subject: [PATCH 165/227] EMBED_OPENSSL before build php --- tools/run_tests/run_tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 591a0bebbde..176c1097ee9 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -533,6 +533,7 @@ class PhpLanguage(object): self.config = config self.args = args _check_compiler(self.args.compiler, ['default']) + self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] def test_specs(self): return [self.config.job_spec(['src/php/bin/run_tests.sh'], @@ -545,7 +546,7 @@ class PhpLanguage(object): return ['static_c', 'shared_c'] def make_options(self): - return [] + return self._make_options; def build_steps(self): return [['tools/run_tests/helper_scripts/build_php.sh']] @@ -569,6 +570,7 @@ class Php7Language(object): self.config = config self.args = args _check_compiler(self.args.compiler, ['default']) + self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] def test_specs(self): return [self.config.job_spec(['src/php/bin/run_tests.sh'], @@ -581,7 +583,7 @@ class Php7Language(object): return ['static_c', 'shared_c'] def make_options(self): - return [] + return self._make_options; def build_steps(self): return [['tools/run_tests/helper_scripts/build_php.sh']] From 3462fd1481e457a110b37c401f49b42bb31a0fb5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 18 Sep 2017 09:00:16 -0700 Subject: [PATCH 166/227] Fix test --- test/core/iomgr/pollset_set_test.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 5750ac0f4b3..70efca8b16f 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -433,8 +434,7 @@ int main(int argc, char **argv) { const char *poll_strategy = grpc_get_poll_strategy_name(); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); - grpc_iomgr_init(&exec_ctx); - grpc_iomgr_start(&exec_ctx); + grpc_init(); if (poll_strategy != NULL && (strcmp(poll_strategy, "epoll") == 0 || @@ -449,8 +449,8 @@ int main(int argc, char **argv) { poll_strategy); } - grpc_iomgr_shutdown(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); + grpc_shutdown(); return 0; } #else /* defined(GRPC_LINUX_EPOLL) */ From 966bf49b9c7b916c4f4a2be1dc7a9e8dae07582d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 18 Sep 2017 09:42:31 -0700 Subject: [PATCH 167/227] Add to closure_list_sched as well --- src/core/lib/iomgr/closure.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index 2253604342e..bb9e437b08f 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -199,7 +199,15 @@ void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { while (c != NULL) { grpc_closure *next = c->next_data.next; #ifndef NDEBUG - GPR_ASSERT(!c->scheduled); + if (c->scheduled) { + gpr_log(GPR_ERROR, + "Closure already scheduled. (closure: %p, created: [%s:%d], " + "previously scheduled at: [%s: %d] run?: %s", + closure, closure->file_created, closure->line_created, + closure->file_initiated, closure->line_initiated, + closure->run ? "true" : "false"); + abort(); + } c->scheduled = true; c->file_initiated = file; c->line_initiated = line; From 0dbfde67507abf83d46bb24098419ce5523cc653 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 18 Sep 2017 10:11:37 -0700 Subject: [PATCH 168/227] Restore no_barrier operations to some places where it is safe --- src/core/lib/surface/completion_queue.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 1329dec35d7..468360fbdef 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -965,7 +965,7 @@ static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx, cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); GPR_ASSERT(cqd->shutdown_called); - GPR_ASSERT(gpr_atm_acq_load(&cqd->pending_events) == 0); + GPR_ASSERT(gpr_atm_no_barrier_load(&cqd->pending_events) == 0); cq->poller_vtable->shutdown(exec_ctx, POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done); @@ -989,6 +989,9 @@ static void cq_shutdown_next(grpc_exec_ctx *exec_ctx, return; } cqd->shutdown_called = true; + /* Doing a full_fetch_add (i.e acq/release) here to match with + * cq_begin_op_for_next and and cq_end_op_for_next functions which read/write + * on this counter without necessarily holding a lock on cq */ if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) { cq_finish_shutdown_next(exec_ctx, cq); } From 87c09525317add9d887cf0580f91feb8b1e970b7 Mon Sep 17 00:00:00 2001 From: Michael Bausor Date: Mon, 18 Sep 2017 10:45:04 -0700 Subject: [PATCH 169/227] Apply formatting tool --- src/compiler/php_generator.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/compiler/php_generator.cc b/src/compiler/php_generator.cc index 9fa14d16840..6d9ff3a29c4 100644 --- a/src/compiler/php_generator.cc +++ b/src/compiler/php_generator.cc @@ -51,24 +51,25 @@ grpc::string PackageName(const FileDescriptor *file) { } } -grpc::string MessageIdentifierName(const grpc::string &name, const FileDescriptor *file) { +grpc::string MessageIdentifierName(const grpc::string &name, + const FileDescriptor *file) { std::vector tokens = grpc_generator::tokenize(name, "."); std::ostringstream oss; - oss << PackageName(file) - << "\\" + oss << PackageName(file) << "\\" << grpc_generator::CapitalizeFirstLetter(tokens[tokens.size() - 1]); return oss.str(); } - void PrintMethod(const MethodDescriptor *method, Printer *out) { const Descriptor *input_type = method->input_type(); const Descriptor *output_type = method->output_type(); map vars; vars["service_name"] = method->service()->full_name(); vars["name"] = method->name(); - vars["input_type_id"] = MessageIdentifierName(input_type->full_name(), input_type->file()); - vars["output_type_id"] = MessageIdentifierName(output_type->full_name(), output_type->file()); + vars["input_type_id"] = + MessageIdentifierName(input_type->full_name(), input_type->file()); + vars["output_type_id"] = + MessageIdentifierName(output_type->full_name(), output_type->file()); out->Print("/**\n"); out->Print(GetPHPComments(method, " *").c_str()); From dc1cd5437c81b57bad315a34dfe46c0eb6e666ad Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 10:51:22 -0700 Subject: [PATCH 170/227] Running generate_projects.sh --- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index c5c848ae447..55a7eed1a47 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -164,10 +164,10 @@ extern census_record_values_type census_record_values_import; typedef int(*grpc_compression_algorithm_parse_type)(grpc_slice value, grpc_compression_algorithm *algorithm); extern grpc_compression_algorithm_parse_type grpc_compression_algorithm_parse_import; #define grpc_compression_algorithm_parse grpc_compression_algorithm_parse_import -typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, char **name); +typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, const char **name); extern grpc_compression_algorithm_name_type grpc_compression_algorithm_name_import; #define grpc_compression_algorithm_name grpc_compression_algorithm_name_import -typedef int(*grpc_stream_compression_algorithm_name_type)(grpc_stream_compression_algorithm algorithm, char **name); +typedef int(*grpc_stream_compression_algorithm_name_type)(grpc_stream_compression_algorithm algorithm, const char **name); extern grpc_stream_compression_algorithm_name_type grpc_stream_compression_algorithm_name_import; #define grpc_stream_compression_algorithm_name grpc_stream_compression_algorithm_name_import typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level, uint32_t accepted_encodings); From 086e95ae657ab87b52de9675140b194c61874683 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Sun, 17 Sep 2017 23:59:11 +0000 Subject: [PATCH 171/227] Use grpc_1_0 flag in beta_python_plugin_test Beta code elements are not generated at all in _pb2_grpc.py files. This duplicates a lot of the in-test code generation done in _split_definitions_test. In a future clean-up we may want to deduplicate the common behavior, put it in a module available to all other tests, and do all of our testing of generated code with in-test code generation. --- .../protoc_plugin/beta_python_plugin_test.py | 341 +++++++++++++----- 1 file changed, 256 insertions(+), 85 deletions(-) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py index 83f21ecbbb7..424b153ff8d 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py @@ -12,19 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse import contextlib -import distutils.spawn -import errno -import itertools +import importlib import os -import pkg_resources +from os import path +import pkgutil import shutil -import subprocess import sys import tempfile import threading -import time import unittest from six import moves @@ -33,12 +29,22 @@ from grpc.beta import implementations from grpc.beta import interfaces from grpc.framework.foundation import future from grpc.framework.interfaces.face import face +from grpc_tools import protoc from tests.unit.framework.common import test_constants -import tests.protoc_plugin.protos.payload.test_payload_pb2 as payload_pb2 -import tests.protoc_plugin.protos.requests.r.test_requests_pb2 as request_pb2 -import tests.protoc_plugin.protos.responses.test_responses_pb2 as response_pb2 -import tests.protoc_plugin.protos.service.test_service_pb2 as service_pb2 +_RELATIVE_PROTO_PATH = 'relative_proto_path' +_RELATIVE_PYTHON_OUT = 'relative_python_out' + +_PROTO_FILES_PATH_COMPONENTS = ( + ('beta_grpc_plugin_test', 'payload', 'test_payload.proto',), + ('beta_grpc_plugin_test', 'requests', 'r', 'test_requests.proto',), + ('beta_grpc_plugin_test', 'responses', 'test_responses.proto',), + ('beta_grpc_plugin_test', 'service', 'test_service.proto',),) + +_PAYLOAD_PB2 = 'beta_grpc_plugin_test.payload.test_payload_pb2' +_REQUESTS_PB2 = 'beta_grpc_plugin_test.requests.r.test_requests_pb2' +_RESPONSES_PB2 = 'beta_grpc_plugin_test.responses.test_responses_pb2' +_SERVICE_PB2 = 'beta_grpc_plugin_test.service.test_service_pb2' # Identifiers of entities we expect to find in the generated module. SERVICER_IDENTIFIER = 'BetaTestServiceServicer' @@ -47,12 +53,50 @@ SERVER_FACTORY_IDENTIFIER = 'beta_create_TestService_server' STUB_FACTORY_IDENTIFIER = 'beta_create_TestService_stub' +@contextlib.contextmanager +def _system_path(path_insertion): + old_system_path = sys.path[:] + sys.path = sys.path[0:1] + path_insertion + sys.path[1:] + yield + sys.path = old_system_path + + +def _create_directory_tree(root, path_components_sequence): + created = set() + for path_components in path_components_sequence: + thus_far = '' + for path_component in path_components: + relative_path = path.join(thus_far, path_component) + if relative_path not in created: + os.makedirs(path.join(root, relative_path)) + created.add(relative_path) + thus_far = path.join(thus_far, path_component) + + +def _massage_proto_content(raw_proto_content): + imports_substituted = raw_proto_content.replace( + b'import "tests/protoc_plugin/protos/', + b'import "beta_grpc_plugin_test/') + package_statement_substituted = imports_substituted.replace( + b'package grpc_protoc_plugin;', b'package beta_grpc_protoc_plugin;') + return package_statement_substituted + + +def _packagify(directory): + for subdirectory, _, _ in os.walk(directory): + init_file_name = path.join(subdirectory, '__init__.py') + with open(init_file_name, 'wb') as init_file: + init_file.write(b'') + + class _ServicerMethods(object): - def __init__(self): + def __init__(self, payload_pb2, responses_pb2): self._condition = threading.Condition() self._paused = False self._fail = False + self._payload_pb2 = payload_pb2 + self._responses_pb2 = responses_pb2 @contextlib.contextmanager def pause(self): # pylint: disable=invalid-name @@ -79,22 +123,22 @@ class _ServicerMethods(object): self._condition.wait() def UnaryCall(self, request, unused_rpc_context): - response = response_pb2.SimpleResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.SimpleResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * request.response_size self._control() return response def StreamingOutputCall(self, request, unused_rpc_context): for parameter in request.response_parameters: - response = response_pb2.StreamingOutputCallResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.StreamingOutputCallResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * parameter.size self._control() yield response def StreamingInputCall(self, request_iter, unused_rpc_context): - response = response_pb2.StreamingInputCallResponse() + response = self._responses_pb2.StreamingInputCallResponse() aggregated_payload_size = 0 for request in request_iter: aggregated_payload_size += len(request.payload.payload_compressable) @@ -105,8 +149,8 @@ class _ServicerMethods(object): def FullDuplexCall(self, request_iter, unused_rpc_context): for request in request_iter: for parameter in request.response_parameters: - response = response_pb2.StreamingOutputCallResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.StreamingOutputCallResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * parameter.size self._control() yield response @@ -115,8 +159,8 @@ class _ServicerMethods(object): responses = [] for request in request_iter: for parameter in request.response_parameters: - response = response_pb2.StreamingOutputCallResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.StreamingOutputCallResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * parameter.size self._control() responses.append(response) @@ -125,7 +169,7 @@ class _ServicerMethods(object): @contextlib.contextmanager -def _CreateService(): +def _CreateService(payload_pb2, responses_pb2, service_pb2): """Provides a servicer backend and a stub. The servicer is just the implementation of the actual servicer passed to the @@ -136,7 +180,7 @@ def _CreateService(): the service bound to the stub and and stub is the stub on which to invoke RPCs. """ - servicer_methods = _ServicerMethods() + servicer_methods = _ServicerMethods(payload_pb2, responses_pb2) class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)): @@ -161,12 +205,12 @@ def _CreateService(): server.start() channel = implementations.insecure_channel('localhost', port) stub = getattr(service_pb2, STUB_FACTORY_IDENTIFIER)(channel) - yield (servicer_methods, stub) + yield servicer_methods, stub, server.stop(0) @contextlib.contextmanager -def _CreateIncompleteService(): +def _CreateIncompleteService(service_pb2): """Provides a servicer backend that fails to implement methods and its stub. The servicer is just the implementation of the actual servicer passed to the @@ -192,16 +236,16 @@ def _CreateIncompleteService(): server.stop(0) -def _streaming_input_request_iterator(): +def _streaming_input_request_iterator(payload_pb2, requests_pb2): for _ in range(3): - request = request_pb2.StreamingInputCallRequest() + request = requests_pb2.StreamingInputCallRequest() request.payload.payload_type = payload_pb2.COMPRESSABLE request.payload.payload_compressable = 'a' yield request -def _streaming_output_request(): - request = request_pb2.StreamingOutputCallRequest() +def _streaming_output_request(requests_pb2): + request = requests_pb2.StreamingOutputCallRequest() sizes = [1, 2, 3] request.response_parameters.add(size=sizes[0], interval_us=0) request.response_parameters.add(size=sizes[1], interval_us=0) @@ -209,11 +253,11 @@ def _streaming_output_request(): return request -def _full_duplex_request_iterator(): - request = request_pb2.StreamingOutputCallRequest() +def _full_duplex_request_iterator(requests_pb2): + request = requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) yield request - request = request_pb2.StreamingOutputCallRequest() + request = requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=2, interval_us=0) request.response_parameters.add(size=3, interval_us=0) yield request @@ -227,22 +271,78 @@ class PythonPluginTest(unittest.TestCase): methods and does not exist for response-streaming methods. """ + def setUp(self): + self._directory = tempfile.mkdtemp(dir='.') + self._proto_path = path.join(self._directory, _RELATIVE_PROTO_PATH) + self._python_out = path.join(self._directory, _RELATIVE_PYTHON_OUT) + + os.makedirs(self._proto_path) + os.makedirs(self._python_out) + + directories_path_components = { + proto_file_path_components[:-1] + for proto_file_path_components in _PROTO_FILES_PATH_COMPONENTS + } + _create_directory_tree(self._proto_path, directories_path_components) + self._proto_file_names = set() + for proto_file_path_components in _PROTO_FILES_PATH_COMPONENTS: + raw_proto_content = pkgutil.get_data( + 'tests.protoc_plugin.protos', + path.join(*proto_file_path_components[1:])) + massaged_proto_content = _massage_proto_content(raw_proto_content) + proto_file_name = path.join(self._proto_path, + *proto_file_path_components) + with open(proto_file_name, 'wb') as proto_file: + proto_file.write(massaged_proto_content) + self._proto_file_names.add(proto_file_name) + + def tearDown(self): + shutil.rmtree(self._directory) + + def _protoc(self): + args = [ + '', + '--proto_path={}'.format(self._proto_path), + '--python_out={}'.format(self._python_out), + '--grpc_python_out=grpc_1_0:{}'.format(self._python_out), + ] + list(self._proto_file_names) + protoc_exit_code = protoc.main(args) + self.assertEqual(0, protoc_exit_code) + + _packagify(self._python_out) + + with _system_path([ + self._python_out, + ]): + self._payload_pb2 = importlib.import_module(_PAYLOAD_PB2) + self._requests_pb2 = importlib.import_module(_REQUESTS_PB2) + self._responses_pb2 = importlib.import_module(_RESPONSES_PB2) + self._service_pb2 = importlib.import_module(_SERVICE_PB2) + def testImportAttributes(self): + self._protoc() + # check that we can access the generated module and its members. - self.assertIsNotNone(getattr(service_pb2, SERVICER_IDENTIFIER, None)) - self.assertIsNotNone(getattr(service_pb2, STUB_IDENTIFIER, None)) self.assertIsNotNone( - getattr(service_pb2, SERVER_FACTORY_IDENTIFIER, None)) + getattr(self._service_pb2, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone(getattr(self._service_pb2, STUB_IDENTIFIER, None)) self.assertIsNotNone( - getattr(service_pb2, STUB_FACTORY_IDENTIFIER, None)) + getattr(self._service_pb2, SERVER_FACTORY_IDENTIFIER, None)) + self.assertIsNotNone( + getattr(self._service_pb2, STUB_FACTORY_IDENTIFIER, None)) def testUpDown(self): - with _CreateService(): - request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2): + self._requests_pb2.SimpleRequest(response_size=13) def testIncompleteServicer(self): - with _CreateIncompleteService() as (_, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateIncompleteService(self._service_pb2) as (_, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) try: stub.UnaryCall(request, test_constants.LONG_TIMEOUT) except face.AbortionError as error: @@ -250,15 +350,21 @@ class PythonPluginTest(unittest.TestCase): error.code) def testUnaryCall(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) response = stub.UnaryCall(request, test_constants.LONG_TIMEOUT) expected_response = methods.UnaryCall(request, 'not a real context!') self.assertEqual(expected_response, response) def testUnaryCallFuture(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) # Check that the call does not block waiting for the server to respond. with methods.pause(): response_future = stub.UnaryCall.future( @@ -268,8 +374,11 @@ class PythonPluginTest(unittest.TestCase): self.assertEqual(expected_response, response) def testUnaryCallFutureExpired(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) with methods.pause(): response_future = stub.UnaryCall.future( request, test_constants.SHORT_TIMEOUT) @@ -277,24 +386,33 @@ class PythonPluginTest(unittest.TestCase): response_future.result() def testUnaryCallFutureCancelled(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) with methods.pause(): response_future = stub.UnaryCall.future(request, 1) response_future.cancel() self.assertTrue(response_future.cancelled()) def testUnaryCallFutureFailed(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) with methods.fail(): response_future = stub.UnaryCall.future( request, test_constants.LONG_TIMEOUT) self.assertIsNotNone(response_future.exception()) def testStreamingOutputCall(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) responses = stub.StreamingOutputCall(request, test_constants.LONG_TIMEOUT) expected_responses = methods.StreamingOutputCall( @@ -304,8 +422,11 @@ class PythonPluginTest(unittest.TestCase): self.assertEqual(expected_response, response) def testStreamingOutputCallExpired(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) with methods.pause(): responses = stub.StreamingOutputCall( request, test_constants.SHORT_TIMEOUT) @@ -313,8 +434,11 @@ class PythonPluginTest(unittest.TestCase): list(responses) def testStreamingOutputCallCancelled(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) responses = stub.StreamingOutputCall(request, test_constants.LONG_TIMEOUT) next(responses) @@ -323,8 +447,11 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testStreamingOutputCallFailed(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) with methods.fail(): responses = stub.StreamingOutputCall(request, 1) self.assertIsNotNone(responses) @@ -332,30 +459,46 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testStreamingInputCall(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): response = stub.StreamingInputCall( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) expected_response = methods.StreamingInputCall( - _streaming_input_request_iterator(), 'not a real RpcContext!') + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), + 'not a real RpcContext!') self.assertEqual(expected_response, response) def testStreamingInputCallFuture(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) response = response_future.result() expected_response = methods.StreamingInputCall( - _streaming_input_request_iterator(), 'not a real RpcContext!') + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), + 'not a real RpcContext!') self.assertEqual(expected_response, response) def testStreamingInputCallFutureExpired(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.SHORT_TIMEOUT) with self.assertRaises(face.ExpirationError): response_future.result() @@ -363,10 +506,14 @@ class PythonPluginTest(unittest.TestCase): face.ExpirationError) def testStreamingInputCallFutureCancelled(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) response_future.cancel() self.assertTrue(response_future.cancelled()) @@ -374,26 +521,38 @@ class PythonPluginTest(unittest.TestCase): response_future.result() def testStreamingInputCallFutureFailed(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.fail(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) self.assertIsNotNone(response_future.exception()) def testFullDuplexCall(self): - with _CreateService() as (methods, stub): - responses = stub.FullDuplexCall(_full_duplex_request_iterator(), - test_constants.LONG_TIMEOUT) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + responses = stub.FullDuplexCall( + _full_duplex_request_iterator(self._requests_pb2), + test_constants.LONG_TIMEOUT) expected_responses = methods.FullDuplexCall( - _full_duplex_request_iterator(), 'not a real RpcContext!') + _full_duplex_request_iterator(self._requests_pb2), + 'not a real RpcContext!') for expected_response, response in moves.zip_longest( expected_responses, responses): self.assertEqual(expected_response, response) def testFullDuplexCallExpired(self): - request_iterator = _full_duplex_request_iterator() - with _CreateService() as (methods, stub): + self._protoc() + + request_iterator = _full_duplex_request_iterator(self._requests_pb2) + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): responses = stub.FullDuplexCall(request_iterator, test_constants.SHORT_TIMEOUT) @@ -401,8 +560,11 @@ class PythonPluginTest(unittest.TestCase): list(responses) def testFullDuplexCallCancelled(self): - with _CreateService() as (methods, stub): - request_iterator = _full_duplex_request_iterator() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request_iterator = _full_duplex_request_iterator(self._requests_pb2) responses = stub.FullDuplexCall(request_iterator, test_constants.LONG_TIMEOUT) next(responses) @@ -411,8 +573,11 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testFullDuplexCallFailed(self): - request_iterator = _full_duplex_request_iterator() - with _CreateService() as (methods, stub): + self._protoc() + + request_iterator = _full_duplex_request_iterator(self._requests_pb2) + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.fail(): responses = stub.FullDuplexCall(request_iterator, test_constants.LONG_TIMEOUT) @@ -421,13 +586,16 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testHalfDuplexCall(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): def half_duplex_request_iterator(): - request = request_pb2.StreamingOutputCallRequest() + request = self._requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) yield request - request = request_pb2.StreamingOutputCallRequest() + request = self._requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=2, interval_us=0) request.response_parameters.add(size=3, interval_us=0) yield request @@ -441,6 +609,8 @@ class PythonPluginTest(unittest.TestCase): self.assertEqual(expected_response, response) def testHalfDuplexCallWedged(self): + self._protoc() + condition = threading.Condition() wait_cell = [False] @@ -455,14 +625,15 @@ class PythonPluginTest(unittest.TestCase): condition.notify_all() def half_duplex_request_iterator(): - request = request_pb2.StreamingOutputCallRequest() + request = self._requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) yield request with condition: while wait_cell[0]: condition.wait() - with _CreateService() as (methods, stub): + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with wait(): responses = stub.HalfDuplexCall(half_duplex_request_iterator(), test_constants.SHORT_TIMEOUT) From 31ad68a7e0d7f2bb6c3e6684782a88887b5190cc Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 18 Sep 2017 11:04:58 -0700 Subject: [PATCH 172/227] fix a bad typo --- src/core/lib/iomgr/closure.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index bb9e437b08f..05401bc2bdc 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -171,9 +171,9 @@ void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *c, gpr_log(GPR_ERROR, "Closure already scheduled. (closure: %p, created: [%s:%d], " "previously scheduled at: [%s: %d] run?: %s", - closure, closure->file_created, closure->line_created, - closure->file_initiated, closure->line_initiated, - closure->run ? "true" : "false"); + c, c->file_created, c->line_created, + c->file_initiated, c->line_initiated, + c->run ? "true" : "false"); abort(); } c->scheduled = true; @@ -203,9 +203,9 @@ void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { gpr_log(GPR_ERROR, "Closure already scheduled. (closure: %p, created: [%s:%d], " "previously scheduled at: [%s: %d] run?: %s", - closure, closure->file_created, closure->line_created, - closure->file_initiated, closure->line_initiated, - closure->run ? "true" : "false"); + c, c->file_created, c->line_created, + c->file_initiated, c->line_initiated, + c->run ? "true" : "false"); abort(); } c->scheduled = true; From 9667e8a57d0e4978f77a7fb35fab175dec2c9e10 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 18 Sep 2017 11:19:54 -0700 Subject: [PATCH 173/227] clang format --- src/core/lib/iomgr/closure.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index 05401bc2bdc..00edefc6ae4 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -171,9 +171,8 @@ void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *c, gpr_log(GPR_ERROR, "Closure already scheduled. (closure: %p, created: [%s:%d], " "previously scheduled at: [%s: %d] run?: %s", - c, c->file_created, c->line_created, - c->file_initiated, c->line_initiated, - c->run ? "true" : "false"); + c, c->file_created, c->line_created, c->file_initiated, + c->line_initiated, c->run ? "true" : "false"); abort(); } c->scheduled = true; @@ -203,9 +202,8 @@ void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { gpr_log(GPR_ERROR, "Closure already scheduled. (closure: %p, created: [%s:%d], " "previously scheduled at: [%s: %d] run?: %s", - c, c->file_created, c->line_created, - c->file_initiated, c->line_initiated, - c->run ? "true" : "false"); + c, c->file_created, c->line_created, c->file_initiated, + c->line_initiated, c->run ? "true" : "false"); abort(); } c->scheduled = true; From 4cd56837465234b7384bd3cb98abebced881f104 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 18 Sep 2017 11:22:03 -0700 Subject: [PATCH 174/227] Split C and C++ basic tests --- .../pull_request/grpc_basictests_c_dbg.cfg | 30 +++++++++++++++++++ .../pull_request/grpc_basictests_c_opt.cfg | 30 +++++++++++++++++++ .../pull_request/grpc_basictests_cpp_dbg.cfg | 30 +++++++++++++++++++ .../pull_request/grpc_basictests_cpp_opt.cfg | 30 +++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg new file mode 100644 index 00000000000..577cb28ae5f --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c dbg --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg new file mode 100644 index 00000000000..9e0b724b2e1 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c opt --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg new file mode 100644 index 00000000000..0fda74cf44e --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c++ dbg --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg new file mode 100644 index 00000000000..199a8905d94 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c++ opt --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} From 998bd2c7f46dc4bdd0e3381fcd1f4430fe3c3e32 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 18 Sep 2017 12:41:07 -0700 Subject: [PATCH 175/227] Fix leak in grpclb's lb_on_response_received_locked --- src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index a776a07d99a..4fcfa745f1d 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -1560,6 +1560,9 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, exec_ctx, glb_policy->lb_call, ops, (size_t)(op - ops), &glb_policy->lb_on_response_received); /* loop */ GPR_ASSERT(GRPC_CALL_OK == call_error); + } else { + GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, + "lb_on_response_received_locked_shutdown"); } } else { /* empty payload: call cancelled. */ /* dispose of the "lb_on_response_received_locked" weak ref taken in From 6a8c0ac2aaecacbdf28a7414109e175297cec003 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Sun, 14 May 2017 23:18:05 -0700 Subject: [PATCH 176/227] Fix ping policy --- .../chttp2/transport/chttp2_transport.c | 4 ++-- .../ext/transport/chttp2/transport/parsing.c | 5 +++++ .../ext/transport/chttp2/transport/writing.c | 16 +++++++++------- test/core/end2end/tests/bad_ping.c | 2 +- test/core/end2end/tests/ping.c | 5 ++++- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 79a9ed827f7..3a28762224d 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -557,8 +557,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } } - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; + /* No pings allowed before receiving a header or data frame. */ + t->ping_state.pings_before_data_required = 0; t->ping_state.is_delayed_ping_timer_set = false; t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 6c12c91365e..e52e19f3d2c 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -383,6 +383,8 @@ error_handler: /* t->parser = grpc_chttp2_data_parser_parse;*/ t->parser = grpc_chttp2_data_parser_parse; t->parser_data = &s->data_parser; + t->ping_state.pings_before_data_required = + t->ping_policy.max_pings_without_data; return GRPC_ERROR_NONE; } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) { /* handle stream errors by closing the stream */ @@ -559,6 +561,9 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx, (t->incoming_frame_flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) != 0; } + t->ping_state.pings_before_data_required = + t->ping_policy.max_pings_without_data; + /* could be a new grpc_chttp2_stream or an existing grpc_chttp2_stream */ s = grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); if (s == NULL) { diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 3ded801985d..d3699301e38 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -66,9 +66,17 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } return; } + if (t->keepalive_permit_without_calls == 0 && + grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { + gpr_log(GPR_DEBUG, "Ping delayed [%p]: no active stream", t->peer_string); + } + return; + } if (t->ping_state.pings_before_data_required == 0 && t->ping_policy.max_pings_without_data != 0) { - /* need to send something of substance before sending a ping again */ + /* need to receive something of substance before sending a ping again */ if (GRPC_TRACER_ON(grpc_http_trace) || GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "Ping delayed [%p]: too many recent pings: %d/%d", @@ -297,8 +305,6 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_window_update_create(s->id, stream_announce, &s->stats.outgoing)); - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; if (!t->is_client) { t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); @@ -375,8 +381,6 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( send_bytes); s->sending_bytes += send_bytes; } - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; if (!t->is_client) { t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); @@ -487,8 +491,6 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_window_update_create(0, transport_announce, &throwaway_stats)); - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; if (!t->is_client) { t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); diff --git a/test/core/end2end/tests/bad_ping.c b/test/core/end2end/tests/bad_ping.c index 12aceda6881..6713597fccf 100644 --- a/test/core/end2end/tests/bad_ping.c +++ b/test/core/end2end/tests/bad_ping.c @@ -71,7 +71,7 @@ static void test_bad_ping(grpc_end2end_test_config config) { .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 20}, + .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_BDP_PROBE, .value.integer = 0}}; diff --git a/test/core/end2end/tests/ping.c b/test/core/end2end/tests/ping.c index 112ad9d7d2a..b4d9b13a93d 100644 --- a/test/core/end2end/tests/ping.c +++ b/test/core/end2end/tests/ping.c @@ -42,7 +42,10 @@ static void test_ping(grpc_end2end_test_config config, .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 20}}; + .value.integer = 0}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, + .value.integer = 1}}; grpc_arg server_a[] = { {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS, From 27411495f6784a79890d676622e1c0d2f2a53584 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 15 May 2017 11:57:30 -0700 Subject: [PATCH 177/227] Fix keepalive_timeout --- test/core/end2end/tests/keepalive_timeout.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/test/core/end2end/tests/keepalive_timeout.c b/test/core/end2end/tests/keepalive_timeout.c index e0ead4ab62a..8d01f23c005 100644 --- a/test/core/end2end/tests/keepalive_timeout.c +++ b/test/core/end2end/tests/keepalive_timeout.c @@ -106,13 +106,13 @@ static void test_keepalive_timeout(grpc_end2end_test_config config) { .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_BDP_PROBE, - .value.integer = 1}}; + .value.integer = 0}}; - grpc_channel_args *client_args = NULL; - client_args = grpc_channel_args_copy_and_add(client_args, keepalive_args, 2); + grpc_channel_args client_args = {.num_args = GPR_ARRAY_SIZE(keepalive_args), + .args = keepalive_args}; grpc_end2end_test_fixture f = - begin_test(config, "keepalive_timeout", client_args, NULL); + begin_test(config, "keepalive_timeout", &client_args, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -216,12 +216,6 @@ static void test_keepalive_timeout(grpc_end2end_test_config config) { grpc_byte_buffer_destroy(response_payload); grpc_byte_buffer_destroy(response_payload_recv); - if (client_args != NULL) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_channel_args_destroy(&exec_ctx, client_args); - grpc_exec_ctx_finish(&exec_ctx); - } - end_test(&f); config.tear_down_data(&f); } From ad32075af84eba51466ef53ae68c2e6e0d899ee6 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 15 May 2017 15:38:06 -0700 Subject: [PATCH 178/227] Set last_ping_sent_time to inf_past after receiving data --- include/grpc/impl/codegen/grpc_types.h | 13 ++-- .../chttp2/transport/chttp2_transport.c | 62 +++++++++++-------- .../transport/chttp2/transport/frame_ping.c | 2 +- .../ext/transport/chttp2/transport/internal.h | 4 +- .../ext/transport/chttp2/transport/parsing.c | 2 + .../ext/transport/chttp2/transport/writing.c | 41 ++++++------ test/core/client_channel/lb_policies_test.c | 2 +- test/core/end2end/tests/bad_ping.c | 21 ++++--- test/core/end2end/tests/ping.c | 21 ++++--- 9 files changed, 95 insertions(+), 73 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 748dc717a33..90f03f49a3e 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -188,9 +188,14 @@ typedef struct { #define GRPC_ARG_HTTP2_MAX_FRAME_SIZE "grpc.http2.max_frame_size" /** Should BDP probing be performed? */ #define GRPC_ARG_HTTP2_BDP_PROBE "grpc.http2.bdp_probe" -/** Minimum time (in milliseconds) between successive ping frames being sent */ -#define GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS \ +/** Minimum time between sending successive ping frames without receiving any + data frame, Int valued, milliseconds. */ +#define GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS \ "grpc.http2.min_time_between_pings_ms" +/** Minimum allowed time between receiving successive ping frames without + sending any data frame. Int valued, milliseconds */ +#define GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS \ + "grpc.http2.min_ping_interval_without_data_ms" /** Channel arg to override the http2 :scheme header */ #define GRPC_ARG_HTTP2_SCHEME "grpc.http2_scheme" /** How many pings can we send before needing to send a data frame or header @@ -202,10 +207,6 @@ typedef struct { closing the transport? (0 indicates that the server can bear an infinite number of misbehaving pings) */ #define GRPC_ARG_HTTP2_MAX_PING_STRIKES "grpc.http2.max_ping_strikes" -/** Minimum allowed time between two pings without sending any data frame. Int - valued, seconds */ -#define GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS \ - "grpc.http2.min_ping_interval_without_data_ms" /** How much data are we willing to queue up per stream if GRPC_WRITE_BUFFER_HINT is set? This is an upper bound */ #define GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE "grpc.http2.write_buffer_size" diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 3a28762224d..c742444902c 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -152,10 +152,10 @@ static void send_ping_locked( static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error); -#define DEFAULT_MIN_TIME_BETWEEN_PINGS_MS 0 -#define DEFAULT_MAX_PINGS_BETWEEN_DATA 3 +#define DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ +#define DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ +#define DEFAULT_MAX_PINGS_BETWEEN_DATA 2 #define DEFAULT_MAX_PING_STRIKES 2 -#define DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ /** keepalive-relevant functions */ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, @@ -364,11 +364,11 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->ping_policy = (grpc_chttp2_repeated_ping_policy){ .max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA, - .min_time_between_pings = - gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN), + .min_sent_ping_interval_without_data = gpr_time_from_millis( + DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), .max_ping_strikes = DEFAULT_MAX_PING_STRIKES, - .min_ping_interval_without_data = gpr_time_from_millis( - DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), + .min_recv_ping_interval_without_data = gpr_time_from_millis( + DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), }; /* Keepalive setting */ @@ -434,23 +434,30 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->ping_policy.max_ping_strikes = grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){DEFAULT_MAX_PING_STRIKES, 0, INT_MAX}); - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS)) { - t->ping_policy.min_time_between_pings = gpr_time_from_millis( - grpc_channel_arg_get_integer( - &channel_args->args[i], - (grpc_integer_options){DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, 0, - INT_MAX}), - GPR_TIMESPAN); } else if (0 == - strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS)) { - t->ping_policy.min_ping_interval_without_data = gpr_time_from_millis( - grpc_channel_arg_get_integer( - &channel_args->args[i], - (grpc_integer_options){ - DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, 0, INT_MAX}), - GPR_TIMESPAN); + strcmp( + channel_args->args[i].key, + GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS)) { + t->ping_policy.min_sent_ping_interval_without_data = + gpr_time_from_millis( + grpc_channel_arg_get_integer( + &channel_args->args[i], + (grpc_integer_options){ + DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, 0, + INT_MAX}), + GPR_TIMESPAN); + } else if (0 == + strcmp( + channel_args->args[i].key, + GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS)) { + t->ping_policy.min_recv_ping_interval_without_data = + gpr_time_from_millis( + grpc_channel_arg_get_integer( + &channel_args->args[i], + (grpc_integer_options){ + DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, 0, + INT_MAX}), + GPR_TIMESPAN); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { t->write_buffer_size = (uint32_t)grpc_channel_arg_get_integer( @@ -625,6 +632,9 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, connectivity_state_set(exec_ctx, t, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error), "close_transport"); grpc_endpoint_shutdown(exec_ctx, t->ep, GRPC_ERROR_REF(error)); + if (t->ping_state.is_delayed_ping_timer_set) { + grpc_timer_cancel(exec_ctx, &t->ping_state.delayed_ping_timer); + } switch (t->keepalive_state) { case GRPC_CHTTP2_KEEPALIVE_STATE_WAITING: grpc_timer_cancel(exec_ctx, &t->keepalive_ping_timer); @@ -1729,8 +1739,10 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)tp; t->ping_state.is_delayed_ping_timer_set = false; - grpc_chttp2_initiate_write(exec_ctx, t, - GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING); + if (error == GRPC_ERROR_NONE) { + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING); + } } void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 81bd02ae702..d431d6b2dfa 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -92,7 +92,7 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); gpr_timespec next_allowed_ping = gpr_time_add(t->ping_recv_state.last_ping_recv_time, - t->ping_policy.min_ping_interval_without_data); + t->ping_policy.min_recv_ping_interval_without_data); if (t->keepalive_permit_without_calls == 0 && grpc_chttp2_stream_map_size(&t->stream_map) == 0) { diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index c2dfce7c9c7..1682be28dd9 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -112,10 +112,10 @@ typedef struct { } grpc_chttp2_ping_queue; typedef struct { - gpr_timespec min_time_between_pings; int max_pings_without_data; int max_ping_strikes; - gpr_timespec min_ping_interval_without_data; + gpr_timespec min_sent_ping_interval_without_data; + gpr_timespec min_recv_ping_interval_without_data; } grpc_chttp2_repeated_ping_policy; typedef struct { diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index e52e19f3d2c..3db1ad41231 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -385,6 +385,7 @@ error_handler: t->parser_data = &s->data_parser; t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; + t->ping_state.last_ping_sent_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); return GRPC_ERROR_NONE; } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) { /* handle stream errors by closing the stream */ @@ -563,6 +564,7 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx, t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; + t->ping_state.last_ping_sent_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); /* could be a new grpc_chttp2_stream or an existing grpc_chttp2_stream */ s = grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index d3699301e38..d61063ed31d 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -66,14 +66,6 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } return; } - if (t->keepalive_permit_without_calls == 0 && - grpc_chttp2_stream_map_size(&t->stream_map) == 0) { - if (GRPC_TRACER_ON(grpc_http_trace) || - GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { - gpr_log(GPR_DEBUG, "Ping delayed [%p]: no active stream", t->peer_string); - } - return; - } if (t->ping_state.pings_before_data_required == 0 && t->ping_policy.max_pings_without_data != 0) { /* need to receive something of substance before sending a ping again */ @@ -86,11 +78,18 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, return; } gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_timespec elapsed = gpr_time_sub(now, t->ping_state.last_ping_sent_time); - /*gpr_log(GPR_DEBUG, "elapsed:%d.%09d min:%d.%09d", (int)elapsed.tv_sec, - elapsed.tv_nsec, (int)t->ping_policy.min_time_between_pings.tv_sec, - (int)t->ping_policy.min_time_between_pings.tv_nsec);*/ - if (gpr_time_cmp(elapsed, t->ping_policy.min_time_between_pings) < 0) { + gpr_timespec next_allowed_ping = + gpr_time_add(t->ping_state.last_ping_sent_time, + t->ping_policy.min_sent_ping_interval_without_data); + if (t->keepalive_permit_without_calls == 0 && + grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + next_allowed_ping = gpr_time_add(t->ping_recv_state.last_ping_recv_time, + gpr_time_from_seconds(7200, GPR_TIMESPAN)); + } + /*gpr_log(GPR_DEBUG, "next_allowed_ping:%d.%09d now:%d.%09d", + (int)next_allowed_ping.tv_sec, (int)next_allowed_pingpsed.tv_nsec, + (int)now.tv_sec, (int)now.tv_nsec);*/ + if (gpr_time_cmp(next_allowed_ping, now) > 0) { /* not enough elapsed time between successive pings */ if (GRPC_TRACER_ON(grpc_http_trace) || GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { @@ -100,11 +99,11 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } if (!t->ping_state.is_delayed_ping_timer_set) { t->ping_state.is_delayed_ping_timer_set = true; - grpc_timer_init(exec_ctx, &t->ping_state.delayed_ping_timer, - gpr_time_add(t->ping_state.last_ping_sent_time, - t->ping_policy.min_time_between_pings), - &t->retry_initiate_ping_locked, - gpr_now(GPR_CLOCK_MONOTONIC)); + grpc_timer_init( + exec_ctx, &t->ping_state.delayed_ping_timer, + gpr_time_add(t->ping_state.last_ping_sent_time, + t->ping_policy.min_sent_ping_interval_without_data), + &t->retry_initiate_ping_locked, gpr_now(GPR_CLOCK_MONOTONIC)); } return; } @@ -127,6 +126,12 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_ping_create(false, pq->inflight_id)); GRPC_STATS_INC_HTTP2_PINGS_SENT(exec_ctx); t->ping_state.last_ping_sent_time = now; + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { + gpr_log(GPR_DEBUG, "Ping sent [%p]: %d/%d", t->peer_string, + t->ping_state.pings_before_data_required, + t->ping_policy.max_pings_without_data); + } t->ping_state.pings_before_data_required -= (t->ping_state.pings_before_data_required != 0); } diff --git a/test/core/client_channel/lb_policies_test.c b/test/core/client_channel/lb_policies_test.c index f70a9fc8808..ba37cd673f6 100644 --- a/test/core/client_channel/lb_policies_test.c +++ b/test/core/client_channel/lb_policies_test.c @@ -519,7 +519,7 @@ static grpc_channel *create_client(const servers_fixture *f) { arg_array[1].key = GRPC_ARG_LB_POLICY_NAME; arg_array[1].value.string = "ROUND_ROBIN"; arg_array[2].type = GRPC_ARG_INTEGER; - arg_array[2].key = GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS; + arg_array[2].key = GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS; arg_array[2].value.integer = 0; args.num_args = GPR_ARRAY_SIZE(arg_array); args.args = arg_array; diff --git a/test/core/end2end/tests/bad_ping.c b/test/core/end2end/tests/bad_ping.c index 6713597fccf..c97d11b3067 100644 --- a/test/core/end2end/tests/bad_ping.c +++ b/test/core/end2end/tests/bad_ping.c @@ -66,18 +66,19 @@ static void end_test(grpc_end2end_test_fixture *f) { static void test_bad_ping(grpc_end2end_test_config config) { grpc_end2end_test_fixture f = config.create_fixture(NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); - grpc_arg client_a[] = {{.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS, - .value.integer = 0}, - {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 0}, - {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_BDP_PROBE, - .value.integer = 0}}; + grpc_arg client_a[] = { + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, + .value.integer = 10}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, + .value.integer = 0}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_BDP_PROBE, + .value.integer = 0}}; grpc_arg server_a[] = { {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS, + .key = GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, .value.integer = 300000 /* 5 minutes */}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MAX_PING_STRIKES, diff --git a/test/core/end2end/tests/ping.c b/test/core/end2end/tests/ping.c index b4d9b13a93d..23c82569ba8 100644 --- a/test/core/end2end/tests/ping.c +++ b/test/core/end2end/tests/ping.c @@ -37,18 +37,19 @@ static void test_ping(grpc_end2end_test_config config, grpc_connectivity_state state = GRPC_CHANNEL_IDLE; int i; - grpc_arg client_a[] = {{.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS, - .value.integer = 0}, - {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 0}, - {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, - .value.integer = 1}}; + grpc_arg client_a[] = { + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, + .value.integer = 10}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, + .value.integer = 0}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, + .value.integer = 1}}; grpc_arg server_a[] = { {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS, + .key = GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, From 1ecbebc08fe578236fbedefe770aa54b157221b4 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 18 May 2017 15:15:46 -0700 Subject: [PATCH 179/227] Set the ping behavior as described in the proposal --- .../transport/chttp2/transport/chttp2_transport.c | 2 +- src/core/ext/transport/chttp2/transport/writing.c | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index c742444902c..94dff495098 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -154,7 +154,7 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, #define DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ #define DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ -#define DEFAULT_MAX_PINGS_BETWEEN_DATA 2 +#define DEFAULT_MAX_PINGS_BETWEEN_DATA 0 /* unlimited */ #define DEFAULT_MAX_PING_STRIKES 2 /** keepalive-relevant functions */ diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index d61063ed31d..8721803267d 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -86,9 +86,9 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, next_allowed_ping = gpr_time_add(t->ping_recv_state.last_ping_recv_time, gpr_time_from_seconds(7200, GPR_TIMESPAN)); } - /*gpr_log(GPR_DEBUG, "next_allowed_ping:%d.%09d now:%d.%09d", - (int)next_allowed_ping.tv_sec, (int)next_allowed_pingpsed.tv_nsec, - (int)now.tv_sec, (int)now.tv_nsec);*/ + /* gpr_log(GPR_DEBUG, "next_allowed_ping:%d.%09d now:%d.%09d", + (int)next_allowed_ping.tv_sec, (int)next_allowed_ping.tv_nsec, + (int)now.tv_sec, (int)now.tv_nsec); */ if (gpr_time_cmp(next_allowed_ping, now) > 0) { /* not enough elapsed time between successive pings */ if (GRPC_TRACER_ON(grpc_http_trace) || @@ -99,11 +99,9 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } if (!t->ping_state.is_delayed_ping_timer_set) { t->ping_state.is_delayed_ping_timer_set = true; - grpc_timer_init( - exec_ctx, &t->ping_state.delayed_ping_timer, - gpr_time_add(t->ping_state.last_ping_sent_time, - t->ping_policy.min_sent_ping_interval_without_data), - &t->retry_initiate_ping_locked, gpr_now(GPR_CLOCK_MONOTONIC)); + grpc_timer_init(exec_ctx, &t->ping_state.delayed_ping_timer, + next_allowed_ping, &t->retry_initiate_ping_locked, + gpr_now(GPR_CLOCK_MONOTONIC)); } return; } From 18ea66fe3787ed27cc444a68451de05e34c4d024 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 22 Jun 2017 11:32:36 -0700 Subject: [PATCH 180/227] Update writing.c with the trailers-only response change --- src/core/ext/transport/chttp2/transport/writing.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 8721803267d..7d9d4867e14 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -268,8 +268,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( .stats = &s->stats.outgoing}; grpc_chttp2_encode_header(exec_ctx, &t->hpack_compressor, NULL, 0, s->send_initial_metadata, &hopt, &t->outbuf); - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; + now_writing = true; if (!t->is_client) { t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); From d7eda6127af92a311f78c57dabc24f679d493ae3 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 29 Jun 2017 18:55:45 -0700 Subject: [PATCH 181/227] Add more options in grpc_chttp2_config_default_keepalive_args --- .../chttp2/transport/chttp2_transport.c | 64 +++++++++++++++---- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 94dff495098..144780b67b0 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -64,6 +64,11 @@ #define DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS false #define KEEPALIVE_TIME_BACKOFF_MULTIPLIER 2 +#define DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ +#define DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ +#define DEFAULT_MAX_PINGS_BETWEEN_DATA 0 /* unlimited */ +#define DEFAULT_MAX_PING_STRIKES 2 + static int g_default_client_keepalive_time_ms = DEFAULT_CLIENT_KEEPALIVE_TIME_MS; static int g_default_client_keepalive_timeout_ms = @@ -75,6 +80,13 @@ static int g_default_server_keepalive_timeout_ms = static bool g_default_keepalive_permit_without_calls = DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS; +static int g_default_min_sent_ping_interval_without_data_ms = + DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS; +static int g_default_min_recv_ping_interval_without_data_ms = + DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS; +static int g_default_max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA; +static int g_default_max_ping_strikes = DEFAULT_MAX_PING_STRIKES; + #define MAX_CLIENT_STREAM_ID 0x7fffffffu grpc_tracer_flag grpc_http_trace = GRPC_TRACER_INITIALIZER(false, "http"); grpc_tracer_flag grpc_flowctl_trace = GRPC_TRACER_INITIALIZER(false, "flowctl"); @@ -152,11 +164,6 @@ static void send_ping_locked( static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error); -#define DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ -#define DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ -#define DEFAULT_MAX_PINGS_BETWEEN_DATA 0 /* unlimited */ -#define DEFAULT_MAX_PING_STRIKES 2 - /** keepalive-relevant functions */ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); @@ -363,12 +370,12 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_CHTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA, 1); t->ping_policy = (grpc_chttp2_repeated_ping_policy){ - .max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA, + .max_pings_without_data = g_default_max_pings_without_data, .min_sent_ping_interval_without_data = gpr_time_from_millis( - DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), - .max_ping_strikes = DEFAULT_MAX_PING_STRIKES, + g_default_min_sent_ping_interval_without_data_ms, GPR_TIMESPAN), + .max_ping_strikes = g_default_max_ping_strikes, .min_recv_ping_interval_without_data = gpr_time_from_millis( - DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), + g_default_min_recv_ping_interval_without_data_ms, GPR_TIMESPAN), }; /* Keepalive setting */ @@ -428,12 +435,13 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA)) { t->ping_policy.max_pings_without_data = grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){DEFAULT_MAX_PINGS_BETWEEN_DATA, 0, INT_MAX}); + (grpc_integer_options){g_default_max_pings_without_data, 0, + INT_MAX}); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_MAX_PING_STRIKES)) { t->ping_policy.max_ping_strikes = grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){DEFAULT_MAX_PING_STRIKES, 0, INT_MAX}); + (grpc_integer_options){g_default_max_ping_strikes, 0, INT_MAX}); } else if (0 == strcmp( channel_args->args[i].key, @@ -443,7 +451,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){ - DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, 0, + g_default_min_sent_ping_interval_without_data_ms, 0, INT_MAX}), GPR_TIMESPAN); } else if (0 == @@ -455,7 +463,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){ - DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, 0, + g_default_min_recv_ping_interval_without_data_ms, 0, INT_MAX}), GPR_TIMESPAN); } else if (0 == strcmp(channel_args->args[i].key, @@ -2643,6 +2651,36 @@ void grpc_chttp2_config_default_keepalive_args(grpc_channel_args *args, &args->args[i], (grpc_integer_options){g_default_keepalive_permit_without_calls, 0, 1}); + } else if (0 == + strcmp(args->args[i].key, GRPC_ARG_HTTP2_MAX_PING_STRIKES)) { + g_default_max_ping_strikes = grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){g_default_max_ping_strikes, 0, INT_MAX}); + } else if (0 == strcmp(args->args[i].key, + GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA)) { + g_default_max_pings_without_data = grpc_channel_arg_get_integer( + &args->args[i], (grpc_integer_options){ + g_default_max_pings_without_data, 0, INT_MAX}); + } else if (0 == + strcmp( + args->args[i].key, + GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS)) { + g_default_min_sent_ping_interval_without_data_ms = + grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){ + g_default_min_sent_ping_interval_without_data_ms, 0, + INT_MAX}); + } else if (0 == + strcmp( + args->args[i].key, + GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS)) { + g_default_min_recv_ping_interval_without_data_ms = + grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){ + g_default_min_recv_ping_interval_without_data_ms, 0, + INT_MAX}); } } } From a594ba0bd7be974cff1a8d1eeb3b21eb049e507e Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 7 Sep 2017 19:04:59 -0700 Subject: [PATCH 182/227] add a client side utility for extracting a google rpc status from a grpc::Status --- grpc.gemspec | 1 + src/ruby/lib/grpc/google_rpc_status_utils.rb | 28 +++ src/ruby/spec/google_rpc_status_utils_spec.rb | 223 ++++++++++++++++++ templates/grpc.gemspec.template | 1 + 4 files changed, 253 insertions(+) create mode 100644 src/ruby/lib/grpc/google_rpc_status_utils.rb create mode 100644 src/ruby/spec/google_rpc_status_utils_spec.rb diff --git a/grpc.gemspec b/grpc.gemspec index 2d0f9fd4504..96323aed106 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -29,6 +29,7 @@ Gem::Specification.new do |s| s.add_dependency 'google-protobuf', '~> 3.1' s.add_dependency 'googleauth', '~> 0.5.1' + s.add_dependency 'googleapis-common-protos-types', '~> 1.0.0' s.add_development_dependency 'bundler', '~> 1.9' s.add_development_dependency 'facter', '~> 2.4' diff --git a/src/ruby/lib/grpc/google_rpc_status_utils.rb b/src/ruby/lib/grpc/google_rpc_status_utils.rb new file mode 100644 index 00000000000..fdadd6b76e4 --- /dev/null +++ b/src/ruby/lib/grpc/google_rpc_status_utils.rb @@ -0,0 +1,28 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative './grpc' +require 'google/rpc/status_pb' + +# GRPC contains the General RPC module. +module GRPC + # GoogleRpcStatusUtils provides utilities to convert between a + # GRPC::Core::Status and a deserialized Google::Rpc::Status proto + class GoogleRpcStatusUtils + def self.extract_google_rpc_status(status) + fail ArgumentError, 'bad type' unless status.is_a? Struct::Status + Google::Rpc::Status.decode(status.metadata['grpc-status-details-bin']) + end + end +end diff --git a/src/ruby/spec/google_rpc_status_utils_spec.rb b/src/ruby/spec/google_rpc_status_utils_spec.rb new file mode 100644 index 00000000000..fe221c30ddd --- /dev/null +++ b/src/ruby/spec/google_rpc_status_utils_spec.rb @@ -0,0 +1,223 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'grpc' +require_relative '../lib/grpc/google_rpc_status_utils' +require_relative '../pb/src/proto/grpc/testing/messages_pb' +require_relative '../pb/src/proto/grpc/testing/messages_pb' +require 'google/protobuf/well_known_types' + +include GRPC::Core + +describe 'conversion from a status struct to a google protobuf status' do + it 'fails if the input is not a status struct' do + begin + GRPC::GoogleRpcStatusUtils.extract_google_rpc_status('string') + rescue => e + exception = e + end + expect(exception.is_a?(ArgumentError)).to be true + expect(exception.message.include?('bad type')).to be true + end + + it 'fails with some error if the header key is missing' do + status = Struct::Status.new(1, 'details', key: 'val') + expect(status.metadata.nil?).to be false + expect do + GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + end.to raise_error(StandardError) + end + + it 'fails with some error if the header key fails to deserialize' do + status = Struct::Status.new(1, 'details', + 'grpc-status-details-bin' => 'string_val') + expect do + GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + end.to raise_error(StandardError) + end + + it 'silently ignores erroneous mismatch between messages in '\ + 'status struct and protobuf status' do + proto = Google::Rpc::Status.new(code: 1, message: 'proto message') + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(1, 'struct message', + 'grpc-status-details-bin' => encoded_proto) + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(rpc_status).to eq(proto) + end + + it 'silently ignores erroneous mismatch between codes in status struct '\ + 'and protobuf status' do + proto = Google::Rpc::Status.new(code: 1, message: 'matching message') + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(2, 'matching message', + 'grpc-status-details-bin' => encoded_proto) + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(rpc_status).to eq(proto) + end + + it 'can succesfully convert a status struct into a google protobuf status '\ + 'when there are no rpcstatus details' do + proto = Google::Rpc::Status.new(code: 1, message: 'matching message') + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(1, 'matching message', + 'grpc-status-details-bin' => encoded_proto) + out = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(out.code).to eq(1) + expect(out.message).to eq('matching message') + expect(out.details).to eq([]) + end + + it 'can succesfully convert a status struct into a google protobuf '\ + 'status when there are multiple rpcstatus details' do + simple_request_any = Google::Protobuf::Any.new + simple_request = Grpc::Testing::SimpleRequest.new( + payload: Grpc::Testing::Payload.new(body: 'request')) + simple_request_any.pack(simple_request) + simple_response_any = Google::Protobuf::Any.new + simple_response = Grpc::Testing::SimpleResponse.new( + payload: Grpc::Testing::Payload.new(body: 'response')) + simple_response_any.pack(simple_response) + payload_any = Google::Protobuf::Any.new + payload = Grpc::Testing::Payload.new(body: 'payload') + payload_any.pack(payload) + proto = Google::Rpc::Status.new(code: 1, + message: 'matching message', + details: [ + simple_request_any, + simple_response_any, + payload_any + ]) + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(1, 'matching message', + 'grpc-status-details-bin' => encoded_proto) + out = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(out.code).to eq(1) + expect(out.message).to eq('matching message') + expect(out.details[0].unpack( + Grpc::Testing::SimpleRequest)).to eq(simple_request) + expect(out.details[1].unpack( + Grpc::Testing::SimpleResponse)).to eq(simple_response) + expect(out.details[2].unpack( + Grpc::Testing::Payload)).to eq(payload) + end +end + +# Test message +class EchoMsg + def self.marshal(_o) + '' + end + + def self.unmarshal(_o) + EchoMsg.new + end +end + +# A test service that fills in the "reserved" grpc-status-details-bin trailer, +# for client-side testing of GoogleRpcStatus protobuf extraction from trailers. +class GoogleRpcStatusTestService + include GRPC::GenericService + rpc :an_rpc, EchoMsg, EchoMsg + + def initialize(encoded_rpc_status) + @encoded_rpc_status = encoded_rpc_status + end + + def an_rpc(_, _) + # TODO: create a server-side utility API for sending a google rpc status. + # Applications are not expected to set the grpc-status-details-bin + # ("grpc"-fixed and reserved for library use) manually. + # Doing so here is only for testing of the client-side api for extracting + # a google rpc status, which is useful + # when the interacting with a server that does fill in this trailer. + fail GRPC::Unknown.new('test message', + 'grpc-status-details-bin' => @encoded_rpc_status) + end +end + +GoogleRpcStatusTestStub = GoogleRpcStatusTestService.rpc_stub_class + +describe 'receving a google rpc status from a remote endpoint' do + def start_server(encoded_rpc_status) + @srv = GRPC::RpcServer.new(pool_size: 1) + @server_port = @srv.add_http2_port('localhost:0', + :this_port_is_insecure) + @srv.handle(GoogleRpcStatusTestService.new(encoded_rpc_status)) + @server_thd = Thread.new { @srv.run } + @srv.wait_till_running + end + + def stop_server + expect(@srv.stopped?).to be(false) + @srv.stop + @server_thd.join + expect(@srv.stopped?).to be(true) + end + + before(:each) do + simple_request_any = Google::Protobuf::Any.new + simple_request = Grpc::Testing::SimpleRequest.new( + payload: Grpc::Testing::Payload.new(body: 'request')) + simple_request_any.pack(simple_request) + simple_response_any = Google::Protobuf::Any.new + simple_response = Grpc::Testing::SimpleResponse.new( + payload: Grpc::Testing::Payload.new(body: 'response')) + simple_response_any.pack(simple_response) + payload_any = Google::Protobuf::Any.new + payload = Grpc::Testing::Payload.new(body: 'payload') + payload_any.pack(payload) + @expected_proto = Google::Rpc::Status.new( + code: StatusCodes::UNKNOWN, + message: 'test message', + details: [simple_request_any, simple_response_any, payload_any]) + start_server(Google::Rpc::Status.encode(@expected_proto)) + end + + after(:each) do + stop_server + end + + it 'should receive be able to extract a google rpc status from the '\ + 'status struct taken from a BadStatus exception' do + stub = GoogleRpcStatusTestStub.new("localhost:#{@server_port}", + :this_channel_is_insecure) + begin + stub.an_rpc(EchoMsg.new) + rescue GRPC::BadStatus => e + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + e.to_status) + end + expect(rpc_status).to eq(@expected_proto) + end + + it 'should receive be able to extract a google rpc status from the '\ + 'status struct taken from the op view of a call' do + stub = GoogleRpcStatusTestStub.new("localhost:#{@server_port}", + :this_channel_is_insecure) + op = stub.an_rpc(EchoMsg.new, return_op: true) + begin + op.execute + rescue GRPC::BadStatus => e + status_from_exception = e.to_status + end + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + op.status) + expect(rpc_status).to eq(@expected_proto) + # "to_status" on the bad status should give the same result + # as "status" on the "op view". + expect(GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + status_from_exception)).to eq(rpc_status) + end +end diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index e62e5b27215..215d5f9df9a 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -31,6 +31,7 @@ s.add_dependency 'google-protobuf', '~> 3.1' s.add_dependency 'googleauth', '~> 0.5.1' + s.add_dependency 'googleapis-common-protos-types', '~> 1.0.0' s.add_development_dependency 'bundler', '~> 1.9' s.add_development_dependency 'facter', '~> 2.4' From 6b56c4d953eb8f46c0d7d2aee9d9e6efa32e06e2 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 00:17:21 -0700 Subject: [PATCH 183/227] Redefine GRPC_SLICE_MALLOC to use grpc_slice_malloc function instead of designated initializers --- include/grpc/slice.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index 9d2a9fbc813..3080d63e2b4 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -65,11 +65,11 @@ GPRAPI grpc_slice grpc_slice_new_with_len(void *p, size_t len, GPRAPI grpc_slice grpc_slice_malloc(size_t length); GPRAPI grpc_slice grpc_slice_malloc_large(size_t length); -#define GRPC_SLICE_MALLOC(len) \ - ((len) <= GRPC_SLICE_INLINED_SIZE \ +#define GRPC_SLICE_MALLOC(len) grpc_slice_malloc(len) +/* ((len) <= GRPC_SLICE_INLINED_SIZE \ ? (grpc_slice){.refcount = NULL, \ .data.inlined = {.length = (uint8_t)(len)}} \ - : grpc_slice_malloc_large((len))) + : grpc_slice_malloc_large((len)))*/ /** Intern a slice: From 533d1184807869dfdd1cd239428be330a28a20d1 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 10:48:22 -0700 Subject: [PATCH 184/227] Removing designated initializers, implicit string literal to char * conversions and solving crosses initialization due to jump warnings --- .../ext/filters/client_channel/http_proxy.c | 4 +- .../resolver/dns/c_ares/grpc_ares_wrapper.c | 25 ++++--- .../transport/chttp2/server/chttp2_server.c | 3 +- .../chttp2/transport/chttp2_transport.c | 14 ++-- .../chttp2/transport/hpack_encoder.c | 34 +++++---- src/core/lib/http/httpcli.c | 2 +- src/core/lib/iomgr/ev_epoll1_linux.c | 69 +++++++++--------- src/core/lib/iomgr/ev_epollex_linux.c | 70 +++++++++---------- src/core/lib/iomgr/ev_epollsig_linux.c | 59 ++++++++-------- src/core/lib/iomgr/ev_poll_posix.c | 56 +++++++-------- src/core/lib/iomgr/iomgr.c | 2 +- .../lib/iomgr/is_epollexclusive_available.c | 12 ++-- src/core/lib/iomgr/socket_factory_posix.c | 4 +- src/core/lib/iomgr/socket_mutator.c | 4 +- src/core/lib/iomgr/tcp_server_posix.c | 4 +- src/core/lib/iomgr/timer_generic.c | 5 +- src/core/lib/surface/call.c | 31 +++++--- src/core/lib/surface/completion_queue.c | 23 ++---- src/core/lib/transport/status_conversion.c | 2 +- src/core/lib/transport/transport.c | 8 ++- 20 files changed, 217 insertions(+), 214 deletions(-) diff --git a/src/core/ext/filters/client_channel/http_proxy.c b/src/core/ext/filters/client_channel/http_proxy.c index 0c3b009e440..c507a2750e2 100644 --- a/src/core/ext/filters/client_channel/http_proxy.c +++ b/src/core/ext/filters/client_channel/http_proxy.c @@ -44,6 +44,8 @@ static char* get_http_proxy_server(grpc_exec_ctx* exec_ctx, char** user_cred) { GPR_ASSERT(user_cred != NULL); char* proxy_name = NULL; char* uri_str = gpr_getenv("http_proxy"); + char** authority_strs = NULL; + size_t authority_nstrs; if (uri_str == NULL) return NULL; grpc_uri* uri = grpc_uri_parse(exec_ctx, uri_str, false /* suppress_errors */); @@ -56,8 +58,6 @@ static char* get_http_proxy_server(grpc_exec_ctx* exec_ctx, char** user_cred) { goto done; } /* Split on '@' to separate user credentials from host */ - char** authority_strs = NULL; - size_t authority_nstrs; gpr_string_split(uri->authority, "@", &authority_strs, &authority_nstrs); GPR_ASSERT(authority_nstrs != 0); /* should have at least 1 string */ if (authority_nstrs == 1) { diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 0ffb38518a2..9975b1ef795 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -275,14 +275,15 @@ static void on_txt_done_cb(void *arg, int status, int timeouts, gpr_log(GPR_DEBUG, "on_txt_done_cb"); char *error_msg; grpc_ares_request *r = (grpc_ares_request *)arg; + const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1; + struct ares_txt_ext *result; + struct ares_txt_ext *reply = NULL; + grpc_error *error = GRPC_ERROR_NONE; gpr_mu_lock(&r->mu); if (status != ARES_SUCCESS) goto fail; - struct ares_txt_ext *reply = NULL; status = ares_parse_txt_reply_ext(buf, len, &reply); if (status != ARES_SUCCESS) goto fail; // Find service config in TXT record. - const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1; - struct ares_txt_ext *result; for (result = reply; result != NULL; result = result->next) { if (result->record_start && memcmp(result->txt, g_service_config_attribute_prefix, prefix_len) == @@ -313,7 +314,7 @@ static void on_txt_done_cb(void *arg, int status, int timeouts, fail: gpr_asprintf(&error_msg, "C-ares TXT lookup status is not ARES_SUCCESS: %s", ares_strerror(status)); - grpc_error *error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg); gpr_free(error_msg); if (r->error == GRPC_ERROR_NONE) { r->error = error; @@ -331,6 +332,9 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb, char **service_config_json) { grpc_error *error = GRPC_ERROR_NONE; + grpc_ares_hostbyname_request *hr = NULL; + grpc_ares_request *r = NULL; + ares_channel *channel = NULL; /* TODO(zyc): Enable tracing after #9603 is checked in */ /* if (grpc_dns_trace) { gpr_log(GPR_DEBUG, "resolve_address (blocking): name=%s, default_port=%s", @@ -360,8 +364,7 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( error = grpc_ares_ev_driver_create(&ev_driver, interested_parties); if (error != GRPC_ERROR_NONE) goto error_cleanup; - grpc_ares_request *r = - (grpc_ares_request *)gpr_zalloc(sizeof(grpc_ares_request)); + r = (grpc_ares_request *)gpr_zalloc(sizeof(grpc_ares_request)); gpr_mu_init(&r->mu); r->ev_driver = ev_driver; r->on_done = on_done; @@ -369,7 +372,7 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( r->service_config_json_out = service_config_json; r->success = false; r->error = GRPC_ERROR_NONE; - ares_channel *channel = grpc_ares_ev_driver_get_channel(r->ev_driver); + channel = grpc_ares_ev_driver_get_channel(r->ev_driver); // If dns_server is specified, use it. if (dns_server != NULL) { @@ -410,12 +413,12 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( } gpr_ref_init(&r->pending_queries, 1); if (grpc_ipv6_loopback_available()) { - grpc_ares_hostbyname_request *hr = create_hostbyname_request( - r, host, strhtons(port), false /* is_balancer */); + hr = create_hostbyname_request(r, host, strhtons(port), + false /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_cb, hr); } - grpc_ares_hostbyname_request *hr = create_hostbyname_request( - r, host, strhtons(port), false /* is_balancer */); + hr = create_hostbyname_request(r, host, strhtons(port), + false /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_cb, hr); if (check_grpclb) { /* Query the SRV record */ diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.c b/src/core/ext/transport/chttp2/server/chttp2_server.c index f5a409a403b..60244e163b2 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.c +++ b/src/core/ext/transport/chttp2/server/chttp2_server.c @@ -201,6 +201,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, grpc_error *err = GRPC_ERROR_NONE; server_state *state = NULL; grpc_error **errors = NULL; + size_t naddrs = 0; *port_num = -1; @@ -225,7 +226,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, state->shutdown = true; gpr_mu_init(&state->mu); - const size_t naddrs = resolved->naddrs; + naddrs = resolved->naddrs; errors = (grpc_error **)gpr_malloc(sizeof(*errors) * naddrs); for (i = 0; i < naddrs; i++) { errors[i] = diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 79a9ed827f7..d18df048511 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -362,14 +362,12 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, queue_setting_update(exec_ctx, t, GRPC_CHTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA, 1); - t->ping_policy = (grpc_chttp2_repeated_ping_policy){ - .max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA, - .min_time_between_pings = - gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN), - .max_ping_strikes = DEFAULT_MAX_PING_STRIKES, - .min_ping_interval_without_data = gpr_time_from_millis( - DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), - }; + t->ping_policy.max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA; + t->ping_policy.min_time_between_pings = + gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN); + t->ping_policy.max_ping_strikes = DEFAULT_MAX_PING_STRIKES; + t->ping_policy.min_ping_interval_without_data = gpr_time_from_millis( + DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN); /* Keepalive setting */ if (t->is_client) { diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 3cd1a7ee5c6..9537c2fa176 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -51,8 +51,9 @@ #define MAX_DECODER_SPACE_USAGE 512 static grpc_slice_refcount terminal_slice_refcount = {NULL, NULL}; -static const grpc_slice terminal_slice = {&terminal_slice_refcount, - .data.refcounted = {0, 0}}; +static const grpc_slice terminal_slice = {&terminal_slice_refcount, /* refcount */ + {{0, 0}} /* refcounted */ +}; extern grpc_tracer_flag grpc_http_trace; @@ -283,29 +284,26 @@ typedef struct { } wire_value; static wire_value get_wire_value(grpc_mdelem elem, bool true_binary_enabled) { + wire_value wire_val; if (grpc_is_binary_header(GRPC_MDKEY(elem))) { if (true_binary_enabled) { - return (wire_value){ - .huffman_prefix = 0x00, - .insert_null_before_wire_value = true, - .data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)), - }; + wire_val.huffman_prefix = 0x00; + wire_val.insert_null_before_wire_value = true; + wire_val.data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)); + } else { - return (wire_value){ - .huffman_prefix = 0x80, - .insert_null_before_wire_value = false, - .data = grpc_chttp2_base64_encode_and_huffman_compress( - GRPC_MDVALUE(elem)), - }; + wire_val.huffman_prefix = 0x80; + wire_val.insert_null_before_wire_value = false; + wire_val.data = + grpc_chttp2_base64_encode_and_huffman_compress(GRPC_MDVALUE(elem)); } } else { /* TODO(ctiller): opportunistically compress non-binary headers */ - return (wire_value){ - .huffman_prefix = 0x00, - .insert_null_before_wire_value = false, - .data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)), - }; + wire_val.huffman_prefix = 0x00; + wire_val.insert_null_before_wire_value = false; + wire_val.data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)); } + return wire_val; } static size_t wire_value_length(wire_value v) { diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c index 84cc39604ca..db995943a96 100644 --- a/src/core/lib/http/httpcli.c +++ b/src/core/lib/http/httpcli.c @@ -217,7 +217,7 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req, GRPC_CLOSURE_INIT(&req->connected, on_connected, req, grpc_schedule_on_exec_ctx); grpc_arg arg = grpc_channel_arg_pointer_create( - GRPC_ARG_RESOURCE_QUOTA, req->resource_quota, + (char *)GRPC_ARG_RESOURCE_QUOTA, req->resource_quota, grpc_resource_quota_arg_vtable()); grpc_channel_args args = {1, &arg}; grpc_tcp_client_connect(exec_ctx, &req->connected, &req->ep, diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 0d57833d859..b5066e5f588 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -280,8 +280,9 @@ static grpc_fd *fd_create(int fd, const char *name) { #endif gpr_free(fd_name); - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET), - .data.ptr = new_fd}; + struct epoll_event ev; + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = new_fd; if (epoll_ctl(g_epoll_set.epfd, EPOLL_CTL_ADD, fd, &ev) != 0) { gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno)); } @@ -435,8 +436,9 @@ static grpc_error *pollset_global_init(void) { global_wakeup_fd.read_fd = -1; grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); if (err != GRPC_ERROR_NONE) return err; - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), - .data.ptr = &global_wakeup_fd}; + struct epoll_event ev; + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = &global_wakeup_fd; if (epoll_ctl(g_epoll_set.epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { return GRPC_OS_ERROR(errno, "epoll_ctl"); @@ -566,8 +568,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = { - .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; + static const gpr_timespec round_up = {0, GPR_NS_PER_MS - 1, GPR_TIMESPAN}; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; @@ -1172,34 +1173,34 @@ static void shutdown_engine(void) { } static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .shutdown_engine = shutdown_engine, + sizeof(grpc_pollset), + + fd_create, + fd_wrapped_fd, + fd_orphan, + fd_shutdown, + fd_notify_on_read, + fd_notify_on_write, + fd_is_shutdown, + fd_get_read_notifier_pollset, + + pollset_init, + pollset_shutdown, + pollset_destroy, + pollset_work, + pollset_kick, + pollset_add_fd, + + pollset_set_create, + pollset_set_destroy, + pollset_set_add_pollset, + pollset_set_del_pollset, + pollset_set_add_pollset_set, + pollset_set_del_pollset_set, + pollset_set_add_fd, + pollset_set_del_fd, + + shutdown_engine, }; /* It is possible that GLIBC has epoll but the underlying kernel doesn't. diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 51200fc0643..671c4cd730d 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -477,8 +477,9 @@ static grpc_error *pollable_materialize(pollable *p) { close(new_epfd); return err; } - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), - .data.ptr = (void *)(1 | (intptr_t)&p->wakeup)}; + struct epoll_event ev; + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = (void *)(1 | (intptr_t)&p->wakeup); if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { err = GRPC_OS_ERROR(errno, "epoll_ctl"); close(new_epfd); @@ -507,9 +508,9 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { gpr_mu_unlock(&fd->orphaned_mu); return GRPC_ERROR_NONE; } - struct epoll_event ev_fd = { - .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE), - .data.ptr = fd}; + struct epoll_event ev_fd; + ev_fd.events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE); + ev_fd.data.ptr = fd; if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { switch (errno) { case EEXIST: @@ -705,8 +706,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = { - .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; + static const gpr_timespec round_up = {0, GPR_NS_PER_MS - 1, GPR_TIMESPAN}; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; @@ -1390,34 +1390,34 @@ static void shutdown_engine(void) { } static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .shutdown_engine = shutdown_engine, + sizeof(grpc_pollset), + + fd_create, + fd_wrapped_fd, + fd_orphan, + fd_shutdown, + fd_notify_on_read, + fd_notify_on_write, + fd_is_shutdown, + fd_get_read_notifier_pollset, + + pollset_init, + pollset_shutdown, + pollset_destroy, + pollset_work, + pollset_kick, + pollset_add_fd, + + pollset_set_create, + pollset_set_destroy, + pollset_set_add_pollset, + pollset_set_del_pollset, + pollset_set_add_pollset_set, + pollset_set_del_pollset_set, + pollset_set_add_fd, + pollset_set_del_fd, + + shutdown_engine, }; const grpc_event_engine_vtable *grpc_init_epollex_linux( diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index b88c3ba111c..87b6f363f8b 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1132,7 +1132,8 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { } static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, - grpc_pollset *ps, char *reason) { + grpc_pollset *ps, + const char *reason) { if (ps->po.pi != NULL) { PI_UNREF(exec_ctx, ps->po.pi, reason); } @@ -1670,34 +1671,34 @@ static void shutdown_engine(void) { } static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .shutdown_engine = shutdown_engine, + sizeof(grpc_pollset), + + fd_create, + fd_wrapped_fd, + fd_orphan, + fd_shutdown, + fd_notify_on_read, + fd_notify_on_write, + fd_is_shutdown, + fd_get_read_notifier_pollset, + + pollset_init, + pollset_shutdown, + pollset_destroy, + pollset_work, + pollset_kick, + pollset_add_fd, + + pollset_set_create, + pollset_set_destroy, + pollset_set_add_pollset, + pollset_set_del_pollset, + pollset_set_add_pollset_set, + pollset_set_del_pollset_set, + pollset_set_add_fd, + pollset_set_del_fd, + + shutdown_engine, }; /* It is possible that GLIBC has epoll but the underlying kernel doesn't. diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 7f44eda1382..53df2d7737a 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -1688,34 +1688,34 @@ static void shutdown_engine(void) { } static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .shutdown_engine = shutdown_engine, + sizeof(grpc_pollset), + + fd_create, + fd_wrapped_fd, + fd_orphan, + fd_shutdown, + fd_notify_on_read, + fd_notify_on_write, + fd_is_shutdown, + fd_get_read_notifier_pollset, + + pollset_init, + pollset_shutdown, + pollset_destroy, + pollset_work, + pollset_kick, + pollset_add_fd, + + pollset_set_create, + pollset_set_destroy, + pollset_set_add_pollset, + pollset_set_del_pollset, + pollset_set_add_pollset_set, + pollset_set_del_pollset_set, + pollset_set_add_fd, + pollset_set_del_fd, + + shutdown_engine, }; const grpc_event_engine_vtable *grpc_init_poll_posix(bool explicit_request) { diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 1feea6d6288..f63f1901551 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -50,7 +50,7 @@ void grpc_iomgr_init(grpc_exec_ctx *exec_ctx) { grpc_executor_init(exec_ctx); grpc_timer_list_init(gpr_now(GPR_CLOCK_MONOTONIC)); g_root_object.next = g_root_object.prev = &g_root_object; - g_root_object.name = "root"; + g_root_object.name = (char *)"root"; grpc_network_status_init(); grpc_iomgr_platform_init(); } diff --git a/src/core/lib/iomgr/is_epollexclusive_available.c b/src/core/lib/iomgr/is_epollexclusive_available.c index e8a7d4d52c6..d08844c0dfd 100644 --- a/src/core/lib/iomgr/is_epollexclusive_available.c +++ b/src/core/lib/iomgr/is_epollexclusive_available.c @@ -57,12 +57,12 @@ bool grpc_is_epollexclusive_available(void) { close(fd); return false; } - struct epoll_event ev = { - /* choose events that should cause an error on - EPOLLEXCLUSIVE enabled kernels - specifically the combination of - EPOLLONESHOT and EPOLLEXCLUSIVE */ - .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT), - .data.ptr = NULL}; + struct epoll_event ev; + /* choose events that should cause an error on + EPOLLEXCLUSIVE enabled kernels - specifically the combination of + EPOLLONESHOT and EPOLLEXCLUSIVE */ + ev.events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT); + ev.data.ptr = NULL; if (epoll_ctl(fd, EPOLL_CTL_ADD, evfd, &ev) != 0) { if (errno != EINVAL) { if (!logged_why_not) { diff --git a/src/core/lib/iomgr/socket_factory_posix.c b/src/core/lib/iomgr/socket_factory_posix.c index c81566575ef..8e907703ae7 100644 --- a/src/core/lib/iomgr/socket_factory_posix.c +++ b/src/core/lib/iomgr/socket_factory_posix.c @@ -85,8 +85,8 @@ static const grpc_arg_pointer_vtable socket_factory_arg_vtable = { socket_factory_arg_copy, socket_factory_arg_destroy, socket_factory_cmp}; grpc_arg grpc_socket_factory_to_arg(grpc_socket_factory *factory) { - return grpc_channel_arg_pointer_create(GRPC_ARG_SOCKET_FACTORY, factory, - &socket_factory_arg_vtable); + return grpc_channel_arg_pointer_create((char *)GRPC_ARG_SOCKET_FACTORY, + factory, &socket_factory_arg_vtable); } #endif diff --git a/src/core/lib/iomgr/socket_mutator.c b/src/core/lib/iomgr/socket_mutator.c index 300ac75b387..b0435d5a076 100644 --- a/src/core/lib/iomgr/socket_mutator.c +++ b/src/core/lib/iomgr/socket_mutator.c @@ -76,6 +76,6 @@ static const grpc_arg_pointer_vtable socket_mutator_arg_vtable = { socket_mutator_arg_copy, socket_mutator_arg_destroy, socket_mutator_cmp}; grpc_arg grpc_socket_mutator_to_arg(grpc_socket_mutator *mutator) { - return grpc_channel_arg_pointer_create(GRPC_ARG_SOCKET_MUTATOR, mutator, - &socket_mutator_arg_vtable); + return grpc_channel_arg_pointer_create((char *)GRPC_ARG_SOCKET_MUTATOR, + mutator, &socket_mutator_arg_vtable); } diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index c3ec3e447a2..06612d639cc 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -198,12 +198,12 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { /* event manager callback when reads are ready */ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { grpc_tcp_listener *sp = (grpc_tcp_listener *)arg; - + grpc_pollset *read_notifier_pollset; if (err != GRPC_ERROR_NONE) { goto error; } - grpc_pollset *read_notifier_pollset = + read_notifier_pollset = sp->server->pollsets[(size_t)gpr_atm_no_barrier_fetch_add( &sp->server->next_pollset_to_assign, 1) % sp->server->pollset_count]; diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index c08bb525b77..c5955c869c2 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -96,7 +96,10 @@ struct shared_mutables { } GPR_ALIGN_STRUCT(GPR_CACHELINE_SIZE); static struct shared_mutables g_shared_mutables = { - .checker_mu = GPR_SPINLOCK_STATIC_INITIALIZER, .initialized = false, + 0, + GPR_SPINLOCK_STATIC_INITIALIZER, + false, + {{0}} }; static gpr_clock_type g_clock_type; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index a41eb336d48..09b107cf6b6 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -1673,6 +1673,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, batch_control *bctl; int num_completion_callbacks_needed = 1; grpc_call_error error = GRPC_CALL_OK; + grpc_transport_stream_op_batch *stream_op; + grpc_transport_stream_op_batch_payload *stream_op_payload; GPR_TIMER_BEGIN("grpc_call_start_batch", 0); GRPC_CALL_LOG_BATCH(GPR_INFO, call, ops, nops, notify_tag); @@ -1699,9 +1701,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, bctl->completion_data.notify_tag.is_closure = (uint8_t)(is_notify_tag_closure != 0); - grpc_transport_stream_op_batch *stream_op = &bctl->op; - grpc_transport_stream_op_batch_payload *stream_op_payload = - &call->stream_op_payload; + stream_op = &bctl->op; + stream_op_payload = &call->stream_op_payload; /* rewrite batch ops into a transport op */ for (i = 0; i < nops; i++) { @@ -1711,7 +1712,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, goto done_with_error; } switch (op->op) { - case GRPC_OP_SEND_INITIAL_METADATA: + case GRPC_OP_SEND_INITIAL_METADATA: { /* Flag validation: currently allow no flags */ if (!are_initial_metadata_flags_valid(op->flags, call->is_client)) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1805,7 +1806,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, &call->peer_string; } break; - case GRPC_OP_SEND_MESSAGE: + } + case GRPC_OP_SEND_MESSAGE: { if (!are_write_flags_valid(op->flags)) { error = GRPC_CALL_ERROR_INVALID_FLAGS; goto done_with_error; @@ -1834,7 +1836,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->send_message.send_message = &call->sending_stream.base; break; - case GRPC_OP_SEND_CLOSE_FROM_CLIENT: + } + case GRPC_OP_SEND_CLOSE_FROM_CLIENT: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1853,7 +1856,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->send_trailing_metadata.send_trailing_metadata = &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; break; - case GRPC_OP_SEND_STATUS_FROM_SERVER: + } + case GRPC_OP_SEND_STATUS_FROM_SERVER: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1915,7 +1919,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->send_trailing_metadata.send_trailing_metadata = &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; break; - case GRPC_OP_RECV_INITIAL_METADATA: + } + case GRPC_OP_RECV_INITIAL_METADATA: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1942,7 +1947,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, } num_completion_callbacks_needed++; break; - case GRPC_OP_RECV_MESSAGE: + } + case GRPC_OP_RECV_MESSAGE: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1963,7 +1969,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, &call->receiving_stream_ready; num_completion_callbacks_needed++; break; - case GRPC_OP_RECV_STATUS_ON_CLIENT: + } + case GRPC_OP_RECV_STATUS_ON_CLIENT: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1990,7 +1997,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->collect_stats.collect_stats = &call->final_info.stats.transport_stream_stats; break; - case GRPC_OP_RECV_CLOSE_ON_SERVER: + } + case GRPC_OP_RECV_CLOSE_ON_SERVER: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -2014,6 +2022,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->collect_stats.collect_stats = &call->final_info.stats.transport_stream_stats; break; + } } } diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 468360fbdef..d60a0cd4515 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -328,25 +328,12 @@ static void cq_destroy_pluck(void *data); /* Completion queue vtables based on the completion-type */ static const cq_vtable g_cq_vtable[] = { /* GRPC_CQ_NEXT */ - {.data_size = sizeof(cq_next_data), - .cq_completion_type = GRPC_CQ_NEXT, - .init = cq_init_next, - .shutdown = cq_shutdown_next, - .destroy = cq_destroy_next, - .begin_op = cq_begin_op_for_next, - .end_op = cq_end_op_for_next, - .next = cq_next, - .pluck = NULL}, + {GRPC_CQ_NEXT, sizeof(cq_next_data), cq_init_next, cq_shutdown_next, + cq_destroy_next, cq_begin_op_for_next, cq_end_op_for_next, cq_next, NULL}, /* GRPC_CQ_PLUCK */ - {.data_size = sizeof(cq_pluck_data), - .cq_completion_type = GRPC_CQ_PLUCK, - .init = cq_init_pluck, - .shutdown = cq_shutdown_pluck, - .destroy = cq_destroy_pluck, - .begin_op = cq_begin_op_for_pluck, - .end_op = cq_end_op_for_pluck, - .next = NULL, - .pluck = cq_pluck}, + {GRPC_CQ_PLUCK, sizeof(cq_pluck_data), cq_init_pluck, cq_shutdown_pluck, + cq_destroy_pluck, cq_begin_op_for_pluck, cq_end_op_for_pluck, NULL, + cq_pluck}, }; #define DATA_FROM_CQ(cq) ((void *)(cq + 1)) diff --git a/src/core/lib/transport/status_conversion.c b/src/core/lib/transport/status_conversion.c index 9a76977e4b8..a40d333284b 100644 --- a/src/core/lib/transport/status_conversion.c +++ b/src/core/lib/transport/status_conversion.c @@ -18,7 +18,7 @@ #include "src/core/lib/transport/status_conversion.h" -int grpc_status_to_http2_error(grpc_status_code status) { +grpc_http2_error_code grpc_status_to_http2_error(grpc_status_code status) { switch (status) { case GRPC_STATUS_OK: return GRPC_HTTP2_NO_ERROR; diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index ae705195f3c..682a820b487 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -102,9 +102,11 @@ static void slice_stream_unref(grpc_exec_ctx *exec_ctx, void *p) { grpc_slice grpc_slice_from_stream_owned_buffer(grpc_stream_refcount *refcount, void *buffer, size_t length) { slice_stream_ref(&refcount->slice_refcount); - return (grpc_slice){ - .refcount = &refcount->slice_refcount, - .data.refcounted = {.bytes = (uint8_t *)buffer, .length = length}}; + grpc_slice res; + res.refcount = &refcount->slice_refcount, + res.data.refcounted.bytes = (uint8_t *)buffer; + res.data.refcounted.length = length; + return res; } static const grpc_slice_refcount_vtable stream_ref_slice_vtable = { From 067611f19214af4a44435016234b65fc3918bbed Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 12:57:42 -0700 Subject: [PATCH 185/227] static metadata and clang format --- .../chttp2/transport/hpack_encoder.c | 5 +- src/core/lib/iomgr/timer_generic.c | 6 +- src/core/lib/transport/static_metadata.c | 816 ++++++------------ tools/codegen/core/gen_static_metadata.py | 4 +- 4 files changed, 278 insertions(+), 553 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 9537c2fa176..52c9e217f1a 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -51,8 +51,9 @@ #define MAX_DECODER_SPACE_USAGE 512 static grpc_slice_refcount terminal_slice_refcount = {NULL, NULL}; -static const grpc_slice terminal_slice = {&terminal_slice_refcount, /* refcount */ - {{0, 0}} /* refcounted */ +static const grpc_slice terminal_slice = { + &terminal_slice_refcount, /* refcount */ + {{0, 0}} /* refcounted */ }; extern grpc_tracer_flag grpc_http_trace; diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index c5955c869c2..57b63cb229a 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -96,11 +96,7 @@ struct shared_mutables { } GPR_ALIGN_STRUCT(GPR_CACHELINE_SIZE); static struct shared_mutables g_shared_mutables = { - 0, - GPR_SPINLOCK_STATIC_INITIALIZER, - false, - {{0}} -}; + 0, GPR_SPINLOCK_STATIC_INITIALIZER, false, {{0}}}; static gpr_clock_type g_clock_type; static gpr_timespec g_start_time; diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c index b20d94aeac9..472cf888eab 100644 --- a/src/core/lib/transport/static_metadata.c +++ b/src/core/lib/transport/static_metadata.c @@ -216,206 +216,106 @@ grpc_slice_refcount grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = { }; const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { - {.refcount = &grpc_static_metadata_refcounts[0], - .data.refcounted = {g_bytes + 0, 5}}, - {.refcount = &grpc_static_metadata_refcounts[1], - .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[3], - .data.refcounted = {g_bytes + 19, 10}}, - {.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[5], - .data.refcounted = {g_bytes + 36, 2}}, - {.refcount = &grpc_static_metadata_refcounts[6], - .data.refcounted = {g_bytes + 38, 12}}, - {.refcount = &grpc_static_metadata_refcounts[7], - .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[8], - .data.refcounted = {g_bytes + 61, 16}}, - {.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[11], - .data.refcounted = {g_bytes + 110, 21}}, - {.refcount = &grpc_static_metadata_refcounts[12], - .data.refcounted = {g_bytes + 131, 13}}, - {.refcount = &grpc_static_metadata_refcounts[13], - .data.refcounted = {g_bytes + 144, 14}}, - {.refcount = &grpc_static_metadata_refcounts[14], - .data.refcounted = {g_bytes + 158, 12}}, - {.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 170, 16}}, - {.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[17], - .data.refcounted = {g_bytes + 201, 30}}, - {.refcount = &grpc_static_metadata_refcounts[18], - .data.refcounted = {g_bytes + 231, 37}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 268, 10}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 278, 4}}, - {.refcount = &grpc_static_metadata_refcounts[21], - .data.refcounted = {g_bytes + 282, 8}}, - {.refcount = &grpc_static_metadata_refcounts[22], - .data.refcounted = {g_bytes + 290, 12}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}, - {.refcount = &grpc_static_metadata_refcounts[24], - .data.refcounted = {g_bytes + 302, 19}}, - {.refcount = &grpc_static_metadata_refcounts[25], - .data.refcounted = {g_bytes + 321, 12}}, - {.refcount = &grpc_static_metadata_refcounts[26], - .data.refcounted = {g_bytes + 333, 30}}, - {.refcount = &grpc_static_metadata_refcounts[27], - .data.refcounted = {g_bytes + 363, 31}}, - {.refcount = &grpc_static_metadata_refcounts[28], - .data.refcounted = {g_bytes + 394, 36}}, - {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 430, 1}}, - {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 431, 1}}, - {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 432, 1}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}, - {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 445, 7}}, - {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 452, 8}}, - {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 460, 16}}, - {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 476, 4}}, - {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 480, 3}}, - {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 483, 3}}, - {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 486, 4}}, - {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 490, 5}}, - {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 495, 4}}, - {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 499, 3}}, - {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 502, 3}}, - {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 505, 1}}, - {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 506, 11}}, - {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 517, 3}}, - {.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 520, 3}}, - {.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 523, 3}}, - {.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 526, 3}}, - {.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 529, 3}}, - {.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 532, 14}}, - {.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 546, 13}}, - {.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 559, 15}}, - {.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 574, 13}}, - {.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 587, 6}}, - {.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 593, 27}}, - {.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 620, 3}}, - {.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 623, 5}}, - {.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 628, 13}}, - {.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 641, 13}}, - {.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 654, 19}}, - {.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 673, 16}}, - {.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 689, 14}}, - {.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 703, 16}}, - {.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 719, 13}}, - {.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 732, 6}}, - {.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 738, 4}}, - {.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 742, 4}}, - {.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 746, 6}}, - {.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 752, 7}}, - {.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 759, 4}}, - {.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 763, 8}}, - {.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 771, 17}}, - {.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 788, 13}}, - {.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 801, 8}}, - {.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 809, 19}}, - {.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 828, 13}}, - {.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 841, 11}}, - {.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 852, 4}}, - {.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 856, 8}}, - {.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 864, 12}}, - {.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 876, 18}}, - {.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 894, 19}}, - {.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 913, 5}}, - {.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 918, 7}}, - {.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 925, 7}}, - {.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 932, 11}}, - {.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 943, 6}}, - {.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 949, 10}}, - {.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 959, 25}}, - {.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 984, 17}}, - {.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 1001, 4}}, - {.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 1005, 3}}, - {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 1008, 16}}, - {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 1024, 16}}, - {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1040, 13}}, - {.refcount = &grpc_static_metadata_refcounts[98], - .data.refcounted = {g_bytes + 1053, 12}}, - {.refcount = &grpc_static_metadata_refcounts[99], - .data.refcounted = {g_bytes + 1065, 21}}, + {&grpc_static_metadata_refcounts[0], {{g_bytes + 0, 5}}}, + {&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, + {&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[3], {{g_bytes + 19, 10}}}, + {&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, + {&grpc_static_metadata_refcounts[5], {{g_bytes + 36, 2}}}, + {&grpc_static_metadata_refcounts[6], {{g_bytes + 38, 12}}}, + {&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, + {&grpc_static_metadata_refcounts[8], {{g_bytes + 61, 16}}}, + {&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, + {&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[11], {{g_bytes + 110, 21}}}, + {&grpc_static_metadata_refcounts[12], {{g_bytes + 131, 13}}}, + {&grpc_static_metadata_refcounts[13], {{g_bytes + 144, 14}}}, + {&grpc_static_metadata_refcounts[14], {{g_bytes + 158, 12}}}, + {&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, + {&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[17], {{g_bytes + 201, 30}}}, + {&grpc_static_metadata_refcounts[18], {{g_bytes + 231, 37}}}, + {&grpc_static_metadata_refcounts[19], {{g_bytes + 268, 10}}}, + {&grpc_static_metadata_refcounts[20], {{g_bytes + 278, 4}}}, + {&grpc_static_metadata_refcounts[21], {{g_bytes + 282, 8}}}, + {&grpc_static_metadata_refcounts[22], {{g_bytes + 290, 12}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}, + {&grpc_static_metadata_refcounts[24], {{g_bytes + 302, 19}}}, + {&grpc_static_metadata_refcounts[25], {{g_bytes + 321, 12}}}, + {&grpc_static_metadata_refcounts[26], {{g_bytes + 333, 30}}}, + {&grpc_static_metadata_refcounts[27], {{g_bytes + 363, 31}}}, + {&grpc_static_metadata_refcounts[28], {{g_bytes + 394, 36}}}, + {&grpc_static_metadata_refcounts[29], {{g_bytes + 430, 1}}}, + {&grpc_static_metadata_refcounts[30], {{g_bytes + 431, 1}}}, + {&grpc_static_metadata_refcounts[31], {{g_bytes + 432, 1}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}, + {&grpc_static_metadata_refcounts[34], {{g_bytes + 445, 7}}}, + {&grpc_static_metadata_refcounts[35], {{g_bytes + 452, 8}}}, + {&grpc_static_metadata_refcounts[36], {{g_bytes + 460, 16}}}, + {&grpc_static_metadata_refcounts[37], {{g_bytes + 476, 4}}}, + {&grpc_static_metadata_refcounts[38], {{g_bytes + 480, 3}}}, + {&grpc_static_metadata_refcounts[39], {{g_bytes + 483, 3}}}, + {&grpc_static_metadata_refcounts[40], {{g_bytes + 486, 4}}}, + {&grpc_static_metadata_refcounts[41], {{g_bytes + 490, 5}}}, + {&grpc_static_metadata_refcounts[42], {{g_bytes + 495, 4}}}, + {&grpc_static_metadata_refcounts[43], {{g_bytes + 499, 3}}}, + {&grpc_static_metadata_refcounts[44], {{g_bytes + 502, 3}}}, + {&grpc_static_metadata_refcounts[45], {{g_bytes + 505, 1}}}, + {&grpc_static_metadata_refcounts[46], {{g_bytes + 506, 11}}}, + {&grpc_static_metadata_refcounts[47], {{g_bytes + 517, 3}}}, + {&grpc_static_metadata_refcounts[48], {{g_bytes + 520, 3}}}, + {&grpc_static_metadata_refcounts[49], {{g_bytes + 523, 3}}}, + {&grpc_static_metadata_refcounts[50], {{g_bytes + 526, 3}}}, + {&grpc_static_metadata_refcounts[51], {{g_bytes + 529, 3}}}, + {&grpc_static_metadata_refcounts[52], {{g_bytes + 532, 14}}}, + {&grpc_static_metadata_refcounts[53], {{g_bytes + 546, 13}}}, + {&grpc_static_metadata_refcounts[54], {{g_bytes + 559, 15}}}, + {&grpc_static_metadata_refcounts[55], {{g_bytes + 574, 13}}}, + {&grpc_static_metadata_refcounts[56], {{g_bytes + 587, 6}}}, + {&grpc_static_metadata_refcounts[57], {{g_bytes + 593, 27}}}, + {&grpc_static_metadata_refcounts[58], {{g_bytes + 620, 3}}}, + {&grpc_static_metadata_refcounts[59], {{g_bytes + 623, 5}}}, + {&grpc_static_metadata_refcounts[60], {{g_bytes + 628, 13}}}, + {&grpc_static_metadata_refcounts[61], {{g_bytes + 641, 13}}}, + {&grpc_static_metadata_refcounts[62], {{g_bytes + 654, 19}}}, + {&grpc_static_metadata_refcounts[63], {{g_bytes + 673, 16}}}, + {&grpc_static_metadata_refcounts[64], {{g_bytes + 689, 14}}}, + {&grpc_static_metadata_refcounts[65], {{g_bytes + 703, 16}}}, + {&grpc_static_metadata_refcounts[66], {{g_bytes + 719, 13}}}, + {&grpc_static_metadata_refcounts[67], {{g_bytes + 732, 6}}}, + {&grpc_static_metadata_refcounts[68], {{g_bytes + 738, 4}}}, + {&grpc_static_metadata_refcounts[69], {{g_bytes + 742, 4}}}, + {&grpc_static_metadata_refcounts[70], {{g_bytes + 746, 6}}}, + {&grpc_static_metadata_refcounts[71], {{g_bytes + 752, 7}}}, + {&grpc_static_metadata_refcounts[72], {{g_bytes + 759, 4}}}, + {&grpc_static_metadata_refcounts[73], {{g_bytes + 763, 8}}}, + {&grpc_static_metadata_refcounts[74], {{g_bytes + 771, 17}}}, + {&grpc_static_metadata_refcounts[75], {{g_bytes + 788, 13}}}, + {&grpc_static_metadata_refcounts[76], {{g_bytes + 801, 8}}}, + {&grpc_static_metadata_refcounts[77], {{g_bytes + 809, 19}}}, + {&grpc_static_metadata_refcounts[78], {{g_bytes + 828, 13}}}, + {&grpc_static_metadata_refcounts[79], {{g_bytes + 841, 11}}}, + {&grpc_static_metadata_refcounts[80], {{g_bytes + 852, 4}}}, + {&grpc_static_metadata_refcounts[81], {{g_bytes + 856, 8}}}, + {&grpc_static_metadata_refcounts[82], {{g_bytes + 864, 12}}}, + {&grpc_static_metadata_refcounts[83], {{g_bytes + 876, 18}}}, + {&grpc_static_metadata_refcounts[84], {{g_bytes + 894, 19}}}, + {&grpc_static_metadata_refcounts[85], {{g_bytes + 913, 5}}}, + {&grpc_static_metadata_refcounts[86], {{g_bytes + 918, 7}}}, + {&grpc_static_metadata_refcounts[87], {{g_bytes + 925, 7}}}, + {&grpc_static_metadata_refcounts[88], {{g_bytes + 932, 11}}}, + {&grpc_static_metadata_refcounts[89], {{g_bytes + 943, 6}}}, + {&grpc_static_metadata_refcounts[90], {{g_bytes + 949, 10}}}, + {&grpc_static_metadata_refcounts[91], {{g_bytes + 959, 25}}}, + {&grpc_static_metadata_refcounts[92], {{g_bytes + 984, 17}}}, + {&grpc_static_metadata_refcounts[93], {{g_bytes + 1001, 4}}}, + {&grpc_static_metadata_refcounts[94], {{g_bytes + 1005, 3}}}, + {&grpc_static_metadata_refcounts[95], {{g_bytes + 1008, 16}}}, + {&grpc_static_metadata_refcounts[96], {{g_bytes + 1024, 16}}}, + {&grpc_static_metadata_refcounts[97], {{g_bytes + 1040, 13}}}, + {&grpc_static_metadata_refcounts[98], {{g_bytes + 1053, 12}}}, + {&grpc_static_metadata_refcounts[99], {{g_bytes + 1065, 21}}}, }; uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { @@ -478,350 +378,178 @@ grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { } grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = { - {{.refcount = &grpc_static_metadata_refcounts[7], - .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 430, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[7], - .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 431, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[7], - .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 432, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 445, 7}}}, - {{.refcount = &grpc_static_metadata_refcounts[5], - .data.refcounted = {g_bytes + 36, 2}}, - {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 452, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[14], - .data.refcounted = {g_bytes + 158, 12}}, - {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 460, 16}}}, - {{.refcount = &grpc_static_metadata_refcounts[1], - .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 476, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 480, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 483, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 486, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 490, 5}}}, - {{.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 495, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[3], - .data.refcounted = {g_bytes + 19, 10}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[1], - .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 499, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[1], - .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 502, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[0], - .data.refcounted = {g_bytes + 0, 5}}, - {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 505, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[0], - .data.refcounted = {g_bytes + 0, 5}}, - {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 506, 11}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 517, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 520, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 523, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 526, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 529, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 532, 14}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 546, 13}}}, - {{.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 559, 15}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 574, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 587, 6}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 593, 27}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 620, 3}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 623, 5}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 628, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 641, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 654, 19}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 170, 16}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 170, 16}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 170, 16}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 673, 16}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 689, 14}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 703, 16}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 719, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[14], - .data.refcounted = {g_bytes + 158, 12}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 732, 6}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 738, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 742, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 746, 6}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 752, 7}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 759, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 278, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 763, 8}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 771, 17}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 788, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 801, 8}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 809, 19}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 828, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[21], - .data.refcounted = {g_bytes + 282, 8}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 841, 11}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 852, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 856, 8}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 864, 12}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 876, 18}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 894, 19}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 913, 5}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 918, 7}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 925, 7}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 932, 11}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 943, 6}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 949, 10}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 959, 25}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 984, 17}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 268, 10}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 1001, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 1005, 3}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 1008, 16}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 445, 7}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 1024, 16}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1040, 13}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[98], - .data.refcounted = {g_bytes + 1053, 12}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[99], - .data.refcounted = {g_bytes + 1065, 21}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1040, 13}}}, + {{&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, + {&grpc_static_metadata_refcounts[29], {{g_bytes + 430, 1}}}}, + {{&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, + {&grpc_static_metadata_refcounts[30], {{g_bytes + 431, 1}}}}, + {{&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, + {&grpc_static_metadata_refcounts[31], {{g_bytes + 432, 1}}}}, + {{&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {{&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {{&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, + {&grpc_static_metadata_refcounts[34], {{g_bytes + 445, 7}}}}, + {{&grpc_static_metadata_refcounts[5], {{g_bytes + 36, 2}}}, + {&grpc_static_metadata_refcounts[35], {{g_bytes + 452, 8}}}}, + {{&grpc_static_metadata_refcounts[14], {{g_bytes + 158, 12}}}, + {&grpc_static_metadata_refcounts[36], {{g_bytes + 460, 16}}}}, + {{&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, + {&grpc_static_metadata_refcounts[37], {{g_bytes + 476, 4}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[38], {{g_bytes + 480, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[39], {{g_bytes + 483, 3}}}}, + {{&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, + {&grpc_static_metadata_refcounts[40], {{g_bytes + 486, 4}}}}, + {{&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, + {&grpc_static_metadata_refcounts[41], {{g_bytes + 490, 5}}}}, + {{&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, + {&grpc_static_metadata_refcounts[42], {{g_bytes + 495, 4}}}}, + {{&grpc_static_metadata_refcounts[3], {{g_bytes + 19, 10}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, + {&grpc_static_metadata_refcounts[43], {{g_bytes + 499, 3}}}}, + {{&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, + {&grpc_static_metadata_refcounts[44], {{g_bytes + 502, 3}}}}, + {{&grpc_static_metadata_refcounts[0], {{g_bytes + 0, 5}}}, + {&grpc_static_metadata_refcounts[45], {{g_bytes + 505, 1}}}}, + {{&grpc_static_metadata_refcounts[0], {{g_bytes + 0, 5}}}, + {&grpc_static_metadata_refcounts[46], {{g_bytes + 506, 11}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[47], {{g_bytes + 517, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[48], {{g_bytes + 520, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[49], {{g_bytes + 523, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[50], {{g_bytes + 526, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[51], {{g_bytes + 529, 3}}}}, + {{&grpc_static_metadata_refcounts[52], {{g_bytes + 532, 14}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[53], {{g_bytes + 546, 13}}}}, + {{&grpc_static_metadata_refcounts[54], {{g_bytes + 559, 15}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[55], {{g_bytes + 574, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[56], {{g_bytes + 587, 6}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[57], {{g_bytes + 593, 27}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[58], {{g_bytes + 620, 3}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[59], {{g_bytes + 623, 5}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[60], {{g_bytes + 628, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[61], {{g_bytes + 641, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[62], {{g_bytes + 654, 19}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {{&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {{&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[63], {{g_bytes + 673, 16}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[64], {{g_bytes + 689, 14}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[65], {{g_bytes + 703, 16}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[66], {{g_bytes + 719, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[14], {{g_bytes + 158, 12}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[67], {{g_bytes + 732, 6}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[68], {{g_bytes + 738, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[69], {{g_bytes + 742, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[70], {{g_bytes + 746, 6}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[71], {{g_bytes + 752, 7}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[72], {{g_bytes + 759, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[20], {{g_bytes + 278, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[73], {{g_bytes + 763, 8}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[74], {{g_bytes + 771, 17}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[75], {{g_bytes + 788, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[76], {{g_bytes + 801, 8}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[77], {{g_bytes + 809, 19}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[78], {{g_bytes + 828, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[21], {{g_bytes + 282, 8}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[79], {{g_bytes + 841, 11}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[80], {{g_bytes + 852, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[81], {{g_bytes + 856, 8}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[82], {{g_bytes + 864, 12}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[83], {{g_bytes + 876, 18}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[84], {{g_bytes + 894, 19}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[85], {{g_bytes + 913, 5}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[86], {{g_bytes + 918, 7}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[87], {{g_bytes + 925, 7}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[88], {{g_bytes + 932, 11}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[89], {{g_bytes + 943, 6}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[90], {{g_bytes + 949, 10}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[91], {{g_bytes + 959, 25}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[92], {{g_bytes + 984, 17}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[19], {{g_bytes + 268, 10}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[93], {{g_bytes + 1001, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[94], {{g_bytes + 1005, 3}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[95], {{g_bytes + 1008, 16}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[34], {{g_bytes + 445, 7}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[96], {{g_bytes + 1024, 16}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[97], {{g_bytes + 1040, 13}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[98], {{g_bytes + 1053, 12}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[99], {{g_bytes + 1065, 21}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[97], {{g_bytes + 1040, 13}}}}, }; bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = { true, // :path diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index 6ee8a7cace3..af7521da363 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -370,8 +370,8 @@ for i, elem in enumerate(all_strs): def slice_def(i): - return ('{.refcount = &grpc_static_metadata_refcounts[%d], .data.refcounted =' - ' {g_bytes+%d, %d}}') % ( + return ('{&grpc_static_metadata_refcounts[%d],' + ' {{g_bytes+%d, %d}}}') % ( i, id2strofs[i], len(all_strs[i])) From 341c7e7eca06e1c66ed2448fb3bcda6d278f5fd6 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 13:37:28 -0700 Subject: [PATCH 186/227] Timer generic correction --- src/core/lib/iomgr/timer_generic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index 57b63cb229a..dc4df807b81 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -95,8 +95,7 @@ struct shared_mutables { gpr_mu mu; } GPR_ALIGN_STRUCT(GPR_CACHELINE_SIZE); -static struct shared_mutables g_shared_mutables = { - 0, GPR_SPINLOCK_STATIC_INITIALIZER, false, {{0}}}; +static struct shared_mutables g_shared_mutables; static gpr_clock_type g_clock_type; static gpr_timespec g_start_time; @@ -154,6 +153,7 @@ void grpc_timer_list_init(gpr_timespec now) { uint32_t i; g_shared_mutables.initialized = true; + g_shared_mutables.checker_mu = GPR_SPINLOCK_STATIC_INITIALIZER; gpr_mu_init(&g_shared_mutables.mu); g_clock_type = now.clock_type; g_start_time = now; From e203067bdcd9522446aa106ce77085c62a932c49 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 14:44:58 -0700 Subject: [PATCH 187/227] initializing checker_mu in timer_list to be safe --- src/core/lib/iomgr/timer_generic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index dc4df807b81..e9a7236c8c2 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -153,7 +153,7 @@ void grpc_timer_list_init(gpr_timespec now) { uint32_t i; g_shared_mutables.initialized = true; - g_shared_mutables.checker_mu = GPR_SPINLOCK_STATIC_INITIALIZER; + g_shared_mutables.checker_mu = GPR_SPINLOCK_INITIALIZER; gpr_mu_init(&g_shared_mutables.mu); g_clock_type = now.clock_type; g_start_time = now; From 06312bdb43c66150e59958479640913c3726ee10 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 15:10:22 -0700 Subject: [PATCH 188/227] Minor formatting, commenting and clang format for C to C++ conversion --- include/grpc/slice.h | 4 ---- .../client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c | 2 +- src/core/ext/transport/chttp2/transport/hpack_encoder.c | 2 +- src/core/lib/iomgr/ev_epoll1_linux.c | 6 +++++- src/core/lib/iomgr/ev_epollex_linux.c | 6 +++++- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index 3080d63e2b4..3f3cff14086 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -66,10 +66,6 @@ GPRAPI grpc_slice grpc_slice_malloc(size_t length); GPRAPI grpc_slice grpc_slice_malloc_large(size_t length); #define GRPC_SLICE_MALLOC(len) grpc_slice_malloc(len) -/* ((len) <= GRPC_SLICE_INLINED_SIZE \ - ? (grpc_slice){.refcount = NULL, \ - .data.inlined = {.length = (uint8_t)(len)}} \ - : grpc_slice_malloc_large((len)))*/ /** Intern a slice: diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 9975b1ef795..04379975e1c 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -276,7 +276,7 @@ static void on_txt_done_cb(void *arg, int status, int timeouts, char *error_msg; grpc_ares_request *r = (grpc_ares_request *)arg; const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1; - struct ares_txt_ext *result; + struct ares_txt_ext *result = NULL; struct ares_txt_ext *reply = NULL; grpc_error *error = GRPC_ERROR_NONE; gpr_mu_lock(&r->mu); diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 52c9e217f1a..9dfbd3c3a9d 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -53,7 +53,7 @@ static grpc_slice_refcount terminal_slice_refcount = {NULL, NULL}; static const grpc_slice terminal_slice = { &terminal_slice_refcount, /* refcount */ - {{0, 0}} /* refcounted */ + {{0, 0}} /* data.refcounted */ }; extern grpc_tracer_flag grpc_http_trace; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index b5066e5f588..7fde45865e2 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -568,7 +568,11 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = {0, GPR_NS_PER_MS - 1, GPR_TIMESPAN}; + static const gpr_timespec round_up = { + 0, /* tv_sec */ + GPR_NS_PER_MS - 1, /* tv_nsec */ + GPR_TIMESPAN /* clock_type */ + }; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 671c4cd730d..cff0c07a2b8 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -706,7 +706,11 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = {0, GPR_NS_PER_MS - 1, GPR_TIMESPAN}; + static const gpr_timespec round_up = { + 0, /* tv_sec */ + GPR_NS_PER_MS - 1, /* tv_nsec */ + GPR_TIMESPAN /* clock_type */ + }; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; From 2c1a7e1375ade318d5925edb7bff63f4ce522a81 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Tue, 19 Sep 2017 00:00:14 +0000 Subject: [PATCH 189/227] Flip gRPC Python Protoc Plug-In default behavior Now the grpc_2_0 flag is no longer required for the only-generate-code- elements-in-_pb2_grpc.py-files behavior and the grpc_1_0 flag is required for the add-generated-gRPC-using-code-elements-to-_pb2.py- files behavior. --- src/compiler/python_generator.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index a60b528e3b5..ef2d90de9e9 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -767,9 +767,9 @@ bool PythonGrpcGenerator::Generate(const FileDescriptor* file, ProtoBufFile pbfile(file); PrivateGenerator generator(config_, &pbfile); - if (parameter == "grpc_2_0") { + if (parameter == "" || parameter == "grpc_2_0") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true); - } else if (parameter == "grpc_1_0" || parameter == "") { + } else if (parameter == "grpc_1_0") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true) && GenerateGrpc(context, generator, pb2_file_name, false); } else { From 75c27789026375122a33043833a25e5b935c906f Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 18 Sep 2017 17:11:48 -0700 Subject: [PATCH 190/227] Update Dockerfiles from ubuntu:15.10 to debian:jessie --- .../tools/dockerfile/clang_format.include | 6 ++-- .../grpc_clang_format/Dockerfile.template | 3 +- .../test/sanity/Dockerfile.template | 27 ++++++++++----- tools/dockerfile/grpc_clang_format/Dockerfile | 8 ++--- tools/dockerfile/test/sanity/Dockerfile | 33 ++++++++++++------- 5 files changed, 48 insertions(+), 29 deletions(-) diff --git a/templates/tools/dockerfile/clang_format.include b/templates/tools/dockerfile/clang_format.include index 9a2b60ba8c4..81bd2be797b 100644 --- a/templates/tools/dockerfile/clang_format.include +++ b/templates/tools/dockerfile/clang_format.include @@ -1,5 +1,5 @@ RUN apt-get update && apt-get -y install wget -RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - RUN apt-get update && apt-get -y install clang-format-3.8 diff --git a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template index 69cd4034b04..1ab667c95d3 100644 --- a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template +++ b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM ubuntu:15.10 + FROM debian:jessie <%include file="../clang_format.include"/> ADD clang_format_all_the_things.sh / CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"] + \ No newline at end of file diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template index 3f9ea322338..bf34c0ab0f1 100644 --- a/templates/tools/dockerfile/test/sanity/Dockerfile.template +++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM ubuntu:15.10 + FROM debian:jessie <%include file="../../apt_get_basic.include"/> <%include file="../../gcp_api_libraries.include"/> @@ -33,14 +33,23 @@ #====================================== # More sanity test dependencies (bazel) - RUN apt-get install -y openjdk-8-jdk - # Check out Bazel version 0.4.1 since this version allows running - # ./compile.sh without a local protoc dependency - # TODO(mattkwong): install dependencies to support latest Bazel version if newer - # version is needed - RUN git clone https://github.com/bazelbuild/bazel.git /bazel && ${"\\"} - cd /bazel && git checkout tags/0.4.1 && ./compile.sh - RUN ln -s /bazel/output/bazel /bin/ + RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list + RUN apt-get update + RUN apt-get install -y -t jessie-backports openjdk-8-jdk + + #======================== + # Bazel installation + RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list + RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - + RUN apt-get -y update + RUN apt-get -y install bazel + + # Pin Bazel to 0.4.4 + # Installing Bazel via apt-get first is required before installing 0.4.4 to + # allow gRPC to build without errors. See https://github.com/grpc/grpc/issues/10553 + RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.4.4/bazel-0.4.4-installer-linux-x86_64.sh + RUN chmod +x ./bazel-0.4.4-installer-linux-x86_64.sh + RUN ./bazel-0.4.4-installer-linux-x86_64.sh <%include file="../../clang_format.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile index 647cb52a7b1..dff07feeb6d 100644 --- a/tools/dockerfile/grpc_clang_format/Dockerfile +++ b/tools/dockerfile/grpc_clang_format/Dockerfile @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:15.10 +FROM debian:jessie RUN apt-get update && apt-get -y install wget -RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - RUN apt-get update && apt-get -y install clang-format-3.8 ADD clang_format_all_the_things.sh / diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 44732a5ae42..dff979dc371 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:15.10 +FROM debian:jessie # Install Git and basic packages. RUN apt-get update && apt-get install -y \ @@ -82,19 +82,28 @@ RUN pip install simplejson mako #====================================== # More sanity test dependencies (bazel) -RUN apt-get install -y openjdk-8-jdk -# Check out Bazel version 0.4.1 since this version allows running -# ./compile.sh without a local protoc dependency -# TODO(mattkwong): install dependencies to support latest Bazel version if newer -# version is needed -RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \ - cd /bazel && git checkout tags/0.4.1 && ./compile.sh -RUN ln -s /bazel/output/bazel /bin/ +RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list +RUN apt-get update +RUN apt-get install -y -t jessie-backports openjdk-8-jdk + +#======================== +# Bazel installation +RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list +RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - +RUN apt-get -y update +RUN apt-get -y install bazel + +# Pin Bazel to 0.4.4 +# Installing Bazel via apt-get first is required before installing 0.4.4 to +# allow gRPC to build without errors. See https://github.com/grpc/grpc/issues/10553 +RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.4.4/bazel-0.4.4-installer-linux-x86_64.sh +RUN chmod +x ./bazel-0.4.4-installer-linux-x86_64.sh +RUN ./bazel-0.4.4-installer-linux-x86_64.sh RUN apt-get update && apt-get -y install wget -RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - RUN apt-get update && apt-get -y install clang-format-3.8 # Prepare ccache From 7d764673ef9a18503339747ab8ee002739f6e329 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 18 Sep 2017 19:10:49 -0700 Subject: [PATCH 191/227] Backends must be up before some grpclb tests start. --- test/cpp/end2end/grpclb_end2end_test.cc | 87 ++++++++++++++++++++----- 1 file changed, 69 insertions(+), 18 deletions(-) diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 570a3d10677..6896b54f1be 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -398,11 +398,40 @@ class GrpclbEnd2endTest : public ::testing::Test { return true; } - void WaitForAllBackends() { + void SendRpcAndCount(int* num_total, int* num_ok, int* num_failure, + int* num_drops) { + const Status status = SendRpc(); + if (status.ok()) { + ++*num_ok; + } else { + if (status.error_message() == "Call dropped by load balancing policy") { + ++*num_drops; + } else { + ++*num_failure; + } + } + ++*num_total; + } + + std::tuple WaitForAllBackends( + int num_requests_multiple_of = 1) { + int num_ok = 0; + int num_failure = 0; + int num_drops = 0; + int num_total = 0; while (!SeenAllBackends()) { - CheckRpcSendOk(); + SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops); + } + while (num_total % num_requests_multiple_of != 0) { + SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops); } ResetBackendCounters(); + gpr_log(GPR_INFO, + "Performed %d warm up requests (a multiple of %d) against the " + "backends. %d succeeded, %d failed, %d dropped.", + num_total, num_requests_multiple_of, num_ok, num_failure, + num_drops); + return std::make_tuple(num_ok, num_failure, num_drops); } void WaitForBackend(size_t backend_idx) { @@ -556,10 +585,8 @@ TEST_F(SingleBalancerTest, Vanilla) { 0); // Make sure that trying to connect works without a call. channel_->GetState(true /* try_to_connect */); - // We need to wait for all backends to come online. WaitForAllBackends(); - // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(kNumRpcsPerAddress * num_backends_); @@ -867,6 +894,8 @@ TEST_F(SingleBalancerTest, Drop) { 0, BalancerServiceImpl::BuildResponseForBackends( GetBackendPorts(), {{"rate_limiting", 1}, {"load_balancing", 2}}), 0); + // Wait until all backends are ready. + WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs for each server and drop address. size_t num_drops = 0; for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { @@ -936,6 +965,11 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) { ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends(GetBackendPorts(), {}), 0); + // Wait until all backends are ready. + int num_ok = 0; + int num_failure = 0; + int num_drops = 0; + std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(kNumRpcsPerAddress * num_backends_); // Each backend should have gotten 100 requests. @@ -950,11 +984,12 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) { EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); const ClientStats client_stats = WaitForLoadReports(); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, client_stats.num_calls_started); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok, + client_stats.num_calls_started); + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok, client_stats.num_calls_finished); EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + (num_ok + num_drops), client_stats.num_calls_finished_known_received); EXPECT_THAT(client_stats.drop_token_counts, ::testing::ElementsAre()); } @@ -965,7 +1000,14 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { 0, BalancerServiceImpl::BuildResponseForBackends( GetBackendPorts(), {{"rate_limiting", 2}, {"load_balancing", 1}}), 0); - + // Wait until all backends are ready. + int num_warmup_ok = 0; + int num_warmup_failure = 0; + int num_warmup_drops = 0; + std::tie(num_warmup_ok, num_warmup_failure, num_warmup_drops) = + WaitForAllBackends(num_backends_ + 3 /* num_requests_multiple_of */); + const int num_total_warmup_requests = + num_warmup_ok + num_warmup_failure + num_warmup_drops; size_t num_drops = 0; for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { EchoResponse response; @@ -980,7 +1022,6 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { } } EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops); - // Each backend should have gotten 100 requests. for (size_t i = 0; i < backends_.size(); ++i) { EXPECT_EQ(kNumRpcsPerAddress, @@ -993,17 +1034,27 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); const ClientStats client_stats = WaitForLoadReports(); - EXPECT_EQ(kNumRpcsPerAddress * (num_backends_ + 3), - client_stats.num_calls_started); - EXPECT_EQ(kNumRpcsPerAddress * (num_backends_ + 3), - client_stats.num_calls_finished); + EXPECT_EQ( + kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + client_stats.num_calls_started); + EXPECT_EQ( + kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + client_stats.num_calls_finished); EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_warmup_ok, client_stats.num_calls_finished_known_received); - EXPECT_THAT(client_stats.drop_token_counts, - ::testing::ElementsAre( - ::testing::Pair("load_balancing", kNumRpcsPerAddress), - ::testing::Pair("rate_limiting", kNumRpcsPerAddress * 2))); + // The number of warmup request is a multiple of the number of addresses. + // Therefore, all addresses in the scheduled balancer response are hit the + // same number of times. + const int num_times_drop_addresses_hit = num_warmup_drops / 3; + EXPECT_THAT( + client_stats.drop_token_counts, + ::testing::ElementsAre( + ::testing::Pair("load_balancing", + (kNumRpcsPerAddress + num_times_drop_addresses_hit)), + ::testing::Pair( + "rate_limiting", + (kNumRpcsPerAddress + num_times_drop_addresses_hit) * 2))); } } // namespace From 588e36a3e3b43276f573c466d88c3dd60bd5be16 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 18 Sep 2017 20:01:27 -0700 Subject: [PATCH 192/227] Get rid of magic numbers --- test/cpp/end2end/grpclb_end2end_test.cc | 45 ++++++++++++++++++------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 6896b54f1be..77ed1552923 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -890,15 +890,22 @@ TEST_F(UpdatesTest, UpdateBalancersDeadUpdate) { TEST_F(SingleBalancerTest, Drop) { const size_t kNumRpcsPerAddress = 100; + const int num_of_drop_by_rate_limiting_addresses = 1; + const int num_of_drop_by_load_balancing_addresses = 2; + const int num_of_drop_addresses = num_of_drop_by_rate_limiting_addresses + + num_of_drop_by_load_balancing_addresses; + const int num_total_addresses = num_backends_ + num_of_drop_addresses; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - GetBackendPorts(), {{"rate_limiting", 1}, {"load_balancing", 2}}), + GetBackendPorts(), + {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 0); // Wait until all backends are ready. WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs for each server and drop address. size_t num_drops = 0; - for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { + for (size_t i = 0; i < kNumRpcsPerAddress * num_total_addresses; ++i) { EchoResponse response; const Status status = SendRpc(&response); if (!status.ok() && @@ -910,7 +917,7 @@ TEST_F(SingleBalancerTest, Drop) { EXPECT_EQ(response.message(), kMessage_); } } - EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops); + EXPECT_EQ(kNumRpcsPerAddress * num_of_drop_addresses, num_drops); // Each backend should have gotten 100 requests. for (size_t i = 0; i < backends_.size(); ++i) { @@ -925,9 +932,12 @@ TEST_F(SingleBalancerTest, Drop) { TEST_F(SingleBalancerTest, DropAllFirst) { // All registered addresses are marked as "drop". + const int num_of_drop_by_rate_limiting_addresses = 1; + const int num_of_drop_by_load_balancing_addresses = 1; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - {}, {{"rate_limiting", 1}, {"load_balancing", 1}}), + {}, {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 0); const Status status = SendRpc(); EXPECT_FALSE(status.ok()); @@ -938,9 +948,12 @@ TEST_F(SingleBalancerTest, DropAll) { ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends(GetBackendPorts(), {}), 0); + const int num_of_drop_by_rate_limiting_addresses = 1; + const int num_of_drop_by_load_balancing_addresses = 1; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - {}, {{"rate_limiting", 1}, {"load_balancing", 1}}), + {}, {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 1000); // First call succeeds. @@ -996,20 +1009,27 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) { TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { const size_t kNumRpcsPerAddress = 3; + const int num_of_drop_by_rate_limiting_addresses = 2; + const int num_of_drop_by_load_balancing_addresses = 1; + const int num_of_drop_addresses = num_of_drop_by_rate_limiting_addresses + + num_of_drop_by_load_balancing_addresses; + const int num_total_addresses = num_backends_ + num_of_drop_addresses; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - GetBackendPorts(), {{"rate_limiting", 2}, {"load_balancing", 1}}), + GetBackendPorts(), + {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 0); // Wait until all backends are ready. int num_warmup_ok = 0; int num_warmup_failure = 0; int num_warmup_drops = 0; std::tie(num_warmup_ok, num_warmup_failure, num_warmup_drops) = - WaitForAllBackends(num_backends_ + 3 /* num_requests_multiple_of */); + WaitForAllBackends(num_total_addresses /* num_requests_multiple_of */); const int num_total_warmup_requests = num_warmup_ok + num_warmup_failure + num_warmup_drops; size_t num_drops = 0; - for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { + for (size_t i = 0; i < kNumRpcsPerAddress * num_total_addresses; ++i) { EchoResponse response; const Status status = SendRpc(&response); if (!status.ok() && @@ -1021,7 +1041,7 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { EXPECT_EQ(response.message(), kMessage_); } } - EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops); + EXPECT_EQ(kNumRpcsPerAddress * num_of_drop_addresses, num_drops); // Each backend should have gotten 100 requests. for (size_t i = 0; i < backends_.size(); ++i) { EXPECT_EQ(kNumRpcsPerAddress, @@ -1035,10 +1055,10 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { const ClientStats client_stats = WaitForLoadReports(); EXPECT_EQ( - kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + kNumRpcsPerAddress * num_total_addresses + num_total_warmup_requests, client_stats.num_calls_started); EXPECT_EQ( - kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + kNumRpcsPerAddress * num_total_addresses + num_total_warmup_requests, client_stats.num_calls_finished); EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send); EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_warmup_ok, @@ -1046,7 +1066,8 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { // The number of warmup request is a multiple of the number of addresses. // Therefore, all addresses in the scheduled balancer response are hit the // same number of times. - const int num_times_drop_addresses_hit = num_warmup_drops / 3; + const int num_times_drop_addresses_hit = + num_warmup_drops / num_of_drop_addresses; EXPECT_THAT( client_stats.drop_token_counts, ::testing::ElementsAre( From 350a4239fe3a56fb20b3eea99badaf307c652ad5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 18 Sep 2017 22:41:24 -0700 Subject: [PATCH 193/227] Fix compile error --- test/cpp/naming/resolver_component_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc index 0857fb6a32c..396f4ceb9e1 100644 --- a/test/cpp/naming/resolver_component_test.cc +++ b/test/cpp/naming/resolver_component_test.cc @@ -270,7 +270,8 @@ void CheckResolverResultLocked(grpc_exec_ctx *exec_ctx, void *argsp, CheckLBPolicyResultLocked(channel_args, args); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } From 67cd21e6c7e1e9347760349c29f32d0d5b42c843 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 19 Sep 2017 16:21:20 +0200 Subject: [PATCH 194/227] adjust sanitizer timeouts --- .../linux/sanitizer/pull_request/grpc_c_asan.cfg | 2 +- .../linux/sanitizer/pull_request/grpc_c_msan.cfg | 2 +- .../linux/sanitizer/pull_request/grpc_c_tsan.cfg | 2 +- .../linux/sanitizer/pull_request/grpc_c_ubsan.cfg | 2 +- .../linux/sanitizer/pull_request/grpc_cpp_asan.cfg | 2 +- tools/run_tests/run_tests_matrix.py | 12 +++++++++++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg index db9cb4f5975..1daf7a514e2 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg index ba651d357c9..a8503b7bcbe 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg index ccbd95ae634..12af4581eba 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg index c240d3a5734..0d3803bf236 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg index 38db2f4f600..557561810bd 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 957e7b569e2..2417c5ae553 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -35,6 +35,9 @@ _DEFAULT_RUNTESTS_TIMEOUT = 1*60*60 # clang docker. _CPP_RUNTESTS_TIMEOUT = 4*60*60 +# C++ TSAN takes longer than other sanitizers +_CPP_TSAN_RUNTESTS_TIMEOUT = 8*60*60 + # Number of jobs assigned to each run_tests.py instance _DEFAULT_INNER_JOBS = 2 @@ -190,12 +193,19 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): inner_jobs=inner_jobs, timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['c++'], - configs=['asan', 'tsan'], + configs=['asan'], platforms=['linux'], labels=['sanitizers', 'corelang'], extra_args=extra_args, inner_jobs=inner_jobs, timeout_seconds=_CPP_RUNTESTS_TIMEOUT) + test_jobs += _generate_jobs(languages=['c++'], + configs=['tsan'], + platforms=['linux'], + labels=['sanitizers', 'corelang'], + extra_args=extra_args, + inner_jobs=inner_jobs, + timeout_seconds=_CPP_TSAN_RUNTESTS_TIMEOUT) return test_jobs From 43b55221e4a109f9dc6845c2c081dd5ea17249ae Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 18 Sep 2017 15:03:53 -0700 Subject: [PATCH 195/227] Cancel outstanding streams when shutting down the client --- test/cpp/qps/client_async.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 1a4438047dd..f5807da81e3 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -56,6 +56,7 @@ class ClientRpcContext { } virtual void Start(CompletionQueue* cq, const ClientConfig& config) = 0; + virtual void TryCancel() = 0; }; template @@ -110,6 +111,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { prepare_req_, callback_); clone->StartInternal(cq); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; @@ -142,8 +144,6 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { } }; -typedef std::forward_list context_list; - template class AsyncClient : public ClientImpl { // Specify which protected members we are using since there is no @@ -247,6 +247,7 @@ class AsyncClient : public ClientImpl { // this thread isn't supposed to shut down std::lock_guard l(shutdown_state_[thread_idx]->mutex); if (shutdown_state_[thread_idx]->shutdown) { + ctx->TryCancel(); delete ctx; return true; } @@ -388,6 +389,7 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; @@ -527,6 +529,7 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; @@ -644,6 +647,7 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; @@ -786,6 +790,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; From 0f7dc4f71d074157263462e49079d95277e83dfe Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 19 Sep 2017 12:17:25 -0700 Subject: [PATCH 196/227] Use shortname for shortname for json_run_localhost tests --- tools/run_tests/run_tests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 176c1097ee9..b38108d456c 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -350,11 +350,12 @@ class CLanguage(object): environ=env)) else: cmdline = [binary] + target['args'] + shortname = target.get('shortname', ' '.join( + pipes.quote(arg) + for arg in cmdline)) + shortname += shortname_ext out.append(self.config.job_spec(cmdline, - shortname=' '.join( - pipes.quote(arg) - for arg in cmdline) + - shortname_ext, + shortname=shortname, cpu_cost=cpu_cost, flaky=target.get('flaky', False), timeout_seconds=target.get('timeout_seconds', _DEFAULT_TIMEOUT_SECONDS) * timeout_scaling, From 2746bd59982a5c186cf12d2007c73bd9fc4c3e33 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 19 Sep 2017 11:12:55 -0700 Subject: [PATCH 197/227] Fix cython definition to match c definition This is required to compile with Cython options that are more strict about const correctness. --- src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi | 2 +- src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 28cf1144510..840af5c43a8 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -523,7 +523,7 @@ cdef extern from "grpc/compression.h": int grpc_compression_algorithm_parse( grpc_slice value, grpc_compression_algorithm *algorithm) nogil int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, - char **name) nogil + const char **name) nogil grpc_compression_algorithm grpc_compression_algorithm_for_level( grpc_compression_level level, uint32_t accepted_encodings) nogil void grpc_compression_options_init(grpc_compression_options *opts) nogil diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi index 0a2a6eee051..d860173b5d4 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi @@ -782,7 +782,7 @@ cdef class CompressionOptions: def compression_algorithm_name(grpc_compression_algorithm algorithm): - cdef char* name + cdef const char* name with nogil: grpc_compression_algorithm_name(algorithm, &name) # Let Cython do the right thing with string casting From 7fcdd7add07a4390c8d2e430d519602206433159 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 19 Sep 2017 14:11:21 -0700 Subject: [PATCH 198/227] Really spell out code generation --- src/core/lib/debug/stats_data.c | 14 ++++++++++++++ tools/codegen/core/gen_stats_data.py | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index e4723dac0a8..8eaf308b71b 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -278,6 +278,7 @@ const uint8_t grpc_stats_table_7[102] = { const int grpc_stats_table_8[9] = {0, 1, 2, 4, 7, 13, 23, 39, 64}; const uint8_t grpc_stats_table_9[9] = {0, 0, 1, 2, 2, 3, 4, 4, 5}; void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 262144); if (value < 6) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, @@ -303,6 +304,7 @@ void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_0, 64)); } void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 29) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), @@ -329,6 +331,7 @@ void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_2, 128)); } void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, @@ -354,6 +357,7 @@ void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), @@ -379,6 +383,7 @@ void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, @@ -404,6 +409,7 @@ void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, @@ -430,6 +436,7 @@ void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) { } void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -457,6 +464,7 @@ void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM( @@ -484,6 +492,7 @@ void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_http2_send_initial_metadata_per_write( grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -513,6 +522,7 @@ void grpc_stats_inc_http2_send_initial_metadata_per_write( } void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -540,6 +550,7 @@ void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_http2_send_trailing_metadata_per_write( grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -569,6 +580,7 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write( } void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -596,6 +608,7 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -622,6 +635,7 @@ void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, (exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 64); if (value < 3) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8359734c848..10ad0cc831f 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -147,7 +147,8 @@ def gen_bucket_code(histogram): shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets) #print first_nontrivial, shift_data, bounds #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]] - code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max + code = '\n/* Automatically generated by tools/codegen/core/gen_stats_data.py */\n' + code += 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data) if first_nontrivial is None: code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n' @@ -407,4 +408,3 @@ with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: for counter in inst_map['Counter']: columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) print >>S, ',\n'.join('%s:%s' % x for x in columns) - From 79c12b9dc02f5558af6868acd2cafdfaa600d89e Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 18 Sep 2017 22:02:52 -0700 Subject: [PATCH 199/227] Drain readable fd --- .../dns/c_ares/grpc_ares_ev_driver_posix.c | 47 +++++++++++++++++-- test/cpp/naming/resolver_component_test.cc | 4 +- .../naming/resolver_component_tests_runner.sh | 8 ++++ .../naming/resolver_test_record_groups.yaml | 5 -- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c index 7f1f57259a9..d38fe66d061 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c @@ -20,6 +20,7 @@ #if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET) #include +#include #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" @@ -54,6 +55,8 @@ typedef struct fd_node { bool readable_registered; /** if the writable closure has been registered */ bool writable_registered; + /** if the fd is being shut down */ + bool shutting_down; } fd_node; struct grpc_ares_ev_driver { @@ -100,7 +103,6 @@ static void fd_node_destroy(grpc_exec_ctx *exec_ctx, fd_node *fdn) { GPR_ASSERT(!fdn->readable_registered); GPR_ASSERT(!fdn->writable_registered); gpr_mu_destroy(&fdn->mu); - grpc_pollset_set_del_fd(exec_ctx, fdn->ev_driver->pollset_set, fdn->fd); /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up immediately by another thread, and should not be closed by the following grpc_fd_orphan. */ @@ -109,6 +111,20 @@ static void fd_node_destroy(grpc_exec_ctx *exec_ctx, fd_node *fdn) { gpr_free(fdn); } +static void fd_node_shutdown(grpc_exec_ctx *exec_ctx, fd_node *fdn) { + gpr_mu_lock(&fdn->mu); + fdn->shutting_down = true; + if (!fdn->readable_registered && !fdn->writable_registered) { + gpr_mu_unlock(&fdn->mu); + fd_node_destroy(exec_ctx, fdn); + } else { + grpc_fd_shutdown( + exec_ctx, fdn->fd, + GRPC_ERROR_CREATE_FROM_STATIC_STRING("c-ares fd shutdown")); + gpr_mu_unlock(&fdn->mu); + } +} + grpc_error *grpc_ares_ev_driver_create(grpc_ares_ev_driver **ev_driver, grpc_pollset_set *pollset_set) { *ev_driver = (grpc_ares_ev_driver *)gpr_malloc(sizeof(grpc_ares_ev_driver)); @@ -175,18 +191,34 @@ static fd_node *pop_fd_node(fd_node **head, int fd) { return NULL; } +/* Check if \a fd is still readable */ +static bool grpc_ares_is_fd_still_readable(grpc_ares_ev_driver *ev_driver, + int fd) { + size_t bytes_available = 0; + return ioctl(fd, FIONREAD, &bytes_available) == 0 && bytes_available > 0; +} + static void on_readable_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { fd_node *fdn = (fd_node *)arg; grpc_ares_ev_driver *ev_driver = fdn->ev_driver; gpr_mu_lock(&fdn->mu); fdn->readable_registered = false; + if (fdn->shutting_down && !fdn->writable_registered) { + gpr_mu_unlock(&fdn->mu); + fd_node_destroy(exec_ctx, fdn); + grpc_ares_ev_driver_unref(ev_driver); + return; + } gpr_mu_unlock(&fdn->mu); gpr_log(GPR_DEBUG, "readable on %d", grpc_fd_wrapped_fd(fdn->fd)); if (error == GRPC_ERROR_NONE) { - ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->fd), - ARES_SOCKET_BAD); + do { + ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->fd), + ARES_SOCKET_BAD); + } while ( + grpc_ares_is_fd_still_readable(ev_driver, grpc_fd_wrapped_fd(fdn->fd))); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or // timed out. The pending lookups made on this ev_driver will be cancelled @@ -208,6 +240,12 @@ static void on_writable_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_ares_ev_driver *ev_driver = fdn->ev_driver; gpr_mu_lock(&fdn->mu); fdn->writable_registered = false; + if (fdn->shutting_down && !fdn->readable_registered) { + gpr_mu_unlock(&fdn->mu); + fd_node_destroy(exec_ctx, fdn); + grpc_ares_ev_driver_unref(ev_driver); + return; + } gpr_mu_unlock(&fdn->mu); gpr_log(GPR_DEBUG, "writable on %d", grpc_fd_wrapped_fd(fdn->fd)); @@ -256,6 +294,7 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, fdn->ev_driver = ev_driver; fdn->readable_registered = false; fdn->writable_registered = false; + fdn->shutting_down = false; gpr_mu_init(&fdn->mu); GRPC_CLOSURE_INIT(&fdn->read_closure, on_readable_cb, fdn, grpc_schedule_on_exec_ctx); @@ -296,7 +335,7 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, while (ev_driver->fds != NULL) { fd_node *cur = ev_driver->fds; ev_driver->fds = ev_driver->fds->next; - fd_node_destroy(exec_ctx, cur); + fd_node_shutdown(exec_ctx, cur); } ev_driver->fds = new_list; // If the ev driver has no working fd, all the tasks are done. diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc index 0857fb6a32c..6a2ce37d00a 100644 --- a/test/cpp/naming/resolver_component_test.cc +++ b/test/cpp/naming/resolver_component_test.cc @@ -267,7 +267,9 @@ void CheckResolverResultLocked(grpc_exec_ctx *exec_ctx, void *argsp, } EXPECT_THAT(args->expected_addrs, UnorderedElementsAreArray(found_lb_addrs)); CheckServiceConfigResultLocked(channel_args, args); - CheckLBPolicyResultLocked(channel_args, args); + if (args->expected_service_config_string == "") { + CheckLBPolicyResultLocked(channel_args, args); + } gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); diff --git a/test/cpp/naming/resolver_component_tests_runner.sh b/test/cpp/naming/resolver_component_tests_runner.sh index 83b03b67a30..cf71c9dcf97 100755 --- a/test/cpp/naming/resolver_component_tests_runner.sh +++ b/test/cpp/naming/resolver_component_tests_runner.sh @@ -168,6 +168,14 @@ $FLAGS_test_bin_path \ --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & wait $! || EXIT_CODE=1 +$FLAGS_test_bin_path \ + --target_name='ipv4-config-causing-fallback-to-tcp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + kill -SIGTERM $DNS_SERVER_PID || true wait exit $EXIT_CODE diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml index 67c611d831f..c2e8ddd0ed2 100644 --- a/test/cpp/naming/resolver_test_record_groups.yaml +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -137,11 +137,6 @@ resolver_component_tests: - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} srv-ipv6-target-has-backend-and-balancer: - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} - -resolver_component_tests_TODO: -- 'TODO: enable this large-txt-record test once working. (it is much longer than 512 - bytes, likely to cause use of TCP even if max payload for UDP is changed somehow, - e.g. via notes in RFC 2671)' - expected_addrs: - {address: '1.2.3.4:443', is_balancer: false} expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' From c6f9cf3158d04359d74d14676c0b1d3b1e504919 Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Tue, 19 Sep 2017 14:34:30 -0700 Subject: [PATCH 200/227] update composer installing protobuf v3 for php-qps --- src/php/tests/qps/composer.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/php/tests/qps/composer.json b/src/php/tests/qps/composer.json index 8c1e7b6c746..f8512648c4a 100644 --- a/src/php/tests/qps/composer.json +++ b/src/php/tests/qps/composer.json @@ -1,8 +1,7 @@ { - "minimum-stability": "dev", "require": { "grpc/grpc": "dev-master", - "google/protobuf": "^v3.3.0" + "google/protobuf": "v3.4.1" }, "autoload": { "psr-4": { From 85629d069638acb5f8834dcff053debde98c69a7 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 19 Sep 2017 16:47:12 -0700 Subject: [PATCH 201/227] Add missing exec_ctx parameter to pollset_uv.c --- src/core/lib/iomgr/pollset_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c index a79fe89d3ef..2651325e257 100644 --- a/src/core/lib/iomgr/pollset_uv.c +++ b/src/core/lib/iomgr/pollset_uv.c @@ -145,7 +145,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return GRPC_ERROR_NONE; } -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { GRPC_UV_ASSERT_SAME_THREAD(); uv_timer_start(dummy_uv_handle, dummy_timer_cb, 0, 0); From b56cf2d3bf6b44be4291b7a9962a10e181a3a753 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 19 Sep 2017 15:01:39 -0700 Subject: [PATCH 202/227] Protect fdn->fd --- .../dns/c_ares/grpc_ares_ev_driver_posix.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c index d38fe66d061..b5be69c4287 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c @@ -38,8 +38,6 @@ typedef struct fd_node { /** the owner of this fd node */ grpc_ares_ev_driver *ev_driver; - /** the grpc_fd owned by this fd node */ - grpc_fd *fd; /** a closure wrapping on_readable_cb, which should be invoked when the grpc_fd in this node becomes readable. */ grpc_closure read_closure; @@ -51,6 +49,8 @@ typedef struct fd_node { /** mutex guarding the rest of the state */ gpr_mu mu; + /** the grpc_fd owned by this fd node */ + grpc_fd *fd; /** if the readable closure has been registered */ bool readable_registered; /** if the writable closure has been registered */ @@ -203,6 +203,7 @@ static void on_readable_cb(grpc_exec_ctx *exec_ctx, void *arg, fd_node *fdn = (fd_node *)arg; grpc_ares_ev_driver *ev_driver = fdn->ev_driver; gpr_mu_lock(&fdn->mu); + const int fd = grpc_fd_wrapped_fd(fdn->fd); fdn->readable_registered = false; if (fdn->shutting_down && !fdn->writable_registered) { gpr_mu_unlock(&fdn->mu); @@ -212,13 +213,11 @@ static void on_readable_cb(grpc_exec_ctx *exec_ctx, void *arg, } gpr_mu_unlock(&fdn->mu); - gpr_log(GPR_DEBUG, "readable on %d", grpc_fd_wrapped_fd(fdn->fd)); + gpr_log(GPR_DEBUG, "readable on %d", fd); if (error == GRPC_ERROR_NONE) { do { - ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->fd), - ARES_SOCKET_BAD); - } while ( - grpc_ares_is_fd_still_readable(ev_driver, grpc_fd_wrapped_fd(fdn->fd))); + ares_process_fd(ev_driver->channel, fd, ARES_SOCKET_BAD); + } while (grpc_ares_is_fd_still_readable(ev_driver, fd)); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or // timed out. The pending lookups made on this ev_driver will be cancelled @@ -239,6 +238,7 @@ static void on_writable_cb(grpc_exec_ctx *exec_ctx, void *arg, fd_node *fdn = (fd_node *)arg; grpc_ares_ev_driver *ev_driver = fdn->ev_driver; gpr_mu_lock(&fdn->mu); + const int fd = grpc_fd_wrapped_fd(fdn->fd); fdn->writable_registered = false; if (fdn->shutting_down && !fdn->readable_registered) { gpr_mu_unlock(&fdn->mu); @@ -248,10 +248,9 @@ static void on_writable_cb(grpc_exec_ctx *exec_ctx, void *arg, } gpr_mu_unlock(&fdn->mu); - gpr_log(GPR_DEBUG, "writable on %d", grpc_fd_wrapped_fd(fdn->fd)); + gpr_log(GPR_DEBUG, "writable on %d", fd); if (error == GRPC_ERROR_NONE) { - ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, - grpc_fd_wrapped_fd(fdn->fd)); + ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, fd); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or // timed out. The pending lookups made on this ev_driver will be cancelled From 4a6b08aa4706b870f51ec61d03fd786a2fc3066a Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 19 Sep 2017 18:49:52 -0700 Subject: [PATCH 203/227] Modifying uv tests to reflect exec_ctx parameter for grpc_pollset_kick --- test/core/iomgr/tcp_client_uv_test.c | 8 ++++---- test/core/iomgr/tcp_server_uv_test.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/core/iomgr/tcp_client_uv_test.c b/test/core/iomgr/tcp_client_uv_test.c index 9927356613b..b0ff11b39a9 100644 --- a/test/core/iomgr/tcp_client_uv_test.c +++ b/test/core/iomgr/tcp_client_uv_test.c @@ -46,11 +46,11 @@ static gpr_timespec test_deadline(void) { return grpc_timeout_seconds_to_deadline(10); } -static void finish_connection() { +static void finish_connection(grpc_exec_ctx *exec_ctx) { gpr_mu_lock(g_mu); g_connections_complete++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -63,7 +63,7 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, GRPC_ERROR_CREATE_FROM_STATIC_STRING("must_succeed called")); grpc_endpoint_destroy(exec_ctx, g_connecting); g_connecting = NULL; - finish_connection(); + finish_connection(exec_ctx); } static void must_fail(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { diff --git a/test/core/iomgr/tcp_server_uv_test.c b/test/core/iomgr/tcp_server_uv_test.c index 8f4d553d1e3..bd8ccb4f603 100644 --- a/test/core/iomgr/tcp_server_uv_test.c +++ b/test/core/iomgr/tcp_server_uv_test.c @@ -111,8 +111,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_mu_lock(g_mu); g_result = temp_result; g_nconnects++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } From 5bb43935e11b2abc623bb278af497a5bd4e16b09 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 19 Sep 2017 18:54:36 -0700 Subject: [PATCH 204/227] Backport #12615 --- src/python/grpcio_testing/grpc_testing/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio_testing/grpc_testing/__init__.py b/src/python/grpcio_testing/grpc_testing/__init__.py index 14e25f09e23..2cf6741fcae 100644 --- a/src/python/grpcio_testing/grpc_testing/__init__.py +++ b/src/python/grpcio_testing/grpc_testing/__init__.py @@ -213,7 +213,7 @@ class StreamStreamChannelRpc(six.with_metaclass(abc.ABCMeta)): raise NotImplementedError() -class Channel(six.with_metaclass(abc.ABCMeta), grpc.Channel): +class Channel(six.with_metaclass(abc.ABCMeta, grpc.Channel)): """A grpc.Channel double with which to test a system that invokes RPCs.""" @abc.abstractmethod From 4b527a7d8c895abe537384b49465cfa6a53b6162 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 20 Sep 2017 13:35:05 -0700 Subject: [PATCH 205/227] Fix test flake of stream_compression_compressed_payload #12307 --- test/core/end2end/tests/compressed_payload.c | 9 +++++---- .../tests/stream_compression_compressed_payload.c | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/test/core/end2end/tests/compressed_payload.c b/test/core/end2end/tests/compressed_payload.c index 639af15454b..ce86e97b83e 100644 --- a/test/core/end2end/tests/compressed_payload.c +++ b/test/core/end2end/tests/compressed_payload.c @@ -151,6 +151,11 @@ static void request_for_disabled_algorithm( grpc_metadata_array_init(&request_metadata_recv); grpc_call_details_init(&call_details); + error = + grpc_server_request_call(f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + memset(ops, 0, sizeof(ops)); op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; @@ -187,10 +192,6 @@ static void request_for_disabled_algorithm( error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); GPR_ASSERT(GRPC_CALL_OK == error); - error = - grpc_server_request_call(f.server, &s, &call_details, - &request_metadata_recv, f.cq, f.cq, tag(101)); - GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(101), true); cq_verify(cqv); diff --git a/test/core/end2end/tests/stream_compression_compressed_payload.c b/test/core/end2end/tests/stream_compression_compressed_payload.c index 5a69091a57f..094c4de1c05 100644 --- a/test/core/end2end/tests/stream_compression_compressed_payload.c +++ b/test/core/end2end/tests/stream_compression_compressed_payload.c @@ -151,6 +151,11 @@ static void request_for_disabled_algorithm( grpc_metadata_array_init(&request_metadata_recv); grpc_call_details_init(&call_details); + error = + grpc_server_request_call(f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + memset(ops, 0, sizeof(ops)); op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; @@ -187,10 +192,6 @@ static void request_for_disabled_algorithm( error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); GPR_ASSERT(GRPC_CALL_OK == error); - error = - grpc_server_request_call(f.server, &s, &call_details, - &request_metadata_recv, f.cq, f.cq, tag(101)); - GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(101), true); cq_verify(cqv); From a17ad0ce3cf83b5d252474e6d9d73e040a4e6f56 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 20 Sep 2017 14:59:49 -0700 Subject: [PATCH 206/227] Missing argument in finish_connection in tcp_client_uv_test --- test/core/iomgr/tcp_client_uv_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/iomgr/tcp_client_uv_test.c b/test/core/iomgr/tcp_client_uv_test.c index b0ff11b39a9..0f1db4705bb 100644 --- a/test/core/iomgr/tcp_client_uv_test.c +++ b/test/core/iomgr/tcp_client_uv_test.c @@ -69,7 +69,7 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, static void must_fail(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { GPR_ASSERT(g_connecting == NULL); GPR_ASSERT(error != GRPC_ERROR_NONE); - finish_connection(); + finish_connection(exec_ctx); } static void close_cb(uv_handle_t *handle) { gpr_free(handle); } From 8b255c7e4764d316adbdc90e49ec86f2336222f6 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 20 Sep 2017 15:47:29 -0700 Subject: [PATCH 207/227] Add line count job to Kokoro --- tools/internal_ci/linux/grpc_line_count.cfg | 19 ++++++++++++++ tools/internal_ci/linux/grpc_line_count.sh | 29 +++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_line_count.cfg create mode 100755 tools/internal_ci/linux/grpc_line_count.sh diff --git a/tools/internal_ci/linux/grpc_line_count.cfg b/tools/internal_ci/linux/grpc_line_count.cfg new file mode 100644 index 00000000000..5854d9c9c2f --- /dev/null +++ b/tools/internal_ci/linux/grpc_line_count.cfg @@ -0,0 +1,19 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_line_count.sh" +timeout_mins: 30 diff --git a/tools/internal_ci/linux/grpc_line_count.sh b/tools/internal_ci/linux/grpc_line_count.sh new file mode 100755 index 00000000000..22e484ba2fe --- /dev/null +++ b/tools/internal_ci/linux/grpc_line_count.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script counts the numbers of line in gRPC's repo and uploads to BQ +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +git submodule update --init + +# Install cloc +git clone -b v1.72 https://github.com/AlDanial/cloc/ ~/cloc +PERL_MM_USE_DEFAULT=1 sudo perl -MCPAN -e 'install Regexp::Common; install Algorithm::Diff' +sudo make install -C ~/cloc/Unix + +./tools/line_count/collect-now.sh From 46c821fe811d0cd2ab18feaec368443d4fd39a75 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 20 Sep 2017 19:23:22 -0700 Subject: [PATCH 208/227] Remove extra record_to_resolve --- test/cpp/naming/resolver_test_record_groups.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml index c2e8ddd0ed2..33d774ca701 100644 --- a/test/cpp/naming/resolver_test_record_groups.yaml +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -141,7 +141,6 @@ resolver_component_tests: - {address: '1.2.3.4:443', is_balancer: false} expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' expected_lb_policy: null - record_to_resolve: srv-ipv6-target-has-backend-and-balancer record_to_resolve: ipv4-config-causing-fallback-to-tcp records: ipv4-config-causing-fallback-to-tcp: From e232f1bd5a97a538bc880282416c39300cb1d62f Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Thu, 21 Sep 2017 05:52:25 +0000 Subject: [PATCH 209/227] Clean up even more uses of gRPC in _pb2.py files --- .../grpcio_tests/tests/http2/negative_http2_client.py | 4 ++-- src/python/grpcio_tests/tests/interop/client.py | 6 +++--- src/python/grpcio_tests/tests/qps/benchmark_client.py | 4 ++-- src/python/grpcio_tests/tests/stress/client.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/python/grpcio_tests/tests/http2/negative_http2_client.py b/src/python/grpcio_tests/tests/http2/negative_http2_client.py index 6d8a6bce77d..8dab5b67f1f 100644 --- a/src/python/grpcio_tests/tests/http2/negative_http2_client.py +++ b/src/python/grpcio_tests/tests/http2/negative_http2_client.py @@ -17,7 +17,7 @@ import argparse import grpc import time -from src.proto.grpc.testing import test_pb2 +from src.proto.grpc.testing import test_pb2_grpc from src.proto.grpc.testing import messages_pb2 @@ -147,7 +147,7 @@ def _stub(server_host, server_port): target = '{}:{}'.format(server_host, server_port) channel = grpc.insecure_channel(target) grpc.channel_ready_future(channel).result() - return test_pb2.TestServiceStub(channel) + return test_pb2_grpc.TestServiceStub(channel) def main(): diff --git a/src/python/grpcio_tests/tests/interop/client.py b/src/python/grpcio_tests/tests/interop/client.py index 47ae96472d0..e520c082903 100644 --- a/src/python/grpcio_tests/tests/interop/client.py +++ b/src/python/grpcio_tests/tests/interop/client.py @@ -19,7 +19,7 @@ import os from google import auth as google_auth from google.auth import jwt as google_auth_jwt import grpc -from src.proto.grpc.testing import test_pb2 +from src.proto.grpc.testing import test_pb2_grpc from tests.interop import methods from tests.interop import resources @@ -106,9 +106,9 @@ def _stub(args): else: channel = grpc.insecure_channel(target) if args.test_case == "unimplemented_service": - return test_pb2.UnimplementedServiceStub(channel) + return test_pb2_grpc.UnimplementedServiceStub(channel) else: - return test_pb2.TestServiceStub(channel) + return test_pb2_grpc.TestServiceStub(channel) def _test_case_from_arg(test_case_arg): diff --git a/src/python/grpcio_tests/tests/qps/benchmark_client.py b/src/python/grpcio_tests/tests/qps/benchmark_client.py index 5f4df79c5bb..17fa61ea366 100644 --- a/src/python/grpcio_tests/tests/qps/benchmark_client.py +++ b/src/python/grpcio_tests/tests/qps/benchmark_client.py @@ -22,7 +22,7 @@ from six.moves import queue import grpc from src.proto.grpc.testing import messages_pb2 -from src.proto.grpc.testing import services_pb2 +from src.proto.grpc.testing import services_pb2_grpc from tests.unit import resources from tests.unit import test_common @@ -58,7 +58,7 @@ class BenchmarkClient: if config.payload_config.WhichOneof('payload') == 'simple_params': self._generic = False - self._stub = services_pb2.BenchmarkServiceStub(channel) + self._stub = services_pb2_grpc.BenchmarkServiceStub(channel) payload = messages_pb2.Payload( body='\0' * config.payload_config.simple_params.req_size) self._request = messages_pb2.SimpleRequest( diff --git a/src/python/grpcio_tests/tests/stress/client.py b/src/python/grpcio_tests/tests/stress/client.py index d5ff0064fd0..40caa3926a5 100644 --- a/src/python/grpcio_tests/tests/stress/client.py +++ b/src/python/grpcio_tests/tests/stress/client.py @@ -20,7 +20,7 @@ import threading import grpc from six.moves import queue from src.proto.grpc.testing import metrics_pb2_grpc -from src.proto.grpc.testing import test_pb2 +from src.proto.grpc.testing import test_pb2_grpc from tests.interop import methods from tests.interop import resources @@ -133,7 +133,7 @@ def run_test(args): for _ in xrange(args.num_channels_per_server): channel = _get_channel(test_server_target, args) for _ in xrange(args.num_stubs_per_channel): - stub = test_pb2.TestServiceStub(channel) + stub = test_pb2_grpc.TestServiceStub(channel) runner = test_runner.TestRunner(stub, test_cases, hist, exception_queue, stop_event) runners.append(runner) From f1a25c7df1f4f18686fdb8de7bc13f103be34f57 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 21 Sep 2017 14:07:02 +0200 Subject: [PATCH 210/227] split windows PR basictests into dbg and opt --- .../pull_request/grpc_basictests_dbg.cfg | 30 +++++++++++++++++++ .../pull_request/grpc_basictests_opt.cfg | 30 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg create mode 100644 tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg new file mode 100644 index 00000000000..81a1e9d3c8f --- /dev/null +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests windows dbg -j 1 --inner_jobs 8 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg new file mode 100644 index 00000000000..3bb6510ca77 --- /dev/null +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests windows opt -j 1 --inner_jobs 8 --internal_ci --max_time=3600" +} From 7862097e5bb7eedd7e5d6af83973703054b3e5ef Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 21 Sep 2017 11:38:32 -0700 Subject: [PATCH 211/227] Add HPACK stats --- .../chttp2/transport/hpack_encoder.c | 61 +++-- .../transport/chttp2/transport/hpack_parser.c | 31 ++- src/core/lib/debug/stats_data.c | 48 ++++ src/core/lib/debug/stats_data.h | 80 +++++++ src/core/lib/debug/stats_data.yaml | 44 ++++ src/core/lib/debug/stats_data_bq_schema.sql | 22 ++ .../performance/massage_qps_stats.py | 22 ++ .../performance/scenario_result_schema.json | 220 ++++++++++++++++++ 8 files changed, 503 insertions(+), 25 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 9dfbd3c3a9d..4e975951736 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -33,6 +33,7 @@ #include "src/core/ext/transport/chttp2/transport/bin_encoder.h" #include "src/core/ext/transport/chttp2/transport/hpack_table.h" #include "src/core/ext/transport/chttp2/transport/varint.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/transport/metadata.h" @@ -271,8 +272,10 @@ static void add_elem(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, } } -static void emit_indexed(grpc_chttp2_hpack_compressor *c, uint32_t elem_index, +static void emit_indexed(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, uint32_t elem_index, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_INDEXED(exec_ctx); uint32_t len = GRPC_CHTTP2_VARINT_LENGTH(elem_index, 1); GRPC_CHTTP2_WRITE_VARINT(elem_index, 1, 0x80, add_tiny_header_data(st, len), len); @@ -284,15 +287,18 @@ typedef struct { bool insert_null_before_wire_value; } wire_value; -static wire_value get_wire_value(grpc_mdelem elem, bool true_binary_enabled) { +static wire_value get_wire_value(grpc_exec_ctx *exec_ctx, grpc_mdelem elem, + bool true_binary_enabled) { wire_value wire_val; if (grpc_is_binary_header(GRPC_MDKEY(elem))) { if (true_binary_enabled) { + GRPC_STATS_INC_HPACK_SEND_BINARY(exec_ctx); wire_val.huffman_prefix = 0x00; wire_val.insert_null_before_wire_value = true; wire_val.data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)); } else { + GRPC_STATS_INC_HPACK_SEND_BINARY_BASE64(exec_ctx); wire_val.huffman_prefix = 0x80; wire_val.insert_null_before_wire_value = false; wire_val.data = @@ -300,6 +306,7 @@ static wire_value get_wire_value(grpc_mdelem elem, bool true_binary_enabled) { } } else { /* TODO(ctiller): opportunistically compress non-binary headers */ + GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED(exec_ctx); wire_val.huffman_prefix = 0x00; wire_val.insert_null_before_wire_value = false; wire_val.data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)); @@ -316,11 +323,14 @@ static void add_wire_value(framer_state *st, wire_value v) { add_header_data(st, v.data); } -static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor *c, +static void emit_lithdr_incidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, uint32_t key_index, grpc_mdelem elem, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX(exec_ctx); uint32_t len_pfx = GRPC_CHTTP2_VARINT_LENGTH(key_index, 2); - wire_value value = get_wire_value(elem, st->use_true_binary_metadata); + wire_value value = + get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); size_t len_val = wire_value_length(value); uint32_t len_val_len; GPR_ASSERT(len_val <= UINT32_MAX); @@ -332,11 +342,14 @@ static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor *c, add_wire_value(st, value); } -static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c, +static void emit_lithdr_noidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, uint32_t key_index, grpc_mdelem elem, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX(exec_ctx); uint32_t len_pfx = GRPC_CHTTP2_VARINT_LENGTH(key_index, 4); - wire_value value = get_wire_value(elem, st->use_true_binary_metadata); + wire_value value = + get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); size_t len_val = wire_value_length(value); uint32_t len_val_len; GPR_ASSERT(len_val <= UINT32_MAX); @@ -348,10 +361,13 @@ static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c, add_wire_value(st, value); } -static void emit_lithdr_incidx_v(grpc_chttp2_hpack_compressor *c, +static void emit_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, grpc_mdelem elem, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX_V(exec_ctx); uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); - wire_value value = get_wire_value(elem, st->use_true_binary_metadata); + wire_value value = + get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); uint32_t len_val = (uint32_t)wire_value_length(value); uint32_t len_key_len = GRPC_CHTTP2_VARINT_LENGTH(len_key, 1); uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1); @@ -366,10 +382,13 @@ static void emit_lithdr_incidx_v(grpc_chttp2_hpack_compressor *c, add_wire_value(st, value); } -static void emit_lithdr_noidx_v(grpc_chttp2_hpack_compressor *c, +static void emit_lithdr_noidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, grpc_mdelem elem, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX_V(exec_ctx); uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); - wire_value value = get_wire_value(elem, st->use_true_binary_metadata); + wire_value value = + get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); uint32_t len_val = (uint32_t)wire_value_length(value); uint32_t len_key_len = GRPC_CHTTP2_VARINT_LENGTH(len_key, 1); uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1); @@ -422,7 +441,7 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, gpr_free(v); } if (!GRPC_MDELEM_IS_INTERNED(elem)) { - emit_lithdr_noidx_v(c, elem, st); + emit_lithdr_noidx_v(exec_ctx, c, elem, st); return; } @@ -444,16 +463,16 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, if (grpc_mdelem_eq(c->entries_elems[HASH_FRAGMENT_2(elem_hash)], elem) && c->indices_elems[HASH_FRAGMENT_2(elem_hash)] > c->tail_remote_index) { /* HIT: complete element (first cuckoo hash) */ - emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_2(elem_hash)]), - st); + emit_indexed(exec_ctx, c, + dynidx(c, c->indices_elems[HASH_FRAGMENT_2(elem_hash)]), st); return; } if (grpc_mdelem_eq(c->entries_elems[HASH_FRAGMENT_3(elem_hash)], elem) && c->indices_elems[HASH_FRAGMENT_3(elem_hash)] > c->tail_remote_index) { /* HIT: complete element (second cuckoo hash) */ - emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_3(elem_hash)]), - st); + emit_indexed(exec_ctx, c, + dynidx(c, c->indices_elems[HASH_FRAGMENT_3(elem_hash)]), st); return; } @@ -471,11 +490,11 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, indices_key > c->tail_remote_index) { /* HIT: key (first cuckoo hash) */ if (should_add_elem) { - emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st); + emit_lithdr_incidx(exec_ctx, c, dynidx(c, indices_key), elem, st); add_elem(exec_ctx, c, elem); return; } else { - emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st); + emit_lithdr_noidx(exec_ctx, c, dynidx(c, indices_key), elem, st); return; } GPR_UNREACHABLE_CODE(return ); @@ -487,11 +506,11 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, indices_key > c->tail_remote_index) { /* HIT: key (first cuckoo hash) */ if (should_add_elem) { - emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st); + emit_lithdr_incidx(exec_ctx, c, dynidx(c, indices_key), elem, st); add_elem(exec_ctx, c, elem); return; } else { - emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st); + emit_lithdr_noidx(exec_ctx, c, dynidx(c, indices_key), elem, st); return; } GPR_UNREACHABLE_CODE(return ); @@ -500,11 +519,11 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, /* no elem, key in the table... fall back to literal emission */ if (should_add_elem) { - emit_lithdr_incidx_v(c, elem, st); + emit_lithdr_incidx_v(exec_ctx, c, elem, st); add_elem(exec_ctx, c, elem); return; } else { - emit_lithdr_noidx_v(c, elem, st); + emit_lithdr_noidx_v(exec_ctx, c, elem, st); return; } GPR_UNREACHABLE_CODE(return ); diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 901e7413c0a..7e3ab565f5d 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -30,6 +30,7 @@ #include #include "src/core/ext/transport/chttp2/transport/bin_encoder.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/support/string.h" @@ -777,8 +778,7 @@ static grpc_error *parse_stream_dep0(grpc_exec_ctx *exec_ctx, return parse_stream_dep1(exec_ctx, p, cur + 1, end); } -/* emit an indexed field; for now just logs it to console; jumps to - begin the next field on completion */ +/* emit an indexed field; jumps to begin the next field on completion */ static grpc_error *finish_indexed_field(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, @@ -792,6 +792,7 @@ static grpc_error *finish_indexed_field(grpc_exec_ctx *exec_ctx, GRPC_ERROR_INT_SIZE, (intptr_t)p->table.num_ents); } GRPC_MDELEM_REF(md); + GRPC_STATS_INC_HPACK_RECV_INDEXED(exec_ctx); grpc_error *err = on_hdr(exec_ctx, p, md, 0); if (err != GRPC_ERROR_NONE) return err; return parse_begin(exec_ctx, p, cur, end); @@ -820,14 +821,14 @@ static grpc_error *parse_indexed_field_x(grpc_exec_ctx *exec_ctx, return parse_value0(exec_ctx, p, cur + 1, end); } -/* finish a literal header with incremental indexing: just log, and jump to ' - begin */ +/* finish a literal header with incremental indexing */ static grpc_error *finish_lithdr_incidx(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index); GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ + GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(md)), @@ -842,6 +843,7 @@ static grpc_error *finish_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { + GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX_V(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, take_string(exec_ctx, p, &p->key, true), @@ -898,6 +900,7 @@ static grpc_error *finish_lithdr_notidx(grpc_exec_ctx *exec_ctx, const uint8_t *end) { grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index); GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ + GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(md)), @@ -912,6 +915,7 @@ static grpc_error *finish_lithdr_notidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { + GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX_V(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, take_string(exec_ctx, p, &p->key, true), @@ -968,6 +972,7 @@ static grpc_error *finish_lithdr_nvridx(grpc_exec_ctx *exec_ctx, const uint8_t *end) { grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index); GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ + GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(md)), @@ -982,6 +987,7 @@ static grpc_error *finish_lithdr_nvridx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { + GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX_V(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, take_string(exec_ctx, p, &p->key, true), @@ -1310,9 +1316,11 @@ static grpc_error *append_string(grpc_exec_ctx *exec_ctx, /* 'true-binary' case */ ++cur; p->binary = NOT_BINARY; + GRPC_STATS_INC_HPACK_RECV_BINARY(exec_ctx); append_bytes(str, cur, (size_t)(end - cur)); return GRPC_ERROR_NONE; } + GRPC_STATS_INC_HPACK_RECV_BINARY_BASE64(exec_ctx); /* fallthrough */ b64_byte0: case B64_BYTE0: @@ -1510,6 +1518,7 @@ static grpc_error *begin_parse_string(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser_string *str) { if (!p->huff && binary == NOT_BINARY && (end - cur) >= (intptr_t)p->strlen && p->current_slice_refcount != NULL) { + GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(exec_ctx); str->copied = false; str->data.referenced.refcount = p->current_slice_refcount; str->data.referenced.data.refcounted.bytes = (uint8_t *)cur; @@ -1523,6 +1532,20 @@ static grpc_error *begin_parse_string(grpc_exec_ctx *exec_ctx, p->parsing.str = str; p->huff_state = 0; p->binary = binary; + switch (p->binary) { + case NOT_BINARY: + if (p->huff) { + GRPC_STATS_INC_HPACK_RECV_HUFFMAN(exec_ctx); + } else { + GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(exec_ctx); + } + break; + case BINARY_BEGIN: + /* stats incremented later: don't know true binary or not */ + break; + default: + abort(); + } return parse_string(exec_ctx, p, cur, end); } diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 8eaf308b71b..fb6055f7957 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -77,6 +77,28 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "http2_initiate_write_due_to_transport_flow_control_unstalled", "http2_initiate_write_due_to_ping_response", "http2_initiate_write_due_to_force_rst_stream", + "hpack_recv_indexed", + "hpack_recv_lithdr_incidx", + "hpack_recv_lithdr_incidx_v", + "hpack_recv_lithdr_notidx", + "hpack_recv_lithdr_notidx_v", + "hpack_recv_lithdr_nvridx", + "hpack_recv_lithdr_nvridx_v", + "hpack_recv_uncompressed", + "hpack_recv_huffman", + "hpack_recv_binary", + "hpack_recv_binary_base64", + "hpack_send_indexed", + "hpack_send_lithdr_incidx", + "hpack_send_lithdr_incidx_v", + "hpack_send_lithdr_notidx", + "hpack_send_lithdr_notidx_v", + "hpack_send_lithdr_nvridx", + "hpack_send_lithdr_nvridx_v", + "hpack_send_uncompressed", + "hpack_send_huffman", + "hpack_send_binary", + "hpack_send_binary_base64", "combiner_locks_initiated", "combiner_locks_scheduled_items", "combiner_locks_scheduled_final_items", @@ -157,6 +179,32 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "'transport_flow_control_unstalled'", "Number of HTTP2 writes initiated due to 'ping_response'", "Number of HTTP2 writes initiated due to 'force_rst_stream'", + "Number of HPACK indexed fields received", + "Number of HPACK literal headers received with incremental indexing", + "Number of HPACK literal headers received with incremental indexing and " + "literal keys", + "Number of HPACK literal headers received with no indexing", + "Number of HPACK literal headers received with no indexing and literal " + "keys", + "Number of HPACK literal headers received with never-indexing", + "Number of HPACK literal headers received with never-indexing and literal " + "keys", + "Number of uncompressed strings received in metadata", + "Number of huffman encoded strings received in metadata", + "Number of binary strings received in metadata", + "Number of binary strings received encoded in base64 in metadata", + "Number of HPACK indexed fields sent", + "Number of HPACK literal headers sent with incremental indexing", + "Number of HPACK literal headers sent with incremental indexing and " + "literal keys", + "Number of HPACK literal headers sent with no indexing", + "Number of HPACK literal headers sent with no indexing and literal keys", + "Number of HPACK literal headers sent with never-indexing", + "Number of HPACK literal headers sent with never-indexing and literal keys", + "Number of uncompressed strings sent in metadata", + "Number of huffman encoded strings sent in metadata", + "Number of binary strings received in metadata", + "Number of binary strings received encoded in base64 in metadata", "Number of combiner lock entries by process (first items queued to a " "combiner)", "Number of items scheduled against combiner locks", diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index 099302f29c0..6c0ad30543f 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -79,6 +79,28 @@ typedef enum { GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED, GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE, GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM, + GRPC_STATS_COUNTER_HPACK_RECV_INDEXED, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_INCIDX, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_INCIDX_V, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NOTIDX, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NOTIDX_V, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NVRIDX, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NVRIDX_V, + GRPC_STATS_COUNTER_HPACK_RECV_UNCOMPRESSED, + GRPC_STATS_COUNTER_HPACK_RECV_HUFFMAN, + GRPC_STATS_COUNTER_HPACK_RECV_BINARY, + GRPC_STATS_COUNTER_HPACK_RECV_BINARY_BASE64, + GRPC_STATS_COUNTER_HPACK_SEND_INDEXED, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_INCIDX, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_INCIDX_V, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NOTIDX, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NOTIDX_V, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NVRIDX, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NVRIDX_V, + GRPC_STATS_COUNTER_HPACK_SEND_UNCOMPRESSED, + GRPC_STATS_COUNTER_HPACK_SEND_HUFFMAN, + GRPC_STATS_COUNTER_HPACK_SEND_BINARY, + GRPC_STATS_COUNTER_HPACK_SEND_BINARY_BASE64, GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_ITEMS, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS, @@ -309,6 +331,64 @@ typedef enum { GRPC_STATS_INC_COUNTER( \ (exec_ctx), \ GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM) +#define GRPC_STATS_INC_HPACK_RECV_INDEXED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_RECV_INDEXED) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_INCIDX) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_INCIDX_V) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NOTIDX) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NOTIDX_V) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NVRIDX) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NVRIDX_V) +#define GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_RECV_UNCOMPRESSED) +#define GRPC_STATS_INC_HPACK_RECV_HUFFMAN(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_RECV_HUFFMAN) +#define GRPC_STATS_INC_HPACK_RECV_BINARY(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_RECV_BINARY) +#define GRPC_STATS_INC_HPACK_RECV_BINARY_BASE64(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_BINARY_BASE64) +#define GRPC_STATS_INC_HPACK_SEND_INDEXED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_SEND_INDEXED) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_INCIDX) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_INCIDX_V) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NOTIDX) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NOTIDX_V) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_NVRIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NVRIDX) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_NVRIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NVRIDX_V) +#define GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_SEND_UNCOMPRESSED) +#define GRPC_STATS_INC_HPACK_SEND_HUFFMAN(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_SEND_HUFFMAN) +#define GRPC_STATS_INC_HPACK_SEND_BINARY(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_SEND_BINARY) +#define GRPC_STATS_INC_HPACK_SEND_BINARY_BASE64(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_BINARY_BASE64) #define GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), \ GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED) diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 2fd23ce6757..de575f01c73 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -189,6 +189,50 @@ doc: Number of HTTP2 writes initiated due to 'ping_response' - counter: http2_initiate_write_due_to_force_rst_stream doc: Number of HTTP2 writes initiated due to 'force_rst_stream' +- counter: hpack_recv_indexed + doc: Number of HPACK indexed fields received +- counter: hpack_recv_lithdr_incidx + doc: Number of HPACK literal headers received with incremental indexing +- counter: hpack_recv_lithdr_incidx_v + doc: Number of HPACK literal headers received with incremental indexing and literal keys +- counter: hpack_recv_lithdr_notidx + doc: Number of HPACK literal headers received with no indexing +- counter: hpack_recv_lithdr_notidx_v + doc: Number of HPACK literal headers received with no indexing and literal keys +- counter: hpack_recv_lithdr_nvridx + doc: Number of HPACK literal headers received with never-indexing +- counter: hpack_recv_lithdr_nvridx_v + doc: Number of HPACK literal headers received with never-indexing and literal keys +- counter: hpack_recv_uncompressed + doc: Number of uncompressed strings received in metadata +- counter: hpack_recv_huffman + doc: Number of huffman encoded strings received in metadata +- counter: hpack_recv_binary + doc: Number of binary strings received in metadata +- counter: hpack_recv_binary_base64 + doc: Number of binary strings received encoded in base64 in metadata +- counter: hpack_send_indexed + doc: Number of HPACK indexed fields sent +- counter: hpack_send_lithdr_incidx + doc: Number of HPACK literal headers sent with incremental indexing +- counter: hpack_send_lithdr_incidx_v + doc: Number of HPACK literal headers sent with incremental indexing and literal keys +- counter: hpack_send_lithdr_notidx + doc: Number of HPACK literal headers sent with no indexing +- counter: hpack_send_lithdr_notidx_v + doc: Number of HPACK literal headers sent with no indexing and literal keys +- counter: hpack_send_lithdr_nvridx + doc: Number of HPACK literal headers sent with never-indexing +- counter: hpack_send_lithdr_nvridx_v + doc: Number of HPACK literal headers sent with never-indexing and literal keys +- counter: hpack_send_uncompressed + doc: Number of uncompressed strings sent in metadata +- counter: hpack_send_huffman + doc: Number of huffman encoded strings sent in metadata +- counter: hpack_send_binary + doc: Number of binary strings received in metadata +- counter: hpack_send_binary_base64 + doc: Number of binary strings received encoded in base64 in metadata # combiner locks - counter: combiner_locks_initiated doc: Number of combiner lock entries by process diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index b10985e4491..0611ccaff0e 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -52,6 +52,28 @@ http2_initiate_write_due_to_keepalive_ping_per_iteration:FLOAT, http2_initiate_write_due_to_transport_flow_control_unstalled_per_iteration:FLOAT, http2_initiate_write_due_to_ping_response_per_iteration:FLOAT, http2_initiate_write_due_to_force_rst_stream_per_iteration:FLOAT, +hpack_recv_indexed_per_iteration:FLOAT, +hpack_recv_lithdr_incidx_per_iteration:FLOAT, +hpack_recv_lithdr_incidx_v_per_iteration:FLOAT, +hpack_recv_lithdr_notidx_per_iteration:FLOAT, +hpack_recv_lithdr_notidx_v_per_iteration:FLOAT, +hpack_recv_lithdr_nvridx_per_iteration:FLOAT, +hpack_recv_lithdr_nvridx_v_per_iteration:FLOAT, +hpack_recv_uncompressed_per_iteration:FLOAT, +hpack_recv_huffman_per_iteration:FLOAT, +hpack_recv_binary_per_iteration:FLOAT, +hpack_recv_binary_base64_per_iteration:FLOAT, +hpack_send_indexed_per_iteration:FLOAT, +hpack_send_lithdr_incidx_per_iteration:FLOAT, +hpack_send_lithdr_incidx_v_per_iteration:FLOAT, +hpack_send_lithdr_notidx_per_iteration:FLOAT, +hpack_send_lithdr_notidx_v_per_iteration:FLOAT, +hpack_send_lithdr_nvridx_per_iteration:FLOAT, +hpack_send_lithdr_nvridx_v_per_iteration:FLOAT, +hpack_send_uncompressed_per_iteration:FLOAT, +hpack_send_huffman_per_iteration:FLOAT, +hpack_send_binary_per_iteration:FLOAT, +hpack_send_binary_base64_per_iteration:FLOAT, combiner_locks_initiated_per_iteration:FLOAT, combiner_locks_scheduled_items_per_iteration:FLOAT, combiner_locks_scheduled_final_items_per_iteration:FLOAT, diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index ef4f90cd930..e1620adad26 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -74,6 +74,28 @@ def massage_qps_stats(scenario_result): stats["core_http2_initiate_write_due_to_transport_flow_control_unstalled"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control_unstalled") stats["core_http2_initiate_write_due_to_ping_response"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_ping_response") stats["core_http2_initiate_write_due_to_force_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_force_rst_stream") + stats["core_hpack_recv_indexed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_indexed") + stats["core_hpack_recv_lithdr_incidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_incidx") + stats["core_hpack_recv_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_incidx_v") + stats["core_hpack_recv_lithdr_notidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_notidx") + stats["core_hpack_recv_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_notidx_v") + stats["core_hpack_recv_lithdr_nvridx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_nvridx") + stats["core_hpack_recv_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_nvridx_v") + stats["core_hpack_recv_uncompressed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_uncompressed") + stats["core_hpack_recv_huffman"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_huffman") + stats["core_hpack_recv_binary"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_binary") + stats["core_hpack_recv_binary_base64"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_binary_base64") + stats["core_hpack_send_indexed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_indexed") + stats["core_hpack_send_lithdr_incidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_incidx") + stats["core_hpack_send_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_incidx_v") + stats["core_hpack_send_lithdr_notidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_notidx") + stats["core_hpack_send_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_notidx_v") + stats["core_hpack_send_lithdr_nvridx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_nvridx") + stats["core_hpack_send_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_nvridx_v") + stats["core_hpack_send_uncompressed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_uncompressed") + stats["core_hpack_send_huffman"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_huffman") + stats["core_hpack_send_binary"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_binary") + stats["core_hpack_send_binary_base64"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_binary_base64") stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index e8fae6989e8..c7b1904bd19 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -380,6 +380,116 @@ "name": "core_http2_initiate_write_due_to_force_rst_stream", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_indexed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_incidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_incidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_notidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_notidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_nvridx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_nvridx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_uncompressed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_huffman", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_binary", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_binary_base64", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_indexed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_incidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_incidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_notidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_notidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_nvridx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_nvridx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_uncompressed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_huffman", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_binary", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_binary_base64", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_combiner_locks_initiated", @@ -1097,6 +1207,116 @@ "name": "core_http2_initiate_write_due_to_force_rst_stream", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_indexed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_incidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_incidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_notidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_notidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_nvridx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_nvridx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_uncompressed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_huffman", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_binary", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_binary_base64", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_indexed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_incidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_incidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_notidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_notidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_nvridx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_nvridx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_uncompressed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_huffman", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_binary", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_binary_base64", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_combiner_locks_initiated", From fb21821a08165628dd1e769cc59e5d38ba78e67d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 21 Sep 2017 11:43:09 -0700 Subject: [PATCH 212/227] Correct uncompressed string count --- src/core/ext/transport/chttp2/transport/hpack_encoder.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 4e975951736..a404b664e33 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -365,6 +365,7 @@ static void emit_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, grpc_mdelem elem, framer_state *st) { GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX_V(exec_ctx); + GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED(exec_ctx); uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); wire_value value = get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); @@ -386,6 +387,7 @@ static void emit_lithdr_noidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, grpc_mdelem elem, framer_state *st) { GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX_V(exec_ctx); + GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED(exec_ctx); uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); wire_value value = get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); From 3b6593679c28afd4455c7d38c417583ddc4341ef Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 21 Sep 2017 12:04:37 -0700 Subject: [PATCH 213/227] Fix format --- .../resolver/dns/c_ares/grpc_ares_ev_driver_posix.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c index b5be69c4287..c30cc93b6ff 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c @@ -118,9 +118,8 @@ static void fd_node_shutdown(grpc_exec_ctx *exec_ctx, fd_node *fdn) { gpr_mu_unlock(&fdn->mu); fd_node_destroy(exec_ctx, fdn); } else { - grpc_fd_shutdown( - exec_ctx, fdn->fd, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("c-ares fd shutdown")); + grpc_fd_shutdown(exec_ctx, fdn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "c-ares fd shutdown")); gpr_mu_unlock(&fdn->mu); } } From d987d95423e7a410190157248f42537657f0b359 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 21 Sep 2017 12:21:17 -0700 Subject: [PATCH 214/227] Don't collect everything after running a build --- tools/internal_ci/linux/grpc_performance_profile_daily.cfg | 1 - tools/internal_ci/linux/grpc_performance_profile_master.cfg | 1 - 2 files changed, 2 deletions(-) diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.cfg b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg index 9831869edb2..7320a19a766 100644 --- a/tools/internal_ci/linux/grpc_performance_profile_daily.cfg +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg @@ -19,7 +19,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_daily.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.cfg b/tools/internal_ci/linux/grpc_performance_profile_master.cfg index e4cefbc3f4e..ccb8aabb389 100644 --- a/tools/internal_ci/linux/grpc_performance_profile_master.cfg +++ b/tools/internal_ci/linux/grpc_performance_profile_master.cfg @@ -19,7 +19,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_master.sh" timeout_mins: 600 action { define_artifacts { - regex: "**" regex: "github/grpc/reports/**" } } From 3121da1f5fe3c1f93fba1cd4a83c7e537852a90d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 21 Sep 2017 12:24:33 -0700 Subject: [PATCH 215/227] Revert "Add automated PHP-to-C++ performance testing" --- tools/gce/linux_performance_worker_init.sh | 5 --- tools/run_tests/performance/run_worker_php.sh | 43 ------------------- .../run_tests/performance/scenario_config.py | 28 ------------ 3 files changed, 76 deletions(-) delete mode 100755 tools/run_tests/performance/run_worker_php.sh diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 88d8de74028..8f0a0f65c34 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -128,11 +128,6 @@ ruby -v # Install bundler (prerequisite for gRPC Ruby) gem install bundler -# PHP dependencies -sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev -curl -sS https://getcomposer.org/installer | php -sudo mv composer.phar /usr/local/bin/composer - # Java dependencies - nothing as we already have Java JDK 8 # Go dependencies diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh deleted file mode 100755 index 1da74cd0dbd..00000000000 --- a/tools/run_tests/performance/run_worker_php.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Copyright 2017, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -source ~/.rvm/scripts/rvm -set -ex - -repo=$(dirname $0)/../../.. - -# First set up all dependences needed for PHP QPS test -cd $repo -cd src/php/tests/qps -composer install -# The proxy worker for PHP is implemented in Ruby -cd ../../../.. -ruby src/ruby/qps/proxy-worker.rb $@ - diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 5efc9f56485..a7560400dd3 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -800,33 +800,6 @@ class RubyLanguage: return 'ruby' -class PhpLanguage: - - def __init__(self): - pass - self.safename = str(self) - - def worker_cmdline(self): - return ['tools/run_tests/performance/run_worker_php.sh'] - - def worker_port_offset(self): - return 800 - - def scenarios(self): - yield _ping_pong_scenario( - 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) - - yield _ping_pong_scenario( - 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) - - def __str__(self): - return 'php' - - class JavaLanguage: def __init__(self): @@ -1024,7 +997,6 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), - 'php' : PhpLanguage(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), From 4917b08a17cd74fbe3c4c4032b9455f00721969e Mon Sep 17 00:00:00 2001 From: Zhouyihai Ding Date: Thu, 21 Sep 2017 12:52:50 -0700 Subject: [PATCH 216/227] Revert "Revert "Add automated PHP-to-C++ performance testing"" --- tools/gce/linux_performance_worker_init.sh | 5 +++ tools/run_tests/performance/run_worker_php.sh | 43 +++++++++++++++++++ .../run_tests/performance/scenario_config.py | 28 ++++++++++++ 3 files changed, 76 insertions(+) create mode 100755 tools/run_tests/performance/run_worker_php.sh diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 8f0a0f65c34..88d8de74028 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -128,6 +128,11 @@ ruby -v # Install bundler (prerequisite for gRPC Ruby) gem install bundler +# PHP dependencies +sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev +curl -sS https://getcomposer.org/installer | php +sudo mv composer.phar /usr/local/bin/composer + # Java dependencies - nothing as we already have Java JDK 8 # Go dependencies diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh new file mode 100755 index 00000000000..1da74cd0dbd --- /dev/null +++ b/tools/run_tests/performance/run_worker_php.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +source ~/.rvm/scripts/rvm +set -ex + +repo=$(dirname $0)/../../.. + +# First set up all dependences needed for PHP QPS test +cd $repo +cd src/php/tests/qps +composer install +# The proxy worker for PHP is implemented in Ruby +cd ../../../.. +ruby src/ruby/qps/proxy-worker.rb $@ + diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index a7560400dd3..5efc9f56485 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -800,6 +800,33 @@ class RubyLanguage: return 'ruby' +class PhpLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_php.sh'] + + def worker_port_offset(self): + return 800 + + def scenarios(self): + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + def __str__(self): + return 'php' + + class JavaLanguage: def __init__(self): @@ -997,6 +1024,7 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), + 'php' : PhpLanguage(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), From efce6e1e5017e8a6c6a81f348a4adcc4825b757d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 14 Sep 2017 09:07:50 -0700 Subject: [PATCH 217/227] Revert "Revert "Allow SerializationTraits to use grpc::ByteBuffer rather than only grpc_byte_buffer"" --- BUILD | 2 + CMakeLists.txt | 15 ++ Makefile | 15 ++ build.yaml | 2 + gRPC-Core.podspec | 1 + grpc.def | 20 +-- grpc.gemspec | 1 + include/grpc++/impl/codegen/byte_buffer.h | 141 ++++++++++++++++++ include/grpc++/impl/codegen/call.h | 114 +++++++++++--- .../grpc++/impl/codegen/method_handler_impl.h | 9 +- .../grpc++/impl/codegen/rpc_service_method.h | 14 +- .../impl/codegen/serialization_traits.h | 25 +++- include/grpc++/impl/codegen/slice.h | 78 ++++++++++ include/grpc++/support/byte_buffer.h | 68 +-------- include/grpc++/support/slice.h | 80 +--------- include/grpc/byte_buffer.h | 64 +------- include/grpc/impl/codegen/byte_buffer.h | 86 +++++++++++ package.xml | 1 + src/core/lib/iomgr/resource_quota.c | 1 + .../health/default_health_check_service.cc | 1 + src/cpp/util/byte_buffer_cc.cc | 37 +++-- src/cpp/util/slice_cc.cc | 2 + src/ruby/ext/grpc/rb_grpc_imports.generated.c | 40 ++--- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 62 ++++---- .../core/surface/public_headers_must_be_c89.c | 1 + test/cpp/util/byte_buffer_test.cc | 5 +- tools/doxygen/Doxyfile.c++ | 2 + tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + .../generated/sources_and_headers.json | 4 + 31 files changed, 565 insertions(+), 330 deletions(-) create mode 100644 include/grpc++/impl/codegen/byte_buffer.h create mode 100644 include/grpc/impl/codegen/byte_buffer.h diff --git a/BUILD b/BUILD index 7b55e663cb9..2135f59341e 100644 --- a/BUILD +++ b/BUILD @@ -989,6 +989,7 @@ grpc_cc_library( name = "grpc_codegen", language = "c", public_hdrs = [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -1487,6 +1488,7 @@ grpc_cc_library( public_hdrs = [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index a5a7fad346c..0a5f4024f9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1250,6 +1250,7 @@ target_link_libraries(grpc ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1556,6 +1557,7 @@ target_link_libraries(grpc_cronet ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1832,6 +1834,7 @@ target_link_libraries(grpc_test_util ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2092,6 +2095,7 @@ target_link_libraries(grpc_test_util_unsecure ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2387,6 +2391,7 @@ target_link_libraries(grpc_unsecure ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2696,6 +2701,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2706,6 +2712,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3188,6 +3195,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3198,6 +3206,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3558,6 +3567,7 @@ target_link_libraries(grpc++_test_util foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3585,6 +3595,7 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3696,6 +3707,7 @@ target_link_libraries(grpc++_test_util_unsecure foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3723,6 +3735,7 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3925,6 +3938,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3935,6 +3949,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h diff --git a/Makefile b/Makefile index 61d6cae6ab5..648689f93e7 100644 --- a/Makefile +++ b/Makefile @@ -3206,6 +3206,7 @@ LIBGRPC_SRC = \ src/core/plugin_registry/grpc_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3512,6 +3513,7 @@ LIBGRPC_CRONET_SRC = \ src/core/plugin_registry/grpc_cronet_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3789,6 +3791,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4040,6 +4043,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4312,6 +4316,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/plugin_registry/grpc_unsecure_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4600,6 +4605,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4610,6 +4616,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5093,6 +5100,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5103,6 +5111,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5456,6 +5465,7 @@ LIBGRPC++_TEST_UTIL_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5483,6 +5493,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5571,6 +5582,7 @@ LIBGRPC++_TEST_UTIL_UNSECURE_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5598,6 +5610,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5805,6 +5818,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5815,6 +5829,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ diff --git a/build.yaml b/build.yaml index d15cf0339ca..92ca2b6237b 100644 --- a/build.yaml +++ b/build.yaml @@ -502,6 +502,7 @@ filegroups: - grpc_deadline_filter - name: grpc_codegen public_headers: + - include/grpc/impl/codegen/byte_buffer.h - include/grpc/impl/codegen/byte_buffer_reader.h - include/grpc/impl/codegen/compression_types.h - include/grpc/impl/codegen/connectivity_state.h @@ -969,6 +970,7 @@ filegroups: public_headers: - include/grpc++/impl/codegen/async_stream.h - include/grpc++/impl/codegen/async_unary_call.h + - include/grpc++/impl/codegen/byte_buffer.h - include/grpc++/impl/codegen/call.h - include/grpc++/impl/codegen/call_hook.h - include/grpc++/impl/codegen/channel_interface.h diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 2f1f4158662..6f84ef235db 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -141,6 +141,7 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_generic.h', 'include/grpc/impl/codegen/sync_posix.h', 'include/grpc/impl/codegen/sync_windows.h', + 'include/grpc/impl/codegen/byte_buffer.h', 'include/grpc/impl/codegen/byte_buffer_reader.h', 'include/grpc/impl/codegen/compression_types.h', 'include/grpc/impl/codegen/connectivity_state.h', diff --git a/grpc.def b/grpc.def index a7a11601ed9..558be60c3c1 100644 --- a/grpc.def +++ b/grpc.def @@ -1,14 +1,4 @@ EXPORTS - grpc_raw_byte_buffer_create - grpc_raw_compressed_byte_buffer_create - grpc_byte_buffer_copy - grpc_byte_buffer_length - grpc_byte_buffer_destroy - grpc_byte_buffer_reader_init - grpc_byte_buffer_reader_destroy - grpc_byte_buffer_reader_next - grpc_byte_buffer_reader_readall - grpc_raw_byte_buffer_from_reader census_initialize census_shutdown census_supported @@ -145,6 +135,16 @@ EXPORTS grpc_server_add_secure_http2_port grpc_call_set_credentials grpc_server_credentials_set_auth_metadata_processor + grpc_raw_byte_buffer_create + grpc_raw_compressed_byte_buffer_create + grpc_byte_buffer_copy + grpc_byte_buffer_length + grpc_byte_buffer_destroy + grpc_byte_buffer_reader_init + grpc_byte_buffer_reader_destroy + grpc_byte_buffer_reader_next + grpc_byte_buffer_reader_readall + grpc_raw_byte_buffer_from_reader grpc_slice_ref grpc_slice_unref grpc_slice_copy diff --git a/grpc.gemspec b/grpc.gemspec index 96323aed106..c00a245693c 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -146,6 +146,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/support/tmpfile_posix.c ) s.files += %w( src/core/lib/support/tmpfile_windows.c ) s.files += %w( src/core/lib/support/wrap_memcpy.c ) + s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h ) s.files += %w( include/grpc/impl/codegen/compression_types.h ) s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h new file mode 100644 index 00000000000..87d390c688a --- /dev/null +++ b/include/grpc++/impl/codegen/byte_buffer.h @@ -0,0 +1,141 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H +#define GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H + +#include + +#include +#include +#include +#include +#include + +#include + +namespace grpc { + +template +class CallOpRecvMessage; +class MethodHandler; +namespace internal { +template +class MessageDeserializer; +} + +/// A sequence of bytes. +class ByteBuffer final { + public: + /// Constuct an empty buffer. + ByteBuffer() : buffer_(nullptr) {} + + /// Construct buffer from \a slices, of which there are \a nslices. + ByteBuffer(const Slice* slices, size_t nslices); + + /// Constuct a byte buffer by referencing elements of existing buffer + /// \a buf. Wrapper of core function grpc_byte_buffer_copy + ByteBuffer(const ByteBuffer& buf); + + ~ByteBuffer() { + if (buffer_) { + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); + } + } + + ByteBuffer& operator=(const ByteBuffer&); + + /// Dump (read) the buffer contents into \a slices. + Status Dump(std::vector* slices) const; + + /// Remove all data. + void Clear() { + if (buffer_) { + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); + buffer_ = nullptr; + } + } + + /// Make a duplicate copy of the internals of this byte + /// buffer so that we have our own owned version of it. + /// bbuf.Duplicate(); is equivalent to bbuf=bbuf; but is actually readable + void Duplicate() { + buffer_ = g_core_codegen_interface->grpc_byte_buffer_copy(buffer_); + } + + /// Forget underlying byte buffer without destroying + /// Use this only for un-owned byte buffers + void Release() { buffer_ = nullptr; } + + /// Buffer size in bytes. + size_t Length() const; + + /// Swap the state of *this and *other. + void Swap(ByteBuffer* other); + + /// Is this ByteBuffer valid? + bool Valid() const { return (buffer_ != nullptr); } + + private: + friend class SerializationTraits; + friend class CallOpSendMessage; + template + friend class CallOpRecvMessage; + friend class CallOpGenericRecvMessage; + friend class MethodHandler; + template + friend class internal::MessageDeserializer; + + // takes ownership + void set_buffer(grpc_byte_buffer* buf) { + if (buffer_) { + Clear(); + } + buffer_ = buf; + } + + grpc_byte_buffer* c_buffer() { return buffer_; } + grpc_byte_buffer** c_buffer_ptr() { return &buffer_; } + + // DEPRECATED: Implicit conversion to transparently + // support deprecated SerializationTraits API + // No need to inline since deprecated + operator grpc_byte_buffer*(); + operator const grpc_byte_buffer*() const; + + grpc_byte_buffer* buffer_; +}; + +template <> +class SerializationTraits { + public: + static Status Deserialize(const ByteBuffer& byte_buffer, ByteBuffer* dest) { + dest->set_buffer(byte_buffer.buffer_); + return Status::OK; + } + static Status Serialize(const ByteBuffer& source, ByteBuffer* buffer, + bool* own_buffer) { + *buffer = source; + *own_buffer = true; + return Status::OK; + } +}; + +} // namespace grpc + +#endif // GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 74ed5cbfb9e..3c418c7ae2b 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -39,8 +40,6 @@ #include #include -struct grpc_byte_buffer; - namespace grpc { class ByteBuffer; @@ -281,7 +280,7 @@ class CallOpSendInitialMetadata { class CallOpSendMessage { public: - CallOpSendMessage() : send_buf_(nullptr) {} + CallOpSendMessage() : send_buf_() {} /// Send \a message using \a options for the write. The \a options are cleared /// after use. @@ -294,33 +293,67 @@ class CallOpSendMessage { protected: void AddOp(grpc_op* ops, size_t* nops) { - if (send_buf_ == nullptr) return; + if (!send_buf_.Valid()) return; grpc_op* op = &ops[(*nops)++]; op->op = GRPC_OP_SEND_MESSAGE; op->flags = write_options_.flags(); op->reserved = NULL; - op->data.send_message.send_message = send_buf_; + op->data.send_message.send_message = send_buf_.c_buffer(); // Flags are per-message: clear them after use. write_options_.Clear(); } - void FinishOp(bool* status) { - g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_); - send_buf_ = nullptr; - } + void FinishOp(bool* status) { send_buf_.Clear(); } private: - grpc_byte_buffer* send_buf_; + template + class MessageSerializer; + + ByteBuffer send_buf_; WriteOptions write_options_; }; +namespace internal { +template +T Example(); +} // namespace internal + +template +class CallOpSendMessage::MessageSerializer< + M, typename std::enable_if::Serialize( + internal::Example(), + internal::Example(), + internal::Example()))>::value>::type> { + public: + static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, + bool* own_buf) { + return SerializationTraits::Serialize(message, bbuf->c_buffer_ptr(), + own_buf); + } +}; + +template +class CallOpSendMessage::MessageSerializer< + M, typename std::enable_if::Serialize( + internal::Example(), + internal::Example<::grpc::ByteBuffer*>(), + internal::Example()))>::value>::type> { + public: + static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, + bool* own_buf) { + return SerializationTraits::Serialize(message, bbuf, own_buf); + } +}; + template Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) { write_options_ = options; bool own_buf; Status result = - SerializationTraits::Serialize(message, &send_buf_, &own_buf); + MessageSerializer::SendMessageInternal(message, &send_buf_, &own_buf); if (!own_buf) { - send_buf_ = g_core_codegen_interface->grpc_byte_buffer_copy(send_buf_); + send_buf_.Duplicate(); } return result; } @@ -330,6 +363,36 @@ Status CallOpSendMessage::SendMessage(const M& message) { return SendMessage(message, WriteOptions()); } +namespace internal { +template +class MessageDeserializer; + +template +class MessageDeserializer< + M, typename std::enable_if::Deserialize( + internal::Example(), + internal::Example()))>::value>::type> { + public: + static Status Deserialize(const ByteBuffer& bbuf, M* message) { + return SerializationTraits::Deserialize(bbuf, message); + } +}; + +template +class MessageDeserializer< + M, typename std::enable_if::Deserialize( + internal::Example(), + internal::Example()))>::value>::type> { + public: + static Status Deserialize(const ByteBuffer& bbuf, M* message) { + return SerializationTraits::Deserialize( + const_cast(bbuf).c_buffer(), message); + } +}; +} // namespace internal + template class CallOpRecvMessage { public: @@ -352,18 +415,20 @@ class CallOpRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = &recv_buf_; + op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); } void FinishOp(bool* status) { if (message_ == nullptr) return; - if (recv_buf_) { + if (recv_buf_.Valid()) { if (*status) { got_message = *status = - SerializationTraits::Deserialize(recv_buf_, message_).ok(); + internal::MessageDeserializer::Deserialize(recv_buf_, message_) + .ok(); + recv_buf_.Release(); } else { got_message = false; - g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); + recv_buf_.Clear(); } } else { got_message = false; @@ -376,14 +441,14 @@ class CallOpRecvMessage { private: R* message_; - grpc_byte_buffer* recv_buf_; + ByteBuffer recv_buf_; bool allow_not_getting_message_; }; namespace CallOpGenericRecvMessageHelper { class DeserializeFunc { public: - virtual Status Deserialize(grpc_byte_buffer* buf) = 0; + virtual Status Deserialize(const ByteBuffer& buf) = 0; virtual ~DeserializeFunc() {} }; @@ -391,8 +456,8 @@ template class DeserializeFuncType final : public DeserializeFunc { public: DeserializeFuncType(R* message) : message_(message) {} - Status Deserialize(grpc_byte_buffer* buf) override { - return SerializationTraits::Deserialize(buf, message_); + Status Deserialize(const ByteBuffer& buf) override { + return grpc::internal::MessageDeserializer::Deserialize(buf, message_); } ~DeserializeFuncType() override {} @@ -428,18 +493,19 @@ class CallOpGenericRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = &recv_buf_; + op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); } void FinishOp(bool* status) { if (!deserialize_) return; - if (recv_buf_) { + if (recv_buf_.Valid()) { if (*status) { got_message = true; *status = deserialize_->Deserialize(recv_buf_).ok(); + recv_buf_.Release(); } else { got_message = false; - g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); + recv_buf_.Clear(); } } else { got_message = false; @@ -452,7 +518,7 @@ class CallOpGenericRecvMessage { private: std::unique_ptr deserialize_; - grpc_byte_buffer* recv_buf_; + ByteBuffer recv_buf_; bool allow_not_getting_message_; }; diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h index 15e24bdcdcb..8125e0a6513 100644 --- a/include/grpc++/impl/codegen/method_handler_impl.h +++ b/include/grpc++/impl/codegen/method_handler_impl.h @@ -19,6 +19,7 @@ #ifndef GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H #define GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H +#include #include #include #include @@ -37,8 +38,8 @@ class RpcMethodHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = - SerializationTraits::Deserialize(param.request, &req); + Status status = internal::MessageDeserializer::Deserialize( + param.request, &req); ResponseType rsp; if (status.ok()) { status = func_(service_, param.server_context, &req, &rsp); @@ -123,8 +124,8 @@ class ServerStreamingHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = - SerializationTraits::Deserialize(param.request, &req); + Status status = internal::MessageDeserializer::Deserialize( + param.request, &req); if (status.ok()) { ServerWriter writer(param.call, param.server_context); diff --git a/include/grpc++/impl/codegen/rpc_service_method.h b/include/grpc++/impl/codegen/rpc_service_method.h index 7165774172c..d356012ad60 100644 --- a/include/grpc++/impl/codegen/rpc_service_method.h +++ b/include/grpc++/impl/codegen/rpc_service_method.h @@ -25,14 +25,11 @@ #include #include +#include #include #include #include -extern "C" { -struct grpc_byte_buffer; -} - namespace grpc { class ServerContext; class StreamContextInterface; @@ -43,11 +40,14 @@ class MethodHandler { virtual ~MethodHandler() {} struct HandlerParameter { HandlerParameter(Call* c, ServerContext* context, grpc_byte_buffer* req) - : call(c), server_context(context), request(req) {} + : call(c), server_context(context) { + request.set_buffer(req); + } + ~HandlerParameter() { request.Release(); } Call* call; ServerContext* server_context; - // Handler required to grpc_byte_buffer_destroy this - grpc_byte_buffer* request; + // Handler required to destroy these contents + ByteBuffer request; }; virtual void RunHandler(const HandlerParameter& param) = 0; }; diff --git a/include/grpc++/impl/codegen/serialization_traits.h b/include/grpc++/impl/codegen/serialization_traits.h index b72d4741267..7fc4e24172f 100644 --- a/include/grpc++/impl/codegen/serialization_traits.h +++ b/include/grpc++/impl/codegen/serialization_traits.h @@ -26,15 +26,24 @@ namespace grpc { /// Used for hooking different message serialization API's into GRPC. /// Each SerializationTraits implementation must provide the following /// functions: -/// static Status Serialize(const Message& msg, -/// grpc_byte_buffer** buffer, -/// bool* own_buffer); -/// static Status Deserialize(grpc_byte_buffer* buffer, -/// Message* msg, -/// int max_receive_message_size); +/// 1. static Status Serialize(const Message& msg, +/// ByteBuffer* buffer, +/// bool* own_buffer); +/// AND/OR +/// static Status Serialize(const Message& msg, +/// grpc_byte_buffer** buffer, +/// bool* own_buffer); +/// The former is preferred; the latter is deprecated /// -/// Serialize is required to convert message to a grpc_byte_buffer, and -/// to store a pointer to that byte buffer at *buffer. *own_buffer should +/// 2. static Status Deserialize(const ByteBuffer& buffer, +/// Message* msg); +/// AND/OR +/// static Status Deserialize(grpc_byte_buffer* buffer, +/// Message* msg); +/// The former is preferred; the latter is deprecated +/// +/// Serialize is required to convert message to a ByteBuffer, and +/// return that byte buffer through *buffer. *own_buffer should /// be set to true if the caller owns said byte buffer, or false if /// ownership is retained elsewhere. /// diff --git a/include/grpc++/impl/codegen/slice.h b/include/grpc++/impl/codegen/slice.h index e682bdef6af..c185bf4fd01 100644 --- a/include/grpc++/impl/codegen/slice.h +++ b/include/grpc++/impl/codegen/slice.h @@ -19,11 +19,89 @@ #ifndef GRPCXX_IMPL_CODEGEN_SLICE_H #define GRPCXX_IMPL_CODEGEN_SLICE_H +#include #include #include +#include + namespace grpc { +/// A wrapper around \a grpc_slice. +/// +/// A slice represents a contiguous reference counted array of bytes. +/// It is cheap to take references to a slice, and it is cheap to create a +/// slice pointing to a subset of another slice. +class Slice final { + public: + /// Construct an empty slice. + Slice(); + /// Destructor - drops one reference. + ~Slice(); + + enum AddRef { ADD_REF }; + /// Construct a slice from \a slice, adding a reference. + Slice(grpc_slice slice, AddRef); + + enum StealRef { STEAL_REF }; + /// Construct a slice from \a slice, stealing a reference. + Slice(grpc_slice slice, StealRef); + + /// Allocate a slice of specified size + Slice(size_t len); + + /// Construct a slice from a copied buffer + Slice(const void* buf, size_t len); + + /// Construct a slice from a copied string + Slice(const grpc::string& str); + + enum StaticSlice { STATIC_SLICE }; + + /// Construct a slice from a static buffer + Slice(const void* buf, size_t len, StaticSlice); + + /// Copy constructor, adds a reference. + Slice(const Slice& other); + + /// Assignment, reference count is unchanged. + Slice& operator=(Slice other) { + std::swap(slice_, other.slice_); + return *this; + } + + /// Create a slice pointing at some data. Calls malloc to allocate a refcount + /// for the object, and arranges that destroy will be called with the + /// user data pointer passed in at destruction. Can be the same as buf or + /// different (e.g., if data is part of a larger structure that must be + /// destroyed when the data is no longer needed) + Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); + + /// Specialization of above for common case where buf == user_data + Slice(void* buf, size_t len, void (*destroy)(void*)) + : Slice(buf, len, destroy, buf) {} + + /// Similar to the above but has a destroy that also takes slice length + Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); + + /// Byte size. + size_t size() const { return GRPC_SLICE_LENGTH(slice_); } + + /// Raw pointer to the beginning (first element) of the slice. + const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } + + /// Raw pointer to the end (one byte \em past the last element) of the slice. + const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } + + /// Raw C slice. Caller needs to call grpc_slice_unref when done. + grpc_slice c_slice() const; + + private: + friend class ByteBuffer; + + grpc_slice slice_; +}; + inline grpc::string_ref StringRefFromSlice(const grpc_slice* slice) { return grpc::string_ref( reinterpret_cast(GRPC_SLICE_START_PTR(*slice)), diff --git a/include/grpc++/support/byte_buffer.h b/include/grpc++/support/byte_buffer.h index df16c6a75e4..81fa3b0a184 100644 --- a/include/grpc++/support/byte_buffer.h +++ b/include/grpc++/support/byte_buffer.h @@ -19,6 +19,7 @@ #ifndef GRPCXX_SUPPORT_BYTE_BUFFER_H #define GRPCXX_SUPPORT_BYTE_BUFFER_H +#include #include #include #include @@ -27,71 +28,4 @@ #include #include -#include - -namespace grpc { - -/// A sequence of bytes. -class ByteBuffer final { - public: - /// Constuct an empty buffer. - ByteBuffer() : buffer_(nullptr) {} - - /// Construct buffer from \a slices, of which there are \a nslices. - ByteBuffer(const Slice* slices, size_t nslices); - - /// Constuct a byte buffer by referencing elements of existing buffer - /// \a buf. Wrapper of core function grpc_byte_buffer_copy - ByteBuffer(const ByteBuffer& buf); - - ~ByteBuffer(); - - ByteBuffer& operator=(const ByteBuffer&); - - /// Dump (read) the buffer contents into \a slices. - Status Dump(std::vector* slices) const; - - /// Remove all data. - void Clear(); - - /// Buffer size in bytes. - size_t Length() const; - - /// Swap the state of *this and *other. - void Swap(ByteBuffer* other); - - private: - friend class SerializationTraits; - - // takes ownership - void set_buffer(grpc_byte_buffer* buf) { - if (buffer_) { - Clear(); - } - buffer_ = buf; - } - - // For \a SerializationTraits's usage. - grpc_byte_buffer* buffer() const { return buffer_; } - - grpc_byte_buffer* buffer_; -}; - -template <> -class SerializationTraits { - public: - static Status Deserialize(grpc_byte_buffer* byte_buffer, ByteBuffer* dest) { - dest->set_buffer(byte_buffer); - return Status::OK; - } - static Status Serialize(const ByteBuffer& source, grpc_byte_buffer** buffer, - bool* own_buffer) { - *buffer = grpc_byte_buffer_copy(source.buffer()); - *own_buffer = true; - return Status::OK; - } -}; - -} // namespace grpc - #endif // GRPCXX_SUPPORT_BYTE_BUFFER_H diff --git a/include/grpc++/support/slice.h b/include/grpc++/support/slice.h index bbf97f280ef..10db10d79c3 100644 --- a/include/grpc++/support/slice.h +++ b/include/grpc++/support/slice.h @@ -19,86 +19,8 @@ #ifndef GRPCXX_SUPPORT_SLICE_H #define GRPCXX_SUPPORT_SLICE_H +#include #include #include -namespace grpc { - -/// A wrapper around \a grpc_slice. -/// -/// A slice represents a contiguous reference counted array of bytes. -/// It is cheap to take references to a slice, and it is cheap to create a -/// slice pointing to a subset of another slice. -class Slice final { - public: - /// Construct an empty slice. - Slice(); - /// Destructor - drops one reference. - ~Slice(); - - enum AddRef { ADD_REF }; - /// Construct a slice from \a slice, adding a reference. - Slice(grpc_slice slice, AddRef); - - enum StealRef { STEAL_REF }; - /// Construct a slice from \a slice, stealing a reference. - Slice(grpc_slice slice, StealRef); - - /// Allocate a slice of specified size - Slice(size_t len); - - /// Construct a slice from a copied buffer - Slice(const void* buf, size_t len); - - /// Construct a slice from a copied string - Slice(const grpc::string& str); - - enum StaticSlice { STATIC_SLICE }; - - /// Construct a slice from a static buffer - Slice(const void* buf, size_t len, StaticSlice); - - /// Copy constructor, adds a reference. - Slice(const Slice& other); - - /// Assignment, reference count is unchanged. - Slice& operator=(Slice other) { - std::swap(slice_, other.slice_); - return *this; - } - - /// Create a slice pointing at some data. Calls malloc to allocate a refcount - /// for the object, and arranges that destroy will be called with the - /// user data pointer passed in at destruction. Can be the same as buf or - /// different (e.g., if data is part of a larger structure that must be - /// destroyed when the data is no longer needed) - Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); - - /// Specialization of above for common case where buf == user_data - Slice(void* buf, size_t len, void (*destroy)(void*)) - : Slice(buf, len, destroy, buf) {} - - /// Similar to the above but has a destroy that also takes slice length - Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); - - /// Byte size. - size_t size() const { return GRPC_SLICE_LENGTH(slice_); } - - /// Raw pointer to the beginning (first element) of the slice. - const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } - - /// Raw pointer to the end (one byte \em past the last element) of the slice. - const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } - - /// Raw C slice. Caller needs to call grpc_slice_unref when done. - grpc_slice c_slice() const { return grpc_slice_ref(slice_); } - - private: - friend class ByteBuffer; - - grpc_slice slice_; -}; - -} // namespace grpc - #endif // GRPCXX_SUPPORT_SLICE_H diff --git a/include/grpc/byte_buffer.h b/include/grpc/byte_buffer.h index 55e191da315..7669582af27 100644 --- a/include/grpc/byte_buffer.h +++ b/include/grpc/byte_buffer.h @@ -19,69 +19,7 @@ #ifndef GRPC_BYTE_BUFFER_H #define GRPC_BYTE_BUFFER_H -#include +#include #include -#ifdef __cplusplus -extern "C" { -#endif - -/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, - size_t nslices); - -/** Returns a *compressed* RAW byte buffer instance over the given slices (up to - * \a nslices). The \a compression argument defines the compression algorithm - * used to generate the data in \a slices. - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( - grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); - -/** Copies input byte buffer \a bb. - * - * Increases the reference count of all the source slices. The user is - * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ -GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); - -/** Returns the size of the given byte buffer, in bytes. */ -GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); - -/** Destroys \a byte_buffer deallocating all its memory. */ -GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); - -/** Reader for byte buffers. Iterates over slices in the byte buffer */ -struct grpc_byte_buffer_reader; -typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; - -/** Initialize \a reader to read over \a buffer. - * Returns 1 upon success, 0 otherwise. */ -GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, - grpc_byte_buffer *buffer); - -/** Cleanup and destroy \a reader */ -GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); - -/** Updates \a slice with the next piece of data from from \a reader and returns - * 1. Returns 0 at the end of the stream. Caller is responsible for calling - * grpc_slice_unref on the result. */ -GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, - grpc_slice *slice); - -/** Merge all data from \a reader into single slice */ -GRPCAPI grpc_slice -grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); - -/** Returns a RAW byte buffer instance from the output of \a reader. */ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( - grpc_byte_buffer_reader *reader); - -#ifdef __cplusplus -} -#endif - #endif /* GRPC_BYTE_BUFFER_H */ diff --git a/include/grpc/impl/codegen/byte_buffer.h b/include/grpc/impl/codegen/byte_buffer.h new file mode 100644 index 00000000000..fc333057134 --- /dev/null +++ b/include/grpc/impl/codegen/byte_buffer.h @@ -0,0 +1,86 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_IMPL_CODEGEN_BYTE_BUFFER_H +#define GRPC_IMPL_CODEGEN_BYTE_BUFFER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, + size_t nslices); + +/** Returns a *compressed* RAW byte buffer instance over the given slices (up to + * \a nslices). The \a compression argument defines the compression algorithm + * used to generate the data in \a slices. + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( + grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); + +/** Copies input byte buffer \a bb. + * + * Increases the reference count of all the source slices. The user is + * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ +GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); + +/** Returns the size of the given byte buffer, in bytes. */ +GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); + +/** Destroys \a byte_buffer deallocating all its memory. */ +GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); + +/** Reader for byte buffers. Iterates over slices in the byte buffer */ +struct grpc_byte_buffer_reader; +typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; + +/** Initialize \a reader to read over \a buffer. + * Returns 1 upon success, 0 otherwise. */ +GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, + grpc_byte_buffer *buffer); + +/** Cleanup and destroy \a reader */ +GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); + +/** Updates \a slice with the next piece of data from from \a reader and returns + * 1. Returns 0 at the end of the stream. Caller is responsible for calling + * grpc_slice_unref on the result. */ +GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, + grpc_slice *slice); + +/** Merge all data from \a reader into single slice */ +GRPCAPI grpc_slice +grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); + +/** Returns a RAW byte buffer instance from the output of \a reader. */ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( + grpc_byte_buffer_reader *reader); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_IMPL_CODEGEN_BYTE_BUFFER_H */ diff --git a/package.xml b/package.xml index 0c201afdaea..7e514bd541d 100644 --- a/package.xml +++ b/package.xml @@ -158,6 +158,7 @@ + diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 4895e0d1c92..4d69986fbc2 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index 815b6070320..d2cba6d6627 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include diff --git a/src/cpp/util/byte_buffer_cc.cc b/src/cpp/util/byte_buffer_cc.cc index b1ff25252a1..1ebe22b6a49 100644 --- a/src/cpp/util/byte_buffer_cc.cc +++ b/src/cpp/util/byte_buffer_cc.cc @@ -16,11 +16,15 @@ * */ +#include #include +#include #include namespace grpc { +static internal::GrpcLibraryInitializer g_gli_initializer; + ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { // The following assertions check that the representation of a grpc::Slice is // identical to that of a grpc_slice: it has a grpc_slice field, and nothing @@ -29,6 +33,7 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { "Slice must have same representation as grpc_slice"); static_assert(sizeof(Slice) == sizeof(grpc_slice), "Slice must have same representation as grpc_slice"); + g_gli_initializer.summon(); // Make sure that initializer linked in // The const_cast is legal if grpc_raw_byte_buffer_create() does no more // than its advertised side effect of increasing the reference count of the // slices it processes, and such an increase does not affect the semantics @@ -37,19 +42,6 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { reinterpret_cast(const_cast(slices)), nslices); } -ByteBuffer::~ByteBuffer() { - if (buffer_) { - grpc_byte_buffer_destroy(buffer_); - } -} - -void ByteBuffer::Clear() { - if (buffer_) { - grpc_byte_buffer_destroy(buffer_); - buffer_ = nullptr; - } -} - Status ByteBuffer::Dump(std::vector* slices) const { slices->clear(); if (!buffer_) { @@ -80,7 +72,9 @@ ByteBuffer::ByteBuffer(const ByteBuffer& buf) : buffer_(grpc_byte_buffer_copy(buf.buffer_)) {} ByteBuffer& ByteBuffer::operator=(const ByteBuffer& buf) { - Clear(); // first remove existing data + if (this != &buf) { + Clear(); // first remove existing data + } if (buf.buffer_) { buffer_ = grpc_byte_buffer_copy(buf.buffer_); // then copy } @@ -93,4 +87,19 @@ void ByteBuffer::Swap(ByteBuffer* other) { buffer_ = tmp; } +ByteBuffer::operator grpc_byte_buffer*() { + // The following assertions check that the representation of a ByteBuffer is + // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, + // and nothing else. + static_assert(std::is_same::value, + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + return buffer_; +} + +ByteBuffer::operator const grpc_byte_buffer*() const { return buffer_; } + } // namespace grpc diff --git a/src/cpp/util/slice_cc.cc b/src/cpp/util/slice_cc.cc index 486d0cdf0ec..3ae17e80520 100644 --- a/src/cpp/util/slice_cc.cc +++ b/src/cpp/util/slice_cc.cc @@ -50,4 +50,6 @@ Slice::Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data) Slice::Slice(void* buf, size_t len, void (*destroy)(void*, size_t)) : slice_(grpc_slice_new_with_len(buf, len, destroy)) {} +grpc_slice Slice::c_slice() const { return grpc_slice_ref(slice_); } + } // namespace grpc diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 57b543967eb..70831494fad 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -22,16 +22,6 @@ #include "rb_grpc_imports.generated.h" -grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; census_initialize_type census_initialize_import; census_shutdown_type census_shutdown_import; census_supported_type census_supported_import; @@ -168,6 +158,16 @@ grpc_ssl_server_credentials_create_ex_type grpc_ssl_server_credentials_create_ex grpc_server_add_secure_http2_port_type grpc_server_add_secure_http2_port_import; grpc_call_set_credentials_type grpc_call_set_credentials_import; grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; +grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; grpc_slice_ref_type grpc_slice_ref_import; grpc_slice_unref_type grpc_slice_unref_import; grpc_slice_copy_type grpc_slice_copy_import; @@ -330,16 +330,6 @@ gpr_sleep_until_type gpr_sleep_until_import; gpr_timespec_to_micros_type gpr_timespec_to_micros_import; void grpc_rb_load_imports(HMODULE library) { - grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); - grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); - grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); - grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); - grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); - grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); - grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); - grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); - grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); - grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize"); census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown"); census_supported_import = (census_supported_type) GetProcAddress(library, "census_supported"); @@ -476,6 +466,16 @@ void grpc_rb_load_imports(HMODULE library) { grpc_server_add_secure_http2_port_import = (grpc_server_add_secure_http2_port_type) GetProcAddress(library, "grpc_server_add_secure_http2_port"); grpc_call_set_credentials_import = (grpc_call_set_credentials_type) GetProcAddress(library, "grpc_call_set_credentials"); grpc_server_credentials_set_auth_metadata_processor_import = (grpc_server_credentials_set_auth_metadata_processor_type) GetProcAddress(library, "grpc_server_credentials_set_auth_metadata_processor"); + grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); + grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); + grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); + grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); + grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); + grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); + grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); + grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); + grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); + grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); grpc_slice_ref_import = (grpc_slice_ref_type) GetProcAddress(library, "grpc_slice_ref"); grpc_slice_unref_import = (grpc_slice_unref_type) GetProcAddress(library, "grpc_slice_unref"); grpc_slice_copy_import = (grpc_slice_copy_type) GetProcAddress(library, "grpc_slice_copy"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 55a7eed1a47..868772cfc85 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -25,12 +25,12 @@ #include -#include #include #include #include #include #include +#include #include #include #include @@ -47,36 +47,6 @@ #include #include -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); -extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); -extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import -typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -#define grpc_byte_buffer_length grpc_byte_buffer_length_import -typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); -extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import -typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); -extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import -typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import -typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); -extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import -typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); -extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; -#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef int(*census_initialize_type)(int features); extern census_initialize_type census_initialize_import; #define census_initialize census_initialize_import @@ -485,6 +455,36 @@ extern grpc_call_set_credentials_type grpc_call_set_credentials_import; typedef void(*grpc_server_credentials_set_auth_metadata_processor_type)(grpc_server_credentials *creds, grpc_auth_metadata_processor processor); extern grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; #define grpc_server_credentials_set_auth_metadata_processor grpc_server_credentials_set_auth_metadata_processor_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); +extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); +extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import +typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +#define grpc_byte_buffer_length grpc_byte_buffer_length_import +typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); +extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import +typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); +extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import +typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import +typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); +extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import +typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); +extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; +#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef grpc_slice(*grpc_slice_ref_type)(grpc_slice s); extern grpc_slice_ref_type grpc_slice_ref_import; #define grpc_slice_ref grpc_slice_ref_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 0d7f68c0add..d36d116afb8 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/test/cpp/util/byte_buffer_test.cc b/test/cpp/util/byte_buffer_test.cc index cac01a73078..8fb51bc6635 100644 --- a/test/cpp/util/byte_buffer_test.cc +++ b/test/cpp/util/byte_buffer_test.cc @@ -93,7 +93,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { std::vector slices; slices.push_back(Slice(hello, Slice::STEAL_REF)); slices.push_back(Slice(world, Slice::STEAL_REF)); - grpc_byte_buffer* send_buffer = nullptr; + ByteBuffer send_buffer; bool owned = false; ByteBuffer buffer(&slices[0], 2); slices.clear(); @@ -101,8 +101,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { buffer, &send_buffer, &owned); EXPECT_TRUE(status.ok()); EXPECT_TRUE(owned); - EXPECT_TRUE(send_buffer != nullptr); - grpc_byte_buffer_destroy(send_buffer); + EXPECT_TRUE(send_buffer.Valid()); } } // namespace diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 62f113907db..d81b7b4d115 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ +include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -881,6 +882,7 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 49919415bd5..263c6c1afbc 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ +include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -882,6 +883,7 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 632735342b3..b8514fe3114 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index e352cb78f10..77cd28e9dfd 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 8d42c3e4328..7cf4d441a00 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8475,6 +8475,7 @@ "gpr_codegen" ], "headers": [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -8488,6 +8489,7 @@ "language": "c", "name": "grpc_codegen", "src": [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -9366,6 +9368,7 @@ "headers": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", @@ -9400,6 +9403,7 @@ "src": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", From a44cffa11ffd66b0f9ff6d2de0e90d782af88841 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 15 Sep 2017 12:32:20 -0700 Subject: [PATCH 218/227] Base the new SerializationTraits API on implicit type conversion rather than SFINAE --- include/grpc++/impl/codegen/byte_buffer.h | 45 +++++++---- include/grpc++/impl/codegen/call.h | 76 ++----------------- .../grpc++/impl/codegen/method_handler_impl.h | 8 +- .../impl/codegen/serialization_traits.h | 10 +-- src/cpp/util/byte_buffer_cc.cc | 24 +++--- 5 files changed, 55 insertions(+), 108 deletions(-) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h index 87d390c688a..57d731be182 100644 --- a/include/grpc++/impl/codegen/byte_buffer.h +++ b/include/grpc++/impl/codegen/byte_buffer.h @@ -34,10 +34,14 @@ namespace grpc { template class CallOpRecvMessage; class MethodHandler; -namespace internal { -template -class MessageDeserializer; -} +template +class RpcMethodHandler; +template +class ServerStreamingHandler; +namespace CallOpGenericRecvMessageHelper { +template +class DeserializeFuncType; +} // namespace CallOpGenericRecvMessageHelper /// A sequence of bytes. class ByteBuffer final { @@ -98,8 +102,14 @@ class ByteBuffer final { friend class CallOpRecvMessage; friend class CallOpGenericRecvMessage; friend class MethodHandler; - template - friend class internal::MessageDeserializer; + template + friend class RpcMethodHandler; + template + friend class ServerStreamingHandler; + template + friend class CallOpGenericRecvMessageHelper::DeserializeFuncType; + + grpc_byte_buffer* buffer_; // takes ownership void set_buffer(grpc_byte_buffer* buf) { @@ -112,20 +122,25 @@ class ByteBuffer final { grpc_byte_buffer* c_buffer() { return buffer_; } grpc_byte_buffer** c_buffer_ptr() { return &buffer_; } - // DEPRECATED: Implicit conversion to transparently - // support deprecated SerializationTraits API - // No need to inline since deprecated - operator grpc_byte_buffer*(); - operator const grpc_byte_buffer*() const; - - grpc_byte_buffer* buffer_; + class ByteBufferPointer { + public: + ByteBufferPointer(const ByteBuffer* b) + : bbuf_(const_cast(b)) {} + operator ByteBuffer*() { return bbuf_; } + operator grpc_byte_buffer*() { return bbuf_->buffer_; } + operator grpc_byte_buffer**() { return &bbuf_->buffer_; } + + private: + ByteBuffer* bbuf_; + }; + ByteBufferPointer bbuf_ptr() const { return ByteBufferPointer(this); } }; template <> class SerializationTraits { public: - static Status Deserialize(const ByteBuffer& byte_buffer, ByteBuffer* dest) { - dest->set_buffer(byte_buffer.buffer_); + static Status Deserialize(ByteBuffer* byte_buffer, ByteBuffer* dest) { + dest->set_buffer(byte_buffer->buffer_); return Status::OK; } static Status Serialize(const ByteBuffer& source, ByteBuffer* buffer, diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 3c418c7ae2b..d9988e51fcc 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -305,9 +305,6 @@ class CallOpSendMessage { void FinishOp(bool* status) { send_buf_.Clear(); } private: - template - class MessageSerializer; - ByteBuffer send_buf_; WriteOptions write_options_; }; @@ -317,41 +314,12 @@ template T Example(); } // namespace internal -template -class CallOpSendMessage::MessageSerializer< - M, typename std::enable_if::Serialize( - internal::Example(), - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, - bool* own_buf) { - return SerializationTraits::Serialize(message, bbuf->c_buffer_ptr(), - own_buf); - } -}; - -template -class CallOpSendMessage::MessageSerializer< - M, typename std::enable_if::Serialize( - internal::Example(), - internal::Example<::grpc::ByteBuffer*>(), - internal::Example()))>::value>::type> { - public: - static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, - bool* own_buf) { - return SerializationTraits::Serialize(message, bbuf, own_buf); - } -}; - template Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) { write_options_ = options; bool own_buf; - Status result = - MessageSerializer::SendMessageInternal(message, &send_buf_, &own_buf); + Status result = SerializationTraits::Serialize( + message, send_buf_.bbuf_ptr(), &own_buf); if (!own_buf) { send_buf_.Duplicate(); } @@ -363,36 +331,6 @@ Status CallOpSendMessage::SendMessage(const M& message) { return SendMessage(message, WriteOptions()); } -namespace internal { -template -class MessageDeserializer; - -template -class MessageDeserializer< - M, typename std::enable_if::Deserialize( - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status Deserialize(const ByteBuffer& bbuf, M* message) { - return SerializationTraits::Deserialize(bbuf, message); - } -}; - -template -class MessageDeserializer< - M, typename std::enable_if::Deserialize( - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status Deserialize(const ByteBuffer& bbuf, M* message) { - return SerializationTraits::Deserialize( - const_cast(bbuf).c_buffer(), message); - } -}; -} // namespace internal - template class CallOpRecvMessage { public: @@ -423,7 +361,7 @@ class CallOpRecvMessage { if (recv_buf_.Valid()) { if (*status) { got_message = *status = - internal::MessageDeserializer::Deserialize(recv_buf_, message_) + SerializationTraits::Deserialize(recv_buf_.bbuf_ptr(), message_) .ok(); recv_buf_.Release(); } else { @@ -448,7 +386,7 @@ class CallOpRecvMessage { namespace CallOpGenericRecvMessageHelper { class DeserializeFunc { public: - virtual Status Deserialize(const ByteBuffer& buf) = 0; + virtual Status Deserialize(ByteBuffer* buf) = 0; virtual ~DeserializeFunc() {} }; @@ -456,8 +394,8 @@ template class DeserializeFuncType final : public DeserializeFunc { public: DeserializeFuncType(R* message) : message_(message) {} - Status Deserialize(const ByteBuffer& buf) override { - return grpc::internal::MessageDeserializer::Deserialize(buf, message_); + Status Deserialize(ByteBuffer* buf) override { + return SerializationTraits::Deserialize(buf->bbuf_ptr(), message_); } ~DeserializeFuncType() override {} @@ -501,7 +439,7 @@ class CallOpGenericRecvMessage { if (recv_buf_.Valid()) { if (*status) { got_message = true; - *status = deserialize_->Deserialize(recv_buf_).ok(); + *status = deserialize_->Deserialize(&recv_buf_).ok(); recv_buf_.Release(); } else { got_message = false; diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h index 8125e0a6513..e14cb0e9265 100644 --- a/include/grpc++/impl/codegen/method_handler_impl.h +++ b/include/grpc++/impl/codegen/method_handler_impl.h @@ -38,8 +38,8 @@ class RpcMethodHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = internal::MessageDeserializer::Deserialize( - param.request, &req); + Status status = SerializationTraits::Deserialize( + param.request.bbuf_ptr(), &req); ResponseType rsp; if (status.ok()) { status = func_(service_, param.server_context, &req, &rsp); @@ -124,8 +124,8 @@ class ServerStreamingHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = internal::MessageDeserializer::Deserialize( - param.request, &req); + Status status = SerializationTraits::Deserialize( + param.request.bbuf_ptr(), &req); if (status.ok()) { ServerWriter writer(param.call, param.server_context); diff --git a/include/grpc++/impl/codegen/serialization_traits.h b/include/grpc++/impl/codegen/serialization_traits.h index 7fc4e24172f..4d91739cc44 100644 --- a/include/grpc++/impl/codegen/serialization_traits.h +++ b/include/grpc++/impl/codegen/serialization_traits.h @@ -24,20 +24,20 @@ namespace grpc { /// Defines how to serialize and deserialize some type. /// /// Used for hooking different message serialization API's into GRPC. -/// Each SerializationTraits implementation must provide the following -/// functions: +/// Each SerializationTraits implementation must provide the +/// following functions: /// 1. static Status Serialize(const Message& msg, /// ByteBuffer* buffer, /// bool* own_buffer); -/// AND/OR +/// OR /// static Status Serialize(const Message& msg, /// grpc_byte_buffer** buffer, /// bool* own_buffer); /// The former is preferred; the latter is deprecated /// -/// 2. static Status Deserialize(const ByteBuffer& buffer, +/// 2. static Status Deserialize(ByteBuffer* buffer, /// Message* msg); -/// AND/OR +/// OR /// static Status Deserialize(grpc_byte_buffer* buffer, /// Message* msg); /// The former is preferred; the latter is deprecated diff --git a/src/cpp/util/byte_buffer_cc.cc b/src/cpp/util/byte_buffer_cc.cc index 1ebe22b6a49..180c8137628 100644 --- a/src/cpp/util/byte_buffer_cc.cc +++ b/src/cpp/util/byte_buffer_cc.cc @@ -33,6 +33,15 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { "Slice must have same representation as grpc_slice"); static_assert(sizeof(Slice) == sizeof(grpc_slice), "Slice must have same representation as grpc_slice"); + // The following assertions check that the representation of a ByteBuffer is + // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, + // and nothing else. + static_assert(std::is_same::value, + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); g_gli_initializer.summon(); // Make sure that initializer linked in // The const_cast is legal if grpc_raw_byte_buffer_create() does no more // than its advertised side effect of increasing the reference count of the @@ -87,19 +96,4 @@ void ByteBuffer::Swap(ByteBuffer* other) { buffer_ = tmp; } -ByteBuffer::operator grpc_byte_buffer*() { - // The following assertions check that the representation of a ByteBuffer is - // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, - // and nothing else. - static_assert(std::is_same::value, - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - return buffer_; -} - -ByteBuffer::operator const grpc_byte_buffer*() const { return buffer_; } - } // namespace grpc From 7115862e98059b3fbd2069ae7ad1bf09232a416e Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Thu, 21 Sep 2017 13:23:35 -0700 Subject: [PATCH 219/227] update license --- tools/run_tests/performance/run_worker_php.sh | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index 1da74cd0dbd..5d0c4fa4fd3 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -1,32 +1,17 @@ #!/bin/bash -# Copyright 2017, Google Inc. -# All rights reserved. +# Copyright 2017 gRPC authors. # -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. +# http://www.apache.org/licenses/LICENSE-2.0 # -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. source ~/.rvm/scripts/rvm set -ex From 2f3e58809911fed67ba4ea79633d820ab6b719ef Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 21 Sep 2017 13:47:20 -0700 Subject: [PATCH 220/227] Dont assume that sigint wasnt masked when invoking ruby tests --- src/ruby/end2end/killed_client_thread_client.rb | 2 +- src/ruby/end2end/killed_client_thread_driver.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ruby/end2end/killed_client_thread_client.rb b/src/ruby/end2end/killed_client_thread_client.rb index 7d6ed8c8d7a..493c0eb56a4 100755 --- a/src/ruby/end2end/killed_client_thread_client.rb +++ b/src/ruby/end2end/killed_client_thread_client.rb @@ -35,7 +35,7 @@ def main :this_channel_is_insecure) stub.echo(Echo::EchoRequest.new(request: 'hello')) fail 'the clients rpc in this test shouldnt complete. ' \ - 'expecting SIGINT to happen in the middle of the call' + 'expecting SIGTERM to happen in the middle of the call' end thd.join end diff --git a/src/ruby/end2end/killed_client_thread_driver.rb b/src/ruby/end2end/killed_client_thread_driver.rb index 09f05a44873..fce5d13e825 100755 --- a/src/ruby/end2end/killed_client_thread_driver.rb +++ b/src/ruby/end2end/killed_client_thread_driver.rb @@ -69,9 +69,9 @@ def main call_started_cv.wait(call_started_mu) until call_started.val end - # SIGINT the child process now that it's + # SIGTERM the child process now that it's # in the middle of an RPC (happening on a non-main thread) - Process.kill('SIGINT', client_pid) + Process.kill('SIGTERM', client_pid) STDERR.puts 'sent shutdown' begin @@ -88,8 +88,8 @@ def main end client_exit_code = $CHILD_STATUS - if client_exit_code.termsig != 2 # SIGINT - fail 'expected client exit from SIGINT ' \ + if client_exit_code.termsig != 15 # SIGTERM + fail 'expected client exit from SIGTERM ' \ "but got child status: #{client_exit_code}" end From 9fa16599510ca4772ef0b2e3631f7eaa3b429695 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 21 Sep 2017 15:43:32 -0700 Subject: [PATCH 221/227] Fix a memory leak point --- test/core/end2end/fixtures/proxy.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/core/end2end/fixtures/proxy.c b/test/core/end2end/fixtures/proxy.c index 9ad862728fe..6a2d75da09f 100644 --- a/test/core/end2end/fixtures/proxy.c +++ b/test/core/end2end/fixtures/proxy.c @@ -227,6 +227,10 @@ static void on_c2p_recv_msg(void *arg, int success) { new_closure(on_p2s_sent_close, pc), NULL); GPR_ASSERT(err == GRPC_CALL_OK); } + } else { + if (pc->c2p_msg != NULL) { + grpc_byte_buffer_destroy(pc->c2p_msg); + } } unrefpc(pc, "on_c2p_recv_msg"); From 2046d0b3c50e555f339062b4fba298e8af8b3a1f Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 19 Sep 2017 15:30:11 -0700 Subject: [PATCH 222/227] Add unary call mode to GenericStub to allow generic RPC with 1 CQ trip --- include/grpc++/generic/generic_stub.h | 10 +++++ src/cpp/client/generic_stub.cc | 10 +++++ test/cpp/end2end/generic_end2end_test.cc | 56 ++++++++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/include/grpc++/generic/generic_stub.h b/include/grpc++/generic/generic_stub.h index 2b3ff59ea24..d5064318cfc 100644 --- a/include/grpc++/generic/generic_stub.h +++ b/include/grpc++/generic/generic_stub.h @@ -20,6 +20,7 @@ #define GRPCXX_GENERIC_GENERIC_STUB_H #include +#include #include namespace grpc { @@ -27,6 +28,7 @@ namespace grpc { class CompletionQueue; typedef ClientAsyncReaderWriter GenericClientAsyncReaderWriter; +typedef ClientAsyncResponseReader GenericClientAsyncResponseReader; /// Generic stubs provide a type-unsafe interface to call gRPC methods /// by name. @@ -51,6 +53,14 @@ class GenericStub final { std::unique_ptr PrepareCall( ClientContext* context, const grpc::string& method, CompletionQueue* cq); + /// Setup a unary call to a named method \a method using \a context, and don't + /// start it. Let it be started explicitly with StartCall. + /// The return value only indicates whether or not registration of the call + /// succeeded (i.e. the call won't proceed if the return value is nullptr). + std::unique_ptr PrepareUnaryCall( + ClientContext* context, const grpc::string& method, + const ByteBuffer& request, CompletionQueue* cq); + private: std::shared_ptr channel_; }; diff --git a/src/cpp/client/generic_stub.cc b/src/cpp/client/generic_stub.cc index de2e449fe8f..693b8bea568 100644 --- a/src/cpp/client/generic_stub.cc +++ b/src/cpp/client/generic_stub.cc @@ -47,4 +47,14 @@ std::unique_ptr GenericStub::PrepareCall( return CallInternal(channel_.get(), context, method, cq, false, nullptr); } +// setup a unary call to a named method +std::unique_ptr GenericStub::PrepareUnaryCall( + ClientContext* context, const grpc::string& method, + const ByteBuffer& request, CompletionQueue* cq) { + return std::unique_ptr( + GenericClientAsyncResponseReader::Create( + channel_.get(), cq, RpcMethod(method.c_str(), RpcMethod::NORMAL_RPC), + context, request, false)); +} + } // namespace grpc diff --git a/test/cpp/end2end/generic_end2end_test.cc b/test/cpp/end2end/generic_end2end_test.cc index b9e6e18ca77..33b35108d22 100644 --- a/test/cpp/end2end/generic_end2end_test.cc +++ b/test/cpp/end2end/generic_end2end_test.cc @@ -196,6 +196,62 @@ TEST_F(GenericEnd2endTest, SequentialRpcs) { SendRpc(10); } +TEST_F(GenericEnd2endTest, SequentialUnaryRpcs) { + ResetStub(); + const int num_rpcs = 10; + const grpc::string kMethodName("/grpc.cpp.test.util.EchoTestService/Echo"); + for (int i = 0; i < num_rpcs; i++) { + EchoRequest send_request; + EchoRequest recv_request; + EchoResponse send_response; + EchoResponse recv_response; + Status recv_status; + + ClientContext cli_ctx; + GenericServerContext srv_ctx; + GenericServerAsyncReaderWriter stream(&srv_ctx); + + // The string needs to be long enough to test heap-based slice. + send_request.set_message("Hello world. Hello world. Hello world."); + + std::unique_ptr cli_send_buffer = + SerializeToByteBuffer(&send_request); + std::unique_ptr call = + generic_stub_->PrepareUnaryCall(&cli_ctx, kMethodName, + *cli_send_buffer.get(), &cli_cq_); + call->StartCall(); + ByteBuffer cli_recv_buffer; + call->Finish(&cli_recv_buffer, &recv_status, tag(1)); + + generic_service_.RequestCall(&srv_ctx, &stream, srv_cq_.get(), + srv_cq_.get(), tag(4)); + + verify_ok(srv_cq_.get(), 4, true); + EXPECT_EQ(server_host_, srv_ctx.host().substr(0, server_host_.length())); + EXPECT_EQ(kMethodName, srv_ctx.method()); + + ByteBuffer srv_recv_buffer; + stream.Read(&srv_recv_buffer, tag(5)); + server_ok(5); + EXPECT_TRUE(ParseFromByteBuffer(&srv_recv_buffer, &recv_request)); + EXPECT_EQ(send_request.message(), recv_request.message()); + + send_response.set_message(recv_request.message()); + std::unique_ptr srv_send_buffer = + SerializeToByteBuffer(&send_response); + stream.Write(*srv_send_buffer, tag(6)); + server_ok(6); + + stream.Finish(Status::OK, tag(7)); + server_ok(7); + + client_ok(1); + EXPECT_TRUE(ParseFromByteBuffer(&cli_recv_buffer, &recv_response)); + EXPECT_EQ(send_response.message(), recv_response.message()); + EXPECT_TRUE(recv_status.ok()); + } +} + // One ping, one pong. TEST_F(GenericEnd2endTest, SimpleBidiStreaming) { ResetStub(); From b8e55626002932d0d4bb10f3050754c1b12f853a Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 21 Sep 2017 14:44:32 -0700 Subject: [PATCH 223/227] Revert "Temporarily take ownership of cpp generator to prevent changes" --- .github/CODEOWNERS | 1 - src/compiler/OWNERS | 1 - 2 files changed, 2 deletions(-) delete mode 100644 src/compiler/OWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2a4eacd9981..094e43e4705 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,5 +3,4 @@ # repository as the source of truth for module ownership. /**/OWNERS @markdroth @nicolasnoble @ctiller /bazel/** @nicolasnoble @dgquintas @ctiller -/src/compiler/cpp_generator.cc @vjpai /src/core/ext/filters/client_channel/** @markdroth @dgquintas @ctiller diff --git a/src/compiler/OWNERS b/src/compiler/OWNERS deleted file mode 100644 index 96b89fc60ff..00000000000 --- a/src/compiler/OWNERS +++ /dev/null @@ -1 +0,0 @@ -@vjpai cpp_generator.cc From af3b4eecbdb6a70b6a3d4b5050c8aaffc259b823 Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Thu, 21 Sep 2017 14:25:07 -0700 Subject: [PATCH 224/227] add php into jenkins performance tests --- tools/jenkins/run_full_performance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jenkins/run_full_performance.sh b/tools/jenkins/run_full_performance.sh index 0f101f6da9c..a9661c7e266 100755 --- a/tools/jenkins/run_full_performance.sh +++ b/tools/jenkins/run_full_performance.sh @@ -21,7 +21,7 @@ cd $(dirname $0)/../.. # run 8core client vs 8core server tools/run_tests/run_performance_tests.py \ - -l c++ csharp node ruby java python go node_express \ + -l c++ csharp node ruby java python go node_express php \ --netperf \ --category scalable \ --bq_result_table performance_test.performance_experiment \ From 604a7ca07b7aeb5ecff210ab4e854526c78ae863 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 22 Sep 2017 09:28:32 -0700 Subject: [PATCH 225/227] Reduce stack size of simple_request e2e test --- test/core/end2end/tests/simple_request.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/core/end2end/tests/simple_request.c b/test/core/end2end/tests/simple_request.c index 82ab0a1cfeb..7ba5bd8f469 100644 --- a/test/core/end2end/tests/simple_request.c +++ b/test/core/end2end/tests/simple_request.c @@ -103,10 +103,10 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_slice details; int was_cancelled = 2; char *peer; - grpc_stats_data before; - grpc_stats_data after; + grpc_stats_data *before = gpr_malloc(sizeof(grpc_stats_data)); + grpc_stats_data *after = gpr_malloc(sizeof(grpc_stats_data)); - grpc_stats_collect(&before); + grpc_stats_collect(before); gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( @@ -214,9 +214,9 @@ static void simple_request_body(grpc_end2end_test_config config, cq_verifier_destroy(cqv); - grpc_stats_collect(&after); + grpc_stats_collect(after); - char *stats = grpc_stats_data_as_json(&after); + char *stats = grpc_stats_data_as_json(after); gpr_log(GPR_DEBUG, "%s", stats); gpr_free(stats); @@ -224,11 +224,11 @@ static void simple_request_body(grpc_end2end_test_config config, if (config.feature_mask & FEATURE_MASK_SUPPORTS_REQUEST_PROXYING) { expected_calls *= 2; } - GPR_ASSERT(after.counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] - - before.counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] == + GPR_ASSERT(after->counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] - + before->counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] == expected_calls); - GPR_ASSERT(after.counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] - - before.counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] == + GPR_ASSERT(after->counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] - + before->counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] == expected_calls); } From 3c5203886838d08f74f20a66d9d00af23f415644 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 22 Sep 2017 09:31:03 -0700 Subject: [PATCH 226/227] Fix memory leak --- test/core/end2end/tests/simple_request.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/core/end2end/tests/simple_request.c b/test/core/end2end/tests/simple_request.c index 7ba5bd8f469..7ce7e1f2855 100644 --- a/test/core/end2end/tests/simple_request.c +++ b/test/core/end2end/tests/simple_request.c @@ -230,6 +230,8 @@ static void simple_request_body(grpc_end2end_test_config config, GPR_ASSERT(after->counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] - before->counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] == expected_calls); + gpr_free(before); + gpr_free(after); } static void test_invoke_simple_request(grpc_end2end_test_config config) { From eec7a917baf37ed26449b106f99027c97aedaba6 Mon Sep 17 00:00:00 2001 From: Shaun McCormick Date: Fri, 11 Aug 2017 14:57:14 -0500 Subject: [PATCH 227/227] Add Ruby server interceptors --- src/ruby/lib/grpc.rb | 1 + src/ruby/lib/grpc/generic/active_call.rb | 43 +++- src/ruby/lib/grpc/generic/bidi_call.rb | 29 ++- src/ruby/lib/grpc/generic/client_stub.rb | 133 ++++++++--- .../lib/grpc/generic/interceptor_registry.rb | 53 +++++ src/ruby/lib/grpc/generic/interceptors.rb | 186 +++++++++++++++ src/ruby/lib/grpc/generic/rpc_desc.rb | 80 +++++-- src/ruby/lib/grpc/generic/rpc_server.rb | 18 +- .../duplicate/echo_duplicate_services_pb.rb | 1 + src/ruby/spec/channel_connection_spec.rb | 35 +-- src/ruby/spec/generic/active_call_spec.rb | 19 +- .../spec/generic/client_interceptors_spec.rb | 153 ++++++++++++ .../spec/generic/interceptor_registry_spec.rb | 65 ++++++ src/ruby/spec/generic/rpc_server_spec.rb | 35 +-- .../spec/generic/server_interceptors_spec.rb | 218 ++++++++++++++++++ src/ruby/spec/spec_helper.rb | 4 + src/ruby/spec/support/helpers.rb | 73 ++++++ src/ruby/spec/support/services.rb | 147 ++++++++++++ 18 files changed, 1143 insertions(+), 150 deletions(-) create mode 100644 src/ruby/lib/grpc/generic/interceptor_registry.rb create mode 100644 src/ruby/lib/grpc/generic/interceptors.rb create mode 100644 src/ruby/spec/generic/client_interceptors_spec.rb create mode 100644 src/ruby/spec/generic/interceptor_registry_spec.rb create mode 100644 src/ruby/spec/generic/server_interceptors_spec.rb create mode 100644 src/ruby/spec/support/helpers.rb create mode 100644 src/ruby/spec/support/services.rb diff --git a/src/ruby/lib/grpc.rb b/src/ruby/lib/grpc.rb index 98bfc0a0fa0..37b03920727 100644 --- a/src/ruby/lib/grpc.rb +++ b/src/ruby/lib/grpc.rb @@ -24,6 +24,7 @@ require_relative 'grpc/generic/active_call' require_relative 'grpc/generic/client_stub' require_relative 'grpc/generic/service' require_relative 'grpc/generic/rpc_server' +require_relative 'grpc/generic/interceptors' begin file = File.open(ssl_roots_path) diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb index 10eb70b4a7f..8c3aa284aae 100644 --- a/src/ruby/lib/grpc/generic/active_call.rb +++ b/src/ruby/lib/grpc/generic/active_call.rb @@ -154,6 +154,15 @@ module GRPC Operation.new(self) end + ## + # Returns a restricted view of this ActiveCall for use in interceptors + # + # @return [InterceptableView] + # + def interceptable + InterceptableView.new(self) + end + def receive_and_check_status batch_result = @call.run_batch(RECV_STATUS_ON_CLIENT => nil) set_input_stream_done @@ -515,15 +524,27 @@ module GRPC # This does not mean that must necessarily be one. E.g, the replies # produced by gen_each_reply could ignore the received_msgs # - # @param gen_each_reply [Proc] generates the BiDi stream replies - def run_server_bidi(gen_each_reply) - bd = BidiCall.new(@call, - @marshal, - @unmarshal, - metadata_received: @metadata_received, - req_view: MultiReqView.new(self)) - - bd.run_on_server(gen_each_reply, proc { set_input_stream_done }) + # @param mth [Proc] generates the BiDi stream replies + # @param interception_ctx [InterceptionContext] + # + def run_server_bidi(mth, interception_ctx) + view = multi_req_view + bidi_call = BidiCall.new( + @call, + @marshal, + @unmarshal, + metadata_received: @metadata_received, + req_view: view + ) + requests = bidi_call.read_next_loop(proc { set_input_stream_done }, false) + interception_ctx.intercept!( + :bidi_streamer, + call: view, + method: mth, + requests: requests + ) do + bidi_call.run_on_server(mth, requests) + end end # Waits till an operation completes @@ -645,5 +666,9 @@ module GRPC Operation = view_class(:cancel, :cancelled?, :deadline, :execute, :metadata, :status, :start_call, :wait, :write_flag, :write_flag=, :trailing_metadata) + + # InterceptableView further limits access to an ActiveCall's methods + # for use in interceptors on the client, exposing only the deadline + InterceptableView = view_class(:deadline) end end diff --git a/src/ruby/lib/grpc/generic/bidi_call.rb b/src/ruby/lib/grpc/generic/bidi_call.rb index c2239d0178e..3bdcc0062ee 100644 --- a/src/ruby/lib/grpc/generic/bidi_call.rb +++ b/src/ruby/lib/grpc/generic/bidi_call.rb @@ -87,23 +87,32 @@ module GRPC # This does not mean that must necessarily be one. E.g, the replies # produced by gen_each_reply could ignore the received_msgs # - # @param gen_each_reply [Proc] generates the BiDi stream replies. - # @param set_input_steam_done [Proc] call back to call when - # the reads have been completely read through. - def run_on_server(gen_each_reply, set_input_stream_done) + # @param [Proc] gen_each_reply generates the BiDi stream replies. + # @param [Enumerable] requests The enumerable of requests to run + def run_on_server(gen_each_reply, requests) + replies = nil + # Pass in the optional call object parameter if possible if gen_each_reply.arity == 1 - replys = gen_each_reply.call( - read_loop(set_input_stream_done, is_client: false)) + replies = gen_each_reply.call(requests) elsif gen_each_reply.arity == 2 - replys = gen_each_reply.call( - read_loop(set_input_stream_done, is_client: false), - @req_view) + replies = gen_each_reply.call(requests, @req_view) else fail 'Illegal arity of reply generator' end - write_loop(replys, is_client: false) + write_loop(replies, is_client: false) + end + + ## + # Read the next stream iteration + # + # @param [Proc] finalize_stream callback to call when the reads have been + # completely read through. + # @param [Boolean] is_client If this is a client or server request + # + def read_next_loop(finalize_stream, is_client = false) + read_loop(finalize_stream, is_client: is_client) end private diff --git a/src/ruby/lib/grpc/generic/client_stub.rb b/src/ruby/lib/grpc/generic/client_stub.rb index 75a95a4e940..9a50f8a99dc 100644 --- a/src/ruby/lib/grpc/generic/client_stub.rb +++ b/src/ruby/lib/grpc/generic/client_stub.rb @@ -89,17 +89,23 @@ module GRPC # used within a gRPC server. # @param channel_args [Hash] the channel arguments. Note: this argument is # ignored if the channel_override argument is provided. + # @param interceptors [Array] An array of + # GRPC::ClientInterceptor objects that will be used for + # intercepting calls before they are executed + # Interceptors are an EXPERIMENTAL API. def initialize(host, creds, channel_override: nil, timeout: nil, propagate_mask: nil, - channel_args: {}) + channel_args: {}, + interceptors: []) @ch = ClientStub.setup_channel(channel_override, host, creds, channel_args) alt_host = channel_args[Core::Channel::SSL_TARGET] @host = alt_host.nil? ? host : alt_host @propagate_mask = propagate_mask @timeout = timeout.nil? ? DEFAULT_TIMEOUT : timeout + @interceptors = InterceptorRegistry.new(interceptors) end # request_response sends a request to a GRPC server, and returns the @@ -149,16 +155,29 @@ module GRPC deadline: deadline, parent: parent, credentials: credentials) - return c.request_response(req, metadata: metadata) unless return_op - - # return the operation view of the active_call; define #execute as a - # new method for this instance that invokes #request_response. - c.merge_metadata_to_send(metadata) - op = c.operation - op.define_singleton_method(:execute) do - c.request_response(req, metadata: metadata) + interception_context = @interceptors.build_context + intercept_args = { + method: method, + request: req, + call: c.interceptable, + metadata: metadata + } + if return_op + # return the operation view of the active_call; define #execute as a + # new method for this instance that invokes #request_response. + c.merge_metadata_to_send(metadata) + op = c.operation + op.define_singleton_method(:execute) do + interception_context.intercept!(:request_response, intercept_args) do + c.request_response(req, metadata: metadata) + end + end + op + else + interception_context.intercept!(:request_response, intercept_args) do + c.request_response(req, metadata: metadata) + end end - op end # client_streamer sends a stream of requests to a GRPC server, and @@ -213,16 +232,29 @@ module GRPC deadline: deadline, parent: parent, credentials: credentials) - return c.client_streamer(requests, metadata: metadata) unless return_op - - # return the operation view of the active_call; define #execute as a - # new method for this instance that invokes #client_streamer. - c.merge_metadata_to_send(metadata) - op = c.operation - op.define_singleton_method(:execute) do - c.client_streamer(requests) + interception_context = @interceptors.build_context + intercept_args = { + method: method, + requests: requests, + call: c.interceptable, + metadata: metadata + } + if return_op + # return the operation view of the active_call; define #execute as a + # new method for this instance that invokes #client_streamer. + c.merge_metadata_to_send(metadata) + op = c.operation + op.define_singleton_method(:execute) do + interception_context.intercept!(:client_streamer, intercept_args) do + c.client_streamer(requests) + end + end + op + else + interception_context.intercept!(:client_streamer, intercept_args) do + c.client_streamer(requests, metadata: metadata) + end end - op end # server_streamer sends one request to the GRPC server, which yields a @@ -292,16 +324,29 @@ module GRPC deadline: deadline, parent: parent, credentials: credentials) - return c.server_streamer(req, metadata: metadata, &blk) unless return_op - - # return the operation view of the active_call; define #execute - # as a new method for this instance that invokes #server_streamer - c.merge_metadata_to_send(metadata) - op = c.operation - op.define_singleton_method(:execute) do - c.server_streamer(req, &blk) + interception_context = @interceptors.build_context + intercept_args = { + method: method, + request: req, + call: c.interceptable, + metadata: metadata + } + if return_op + # return the operation view of the active_call; define #execute + # as a new method for this instance that invokes #server_streamer + c.merge_metadata_to_send(metadata) + op = c.operation + op.define_singleton_method(:execute) do + interception_context.intercept!(:server_streamer, intercept_args) do + c.server_streamer(req, &blk) + end + end + op + else + interception_context.intercept!(:server_streamer, intercept_args) do + c.server_streamer(req, metadata: metadata, &blk) + end end - op end # bidi_streamer sends a stream of requests to the GRPC server, and yields @@ -405,17 +450,29 @@ module GRPC deadline: deadline, parent: parent, credentials: credentials) - return c.bidi_streamer(requests, metadata: metadata, - &blk) unless return_op - - # return the operation view of the active_call; define #execute - # as a new method for this instance that invokes #bidi_streamer - c.merge_metadata_to_send(metadata) - op = c.operation - op.define_singleton_method(:execute) do - c.bidi_streamer(requests, &blk) + interception_context = @interceptors.build_context + intercept_args = { + method: method, + requests: requests, + call: c.interceptable, + metadata: metadata + } + if return_op + # return the operation view of the active_call; define #execute + # as a new method for this instance that invokes #bidi_streamer + c.merge_metadata_to_send(metadata) + op = c.operation + op.define_singleton_method(:execute) do + interception_context.intercept!(:bidi_streamer, intercept_args) do + c.bidi_streamer(requests, &blk) + end + end + op + else + interception_context.intercept!(:bidi_streamer, intercept_args) do + c.bidi_streamer(requests, metadata: metadata, &blk) + end end - op end private diff --git a/src/ruby/lib/grpc/generic/interceptor_registry.rb b/src/ruby/lib/grpc/generic/interceptor_registry.rb new file mode 100644 index 00000000000..b241eb9a86d --- /dev/null +++ b/src/ruby/lib/grpc/generic/interceptor_registry.rb @@ -0,0 +1,53 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GRPC contains the General RPC module. +module GRPC + ## + # Represents a registry of added interceptors available for enumeration. + # The registry can be used for both server and client interceptors. + # This class is internal to gRPC and not meant for public usage. + # + class InterceptorRegistry + ## + # An error raised when an interceptor is attempted to be added + # that does not extend GRPC::Interceptor + # + class DescendantError < StandardError; end + + ## + # Initialize the registry with an empty interceptor list + # This is an EXPERIMENTAL API. + # + def initialize(interceptors = []) + @interceptors = [] + interceptors.each do |i| + base = GRPC::Interceptor + unless i.class.ancestors.include?(base) + fail DescendantError, "Interceptors must descend from #{base}" + end + @interceptors << i + end + end + + ## + # Builds an interception context from this registry + # + # @return [InterceptionContext] + # + def build_context + InterceptionContext.new(@interceptors) + end + end +end diff --git a/src/ruby/lib/grpc/generic/interceptors.rb b/src/ruby/lib/grpc/generic/interceptors.rb new file mode 100644 index 00000000000..73faec4b9c7 --- /dev/null +++ b/src/ruby/lib/grpc/generic/interceptors.rb @@ -0,0 +1,186 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require_relative 'interceptor_registry' + +# GRPC contains the General RPC module. +module GRPC + ## + # Base class for interception in GRPC + # + class Interceptor + ## + # @param [Hash] options A hash of options that will be used + # by the interceptor. This is an EXPERIMENTAL API. + # + def initialize(options = {}) + @options = options || {} + end + end + + ## + # ClientInterceptor allows for wrapping outbound gRPC client stub requests. + # This is an EXPERIMENTAL API. + # + class ClientInterceptor < Interceptor + ## + # Intercept a unary request response call + # + # @param [Object] request + # @param [GRPC::ActiveCall] call + # @param [Method] method + # @param [Hash] metadata + # + def request_response(request:, call:, method:, metadata:) + GRPC.logger.debug "Intercepting request response method #{method}" \ + " for request #{request} with call #{call} and metadata: #{metadata}" + yield + end + + ## + # Intercept a client streaming call + # + # @param [Enumerable] requests + # @param [GRPC::ActiveCall] call + # @param [Method] method + # @param [Hash] metadata + # + def client_streamer(requests:, call:, method:, metadata:) + GRPC.logger.debug "Intercepting client streamer method #{method}" \ + " for requests #{requests} with call #{call} and metadata: #{metadata}" + yield + end + + ## + # Intercept a server streaming call + # + # @param [Object] request + # @param [GRPC::ActiveCall] call + # @param [Method] method + # @param [Hash] metadata + # + def server_streamer(request:, call:, method:, metadata:) + GRPC.logger.debug "Intercepting server streamer method #{method}" \ + " for request #{request} with call #{call} and metadata: #{metadata}" + yield + end + + ## + # Intercept a BiDi streaming call + # + # @param [Enumerable] requests + # @param [GRPC::ActiveCall] call + # @param [Method] method + # @param [Hash] metadata + # + def bidi_streamer(requests:, call:, method:, metadata:) + GRPC.logger.debug "Intercepting bidi streamer method #{method}" \ + " for requests #{requests} with call #{call} and metadata: #{metadata}" + yield + end + end + + ## + # ServerInterceptor allows for wrapping gRPC server execution handling. + # This is an EXPERIMENTAL API. + # + class ServerInterceptor < Interceptor + ## + # Intercept a unary request response call. + # + # @param [Object] request + # @param [GRPC::ActiveCall::SingleReqView] call + # @param [Method] method + # + def request_response(request:, call:, method:) + GRPC.logger.debug "Intercepting request response method #{method}" \ + " for request #{request} with call #{call}" + yield + end + + ## + # Intercept a client streaming call + # + # @param [GRPC::ActiveCall::MultiReqView] call + # @param [Method] method + # + def client_streamer(call:, method:) + GRPC.logger.debug "Intercepting client streamer method #{method}" \ + " with call #{call}" + yield + end + + ## + # Intercept a server streaming call + # + # @param [Object] request + # @param [GRPC::ActiveCall::SingleReqView] call + # @param [Method] method + # + def server_streamer(request:, call:, method:) + GRPC.logger.debug "Intercepting server streamer method #{method}" \ + " for request #{request} with call #{call}" + yield + end + + ## + # Intercept a BiDi streaming call + # + # @param [Enumerable] requests + # @param [GRPC::ActiveCall::MultiReqView] call + # @param [Method] method + # + def bidi_streamer(requests:, call:, method:) + GRPC.logger.debug "Intercepting bidi streamer method #{method}" \ + " for requests #{requests} with call #{call}" + yield + end + end + + ## + # Represents the context in which an interceptor runs. Used to provide an + # injectable mechanism for handling interception. This is an EXPERIMENTAL API. + # + class InterceptionContext + ## + # @param [Array] + # + def initialize(interceptors = []) + @interceptors = interceptors.dup + end + + ## + # Intercept the call and fire out to interceptors in a FIFO execution. + # This is an EXPERIMENTAL API. + # + # @param [Symbol] type The request type + # @param [Hash] args The arguments for the call + # + def intercept!(type, args = {}) + return yield if @interceptors.none? + + i = @interceptors.pop + return yield unless i + + i.send(type, args) do + if @interceptors.any? + intercept!(type, args) do + yield + end + else + yield + end + end + end + end +end diff --git a/src/ruby/lib/grpc/generic/rpc_desc.rb b/src/ruby/lib/grpc/generic/rpc_desc.rb index 6fb6c412fb1..5fd1805aabf 100644 --- a/src/ruby/lib/grpc/generic/rpc_desc.rb +++ b/src/ruby/lib/grpc/generic/rpc_desc.rb @@ -47,43 +47,85 @@ module GRPC proc { |o| unmarshal_class.method(unmarshal_method).call(o) } end - def handle_request_response(active_call, mth) + def handle_request_response(active_call, mth, inter_ctx) req = active_call.read_unary_request - resp = mth.call(req, active_call.single_req_view) - active_call.server_unary_response( - resp, trailing_metadata: active_call.output_metadata) + call = active_call.single_req_view + + inter_ctx.intercept!( + :request_response, + method: mth, + call: call, + request: req + ) do + resp = mth.call(req, call) + active_call.server_unary_response( + resp, + trailing_metadata: active_call.output_metadata + ) + end end - def handle_client_streamer(active_call, mth) - resp = mth.call(active_call.multi_req_view) - active_call.server_unary_response( - resp, trailing_metadata: active_call.output_metadata) + def handle_client_streamer(active_call, mth, inter_ctx) + call = active_call.multi_req_view + + inter_ctx.intercept!( + :client_streamer, + method: mth, + call: call + ) do + resp = mth.call(call) + active_call.server_unary_response( + resp, + trailing_metadata: active_call.output_metadata + ) + end end - def handle_server_streamer(active_call, mth) + def handle_server_streamer(active_call, mth, inter_ctx) req = active_call.read_unary_request - replys = mth.call(req, active_call.single_req_view) - replys.each { |r| active_call.remote_send(r) } - send_status(active_call, OK, 'OK', active_call.output_metadata) + call = active_call.single_req_view + + inter_ctx.intercept!( + :server_streamer, + method: mth, + call: call, + request: req + ) do + replies = mth.call(req, call) + replies.each { |r| active_call.remote_send(r) } + send_status(active_call, OK, 'OK', active_call.output_metadata) + end end - def handle_bidi_streamer(active_call, mth) - active_call.run_server_bidi(mth) + ## + # @param [GRPC::ActiveCall] active_call + # @param [Method] mth + # @param [Array] inter_ctx + # + def handle_bidi_streamer(active_call, mth, inter_ctx) + active_call.run_server_bidi(mth, inter_ctx) send_status(active_call, OK, 'OK', active_call.output_metadata) end - def run_server_method(active_call, mth) + ## + # @param [GRPC::ActiveCall] active_call The current active call object + # for the request + # @param [Method] mth The current RPC method being called + # @param [GRPC::InterceptionContext] inter_ctx The interception context + # being executed + # + def run_server_method(active_call, mth, inter_ctx = InterceptionContext.new) # While a server method is running, it might be cancelled, its deadline # might be reached, the handler could throw an unknown error, or a # well-behaved handler could throw a StatusError. if request_response? - handle_request_response(active_call, mth) + handle_request_response(active_call, mth, inter_ctx) elsif client_streamer? - handle_client_streamer(active_call, mth) + handle_client_streamer(active_call, mth, inter_ctx) elsif server_streamer? - handle_server_streamer(active_call, mth) + handle_server_streamer(active_call, mth, inter_ctx) else # is a bidi_stream - handle_bidi_streamer(active_call, mth) + handle_bidi_streamer(active_call, mth, inter_ctx) end rescue BadStatus => e # this is raised by handlers that want GRPC to send an application error diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index 33b3cea1fc3..d5fc11dc1ca 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -196,11 +196,18 @@ module GRPC # # * server_args: # A server arguments hash to be passed down to the underlying core server + # + # * interceptors: + # Am array of GRPC::ServerInterceptor objects that will be used for + # intercepting server handlers to provide extra functionality. + # Interceptors are an EXPERIMENTAL API. + # def initialize(pool_size:DEFAULT_POOL_SIZE, max_waiting_requests:DEFAULT_MAX_WAITING_REQUESTS, poll_period:DEFAULT_POLL_PERIOD, connect_md_proc:nil, - server_args:{}) + server_args:{}, + interceptors:[]) @connect_md_proc = RpcServer.setup_connect_md_proc(connect_md_proc) @max_waiting_requests = max_waiting_requests @poll_period = poll_period @@ -212,6 +219,7 @@ module GRPC # :stopped. State transitions can only proceed in that order. @running_state = :not_started @server = Core::Server.new(server_args) + @interceptors = InterceptorRegistry.new(interceptors) end # stops a running server @@ -374,7 +382,11 @@ module GRPC @pool.schedule(active_call) do |ac| c, mth = ac begin - rpc_descs[mth].run_server_method(c, rpc_handlers[mth]) + rpc_descs[mth].run_server_method( + c, + rpc_handlers[mth], + @interceptors.build_context + ) rescue StandardError c.send_status(GRPC::Core::StatusCodes::INTERNAL, 'Server handler failed') @@ -382,7 +394,7 @@ module GRPC end end rescue Core::CallError, RuntimeError => e - # these might happen for various reasonse. The correct behaviour of + # these might happen for various reasons. The correct behavior of # the server is to log them and continue, if it's not shutting down. if running_state == :running GRPC.logger.warn("server call failed: #{e}") diff --git a/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb b/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb index 683370121ea..ab50d9b3a58 100644 --- a/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb +++ b/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb @@ -34,6 +34,7 @@ module Grpc self.service_name = 'grpc.testing.duplicate.EchoTestService' rpc :Echo, Grpc::Testing::EchoRequest, Grpc::Testing::EchoResponse + rpc :ResponseStream, Grpc::Testing::EchoRequest, stream(Grpc::Testing::EchoResponse) end Stub = Service.rpc_stub_class diff --git a/src/ruby/spec/channel_connection_spec.rb b/src/ruby/spec/channel_connection_spec.rb index c76056606bd..ce3e3b1c935 100644 --- a/src/ruby/spec/channel_connection_spec.rb +++ b/src/ruby/spec/channel_connection_spec.rb @@ -11,45 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -require 'grpc' +require 'spec_helper' require 'timeout' include Timeout include GRPC::Core -# A test message -class EchoMsg - def self.marshal(_o) - '' - end - - def self.unmarshal(_o) - EchoMsg.new - end -end - -# A test service with an echo implementation. -class EchoService - include GRPC::GenericService - rpc :an_rpc, EchoMsg, EchoMsg - attr_reader :received_md - - def initialize(**kw) - @trailing_metadata = kw - @received_md = [] - end - - def an_rpc(req, call) - GRPC.logger.info('echo service received a request') - call.output_metadata.update(@trailing_metadata) - @received_md << call.metadata unless call.metadata.nil? - req - end -end - -EchoStub = EchoService.rpc_stub_class - def start_server(port = 0) @srv = GRPC::RpcServer.new(pool_size: 1) server_port = @srv.add_http2_port("localhost:#{port}", :this_port_is_insecure) diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb index a00df9236d2..120acc35afb 100644 --- a/src/ruby/spec/generic/active_call_spec.rb +++ b/src/ruby/spec/generic/active_call_spec.rb @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'grpc' +require 'spec_helper' include GRPC::Core::StatusCodes @@ -82,6 +82,16 @@ describe GRPC::ActiveCall do end end end + + describe '#interceptable' do + it 'exposes a fixed subset of the ActiveCall.methods' do + want = %w(deadline) + v = @client_call.interceptable + want.each do |w| + expect(v.methods.include?(w)) + end + end + end end describe '#remote_send' do @@ -609,9 +619,11 @@ describe GRPC::ActiveCall do msgs end + int_ctx = GRPC::InterceptionContext.new + @server_thread = Thread.new do @server_call.run_server_bidi( - fake_gen_each_reply_with_no_call_param) + fake_gen_each_reply_with_no_call_param, int_ctx) @server_call.send_status(@server_status) end end @@ -624,10 +636,11 @@ describe GRPC::ActiveCall do call_param.send_initial_metadata msgs end + int_ctx = GRPC::InterceptionContext.new @server_thread = Thread.new do @server_call.run_server_bidi( - fake_gen_each_reply_with_call_param) + fake_gen_each_reply_with_call_param, int_ctx) @server_call.send_status(@server_status) end end diff --git a/src/ruby/spec/generic/client_interceptors_spec.rb b/src/ruby/spec/generic/client_interceptors_spec.rb new file mode 100644 index 00000000000..f292715e4df --- /dev/null +++ b/src/ruby/spec/generic/client_interceptors_spec.rb @@ -0,0 +1,153 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require 'spec_helper' + +describe 'Client Interceptors' do + let(:interceptor) { TestClientInterceptor.new } + let(:interceptors_opts) { { interceptors: [interceptor] } } + let(:request) { EchoMsg.new } + let(:service) { EchoService } + + before(:each) do + build_rpc_server + end + + context 'when a client interceptor is added' do + context 'with a request/response call' do + it 'should be called', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + expect_any_instance_of(GRPC::ActiveCall).to receive(:request_response) + .once.and_call_original + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + + it 'can modify outgoing metadata', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + expect_any_instance_of(GRPC::ActiveCall).to receive(:request_response) + .with(request, metadata: { 'foo' => 'bar_from_request_response' }) + .once.and_call_original + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + end + + context 'with a client streaming call' do + it 'should be called', server: true do + expect(interceptor).to receive(:client_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + expect_any_instance_of(GRPC::ActiveCall).to receive(:client_streamer) + .once.and_call_original + requests = [EchoMsg.new, EchoMsg.new] + expect(stub.a_client_streaming_rpc(requests)).to be_a(EchoMsg) + end + end + + it 'can modify outgoing metadata', server: true do + expect(interceptor).to receive(:client_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + requests = [EchoMsg.new, EchoMsg.new] + expect_any_instance_of(GRPC::ActiveCall).to receive(:client_streamer) + .with(requests, metadata: { 'foo' => 'bar_from_client_streamer' }) + .once.and_call_original + expect(stub.a_client_streaming_rpc(requests)).to be_a(EchoMsg) + end + end + end + + context 'with a server streaming call' do + it 'should be called', server: true do + expect(interceptor).to receive(:server_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + request = EchoMsg.new + expect_any_instance_of(GRPC::ActiveCall).to receive(:server_streamer) + .once.and_call_original + responses = stub.a_server_streaming_rpc(request) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + + it 'can modify outgoing metadata', server: true do + expect(interceptor).to receive(:server_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + request = EchoMsg.new + expect_any_instance_of(GRPC::ActiveCall).to receive(:server_streamer) + .with(request, metadata: { 'foo' => 'bar_from_server_streamer' }) + .once.and_call_original + responses = stub.a_server_streaming_rpc(request) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + end + + context 'with a bidi call' do + it 'should be called', server: true do + expect(interceptor).to receive(:bidi_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + expect_any_instance_of(GRPC::ActiveCall).to receive(:bidi_streamer) + .once.and_call_original + requests = [EchoMsg.new, EchoMsg.new] + responses = stub.a_bidi_rpc(requests) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + + it 'can modify outgoing metadata', server: true do + expect(interceptor).to receive(:bidi_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + requests = [EchoMsg.new, EchoMsg.new] + expect_any_instance_of(GRPC::ActiveCall).to receive(:bidi_streamer) + .with(requests, metadata: { 'foo' => 'bar_from_bidi_streamer' }) + .once.and_call_original + responses = stub.a_bidi_rpc(requests) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + end + end +end diff --git a/src/ruby/spec/generic/interceptor_registry_spec.rb b/src/ruby/spec/generic/interceptor_registry_spec.rb new file mode 100644 index 00000000000..f93f5cec096 --- /dev/null +++ b/src/ruby/spec/generic/interceptor_registry_spec.rb @@ -0,0 +1,65 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require 'spec_helper' + +describe GRPC::InterceptorRegistry do + let(:server) { RpcServer.new } + let(:interceptor) { TestServerInterceptor.new } + let(:interceptors) { [interceptor] } + let(:registry) { described_class.new(interceptors) } + + describe 'initialization' do + subject { registry } + + context 'with an interceptor extending GRPC::ServerInterceptor' do + it 'should add the interceptor to the registry' do + subject + is = registry.instance_variable_get('@interceptors') + expect(is.count).to eq 1 + expect(is.first).to eq interceptor + end + end + + context 'with multiple interceptors' do + let(:interceptor2) { TestServerInterceptor.new } + let(:interceptor3) { TestServerInterceptor.new } + let(:interceptors) { [interceptor, interceptor2, interceptor3] } + + it 'should maintain order of insertion when iterated against' do + subject + is = registry.instance_variable_get('@interceptors') + expect(is.count).to eq 3 + is.each_with_index do |i, idx| + case idx + when 0 + expect(i).to eq interceptor + when 1 + expect(i).to eq interceptor2 + when 2 + expect(i).to eq interceptor3 + end + end + end + end + + context 'with an interceptor not extending GRPC::ServerInterceptor' do + let(:interceptor) { Class } + let(:err) { GRPC::InterceptorRegistry::DescendantError } + + it 'should raise an InvalidArgument exception' do + expect { subject }.to raise_error(err) + end + end + end +end diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index b887eaaf4e9..05059fbecf0 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -11,8 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -require 'grpc' +require 'spec_helper' def load_test_certs test_root = File.join(File.dirname(File.dirname(__FILE__)), 'testdata') @@ -28,17 +27,6 @@ def check_md(wanted_md, received_md) end end -# A test message -class EchoMsg - def self.marshal(_o) - '' - end - - def self.unmarshal(_o) - EchoMsg.new - end -end - # A test service with no methods. class EmptyService include GRPC::GenericService @@ -50,27 +38,6 @@ class NoRpcImplementation rpc :an_rpc, EchoMsg, EchoMsg end -# A test service with an echo implementation. -class EchoService - include GRPC::GenericService - rpc :an_rpc, EchoMsg, EchoMsg - attr_reader :received_md - - def initialize(**kw) - @trailing_metadata = kw - @received_md = [] - end - - def an_rpc(req, call) - GRPC.logger.info('echo service received a request') - call.output_metadata.update(@trailing_metadata) - @received_md << call.metadata unless call.metadata.nil? - req - end -end - -EchoStub = EchoService.rpc_stub_class - # A test service with an implementation that fails with BadStatus class FailingService include GRPC::GenericService diff --git a/src/ruby/spec/generic/server_interceptors_spec.rb b/src/ruby/spec/generic/server_interceptors_spec.rb new file mode 100644 index 00000000000..eb866860843 --- /dev/null +++ b/src/ruby/spec/generic/server_interceptors_spec.rb @@ -0,0 +1,218 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require 'spec_helper' + +describe 'Server Interceptors' do + let(:interceptor) { TestServerInterceptor.new } + let(:request) { EchoMsg.new } + let(:trailing_metadata) { {} } + let(:service) { EchoService.new(trailing_metadata) } + let(:interceptors) { [] } + + before(:each) do + build_rpc_server(server_opts: { interceptors: interceptors }) + end + + context 'when a server interceptor is added' do + let(:interceptors) { [interceptor] } + let(:client_metadata) { { client_md: 'test' } } + let(:client_call_opts) { { metadata: client_metadata, return_op: true } } + + context 'with a request/response call' do + let(:trailing_metadata) { { server_om: 'from_request_response' } } + + it 'should be called', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + + it 'can modify trailing metadata', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect_any_instance_of(GRPC::ActiveCall).to( + receive(:request_response).with(request, metadata: client_metadata) + .once.and_call_original + ) + op = stub.an_rpc(request, client_call_opts) + msg = op.execute + expect(op.trailing_metadata).to eq( + 'interc' => 'from_request_response', + 'server_om' => 'from_request_response' + ) + expect(msg).to be_a(EchoMsg) + end + end + end + + context 'with a client streaming call' do + let(:trailing_metadata) { { server_om: 'from_client_streamer' } } + let(:requests) { [EchoMsg.new, EchoMsg.new] } + + it 'should be called', server: true do + expect(interceptor).to receive(:client_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect(stub.a_client_streaming_rpc(requests)).to be_a(EchoMsg) + end + end + + it 'can modify trailing metadata', server: true do + expect(interceptor).to receive(:client_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect_any_instance_of(GRPC::ActiveCall).to( + receive(:client_streamer).with(requests) + .once.and_call_original + ) + op = stub.a_client_streaming_rpc(requests, client_call_opts) + msg = op.execute + expect(op.trailing_metadata).to eq( + 'interc' => 'from_client_streamer', + 'server_om' => 'from_client_streamer' + ) + expect(msg).to be_a(EchoMsg) + end + end + end + + context 'with a server streaming call' do + let(:trailing_metadata) { { server_om: 'from_server_streamer' } } + let(:request) { EchoMsg.new } + + it 'should be called', server: true do + expect(interceptor).to receive(:server_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + responses = stub.a_server_streaming_rpc(request) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + + it 'can modify trailing metadata', server: true do + expect(interceptor).to receive(:server_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect_any_instance_of(GRPC::ActiveCall).to( + receive(:server_streamer).with(request) + .once.and_call_original + ) + op = stub.a_server_streaming_rpc(request, client_call_opts) + responses = op.execute + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + expect(op.trailing_metadata).to eq( + 'interc' => 'from_server_streamer', + 'server_om' => 'from_server_streamer' + ) + end + end + end + + context 'with a bidi call' do + let(:trailing_metadata) { { server_om: 'from_bidi_streamer' } } + let(:requests) { [EchoMsg.new, EchoMsg.new] } + + it 'should be called', server: true do + expect(interceptor).to receive(:bidi_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + responses = stub.a_bidi_rpc(requests) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + + it 'can modify trailing metadata', server: true do + expect(interceptor).to receive(:bidi_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect_any_instance_of(GRPC::ActiveCall).to( + receive(:bidi_streamer).with(requests) + .once.and_call_original + ) + op = stub.a_bidi_rpc(requests, client_call_opts) + responses = op.execute + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + expect(op.trailing_metadata).to eq( + 'interc' => 'from_bidi_streamer', + 'server_om' => 'from_bidi_streamer' + ) + end + end + end + end + + context 'when multiple interceptors are added' do + let(:interceptor2) { TestServerInterceptor.new } + let(:interceptor3) { TestServerInterceptor.new } + let(:interceptors) do + [ + interceptor, + interceptor2, + interceptor3 + ] + end + + it 'each should be called', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + expect(interceptor2).to receive(:request_response) + .once.and_call_original + expect(interceptor3).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + end + + context 'when an interceptor is not added' do + it 'should not be called', server: true do + expect(interceptor).to_not receive(:call) + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + end +end diff --git a/src/ruby/spec/spec_helper.rb b/src/ruby/spec/spec_helper.rb index 6e1eba19455..8fe9e6e808c 100644 --- a/src/ruby/spec/spec_helper.rb +++ b/src/ruby/spec/spec_helper.rb @@ -32,6 +32,9 @@ require 'rspec' require 'logging' require 'rspec/logging_helper' +require_relative 'support/services' +require_relative 'support/helpers' + # GRPC is the general RPC module # # Configure its logging for fine-grained log control during test runs @@ -49,6 +52,7 @@ Logging.logger['GRPC::BidiCall'].level = :info RSpec.configure do |config| include RSpec::LoggingHelper config.capture_log_messages # comment this out to see logs during test runs + include GRPC::Spec::Helpers end RSpec::Expectations.configuration.warn_about_potential_false_positives = false diff --git a/src/ruby/spec/support/helpers.rb b/src/ruby/spec/support/helpers.rb new file mode 100644 index 00000000000..65fffff9e7e --- /dev/null +++ b/src/ruby/spec/support/helpers.rb @@ -0,0 +1,73 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GRPC contains the General RPC module. +module GRPC + ## + # GRPC RSpec base module + # + module Spec + ## + # A module that is used for providing generic helpers across the + # GRPC test suite + # + module Helpers + # Shortcut syntax for a GRPC RPC Server + RpcServer = GRPC::RpcServer + + ## + # Build an RPC server used for testing + # + def build_rpc_server(server_opts: {}, + client_opts: {}) + @server = RpcServer.new({ poll_period: 1 }.merge(server_opts)) + @port = @server.add_http2_port('0.0.0.0:0', :this_port_is_insecure) + @host = "0.0.0.0:#{@port}" + @client_opts = client_opts + @server + end + + ## + # Run services on an RPC server, yielding to allow testing within + # + # @param [RpcServer] server + # @param [Array] services + # + def run_services_on_server(server, services: []) + services.each do |s| + server.handle(s) + end + t = Thread.new { server.run } + server.wait_till_running + + yield + + server.stop + t.join + end + + ## + # Build an insecure stub from a given stub class + # + # @param [Class] klass + # @param [String] host + # + def build_insecure_stub(klass, host: nil, opts: nil) + host ||= @host + opts ||= @client_opts + klass.new(host, :this_channel_is_insecure, **opts) + end + end + end +end diff --git a/src/ruby/spec/support/services.rb b/src/ruby/spec/support/services.rb new file mode 100644 index 00000000000..27cc8e61acf --- /dev/null +++ b/src/ruby/spec/support/services.rb @@ -0,0 +1,147 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test stubs for various scenarios +require 'grpc' + +# A test message +class EchoMsg + def self.marshal(_o) + '' + end + + def self.unmarshal(_o) + EchoMsg.new + end +end + +# A test service with an echo implementation. +class EchoService + include GRPC::GenericService + rpc :an_rpc, EchoMsg, EchoMsg + rpc :a_client_streaming_rpc, stream(EchoMsg), EchoMsg + rpc :a_server_streaming_rpc, EchoMsg, stream(EchoMsg) + rpc :a_bidi_rpc, stream(EchoMsg), stream(EchoMsg) + attr_reader :received_md + + def initialize(**kw) + @trailing_metadata = kw + @received_md = [] + end + + def an_rpc(req, call) + GRPC.logger.info('echo service received a request') + call.output_metadata.update(@trailing_metadata) + @received_md << call.metadata unless call.metadata.nil? + req + end + + def a_client_streaming_rpc(call) + # iterate through requests so call can complete + call.output_metadata.update(@trailing_metadata) + call.each_remote_read.each { |r| p r } + EchoMsg.new + end + + def a_server_streaming_rpc(_req, call) + call.output_metadata.update(@trailing_metadata) + [EchoMsg.new, EchoMsg.new] + end + + def a_bidi_rpc(requests, call) + call.output_metadata.update(@trailing_metadata) + requests.each { |r| p r } + [EchoMsg.new, EchoMsg.new] + end +end + +EchoStub = EchoService.rpc_stub_class + +# For testing server interceptors +class TestServerInterceptor < GRPC::ServerInterceptor + def request_response(request:, call:, method:) + p "Received request/response call at method #{method}" \ + " with request #{request} for call #{call}" + call.output_metadata[:interc] = 'from_request_response' + p "[GRPC::Ok] (#{method.owner.name}.#{method.name})" + yield + end + + def client_streamer(call:, method:) + call.output_metadata[:interc] = 'from_client_streamer' + call.each_remote_read.each do |r| + p "In interceptor: #{r}" + end + p "Received client streamer call at method #{method} for call #{call}" + yield + end + + def server_streamer(request:, call:, method:) + p "Received server streamer call at method #{method} with request" \ + " #{request} for call #{call}" + call.output_metadata[:interc] = 'from_server_streamer' + yield + end + + def bidi_streamer(requests:, call:, method:) + requests.each do |r| + p "Bidi request: #{r}" + end + p "Received bidi streamer call at method #{method} with requests" \ + " #{requests} for call #{call}" + call.output_metadata[:interc] = 'from_bidi_streamer' + yield + end +end + +# For testing client interceptors +class TestClientInterceptor < GRPC::ClientInterceptor + def request_response(request:, call:, method:, metadata: {}) + p "Intercepted request/response call at method #{method}" \ + " with request #{request} for call #{call}" \ + " and metadata: #{metadata}" + metadata['foo'] = 'bar_from_request_response' + yield + end + + def client_streamer(requests:, call:, method:, metadata: {}) + p "Received client streamer call at method #{method}" \ + " with requests #{requests} for call #{call}" \ + " and metadata: #{metadata}" + requests.each do |r| + p "In client interceptor: #{r}" + end + metadata['foo'] = 'bar_from_client_streamer' + yield + end + + def server_streamer(request:, call:, method:, metadata: {}) + p "Received server streamer call at method #{method}" \ + " with request #{request} for call #{call}" \ + " and metadata: #{metadata}" + metadata['foo'] = 'bar_from_server_streamer' + yield + end + + def bidi_streamer(requests:, call:, method:, metadata: {}) + p "Received bidi streamer call at method #{method}" \ + "with requests #{requests} for call #{call}" \ + " and metadata: #{metadata}" + requests.each do |r| + p "In client interceptor: #{r}" + end + metadata['foo'] = 'bar_from_bidi_streamer' + yield + end +end