From c65ceefbe0681559a0bc3e745ad619d5c577fc0e Mon Sep 17 00:00:00 2001 From: Chris Bacon Date: Fri, 5 Feb 2016 09:04:46 +0000 Subject: [PATCH 001/236] Support coreclr platform detection Coreclr uses different platform detection primitives to full .NET --- src/csharp/Grpc.Core/Internal/PlatformApis.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs index f0c5b0f63f6..fb1acfb607b 100644 --- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs +++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs @@ -53,12 +53,18 @@ namespace Grpc.Core.Internal static PlatformApis() { +#if DNXCORE50 + isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + isMacOSX = RuntimeInformation.IsOSPlatform(OSPlatform.OSX); + isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); +#else var platform = Environment.OSVersion.Platform; // PlatformID.MacOSX is never returned, commonly used trick is to identify Mac is by using uname. isMacOSX = (platform == PlatformID.Unix && GetUname() == "Darwin"); isLinux = (platform == PlatformID.Unix && !isMacOSX); isWindows = (platform == PlatformID.Win32NT || platform == PlatformID.Win32S || platform == PlatformID.Win32Windows); +#endif isMono = Type.GetType("Mono.Runtime") != null; } From 1c755d5209b0da68196a18a637d7e52fd28c7302 Mon Sep 17 00:00:00 2001 From: Chris Bacon Date: Fri, 5 Feb 2016 10:33:41 +0000 Subject: [PATCH 002/236] Coreclr compatible DefaultSslRootsOverride.cs Coreclr doesn't support Assembly.GetExecutingAssembly(). Use TypeInfo.Assembly instead, which is supported on all platform. --- src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs b/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs index eeaa7add813..dfaee5d9d7c 100644 --- a/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs +++ b/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs @@ -56,7 +56,7 @@ namespace Grpc.Core.Internal { lock (staticLock) { - var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(RootsPemResourceName); + var stream = typeof(DefaultSslRootsOverride).GetTypeInfo().Assembly.GetManifestResourceStream(RootsPemResourceName); if (stream == null) { throw new IOException(string.Format("Error loading the embedded resource \"{0}\"", RootsPemResourceName)); From 95e8187c74e28e622aacf9a8e962977243f90bf8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 8 Feb 2016 15:48:55 -0800 Subject: [PATCH 003/236] Sanity check for version metadata --- tools/run_tests/sanity/check_version.py | 97 ++++++++++++++++++++++++ tools/run_tests/sanity/sanity_tests.yaml | 3 +- 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100755 tools/run_tests/sanity/check_version.py diff --git a/tools/run_tests/sanity/check_version.py b/tools/run_tests/sanity/check_version.py new file mode 100755 index 00000000000..41dd5efe388 --- /dev/null +++ b/tools/run_tests/sanity/check_version.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python2.7 + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys +import yaml +import os +import re +import subprocess + +errors = 0 + +os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) + +# hack import paths to pick up extra code +sys.path.insert(0, os.path.abspath('tools/buildgen/plugins')) +from expand_version import Version + +try: + branch_name = subprocess.check_output( + 'git rev-parse --abbrev-ref HEAD', + shell=True) +except: + print 'WARNING: not a git repository' + branch_name = None + +if branch_name is not None: + m = re.match(r'^release-([0-9]+)_([0-9]+)$', branch_name) + if m: + print 'RELEASE branch' + # version number should align with the branched version + check_version = lambda version: ( + version.major == int(m.group(1)) and + version.minor == int(m.group(2))) + warning = 'Version key "%%s" value "%%s" should have a major version %s and minor version %s' % (m.group(1), m.group(2)) + elif re.match(r'^debian/.*$', branch_name): + # no additional version checks for debian branches + check_version = lambda version: True + else: + # all other branches should have a -dev tag + check_version = lambda version: version.tag == 'dev' + warning = 'Version key "%s" value "%s" should have a -dev tag' +else: + check_version = lambda version: True + +with open('build.yaml', 'r') as f: + build_yaml = yaml.load(f.read()) + +settings = build_yaml['settings'] + +top_version = Version(settings['version']) +if not check_version(top_version): + errors += 1 + print warning % ('version', top_version) + +for tag, value in settings.iteritems(): + if re.match(r'^[a-z]+_version$', tag): + value = Version(value) + if value.major != top_version.major: + errors += 1 + print 'major version mismatch on %s: %d vs %d' % (tag, value.major, top_version.major) + if value.minor != top_version.minor: + errors += 1 + print 'minor version mismatch on %s: %d vs %d' % (tag, value.minor, top_version.minor) + if not check_version(value): + errors += 1 + print warning % (tag, value) + +sys.exit(errors) + diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index 809e6ce6454..50864b5d879 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -1,7 +1,8 @@ # a set of tests that are run in parallel for sanity tests +- script: tools/run_tests/sanity/check_cache_mk.sh - script: tools/run_tests/sanity/check_sources_and_headers.py - script: tools/run_tests/sanity/check_submodules.sh -- script: tools/run_tests/sanity/check_cache_mk.sh +- script: tools/run_tests/sanity/check_version.py - script: tools/buildgen/generate_projects.sh -j 3 cpu_cost: 3 - script: tools/distrib/check_copyright.py From eb841e20108e0e54f99800f099c044f9f183a632 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 11 Feb 2016 15:49:16 -0800 Subject: [PATCH 004/236] Revert "Revert "Proto API for LB request/responses"" --- .gitmodules | 3 + BUILD | 33 +++ Makefile | 76 ++++++ binding.gyp | 5 + build.yaml | 27 +++ gRPC.podspec | 21 +- grpc.gemspec | 11 + package.json | 11 + .../lb_policies/load_balancer_api.c | 163 +++++++++++++ .../lb_policies/load_balancer_api.h | 85 +++++++ src/core/proto/grpc/lb/v0/load_balancer.pb.c | 150 ++++++++++++ src/core/proto/grpc/lb/v0/load_balancer.pb.h | 221 ++++++++++++++++++ src/proto/grpc/lb/v0/load_balancer.options | 6 + src/proto/grpc/lb/v0/load_balancer.proto | 144 ++++++++++++ src/python/grpcio/grpc_core_dependencies.py | 5 + .../sources_and_headers.json.template | 21 +- test/cpp/grpclb/grpclb_api_test.cc | 133 +++++++++++ third_party/nanopb | 1 + .../codegen/core/gen_load_balancing_proto.sh | 133 +++++++++++ tools/distrib/check_nanopb_output.sh | 86 +++++++ tools/doxygen/Doxyfile.core.internal | 11 + tools/jenkins/build_docker_and_run_tests.sh | 6 +- tools/run_tests/sanity/check_submodules.sh | 1 + tools/run_tests/sanity/sanity_tests.yaml | 1 + tools/run_tests/sources_and_headers.json | 41 +++- tools/run_tests/tests.json | 20 ++ vsprojects/vcxproj/grpc/grpc.vcxproj | 16 ++ vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 51 ++++ .../grpc_unsecure/grpc_unsecure.vcxproj | 16 ++ .../grpc_unsecure.vcxproj.filters | 51 ++++ .../grpclb_api_test/grpclb_api_test.vcxproj | 209 +++++++++++++++++ .../grpclb_api_test.vcxproj.filters | 39 ++++ 32 files changed, 1785 insertions(+), 12 deletions(-) create mode 100644 src/core/client_config/lb_policies/load_balancer_api.c create mode 100644 src/core/client_config/lb_policies/load_balancer_api.h create mode 100644 src/core/proto/grpc/lb/v0/load_balancer.pb.c create mode 100644 src/core/proto/grpc/lb/v0/load_balancer.pb.h create mode 100644 src/proto/grpc/lb/v0/load_balancer.options create mode 100644 src/proto/grpc/lb/v0/load_balancer.proto create mode 100644 test/cpp/grpclb/grpclb_api_test.cc create mode 160000 third_party/nanopb create mode 100755 tools/codegen/core/gen_load_balancing_proto.sh create mode 100755 tools/distrib/check_nanopb_output.sh create mode 100644 vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj create mode 100644 vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj.filters diff --git a/.gitmodules b/.gitmodules index 008bc5fae80..c37d0abdf0b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,6 @@ [submodule "third_party/boringssl"] path = third_party/boringssl url = https://boringssl.googlesource.com/boringssl +[submodule "third_party/nanopb"] + path = third_party/nanopb + url = https://github.com/nanopb/nanopb.git diff --git a/BUILD b/BUILD index 72d5caa8d43..31e0f3aacc6 100644 --- a/BUILD +++ b/BUILD @@ -182,6 +182,7 @@ cc_library( "src/core/client_config/client_config.h", "src/core/client_config/connector.h", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.h", "src/core/client_config/lb_policy.h", @@ -242,6 +243,7 @@ cc_library( "src/core/json/json_common.h", "src/core/json/json_reader.h", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/statistics/census_interface.h", "src/core/statistics/census_rpc_stats.h", "src/core/surface/api_trace.h", @@ -283,6 +285,10 @@ cc_library( "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", "src/core/census/rpc_metric_id.h", + "third_party/nanopb/pb.h", + "third_party/nanopb/pb_common.h", + "third_party/nanopb/pb_decode.h", + "third_party/nanopb/pb_encode.h", "src/core/httpcli/httpcli_security_connector.c", "src/core/security/base64.c", "src/core/security/client_auth_filter.c", @@ -319,6 +325,7 @@ cc_library( "src/core/client_config/connector.c", "src/core/client_config/default_initial_connect_string.c", "src/core/client_config/initial_connect_string.c", + "src/core/client_config/lb_policies/load_balancer_api.c", "src/core/client_config/lb_policies/pick_first.c", "src/core/client_config/lb_policies/round_robin.c", "src/core/client_config/lb_policy.c", @@ -382,6 +389,7 @@ cc_library( "src/core/json/json_reader.c", "src/core/json/json_string.c", "src/core/json/json_writer.c", + "src/core/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/surface/alarm.c", "src/core/surface/api_trace.c", "src/core/surface/byte_buffer.c", @@ -436,6 +444,9 @@ cc_library( "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tracing.c", + "third_party/nanopb/pb_common.c", + "third_party/nanopb/pb_decode.c", + "third_party/nanopb/pb_encode.c", ], hdrs = [ "include/grpc/grpc_security.h", @@ -484,6 +495,7 @@ cc_library( "src/core/client_config/client_config.h", "src/core/client_config/connector.h", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.h", "src/core/client_config/lb_policy.h", @@ -544,6 +556,7 @@ cc_library( "src/core/json/json_common.h", "src/core/json/json_reader.h", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/statistics/census_interface.h", "src/core/statistics/census_rpc_stats.h", "src/core/surface/api_trace.h", @@ -585,6 +598,10 @@ cc_library( "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", "src/core/census/rpc_metric_id.h", + "third_party/nanopb/pb.h", + "third_party/nanopb/pb_common.h", + "third_party/nanopb/pb_decode.h", + "third_party/nanopb/pb_encode.h", "src/core/surface/init_unsecure.c", "src/core/census/grpc_context.c", "src/core/census/grpc_filter.c", @@ -601,6 +618,7 @@ cc_library( "src/core/client_config/connector.c", "src/core/client_config/default_initial_connect_string.c", "src/core/client_config/initial_connect_string.c", + "src/core/client_config/lb_policies/load_balancer_api.c", "src/core/client_config/lb_policies/pick_first.c", "src/core/client_config/lb_policies/round_robin.c", "src/core/client_config/lb_policy.c", @@ -664,6 +682,7 @@ cc_library( "src/core/json/json_reader.c", "src/core/json/json_string.c", "src/core/json/json_writer.c", + "src/core/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/surface/alarm.c", "src/core/surface/api_trace.c", "src/core/surface/byte_buffer.c", @@ -718,6 +737,9 @@ cc_library( "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tracing.c", + "third_party/nanopb/pb_common.c", + "third_party/nanopb/pb_decode.c", + "third_party/nanopb/pb_encode.c", ], hdrs = [ "include/grpc/byte_buffer.h", @@ -1281,6 +1303,7 @@ objc_library( "src/core/client_config/connector.c", "src/core/client_config/default_initial_connect_string.c", "src/core/client_config/initial_connect_string.c", + "src/core/client_config/lb_policies/load_balancer_api.c", "src/core/client_config/lb_policies/pick_first.c", "src/core/client_config/lb_policies/round_robin.c", "src/core/client_config/lb_policy.c", @@ -1344,6 +1367,7 @@ objc_library( "src/core/json/json_reader.c", "src/core/json/json_string.c", "src/core/json/json_writer.c", + "src/core/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/surface/alarm.c", "src/core/surface/api_trace.c", "src/core/surface/byte_buffer.c", @@ -1398,6 +1422,9 @@ objc_library( "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tracing.c", + "third_party/nanopb/pb_common.c", + "third_party/nanopb/pb_decode.c", + "third_party/nanopb/pb_encode.c", ], hdrs = [ "include/grpc/grpc_security.h", @@ -1441,6 +1468,7 @@ objc_library( "src/core/client_config/client_config.h", "src/core/client_config/connector.h", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.h", "src/core/client_config/lb_policy.h", @@ -1501,6 +1529,7 @@ objc_library( "src/core/json/json_common.h", "src/core/json/json_reader.h", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/statistics/census_interface.h", "src/core/statistics/census_rpc_stats.h", "src/core/surface/api_trace.h", @@ -1542,6 +1571,10 @@ objc_library( "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", "src/core/census/rpc_metric_id.h", + "third_party/nanopb/pb.h", + "third_party/nanopb/pb_common.h", + "third_party/nanopb/pb_decode.h", + "third_party/nanopb/pb_encode.h", ], includes = [ "include", diff --git a/Makefile b/Makefile index 3c215b35b64..01bbb2cb3c4 100644 --- a/Makefile +++ b/Makefile @@ -940,6 +940,7 @@ grpc_csharp_plugin: $(BINDIR)/$(CONFIG)/grpc_csharp_plugin grpc_objective_c_plugin: $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin grpc_python_plugin: $(BINDIR)/$(CONFIG)/grpc_python_plugin grpc_ruby_plugin: $(BINDIR)/$(CONFIG)/grpc_ruby_plugin +grpclb_api_test: $(BINDIR)/$(CONFIG)/grpclb_api_test hybrid_end2end_test: $(BINDIR)/$(CONFIG)/hybrid_end2end_test interop_client: $(BINDIR)/$(CONFIG)/interop_client interop_server: $(BINDIR)/$(CONFIG)/interop_server @@ -1283,6 +1284,7 @@ buildtests_cxx: buildtests_zookeeper privatelibs_cxx \ $(BINDIR)/$(CONFIG)/generic_async_streaming_ping_pong_test \ $(BINDIR)/$(CONFIG)/generic_end2end_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/hybrid_end2end_test \ $(BINDIR)/$(CONFIG)/interop_client \ $(BINDIR)/$(CONFIG)/interop_server \ @@ -1591,6 +1593,8 @@ test_cxx: test_zookeeper buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/generic_async_streaming_ping_pong_test || ( echo test generic_async_streaming_ping_pong_test failed ; exit 1 ) $(E) "[RUN] Testing generic_end2end_test" $(Q) $(BINDIR)/$(CONFIG)/generic_end2end_test || ( echo test generic_end2end_test failed ; exit 1 ) + $(E) "[RUN] Testing grpclb_api_test" + $(Q) $(BINDIR)/$(CONFIG)/grpclb_api_test || ( echo test grpclb_api_test failed ; exit 1 ) $(E) "[RUN] Testing hybrid_end2end_test" $(Q) $(BINDIR)/$(CONFIG)/hybrid_end2end_test || ( echo test hybrid_end2end_test failed ; exit 1 ) $(E) "[RUN] Testing interop_test" @@ -1740,6 +1744,21 @@ $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc: $(Q) mkdir -p $(@D) $(Q) echo "$(GRPCXX_UNSECURE_PC_FILE)" | tr , '\n' >$@ +ifeq ($(NO_PROTOC),true) +$(GENDIR)/src/proto/grpc/lb/v0/load_balancer.pb.cc: protoc_dep_error +$(GENDIR)/src/proto/grpc/lb/v0/load_balancer.grpc.pb.cc: protoc_dep_error +else +$(GENDIR)/src/proto/grpc/lb/v0/load_balancer.pb.cc: src/proto/grpc/lb/v0/load_balancer.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) + $(E) "[PROTOC] Generating protobuf CC file from $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(PROTOC) --cpp_out=$(GENDIR) $< + +$(GENDIR)/src/proto/grpc/lb/v0/load_balancer.grpc.pb.cc: src/proto/grpc/lb/v0/load_balancer.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) + $(E) "[GRPC] Generating gRPC's protobuf service CC file from $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(PROTOC) --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $< +endif + ifeq ($(NO_PROTOC),true) $(GENDIR)/src/proto/grpc/testing/control.pb.cc: protoc_dep_error $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc: protoc_dep_error @@ -2348,6 +2367,7 @@ LIBGRPC_SRC = \ src/core/client_config/connector.c \ src/core/client_config/default_initial_connect_string.c \ src/core/client_config/initial_connect_string.c \ + src/core/client_config/lb_policies/load_balancer_api.c \ src/core/client_config/lb_policies/pick_first.c \ src/core/client_config/lb_policies/round_robin.c \ src/core/client_config/lb_policy.c \ @@ -2411,6 +2431,7 @@ LIBGRPC_SRC = \ src/core/json/json_reader.c \ src/core/json/json_string.c \ src/core/json/json_writer.c \ + src/core/proto/grpc/lb/v0/load_balancer.pb.c \ src/core/surface/alarm.c \ src/core/surface/api_trace.c \ src/core/surface/byte_buffer.c \ @@ -2465,6 +2486,9 @@ LIBGRPC_SRC = \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tracing.c \ + third_party/nanopb/pb_common.c \ + third_party/nanopb/pb_decode.c \ + third_party/nanopb/pb_encode.c \ PUBLIC_HEADERS_C += \ include/grpc/grpc_security.h \ @@ -2632,6 +2656,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/client_config/connector.c \ src/core/client_config/default_initial_connect_string.c \ src/core/client_config/initial_connect_string.c \ + src/core/client_config/lb_policies/load_balancer_api.c \ src/core/client_config/lb_policies/pick_first.c \ src/core/client_config/lb_policies/round_robin.c \ src/core/client_config/lb_policy.c \ @@ -2695,6 +2720,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/json/json_reader.c \ src/core/json/json_string.c \ src/core/json/json_writer.c \ + src/core/proto/grpc/lb/v0/load_balancer.pb.c \ src/core/surface/alarm.c \ src/core/surface/api_trace.c \ src/core/surface/byte_buffer.c \ @@ -2749,6 +2775,9 @@ LIBGRPC_UNSECURE_SRC = \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tracing.c \ + third_party/nanopb/pb_common.c \ + third_party/nanopb/pb_decode.c \ + third_party/nanopb/pb_encode.c \ PUBLIC_HEADERS_C += \ include/grpc/byte_buffer.h \ @@ -9670,6 +9699,53 @@ ifneq ($(NO_DEPS),true) endif +GRPCLB_API_TEST_SRC = \ + $(GENDIR)/src/proto/grpc/lb/v0/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v0/load_balancer.grpc.pb.cc \ + test/cpp/grpclb/grpclb_api_test.cc \ + +GRPCLB_API_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPCLB_API_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpclb_api_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/grpclb_api_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/grpclb_api_test: $(PROTOBUF_DEP) $(GRPCLB_API_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(GRPCLB_API_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpclb_api_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/lb/v0/load_balancer.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a + +$(OBJDIR)/$(CONFIG)/test/cpp/grpclb/grpclb_api_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a + +deps_grpclb_api_test: $(GRPCLB_API_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPCLB_API_TEST_OBJS:.o=.dep) +endif +endif +$(OBJDIR)/$(CONFIG)/test/cpp/grpclb/grpclb_api_test.o: $(GENDIR)/src/proto/grpc/lb/v0/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v0/load_balancer.grpc.pb.cc + + HYBRID_END2END_TEST_SRC = \ test/cpp/end2end/hybrid_end2end_test.cc \ diff --git a/binding.gyp b/binding.gyp index 3659bfcf13e..f7a21e56cd2 100644 --- a/binding.gyp +++ b/binding.gyp @@ -591,6 +591,7 @@ 'src/core/client_config/connector.c', 'src/core/client_config/default_initial_connect_string.c', 'src/core/client_config/initial_connect_string.c', + 'src/core/client_config/lb_policies/load_balancer_api.c', 'src/core/client_config/lb_policies/pick_first.c', 'src/core/client_config/lb_policies/round_robin.c', 'src/core/client_config/lb_policy.c', @@ -654,6 +655,7 @@ 'src/core/json/json_reader.c', 'src/core/json/json_string.c', 'src/core/json/json_writer.c', + 'src/core/proto/grpc/lb/v0/load_balancer.pb.c', 'src/core/surface/alarm.c', 'src/core/surface/api_trace.c', 'src/core/surface/byte_buffer.c', @@ -708,6 +710,9 @@ 'src/core/census/operation.c', 'src/core/census/placeholders.c', 'src/core/census/tracing.c', + 'third_party/nanopb/pb_common.c', + 'third_party/nanopb/pb_decode.c', + 'third_party/nanopb/pb_encode.c', ], "conditions": [ ['OS == "mac"', { diff --git a/build.yaml b/build.yaml index 7f33ef3f0e5..ffa16675d82 100644 --- a/build.yaml +++ b/build.yaml @@ -258,6 +258,7 @@ filegroups: - src/core/client_config/client_config.h - src/core/client_config/connector.h - src/core/client_config/initial_connect_string.h + - src/core/client_config/lb_policies/load_balancer_api.h - src/core/client_config/lb_policies/pick_first.h - src/core/client_config/lb_policies/round_robin.h - src/core/client_config/lb_policy.h @@ -318,6 +319,7 @@ filegroups: - src/core/json/json_common.h - src/core/json/json_reader.h - src/core/json/json_writer.h + - src/core/proto/grpc/lb/v0/load_balancer.pb.h - src/core/statistics/census_interface.h - src/core/statistics/census_rpc_stats.h - src/core/surface/api_trace.h @@ -373,6 +375,7 @@ filegroups: - src/core/client_config/connector.c - src/core/client_config/default_initial_connect_string.c - src/core/client_config/initial_connect_string.c + - src/core/client_config/lb_policies/load_balancer_api.c - src/core/client_config/lb_policies/pick_first.c - src/core/client_config/lb_policies/round_robin.c - src/core/client_config/lb_policy.c @@ -436,6 +439,7 @@ filegroups: - src/core/json/json_reader.c - src/core/json/json_string.c - src/core/json/json_writer.c + - src/core/proto/grpc/lb/v0/load_balancer.pb.c - src/core/surface/alarm.c - src/core/surface/api_trace.c - src/core/surface/byte_buffer.c @@ -511,6 +515,16 @@ filegroups: - test/core/util/port_posix.c - test/core/util/port_windows.c - test/core/util/slice_splitter.c +- name: nanopb + headers: + - third_party/nanopb/pb.h + - third_party/nanopb/pb_common.h + - third_party/nanopb/pb_decode.h + - third_party/nanopb/pb_encode.h + src: + - third_party/nanopb/pb_common.c + - third_party/nanopb/pb_decode.c + - third_party/nanopb/pb_encode.c libs: - name: gpr build: all @@ -582,6 +596,7 @@ libs: - grpc_codegen - grpc_base - census + - nanopb secure: true vs_packages: - grpc.dependencies.openssl @@ -630,6 +645,7 @@ libs: - grpc_base - grpc_codegen - census + - nanopb secure: false vs_project_guid: '{46CEDFFF-9692-456A-AA24-38B5D6BCF4C5}' - name: grpc_zookeeper @@ -2174,6 +2190,17 @@ targets: secure: false vs_config_type: Application vs_project_guid: '{069E9D05-B78B-4751-9252-D21EBAE7DE8E}' +- name: grpclb_api_test + build: test + language: c++ + src: + - src/proto/grpc/lb/v0/load_balancer.proto + - test/cpp/grpclb/grpclb_api_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc - name: hybrid_end2end_test build: test language: c++ diff --git a/gRPC.podspec b/gRPC.podspec index 5b4d24e4820..05e3802b77f 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -186,6 +186,7 @@ Pod::Spec.new do |s| 'src/core/client_config/client_config.h', 'src/core/client_config/connector.h', 'src/core/client_config/initial_connect_string.h', + 'src/core/client_config/lb_policies/load_balancer_api.h', 'src/core/client_config/lb_policies/pick_first.h', 'src/core/client_config/lb_policies/round_robin.h', 'src/core/client_config/lb_policy.h', @@ -246,6 +247,7 @@ Pod::Spec.new do |s| 'src/core/json/json_common.h', 'src/core/json/json_reader.h', 'src/core/json/json_writer.h', + 'src/core/proto/grpc/lb/v0/load_balancer.pb.h', 'src/core/statistics/census_interface.h', 'src/core/statistics/census_rpc_stats.h', 'src/core/surface/api_trace.h', @@ -287,6 +289,10 @@ Pod::Spec.new do |s| 'src/core/transport/transport_impl.h', 'src/core/census/aggregation.h', 'src/core/census/rpc_metric_id.h', + 'third_party/nanopb/pb.h', + 'third_party/nanopb/pb_common.h', + 'third_party/nanopb/pb_decode.h', + 'third_party/nanopb/pb_encode.h', 'include/grpc/grpc_security.h', 'include/grpc/impl/codegen/byte_buffer.h', 'include/grpc/impl/codegen/compression_types.h', @@ -336,6 +342,7 @@ Pod::Spec.new do |s| 'src/core/client_config/connector.c', 'src/core/client_config/default_initial_connect_string.c', 'src/core/client_config/initial_connect_string.c', + 'src/core/client_config/lb_policies/load_balancer_api.c', 'src/core/client_config/lb_policies/pick_first.c', 'src/core/client_config/lb_policies/round_robin.c', 'src/core/client_config/lb_policy.c', @@ -399,6 +406,7 @@ Pod::Spec.new do |s| 'src/core/json/json_reader.c', 'src/core/json/json_string.c', 'src/core/json/json_writer.c', + 'src/core/proto/grpc/lb/v0/load_balancer.pb.c', 'src/core/surface/alarm.c', 'src/core/surface/api_trace.c', 'src/core/surface/byte_buffer.c', @@ -452,7 +460,10 @@ Pod::Spec.new do |s| 'src/core/census/initialize.c', 'src/core/census/operation.c', 'src/core/census/placeholders.c', - 'src/core/census/tracing.c' + 'src/core/census/tracing.c', + 'third_party/nanopb/pb_common.c', + 'third_party/nanopb/pb_decode.c', + 'third_party/nanopb/pb_encode.c' ss.private_header_files = 'src/core/profiling/timers.h', 'src/core/support/block_annotate.h', @@ -492,6 +503,7 @@ Pod::Spec.new do |s| 'src/core/client_config/client_config.h', 'src/core/client_config/connector.h', 'src/core/client_config/initial_connect_string.h', + 'src/core/client_config/lb_policies/load_balancer_api.h', 'src/core/client_config/lb_policies/pick_first.h', 'src/core/client_config/lb_policies/round_robin.h', 'src/core/client_config/lb_policy.h', @@ -552,6 +564,7 @@ Pod::Spec.new do |s| 'src/core/json/json_common.h', 'src/core/json/json_reader.h', 'src/core/json/json_writer.h', + 'src/core/proto/grpc/lb/v0/load_balancer.pb.h', 'src/core/statistics/census_interface.h', 'src/core/statistics/census_rpc_stats.h', 'src/core/surface/api_trace.h', @@ -592,7 +605,11 @@ Pod::Spec.new do |s| 'src/core/transport/transport.h', 'src/core/transport/transport_impl.h', 'src/core/census/aggregation.h', - 'src/core/census/rpc_metric_id.h' + 'src/core/census/rpc_metric_id.h', + 'third_party/nanopb/pb.h', + 'third_party/nanopb/pb_common.h', + 'third_party/nanopb/pb_decode.h', + 'third_party/nanopb/pb_encode.h' ss.header_mappings_dir = '.' # This isn't officially supported in Cocoapods. We've asked for an alternative: diff --git a/grpc.gemspec b/grpc.gemspec index 9e4683f41c7..1ef26761b72 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -182,6 +182,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/client_config/client_config.h ) s.files += %w( src/core/client_config/connector.h ) s.files += %w( src/core/client_config/initial_connect_string.h ) + s.files += %w( src/core/client_config/lb_policies/load_balancer_api.h ) s.files += %w( src/core/client_config/lb_policies/pick_first.h ) s.files += %w( src/core/client_config/lb_policies/round_robin.h ) s.files += %w( src/core/client_config/lb_policy.h ) @@ -242,6 +243,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/json/json_common.h ) s.files += %w( src/core/json/json_reader.h ) s.files += %w( src/core/json/json_writer.h ) + s.files += %w( src/core/proto/grpc/lb/v0/load_balancer.pb.h ) s.files += %w( src/core/statistics/census_interface.h ) s.files += %w( src/core/statistics/census_rpc_stats.h ) s.files += %w( src/core/surface/api_trace.h ) @@ -283,6 +285,10 @@ Gem::Specification.new do |s| s.files += %w( src/core/transport/transport_impl.h ) s.files += %w( src/core/census/aggregation.h ) s.files += %w( src/core/census/rpc_metric_id.h ) + s.files += %w( third_party/nanopb/pb.h ) + s.files += %w( third_party/nanopb/pb_common.h ) + s.files += %w( third_party/nanopb/pb_decode.h ) + s.files += %w( third_party/nanopb/pb_encode.h ) s.files += %w( src/core/httpcli/httpcli_security_connector.c ) s.files += %w( src/core/security/base64.c ) s.files += %w( src/core/security/client_auth_filter.c ) @@ -319,6 +325,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/client_config/connector.c ) s.files += %w( src/core/client_config/default_initial_connect_string.c ) s.files += %w( src/core/client_config/initial_connect_string.c ) + s.files += %w( src/core/client_config/lb_policies/load_balancer_api.c ) s.files += %w( src/core/client_config/lb_policies/pick_first.c ) s.files += %w( src/core/client_config/lb_policies/round_robin.c ) s.files += %w( src/core/client_config/lb_policy.c ) @@ -382,6 +389,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/json/json_reader.c ) s.files += %w( src/core/json/json_string.c ) s.files += %w( src/core/json/json_writer.c ) + s.files += %w( src/core/proto/grpc/lb/v0/load_balancer.pb.c ) s.files += %w( src/core/surface/alarm.c ) s.files += %w( src/core/surface/api_trace.c ) s.files += %w( src/core/surface/byte_buffer.c ) @@ -436,6 +444,9 @@ Gem::Specification.new do |s| s.files += %w( src/core/census/operation.c ) s.files += %w( src/core/census/placeholders.c ) s.files += %w( src/core/census/tracing.c ) + s.files += %w( third_party/nanopb/pb_common.c ) + s.files += %w( third_party/nanopb/pb_decode.c ) + s.files += %w( third_party/nanopb/pb_encode.c ) s.files += %w( third_party/boringssl/crypto/aes/internal.h ) s.files += %w( third_party/boringssl/crypto/asn1/asn1_locl.h ) s.files += %w( third_party/boringssl/crypto/bio/internal.h ) diff --git a/package.json b/package.json index 1b79ecf92dc..b8f8c0a256e 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "src/core/client_config/client_config.h", "src/core/client_config/connector.h", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.h", "src/core/client_config/lb_policy.h", @@ -188,6 +189,7 @@ "src/core/json/json_common.h", "src/core/json/json_reader.h", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/statistics/census_interface.h", "src/core/statistics/census_rpc_stats.h", "src/core/surface/api_trace.h", @@ -229,6 +231,10 @@ "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", "src/core/census/rpc_metric_id.h", + "third_party/nanopb/pb.h", + "third_party/nanopb/pb_common.h", + "third_party/nanopb/pb_decode.h", + "third_party/nanopb/pb_encode.h", "src/core/httpcli/httpcli_security_connector.c", "src/core/security/base64.c", "src/core/security/client_auth_filter.c", @@ -265,6 +271,7 @@ "src/core/client_config/connector.c", "src/core/client_config/default_initial_connect_string.c", "src/core/client_config/initial_connect_string.c", + "src/core/client_config/lb_policies/load_balancer_api.c", "src/core/client_config/lb_policies/pick_first.c", "src/core/client_config/lb_policies/round_robin.c", "src/core/client_config/lb_policy.c", @@ -328,6 +335,7 @@ "src/core/json/json_reader.c", "src/core/json/json_string.c", "src/core/json/json_writer.c", + "src/core/proto/grpc/lb/v0/load_balancer.pb.c", "src/core/surface/alarm.c", "src/core/surface/api_trace.c", "src/core/surface/byte_buffer.c", @@ -382,6 +390,9 @@ "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tracing.c", + "third_party/nanopb/pb_common.c", + "third_party/nanopb/pb_decode.c", + "third_party/nanopb/pb_encode.c", "third_party/zlib/crc32.h", "third_party/zlib/deflate.h", "third_party/zlib/gzguts.h", diff --git a/src/core/client_config/lb_policies/load_balancer_api.c b/src/core/client_config/lb_policies/load_balancer_api.c new file mode 100644 index 00000000000..a6b5785fe4e --- /dev/null +++ b/src/core/client_config/lb_policies/load_balancer_api.c @@ -0,0 +1,163 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/client_config/lb_policies/load_balancer_api.h" +#include "third_party/nanopb/pb_decode.h" +#include "third_party/nanopb/pb_encode.h" + +#include + +typedef struct decode_serverlist_arg { + int first_pass; + int i; + size_t num_servers; + grpc_grpclb_server **servers; +} decode_serverlist_arg; + +/* invoked once for every Server in ServerList */ +static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field, + void **arg) { + decode_serverlist_arg *dec_arg = *arg; + if (dec_arg->first_pass != 0) { /* first pass */ + grpc_grpclb_server server; + if (!pb_decode(stream, grpc_lb_v0_Server_fields, &server)) { + return false; + } + dec_arg->num_servers++; + } else { /* second pass */ + grpc_grpclb_server *server = gpr_malloc(sizeof(grpc_grpclb_server)); + GPR_ASSERT(dec_arg->num_servers > 0); + if (dec_arg->i == 0) { /* first iteration of second pass */ + dec_arg->servers = + gpr_malloc(sizeof(grpc_grpclb_server *) * dec_arg->num_servers); + } + if (!pb_decode(stream, grpc_lb_v0_Server_fields, server)) { + return false; + } + dec_arg->servers[dec_arg->i++] = server; + } + + return true; +} + +grpc_grpclb_request *grpc_grpclb_request_create(const char *lb_service_name) { + grpc_grpclb_request *req = gpr_malloc(sizeof(grpc_grpclb_request)); + + req->has_client_stats = 0; /* TODO(dgq): add support for stats once defined */ + req->has_initial_request = 1; + req->initial_request.has_name = 1; + strncpy(req->initial_request.name, lb_service_name, + GRPC_GRPCLB_SERVICE_NAME_MAX_LENGTH); + return req; +} + +gpr_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request) { + size_t encoded_length; + pb_ostream_t sizestream; + pb_ostream_t outputstream; + gpr_slice slice; + memset(&sizestream, 0, sizeof(pb_ostream_t)); + pb_encode(&sizestream, grpc_lb_v0_LoadBalanceRequest_fields, request); + encoded_length = sizestream.bytes_written; + + slice = gpr_slice_malloc(encoded_length); + outputstream = + pb_ostream_from_buffer(GPR_SLICE_START_PTR(slice), encoded_length); + GPR_ASSERT(pb_encode(&outputstream, grpc_lb_v0_LoadBalanceRequest_fields, + request) != 0); + return slice; +} + +void grpc_grpclb_request_destroy(grpc_grpclb_request *request) { + gpr_free(request); +} + +grpc_grpclb_response *grpc_grpclb_response_parse(gpr_slice encoded_response) { + bool status; + pb_istream_t stream = + pb_istream_from_buffer(GPR_SLICE_START_PTR(encoded_response), + GPR_SLICE_LENGTH(encoded_response)); + grpc_grpclb_response *res = gpr_malloc(sizeof(grpc_grpclb_response)); + memset(res, 0, sizeof(*res)); + status = pb_decode(&stream, grpc_lb_v0_LoadBalanceResponse_fields, res); + GPR_ASSERT(status == true); + return res; +} + +grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist( + gpr_slice encoded_response) { + grpc_grpclb_serverlist *sl = gpr_malloc(sizeof(grpc_grpclb_serverlist)); + bool status; + decode_serverlist_arg arg; + pb_istream_t stream = + pb_istream_from_buffer(GPR_SLICE_START_PTR(encoded_response), + GPR_SLICE_LENGTH(encoded_response)); + pb_istream_t stream_at_start = stream; + grpc_grpclb_response *res = gpr_malloc(sizeof(grpc_grpclb_response)); + memset(res, 0, sizeof(*res)); + memset(&arg, 0, sizeof(decode_serverlist_arg)); + + res->server_list.servers.funcs.decode = decode_serverlist; + res->server_list.servers.arg = &arg; + arg.first_pass = 1; + status = pb_decode(&stream, grpc_lb_v0_LoadBalanceResponse_fields, res); + GPR_ASSERT(status == true); + GPR_ASSERT(arg.num_servers > 0); + + arg.first_pass = 0; + status = + pb_decode(&stream_at_start, grpc_lb_v0_LoadBalanceResponse_fields, res); + GPR_ASSERT(status == true); + GPR_ASSERT(arg.servers != NULL); + + sl->num_servers = arg.num_servers; + sl->servers = arg.servers; + if (res->server_list.has_expiration_interval) { + sl->expiration_interval = res->server_list.expiration_interval; + } + grpc_grpclb_response_destroy(res); + return sl; +} + +void grpc_grpclb_destroy_serverlist(grpc_grpclb_serverlist *serverlist) { + size_t i; + for (i = 0; i < serverlist->num_servers; i++) { + gpr_free(serverlist->servers[i]); + } + gpr_free(serverlist->servers); + gpr_free(serverlist); +} + +void grpc_grpclb_response_destroy(grpc_grpclb_response *response) { + gpr_free(response); +} diff --git a/src/core/client_config/lb_policies/load_balancer_api.h b/src/core/client_config/lb_policies/load_balancer_api.h new file mode 100644 index 00000000000..4dbe1d6c224 --- /dev/null +++ b/src/core/client_config/lb_policies/load_balancer_api.h @@ -0,0 +1,85 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H +#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H + +#include + +#include "src/core/client_config/lb_policy_factory.h" +#include "src/core/proto/grpc/lb/v0/load_balancer.pb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GRPC_GRPCLB_SERVICE_NAME_MAX_LENGTH 128 + +typedef grpc_lb_v0_LoadBalanceRequest grpc_grpclb_request; +typedef grpc_lb_v0_LoadBalanceResponse grpc_grpclb_response; +typedef grpc_lb_v0_Server grpc_grpclb_server; +typedef grpc_lb_v0_Duration grpc_grpclb_duration; +typedef struct grpc_grpclb_serverlist { + grpc_grpclb_server **servers; + size_t num_servers; + grpc_grpclb_duration expiration_interval; +} grpc_grpclb_serverlist; + +/** Create a request for a gRPC LB service under \a lb_service_name */ +grpc_grpclb_request *grpc_grpclb_request_create(const char *lb_service_name); + +/** Protocol Buffers v3-encode \a request */ +gpr_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request); + +/** Destroy \a request */ +void grpc_grpclb_request_destroy(grpc_grpclb_request *request); + +/** Parse (ie, decode) the bytes in \a encoded_response as a \a + * grpc_grpclb_response */ +grpc_grpclb_response *grpc_grpclb_response_parse(gpr_slice encoded_response); + +/** Destroy \a serverlist */ +void grpc_grpclb_destroy_serverlist(grpc_grpclb_serverlist *serverlist); + +/** Parse the list of servers from an encoded \a grpc_grpclb_response */ +grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist( + gpr_slice encoded_response); + +/** Destroy \a response */ +void grpc_grpclb_response_destroy(grpc_grpclb_response *response); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H */ diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.c b/src/core/proto/grpc/lb/v0/load_balancer.pb.c new file mode 100644 index 00000000000..bbb6fa82249 --- /dev/null +++ b/src/core/proto/grpc/lb/v0/load_balancer.pb.c @@ -0,0 +1,150 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +/* Automatically generated nanopb constant definitions */ +/* Generated by nanopb-0.3.4-dev */ + +#include "src/core/proto/grpc/lb/v0/load_balancer.pb.h" + +#if PB_PROTO_HEADER_VERSION != 30 +#error Regenerate this file with the current version of nanopb generator. +#endif + +const pb_field_t grpc_lb_v0_Duration_fields[3] = { + PB_FIELD(1, INT64, OPTIONAL, STATIC, FIRST, grpc_lb_v0_Duration, seconds, + seconds, 0), + PB_FIELD(2, INT32, OPTIONAL, STATIC, OTHER, grpc_lb_v0_Duration, nanos, + seconds, 0), + PB_LAST_FIELD}; + +const pb_field_t grpc_lb_v0_LoadBalanceRequest_fields[3] = { + PB_FIELD(1, MESSAGE, OPTIONAL, STATIC, FIRST, grpc_lb_v0_LoadBalanceRequest, + initial_request, initial_request, + &grpc_lb_v0_InitialLoadBalanceRequest_fields), + PB_FIELD(2, MESSAGE, OPTIONAL, STATIC, OTHER, grpc_lb_v0_LoadBalanceRequest, + client_stats, initial_request, &grpc_lb_v0_ClientStats_fields), + PB_LAST_FIELD}; + +const pb_field_t grpc_lb_v0_InitialLoadBalanceRequest_fields[2] = { + PB_FIELD(1, STRING, OPTIONAL, STATIC, FIRST, + grpc_lb_v0_InitialLoadBalanceRequest, name, name, 0), + PB_LAST_FIELD}; + +const pb_field_t grpc_lb_v0_ClientStats_fields[4] = { + PB_FIELD(1, INT64, OPTIONAL, STATIC, FIRST, grpc_lb_v0_ClientStats, + total_requests, total_requests, 0), + PB_FIELD(2, INT64, OPTIONAL, STATIC, OTHER, grpc_lb_v0_ClientStats, + client_rpc_errors, total_requests, 0), + PB_FIELD(3, INT64, OPTIONAL, STATIC, OTHER, grpc_lb_v0_ClientStats, + dropped_requests, client_rpc_errors, 0), + PB_LAST_FIELD}; + +const pb_field_t grpc_lb_v0_LoadBalanceResponse_fields[3] = { + PB_FIELD(1, MESSAGE, OPTIONAL, STATIC, FIRST, + grpc_lb_v0_LoadBalanceResponse, initial_response, initial_response, + &grpc_lb_v0_InitialLoadBalanceResponse_fields), + PB_FIELD(2, MESSAGE, OPTIONAL, STATIC, OTHER, + grpc_lb_v0_LoadBalanceResponse, server_list, initial_response, + &grpc_lb_v0_ServerList_fields), + PB_LAST_FIELD}; + +const pb_field_t grpc_lb_v0_InitialLoadBalanceResponse_fields[4] = { + PB_FIELD(1, STRING, OPTIONAL, STATIC, FIRST, + grpc_lb_v0_InitialLoadBalanceResponse, client_config, + client_config, 0), + PB_FIELD(2, STRING, OPTIONAL, STATIC, OTHER, + grpc_lb_v0_InitialLoadBalanceResponse, load_balancer_delegate, + client_config, 0), + PB_FIELD(3, MESSAGE, OPTIONAL, STATIC, OTHER, + grpc_lb_v0_InitialLoadBalanceResponse, + client_stats_report_interval, load_balancer_delegate, + &grpc_lb_v0_Duration_fields), + PB_LAST_FIELD}; + +const pb_field_t grpc_lb_v0_ServerList_fields[3] = { + PB_FIELD(1, MESSAGE, REPEATED, CALLBACK, FIRST, grpc_lb_v0_ServerList, + servers, servers, &grpc_lb_v0_Server_fields), + PB_FIELD(3, MESSAGE, OPTIONAL, STATIC, OTHER, grpc_lb_v0_ServerList, + expiration_interval, servers, &grpc_lb_v0_Duration_fields), + PB_LAST_FIELD}; + +const pb_field_t grpc_lb_v0_Server_fields[5] = { + PB_FIELD(1, STRING, OPTIONAL, STATIC, FIRST, grpc_lb_v0_Server, ip_address, + ip_address, 0), + PB_FIELD(2, INT32, OPTIONAL, STATIC, OTHER, grpc_lb_v0_Server, port, + ip_address, 0), + PB_FIELD(3, BYTES, OPTIONAL, STATIC, OTHER, grpc_lb_v0_Server, + load_balance_token, port, 0), + PB_FIELD(4, BOOL, OPTIONAL, STATIC, OTHER, grpc_lb_v0_Server, drop_request, + load_balance_token, 0), + PB_LAST_FIELD}; + +/* Check that field information fits in pb_field_t */ +#if !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_32BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in 8 or 16 bit + * field descriptors. + */ +PB_STATIC_ASSERT( + (pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 65536 && + pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 65536 && + pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 65536 && + pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 65536 && + pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, + client_stats_report_interval) < 65536 && + pb_membersize(grpc_lb_v0_ServerList, servers) < 65536 && + pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 65536), + YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server) +#endif + +#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_16BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in the default + * 8 bit descriptors. + */ +PB_STATIC_ASSERT( + (pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 256 && + pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 256 && + pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 256 && + pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 256 && + pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, + client_stats_report_interval) < 256 && + pb_membersize(grpc_lb_v0_ServerList, servers) < 256 && + pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 256), + YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server) +#endif diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.h b/src/core/proto/grpc/lb/v0/load_balancer.pb.h new file mode 100644 index 00000000000..ac836658399 --- /dev/null +++ b/src/core/proto/grpc/lb/v0/load_balancer.pb.h @@ -0,0 +1,221 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +/* Automatically generated nanopb header */ +/* Generated by nanopb-0.3.4-dev */ + +#ifndef PB_LOAD_BALANCER_PB_H_INCLUDED +#define PB_LOAD_BALANCER_PB_H_INCLUDED +#include "third_party/nanopb/pb.h" +#if PB_PROTO_HEADER_VERSION != 30 +#error Regenerate this file with the current version of nanopb generator. +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Enum definitions */ +/* Struct definitions */ +typedef struct _grpc_lb_v0_ClientStats { + bool has_total_requests; + int64_t total_requests; + bool has_client_rpc_errors; + int64_t client_rpc_errors; + bool has_dropped_requests; + int64_t dropped_requests; +} grpc_lb_v0_ClientStats; + +typedef struct _grpc_lb_v0_Duration { + bool has_seconds; + int64_t seconds; + bool has_nanos; + int32_t nanos; +} grpc_lb_v0_Duration; + +typedef struct _grpc_lb_v0_InitialLoadBalanceRequest { + bool has_name; + char name[128]; +} grpc_lb_v0_InitialLoadBalanceRequest; + +typedef PB_BYTES_ARRAY_T(64) grpc_lb_v0_Server_load_balance_token_t; +typedef struct _grpc_lb_v0_Server { + bool has_ip_address; + char ip_address[46]; + bool has_port; + int32_t port; + bool has_load_balance_token; + grpc_lb_v0_Server_load_balance_token_t load_balance_token; + bool has_drop_request; + bool drop_request; +} grpc_lb_v0_Server; + +typedef struct _grpc_lb_v0_InitialLoadBalanceResponse { + bool has_client_config; + char client_config[64]; + bool has_load_balancer_delegate; + char load_balancer_delegate[64]; + bool has_client_stats_report_interval; + grpc_lb_v0_Duration client_stats_report_interval; +} grpc_lb_v0_InitialLoadBalanceResponse; + +typedef struct _grpc_lb_v0_LoadBalanceRequest { + bool has_initial_request; + grpc_lb_v0_InitialLoadBalanceRequest initial_request; + bool has_client_stats; + grpc_lb_v0_ClientStats client_stats; +} grpc_lb_v0_LoadBalanceRequest; + +typedef struct _grpc_lb_v0_ServerList { + pb_callback_t servers; + bool has_expiration_interval; + grpc_lb_v0_Duration expiration_interval; +} grpc_lb_v0_ServerList; + +typedef struct _grpc_lb_v0_LoadBalanceResponse { + bool has_initial_response; + grpc_lb_v0_InitialLoadBalanceResponse initial_response; + bool has_server_list; + grpc_lb_v0_ServerList server_list; +} grpc_lb_v0_LoadBalanceResponse; + +/* Default values for struct fields */ + +/* Initializer values for message structs */ +#define grpc_lb_v0_Duration_init_default \ + { false, 0, false, 0 } +#define grpc_lb_v0_LoadBalanceRequest_init_default \ + { \ + false, grpc_lb_v0_InitialLoadBalanceRequest_init_default, false, \ + grpc_lb_v0_ClientStats_init_default \ + } +#define grpc_lb_v0_InitialLoadBalanceRequest_init_default \ + { false, "" } +#define grpc_lb_v0_ClientStats_init_default \ + { false, 0, false, 0, false, 0 } +#define grpc_lb_v0_LoadBalanceResponse_init_default \ + { \ + false, grpc_lb_v0_InitialLoadBalanceResponse_init_default, false, \ + grpc_lb_v0_ServerList_init_default \ + } +#define grpc_lb_v0_InitialLoadBalanceResponse_init_default \ + { false, "", false, "", false, grpc_lb_v0_Duration_init_default } +#define grpc_lb_v0_ServerList_init_default \ + { \ + { \ + { NULL } \ + , NULL \ + } \ + , false, grpc_lb_v0_Duration_init_default \ + } +#define grpc_lb_v0_Server_init_default \ + { false, "", false, 0, false, {0, {0}}, false, 0 } +#define grpc_lb_v0_Duration_init_zero \ + { false, 0, false, 0 } +#define grpc_lb_v0_LoadBalanceRequest_init_zero \ + { \ + false, grpc_lb_v0_InitialLoadBalanceRequest_init_zero, false, \ + grpc_lb_v0_ClientStats_init_zero \ + } +#define grpc_lb_v0_InitialLoadBalanceRequest_init_zero \ + { false, "" } +#define grpc_lb_v0_ClientStats_init_zero \ + { false, 0, false, 0, false, 0 } +#define grpc_lb_v0_LoadBalanceResponse_init_zero \ + { \ + false, grpc_lb_v0_InitialLoadBalanceResponse_init_zero, false, \ + grpc_lb_v0_ServerList_init_zero \ + } +#define grpc_lb_v0_InitialLoadBalanceResponse_init_zero \ + { false, "", false, "", false, grpc_lb_v0_Duration_init_zero } +#define grpc_lb_v0_ServerList_init_zero \ + { \ + { \ + { NULL } \ + , NULL \ + } \ + , false, grpc_lb_v0_Duration_init_zero \ + } +#define grpc_lb_v0_Server_init_zero \ + { false, "", false, 0, false, {0, {0}}, false, 0 } + +/* Field tags (for use in manual encoding/decoding) */ +#define grpc_lb_v0_ClientStats_total_requests_tag 1 +#define grpc_lb_v0_ClientStats_client_rpc_errors_tag 2 +#define grpc_lb_v0_ClientStats_dropped_requests_tag 3 +#define grpc_lb_v0_Duration_seconds_tag 1 +#define grpc_lb_v0_Duration_nanos_tag 2 +#define grpc_lb_v0_InitialLoadBalanceRequest_name_tag 1 +#define grpc_lb_v0_Server_ip_address_tag 1 +#define grpc_lb_v0_Server_port_tag 2 +#define grpc_lb_v0_Server_load_balance_token_tag 3 +#define grpc_lb_v0_Server_drop_request_tag 4 +#define grpc_lb_v0_InitialLoadBalanceResponse_client_config_tag 1 +#define grpc_lb_v0_InitialLoadBalanceResponse_load_balancer_delegate_tag 2 +#define grpc_lb_v0_InitialLoadBalanceResponse_client_stats_report_interval_tag 3 +#define grpc_lb_v0_LoadBalanceRequest_initial_request_tag 1 +#define grpc_lb_v0_LoadBalanceRequest_client_stats_tag 2 +#define grpc_lb_v0_ServerList_servers_tag 1 +#define grpc_lb_v0_ServerList_expiration_interval_tag 3 +#define grpc_lb_v0_LoadBalanceResponse_initial_response_tag 1 +#define grpc_lb_v0_LoadBalanceResponse_server_list_tag 2 + +/* Struct field encoding specification for nanopb */ +extern const pb_field_t grpc_lb_v0_Duration_fields[3]; +extern const pb_field_t grpc_lb_v0_LoadBalanceRequest_fields[3]; +extern const pb_field_t grpc_lb_v0_InitialLoadBalanceRequest_fields[2]; +extern const pb_field_t grpc_lb_v0_ClientStats_fields[4]; +extern const pb_field_t grpc_lb_v0_LoadBalanceResponse_fields[3]; +extern const pb_field_t grpc_lb_v0_InitialLoadBalanceResponse_fields[4]; +extern const pb_field_t grpc_lb_v0_ServerList_fields[3]; +extern const pb_field_t grpc_lb_v0_Server_fields[5]; + +/* Maximum encoded size of messages (where known) */ +#define grpc_lb_v0_Duration_size 22 +#define grpc_lb_v0_LoadBalanceRequest_size 169 +#define grpc_lb_v0_InitialLoadBalanceRequest_size 131 +#define grpc_lb_v0_ClientStats_size 33 +#define grpc_lb_v0_InitialLoadBalanceResponse_size 156 +#define grpc_lb_v0_Server_size 127 + +/* Message IDs (where set with "msgid" option) */ +#ifdef PB_MSGID + +#define LOAD_BALANCER_MESSAGES + +#endif + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/src/proto/grpc/lb/v0/load_balancer.options b/src/proto/grpc/lb/v0/load_balancer.options new file mode 100644 index 00000000000..6d4528f838a --- /dev/null +++ b/src/proto/grpc/lb/v0/load_balancer.options @@ -0,0 +1,6 @@ +grpc.lb.v0.InitialLoadBalanceRequest.name max_size:128 +grpc.lb.v0.InitialLoadBalanceResponse.client_config max_size:64 +grpc.lb.v0.InitialLoadBalanceResponse.load_balancer_delegate max_size:64 +grpc.lb.v0.Server.ip_address max_size:46 +grpc.lb.v0.Server.load_balance_token max_size:64 +load_balancer.proto no_unions:true diff --git a/src/proto/grpc/lb/v0/load_balancer.proto b/src/proto/grpc/lb/v0/load_balancer.proto new file mode 100644 index 00000000000..e88a4f8c4a7 --- /dev/null +++ b/src/proto/grpc/lb/v0/load_balancer.proto @@ -0,0 +1,144 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package grpc.lb.v0; + +message Duration { + + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} + +service LoadBalancer { + // Bidirectional rpc to get a list of servers. + rpc BalanceLoad(stream LoadBalanceRequest) + returns (stream LoadBalanceResponse); +} + +message LoadBalanceRequest { + oneof load_balance_request_type { + // This message should be sent on the first request to the load balancer. + InitialLoadBalanceRequest initial_request = 1; + + // The client stats should be periodically reported to the load balancer + // based on the duration defined in the InitialLoadBalanceResponse. + ClientStats client_stats = 2; + } +} + +message InitialLoadBalanceRequest { + // Name of load balanced service (IE, service.grpc.gslb.google.com) + string name = 1; +} + +// Contains client level statistics that are useful to load balancing. Each +// count should be reset to zero after reporting the stats. +message ClientStats { + // The total number of requests sent by the client since the last report. + int64 total_requests = 1; + + // The number of client rpc errors since the last report. + int64 client_rpc_errors = 2; + + // The number of dropped requests since the last report. + int64 dropped_requests = 3; +} + +message LoadBalanceResponse { + oneof load_balance_response_type { + // This message should be sent on the first response to the client. + InitialLoadBalanceResponse initial_response = 1; + + // Contains the list of servers selected by the load balancer. The client + // should send requests to these servers in the specified order. + ServerList server_list = 2; + } +} + +message InitialLoadBalanceResponse { + oneof initial_response_type { + // Contains gRPC config options like RPC deadline or flow control. + // TODO(yetianx): Change to ClientConfig after it is defined. + string client_config = 1; + + // This is an application layer redirect that indicates the client should + // use the specified server for load balancing. When this field is set in + // the response, the client should open a separate connection to the + // load_balancer_delegate and call the BalanceLoad method. + string load_balancer_delegate = 2; + } + + // This interval defines how often the client should send the client stats + // to the load balancer. Stats should only be reported when the duration is + // positive. + Duration client_stats_report_interval = 3; +} + +message ServerList { + // Contains a list of servers selected by the load balancer. The list will + // be updated when server resolutions change or as needed to balance load + // across more servers. The client should consume the server list in order + // unless instructed otherwise via the client_config. + repeated Server servers = 1; + + // Indicates the amount of time that the client should consider this server + // list as valid. It may be considered stale after waiting this interval of + // time after receiving the list. If the interval is not positive, the + // client can assume the list is valid until the next list is received. + Duration expiration_interval = 3; +} + +message Server { + // A resolved address and port for the server. The IP address string may + // either be an IPv4 or IPv6 address. + string ip_address = 1; + int32 port = 2; + + // An opaque token that is passed from the client to the server in metadata. + // The server may expect this token to indicate that the request from the + // client was load balanced. + // TODO(yetianx): Not used right now, and will be used after implementing + // load report. + bytes load_balance_token = 3; + + // Indicates whether this particular request should be dropped by the client + // when this server is chosen from the list. + bool drop_request = 4; +} diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 8e90f7a61d2..60af8623055 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -109,6 +109,7 @@ CORE_SOURCE_FILES = [ 'src/core/client_config/connector.c', 'src/core/client_config/default_initial_connect_string.c', 'src/core/client_config/initial_connect_string.c', + 'src/core/client_config/lb_policies/load_balancer_api.c', 'src/core/client_config/lb_policies/pick_first.c', 'src/core/client_config/lb_policies/round_robin.c', 'src/core/client_config/lb_policy.c', @@ -172,6 +173,7 @@ CORE_SOURCE_FILES = [ 'src/core/json/json_reader.c', 'src/core/json/json_string.c', 'src/core/json/json_writer.c', + 'src/core/proto/grpc/lb/v0/load_balancer.pb.c', 'src/core/surface/alarm.c', 'src/core/surface/api_trace.c', 'src/core/surface/byte_buffer.c', @@ -226,6 +228,9 @@ CORE_SOURCE_FILES = [ 'src/core/census/operation.c', 'src/core/census/placeholders.c', 'src/core/census/tracing.c', + 'third_party/nanopb/pb_common.c', + 'third_party/nanopb/pb_decode.c', + 'third_party/nanopb/pb_encode.c', 'src/boringssl/err_data.c', 'third_party/boringssl/crypto/aes/aes.c', 'third_party/boringssl/crypto/aes/mode_wrappers.c', diff --git a/templates/tools/run_tests/sources_and_headers.json.template b/templates/tools/run_tests/sources_and_headers.json.template index 04802772af5..78363ff1fae 100644 --- a/templates/tools/run_tests/sources_and_headers.json.template +++ b/templates/tools/run_tests/sources_and_headers.json.template @@ -12,20 +12,27 @@ out.extend(fmt % name for fmt in ['%s.grpc.pb.h', '%s.pb.h']) return out - def no_protos(src): + def no_protos_filter(src): + return os.path.splitext(src)[1] != '.proto' + + def no_third_party_filter(src): + return not src.startswith('third_party/') + + def filter_srcs(srcs, filters): out = [] - for f in src: - if os.path.splitext(f)[1] != '.proto': - out.append(f) + for s in srcs: + filter_passes = (f(s) for f in filters) + if all(filter_passes): + out.append(s) return out %> ${json.dumps([{"name": tgt.name, "language": tgt.language, "src": sorted( - no_protos(tgt.src) + - tgt.get('public_headers', []) + - tgt.get('headers', [])), + filter_srcs(tgt.src, (no_protos_filter, no_third_party_filter)) + + filter_srcs(tgt.get('public_headers', []), (no_protos_filter, no_third_party_filter)) + + filter_srcs(tgt.get('headers', []), (no_third_party_filter,))), "headers": sorted( tgt.get('public_headers', []) + tgt.get('headers', []) + diff --git a/test/cpp/grpclb/grpclb_api_test.cc b/test/cpp/grpclb/grpclb_api_test.cc new file mode 100644 index 00000000000..bd4885fb4cd --- /dev/null +++ b/test/cpp/grpclb/grpclb_api_test.cc @@ -0,0 +1,133 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +#include "src/core/client_config/lb_policies/load_balancer_api.h" +#include "src/proto/grpc/lb/v0/load_balancer.pb.h" // C++ version + +namespace grpc { +namespace { + +using grpc::lb::v0::LoadBalanceRequest; +using grpc::lb::v0::LoadBalanceResponse; + +class GrpclbTest : public ::testing::Test {}; + +TEST_F(GrpclbTest, CreateRequest) { + const std::string service_name = "AServiceName"; + LoadBalanceRequest request; + grpc_grpclb_request* c_req = grpc_grpclb_request_create(service_name.c_str()); + gpr_slice slice = grpc_grpclb_request_encode(c_req); + const int num_bytes_written = GPR_SLICE_LENGTH(slice); + EXPECT_GT(num_bytes_written, 0); + request.ParseFromArray(GPR_SLICE_START_PTR(slice), num_bytes_written); + EXPECT_EQ(request.initial_request().name(), service_name); + gpr_slice_unref(slice); + grpc_grpclb_request_destroy(c_req); +} + +TEST_F(GrpclbTest, ParseResponse) { + LoadBalanceResponse response; + const std::string client_config_str = "I'm a client config"; + auto* initial_response = response.mutable_initial_response(); + initial_response->set_client_config(client_config_str); + auto* client_stats_report_interval = + initial_response->mutable_client_stats_report_interval(); + client_stats_report_interval->set_seconds(123); + client_stats_report_interval->set_nanos(456); + + const std::string encoded_response = response.SerializeAsString(); + gpr_slice encoded_slice = + gpr_slice_from_copied_string(encoded_response.c_str()); + grpc_grpclb_response* c_response = grpc_grpclb_response_parse(encoded_slice); + EXPECT_TRUE(c_response->has_initial_response); + EXPECT_TRUE(c_response->initial_response.has_client_config); + EXPECT_FALSE(c_response->initial_response.has_load_balancer_delegate); + EXPECT_TRUE(strcmp(c_response->initial_response.client_config, + client_config_str.c_str()) == 0); + EXPECT_EQ(c_response->initial_response.client_stats_report_interval.seconds, + 123); + EXPECT_EQ(c_response->initial_response.client_stats_report_interval.nanos, + 456); + gpr_slice_unref(encoded_slice); + grpc_grpclb_response_destroy(c_response); +} + +TEST_F(GrpclbTest, ParseResponseServerList) { + LoadBalanceResponse response; + auto* serverlist = response.mutable_server_list(); + auto* server = serverlist->add_servers(); + server->set_ip_address("127.0.0.1"); + server->set_port(12345); + server->set_drop_request(true); + server = response.mutable_server_list()->add_servers(); + server->set_ip_address("10.0.0.1"); + server->set_port(54321); + server->set_drop_request(false); + auto* expiration_interval = serverlist->mutable_expiration_interval(); + expiration_interval->set_seconds(888); + expiration_interval->set_nanos(999); + + const std::string encoded_response = response.SerializeAsString(); + gpr_slice encoded_slice = + gpr_slice_from_copied_string(encoded_response.c_str()); + grpc_grpclb_serverlist* c_serverlist = + grpc_grpclb_response_parse_serverlist(encoded_slice); + ASSERT_EQ(c_serverlist->num_servers, 2ul); + EXPECT_TRUE(c_serverlist->servers[0]->has_ip_address); + EXPECT_TRUE(strcmp(c_serverlist->servers[0]->ip_address, "127.0.0.1") == 0); + EXPECT_EQ(c_serverlist->servers[0]->port, 12345); + EXPECT_TRUE(c_serverlist->servers[0]->drop_request); + EXPECT_TRUE(c_serverlist->servers[1]->has_ip_address); + EXPECT_TRUE(strcmp(c_serverlist->servers[1]->ip_address, "10.0.0.1") == 0); + EXPECT_EQ(c_serverlist->servers[1]->port, 54321); + EXPECT_FALSE(c_serverlist->servers[1]->drop_request); + + EXPECT_TRUE(c_serverlist->expiration_interval.has_seconds); + EXPECT_EQ(c_serverlist->expiration_interval.seconds, 888); + EXPECT_TRUE(c_serverlist->expiration_interval.has_nanos); + EXPECT_EQ(c_serverlist->expiration_interval.nanos, 999); + + gpr_slice_unref(encoded_slice); + grpc_grpclb_destroy_serverlist(c_serverlist); +} + +} // namespace +} // namespace grpc + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/third_party/nanopb b/third_party/nanopb new file mode 160000 index 00000000000..5497a1dfc91 --- /dev/null +++ b/third_party/nanopb @@ -0,0 +1 @@ +Subproject commit 5497a1dfc91a86965383cdd1652e348345400435 diff --git a/tools/codegen/core/gen_load_balancing_proto.sh b/tools/codegen/core/gen_load_balancing_proto.sh new file mode 100755 index 00000000000..114dd9d70d8 --- /dev/null +++ b/tools/codegen/core/gen_load_balancing_proto.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# Example usage: +# tools/codegen/core/gen_load_balancing_proto.sh \ +# src/proto/grpc/lb/v0/load_balancer.proto + +read -r -d '' COPYRIGHT <<'EOF' +/* + * + * Copyright , Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +EOF + +# build clang-format docker image +docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format + +CURRENT_YEAR=$(date +%Y) +COPYRIGHT_FILE=$(mktemp) +echo "${COPYRIGHT//$CURRENT_YEAR}" > $COPYRIGHT_FILE + +set -ex +if [ $# -eq 0 ]; then + echo "Usage: $0 [output dir]" + exit 1 +fi + +readonly GRPC_ROOT=$PWD + +OUTPUT_DIR="$GRPC_ROOT/src/core/proto/grpc/lb/v0" +if [ $# -eq 2 ]; then + mkdir -p "$2" + if [ $? != 0 ]; then + echo "Error creating output directory $2" + exit 2 + fi + OUTPUT_DIR="$2" +fi + +readonly EXPECTED_OPTIONS_FILE_PATH="${1%.*}.options" + +if [[ ! -f "$1" ]]; then + echo "Input proto file '$1' doesn't exist." + exit 3 +fi +if [[ ! -f "${EXPECTED_OPTIONS_FILE_PATH}" ]]; then + echo "Expected nanopb options file '${EXPECTED_OPTIONS_FILE_PATH}' missing" + exit 4 +fi + +pushd "$(dirname $1)" > /dev/null + +protoc \ +--plugin=protoc-gen-nanopb="$GRPC_ROOT/third_party/nanopb/generator/protoc-gen-nanopb" \ +--nanopb_out='-T -L#include\ \"third_party/nanopb/pb.h\"'":$OUTPUT_DIR" \ +"$(basename $1)" + +readonly PROTO_BASENAME=$(basename $1 .proto) +sed -i "s:$PROTO_BASENAME.pb.h:src/core/proto/grpc/lb/v0/$PROTO_BASENAME.pb.h:g" \ + "$OUTPUT_DIR/$PROTO_BASENAME.pb.c" + +# prepend copyright +TMPFILE=$(mktemp) +cat $COPYRIGHT_FILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.c" > $TMPFILE +mv -v $TMPFILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.c" +cat $COPYRIGHT_FILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.h" > $TMPFILE +mv -v $TMPFILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.h" + +readonly MOUNTPOINT='/protos' +docker run --rm=true -v ${HOST_GIT_ROOT}/gens/src/proto/grpc/lb/v0:$MOUNTPOINT \ + -t grpc_clang_format \ + clang-format-3.6 -style="{BasedOnStyle: Google, Language: Cpp}" \ + -i $MOUNTPOINT/load_balancer.pb.c $MOUNTPOINT/load_balancer.pb.h + +popd > /dev/null diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh new file mode 100755 index 00000000000..78d3a734e96 --- /dev/null +++ b/tools/distrib/check_nanopb_output.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +apt-get install -y autoconf automake libtool curl python-virtualenv + +readonly NANOPB_TMP_OUTPUT="${LOCAL_GIT_ROOT}/gens/src/proto/grpc/lb/v0" +readonly VENV_DIR=$(mktemp -d) +# create a virtualenv for nanopb's compiler +pushd $VENV_DIR +readonly VENV_NAME="nanopb-$(date '+%Y%m%d_%H%M%S_%N')" +virtualenv $VENV_NAME +. $VENV_NAME/bin/activate +popd + +# install proto3 +pip install protobuf==3.0.0b2 + +# change to root directory +cd $(dirname $0)/../.. + +# build clang-format docker image +docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format + +# install protoc version 3 +pushd third_party/protobuf +./autogen.sh +./configure +make +make install +ldconfig +popd + +if [ ! -x "/usr/local/bin/protoc" ]; then + echo "Error: protoc not found in path" + exit 1 +fi +readonly PROTOC_PATH='/usr/local/bin' +# stack up and change to nanopb's proto generator directory +pushd third_party/nanopb/generator/proto +PATH="$PROTOC_PATH:$PATH" make + +# back to the root directory +popd + + +# nanopb-compile the proto to a temp location +PATH="$PROTOC_PATH:$PATH" ./tools/codegen/core/gen_load_balancing_proto.sh \ + src/proto/grpc/lb/v0/load_balancer.proto \ + $NANOPB_TMP_OUTPUT + +# compare outputs to checked compiled code +diff -rq $NANOPB_TMP_OUTPUT src/core/proto/grpc/lb/v0 +if [ $? != 0 ]; then + echo "Outputs differ: $NANOPB_TMP_OUTPUT vs src/core/proto/grpc/lb/v0" + exit 1 +fi +deactivate diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index b6268432335..c6f1b6ae7c2 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -801,6 +801,7 @@ src/core/channel/subchannel_call_holder.h \ src/core/client_config/client_config.h \ src/core/client_config/connector.h \ src/core/client_config/initial_connect_string.h \ +src/core/client_config/lb_policies/load_balancer_api.h \ src/core/client_config/lb_policies/pick_first.h \ src/core/client_config/lb_policies/round_robin.h \ src/core/client_config/lb_policy.h \ @@ -861,6 +862,7 @@ src/core/json/json.h \ src/core/json/json_common.h \ src/core/json/json_reader.h \ src/core/json/json_writer.h \ +src/core/proto/grpc/lb/v0/load_balancer.pb.h \ src/core/statistics/census_interface.h \ src/core/statistics/census_rpc_stats.h \ src/core/surface/api_trace.h \ @@ -902,6 +904,10 @@ src/core/transport/transport.h \ src/core/transport/transport_impl.h \ src/core/census/aggregation.h \ src/core/census/rpc_metric_id.h \ +third_party/nanopb/pb.h \ +third_party/nanopb/pb_common.h \ +third_party/nanopb/pb_decode.h \ +third_party/nanopb/pb_encode.h \ src/core/httpcli/httpcli_security_connector.c \ src/core/security/base64.c \ src/core/security/client_auth_filter.c \ @@ -938,6 +944,7 @@ src/core/client_config/client_config.c \ src/core/client_config/connector.c \ src/core/client_config/default_initial_connect_string.c \ src/core/client_config/initial_connect_string.c \ +src/core/client_config/lb_policies/load_balancer_api.c \ src/core/client_config/lb_policies/pick_first.c \ src/core/client_config/lb_policies/round_robin.c \ src/core/client_config/lb_policy.c \ @@ -1001,6 +1008,7 @@ src/core/json/json.c \ src/core/json/json_reader.c \ src/core/json/json_string.c \ src/core/json/json_writer.c \ +src/core/proto/grpc/lb/v0/load_balancer.pb.c \ src/core/surface/alarm.c \ src/core/surface/api_trace.c \ src/core/surface/byte_buffer.c \ @@ -1055,6 +1063,9 @@ src/core/census/initialize.c \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tracing.c \ +third_party/nanopb/pb_common.c \ +third_party/nanopb/pb_decode.c \ +third_party/nanopb/pb_encode.c \ include/grpc/support/alloc.h \ include/grpc/support/atm.h \ include/grpc/support/atm_gcc_atomic.h \ diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh index e2ac7518f09..daa29b2f94b 100755 --- a/tools/jenkins/build_docker_and_run_tests.sh +++ b/tools/jenkins/build_docker_and_run_tests.sh @@ -60,6 +60,9 @@ docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR # Choose random name for docker container CONTAINER_NAME="run_tests_$(uuidgen)" +# Git root as seen by the docker instance +docker_instance_git_root=/var/local/jenkins/grpc + # Run tests inside docker docker run \ -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND" \ @@ -69,9 +72,10 @@ docker run \ -e XDG_CACHE_HOME=/tmp/xdg-cache-home \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ -e HOST_GIT_ROOT=$git_root \ + -e LOCAL_GIT_ROOT=$docker_instance_git_root \ -e "BUILD_ID=$BUILD_ID" \ -i $TTY_FLAG \ - -v "$git_root:/var/local/jenkins/grpc" \ + -v "$git_root:$docker_instance_git_root" \ -v /tmp/ccache:/tmp/ccache \ -v /tmp/npm-cache:/tmp/npm-cache \ -v /tmp/xdg-cache-home:/tmp/xdg-cache-home \ diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index f49230e49aa..c08b382638f 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -44,6 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules 9f897b25800d2f54f5c442ef01a60721aeca6d87 third_party/boringssl (version_for_cocoapods_1.0-67-g9f897b2) 05b155ff59114735ec8cd089f669c4c3d8f59029 third_party/gflags (v2.1.0-45-g05b155f) c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0) + 5497a1dfc91a86965383cdd1652e348345400435 third_party/nanopb (nanopb-0.3.3-10-g5497a1d) d5fb408ddc281ffcadeb08699e65bb694656d0bd third_party/protobuf (v3.0.0-beta-2) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) EOF diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index 809e6ce6454..1fc0525e029 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -7,3 +7,4 @@ - script: tools/distrib/check_copyright.py - script: tools/distrib/clang_format_code.sh - script: tools/distrib/check_trailing_newlines.sh +- script: tools/distrib/check_nanopb_output.sh diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 6538ddc37e1..76fa3d07764 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1644,6 +1644,23 @@ "src/compiler/ruby_plugin.cc" ] }, + { + "deps": [ + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [ + "src/proto/grpc/lb/v0/load_balancer.grpc.pb.h", + "src/proto/grpc/lb/v0/load_balancer.pb.h" + ], + "language": "c++", + "name": "grpclb_api_test", + "src": [ + "test/cpp/grpclb/grpclb_api_test.cc" + ] + }, { "deps": [ "gpr", @@ -2983,6 +3000,7 @@ "src/core/client_config/client_config.h", "src/core/client_config/connector.h", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.h", "src/core/client_config/lb_policy.h", @@ -3043,6 +3061,7 @@ "src/core/json/json_common.h", "src/core/json/json_reader.h", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/security/auth_filters.h", "src/core/security/base64.h", "src/core/security/credentials.h", @@ -3095,7 +3114,11 @@ "src/core/tsi/ssl_transport_security.h", "src/core/tsi/ssl_types.h", "src/core/tsi/transport_security.h", - "src/core/tsi/transport_security_interface.h" + "src/core/tsi/transport_security_interface.h", + "third_party/nanopb/pb.h", + "third_party/nanopb/pb_common.h", + "third_party/nanopb/pb_decode.h", + "third_party/nanopb/pb_encode.h" ], "language": "c", "name": "grpc", @@ -3149,6 +3172,8 @@ "src/core/client_config/default_initial_connect_string.c", "src/core/client_config/initial_connect_string.c", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.c", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.c", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.c", @@ -3273,6 +3298,8 @@ "src/core/json/json_string.c", "src/core/json/json_writer.c", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.c", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/security/auth_filters.h", "src/core/security/base64.c", "src/core/security/base64.h", @@ -3508,6 +3535,7 @@ "src/core/client_config/client_config.h", "src/core/client_config/connector.h", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.h", "src/core/client_config/lb_policy.h", @@ -3568,6 +3596,7 @@ "src/core/json/json_common.h", "src/core/json/json_reader.h", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/statistics/census_interface.h", "src/core/statistics/census_rpc_stats.h", "src/core/surface/api_trace.h", @@ -3606,7 +3635,11 @@ "src/core/transport/metadata_batch.h", "src/core/transport/static_metadata.h", "src/core/transport/transport.h", - "src/core/transport/transport_impl.h" + "src/core/transport/transport_impl.h", + "third_party/nanopb/pb.h", + "third_party/nanopb/pb_common.h", + "third_party/nanopb/pb_decode.h", + "third_party/nanopb/pb_encode.h" ], "language": "c", "name": "grpc_unsecure", @@ -3659,6 +3692,8 @@ "src/core/client_config/default_initial_connect_string.c", "src/core/client_config/initial_connect_string.c", "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/load_balancer_api.c", + "src/core/client_config/lb_policies/load_balancer_api.h", "src/core/client_config/lb_policies/pick_first.c", "src/core/client_config/lb_policies/pick_first.h", "src/core/client_config/lb_policies/round_robin.c", @@ -3782,6 +3817,8 @@ "src/core/json/json_string.c", "src/core/json/json_writer.c", "src/core/json/json_writer.h", + "src/core/proto/grpc/lb/v0/load_balancer.pb.c", + "src/core/proto/grpc/lb/v0/load_balancer.pb.h", "src/core/statistics/census_interface.h", "src/core/statistics/census_rpc_stats.h", "src/core/surface/alarm.c", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 2c73c40d1f5..3b9bce1ec38 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -2019,6 +2019,26 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "grpclb_api_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 76975322be8..4e1155f84b5 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -310,6 +310,7 @@ + @@ -370,6 +371,7 @@ + @@ -411,6 +413,10 @@ + + + + @@ -485,6 +491,8 @@ + + @@ -611,6 +619,8 @@ + + @@ -719,6 +729,12 @@ + + + + + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 4660572f979..e0cfd8295b1 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -109,6 +109,9 @@ src\core\client_config + + src\core\client_config\lb_policies + src\core\client_config\lb_policies @@ -298,6 +301,9 @@ src\core\json + + src\core\proto\grpc\lb\v0 + src\core\surface @@ -460,6 +466,15 @@ src\core\census + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + @@ -587,6 +602,9 @@ src\core\client_config + + src\core\client_config\lb_policies + src\core\client_config\lb_policies @@ -767,6 +785,9 @@ src\core\json + + src\core\proto\grpc\lb\v0 + src\core\statistics @@ -890,6 +911,18 @@ src\core\census + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + @@ -941,6 +974,18 @@ {e665cc0e-b994-d7c5-cc18-2007392019f0} + + {1ff04466-0905-8a5d-d6f4-7ff2df4c13b5} + + + {7c7ad0b3-bf85-5bd3-e0c8-4f5468a8e2e6} + + + {3d533dad-8100-e8a3-b7c3-1fc13a4d60da} + + + {0ffcf868-7617-5fed-b6ce-2162d9d09148} + {1d850ac6-e639-4eab-5338-4ba40272fcc9} @@ -959,6 +1004,12 @@ {0b0f9ab1-efa4-7f03-e446-6fb9b5227e84} + + {aaab30a4-2a15-732e-c141-3fbc0f0f5a7a} + + + {93d6596d-330c-1d27-6f84-3c840e57869e} + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 541000af404..3820d89968b 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -286,6 +286,7 @@ + @@ -346,6 +347,7 @@ + @@ -387,6 +389,10 @@ + + + + @@ -421,6 +427,8 @@ + + @@ -547,6 +555,8 @@ + + @@ -655,6 +665,12 @@ + + + + + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 48814f997e1..c4c957d36b2 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -49,6 +49,9 @@ src\core\client_config + + src\core\client_config\lb_policies + src\core\client_config\lb_policies @@ -238,6 +241,9 @@ src\core\json + + src\core\proto\grpc\lb\v0 + src\core\surface @@ -400,6 +406,15 @@ src\core\census + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + @@ -482,6 +497,9 @@ src\core\client_config + + src\core\client_config\lb_policies + src\core\client_config\lb_policies @@ -662,6 +680,9 @@ src\core\json + + src\core\proto\grpc\lb\v0 + src\core\statistics @@ -785,6 +806,18 @@ src\core\census + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + @@ -836,6 +869,18 @@ {443ffc61-1bea-2477-6e54-1ddf8c139264} + + {7f4bb22a-65ba-0f8f-6387-66b1f6677a80} + + + {9c2bd164-c317-8a13-564d-3b28b0fd79cf} + + + {2bad8e10-4fc5-d8b3-e026-4abbd0c25cda} + + + {4475c8ed-e01b-8906-47d0-8a504189c0d5} + {e084164c-a069-00e3-db35-4e0b1cd6f0b7} @@ -848,6 +893,12 @@ {5fcd6206-f774-9ae6-4b85-305d6a723843} + + {025c051e-8eba-125b-67f9-173f95176eb2} + + + {6511f77d-f28c-80e0-0889-8975e688e344} + diff --git a/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj b/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj new file mode 100644 index 00000000000..1509ece9f96 --- /dev/null +++ b/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj @@ -0,0 +1,209 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {990AF023-17D7-8DBF-EB6E-14C7C016C77E} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + grpclb_api_test + static + Debug + static + Debug + + + grpclb_api_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + + + + + + + + + {0BE77741-552A-929B-A497-4EF7ECE17A64} + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj.filters b/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj.filters new file mode 100644 index 00000000000..6c57b8c162a --- /dev/null +++ b/vsprojects/vcxproj/test/grpclb_api_test/grpclb_api_test.vcxproj.filters @@ -0,0 +1,39 @@ + + + + + src\proto\grpc\lb\v0 + + + test\cpp\grpclb + + + + + + {a31d21fb-c6ab-75ce-43dc-7d6f506765e6} + + + {10d49c90-8503-9b10-6678-eed983bc25d9} + + + {8b6be783-e071-44cc-2096-f1c476012556} + + + {2981699e-c196-c599-bc17-c177770f89ee} + + + {3d04774a-1c2f-e100-435e-08af5d539250} + + + {64736e1d-eb77-664f-34ab-6cf41263d3d8} + + + {c86e9cb1-bed4-3697-40f2-9ecff6297fa5} + + + {6b5ba83a-6cf2-5a7b-0ab8-62de31882705} + + + + From 00c48296693d8c1d4fc5c600bcb318abeb4e4c3d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 11 Feb 2016 16:20:28 -0800 Subject: [PATCH 005/236] Ignore pb.h, pb.c for clang-fmt --- src/core/proto/grpc/lb/v0/load_balancer.pb.c | 119 +++++------- src/core/proto/grpc/lb/v0/load_balancer.pb.h | 169 +++++++----------- .../codegen/core/gen_load_balancing_proto.sh | 19 +- tools/distrib/check_nanopb_output.sh | 16 -- .../clang_format_all_the_things.sh | 2 +- 5 files changed, 121 insertions(+), 204 deletions(-) diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.c b/src/core/proto/grpc/lb/v0/load_balancer.pb.c index bbb6fa82249..abd4b1cefc6 100644 --- a/src/core/proto/grpc/lb/v0/load_balancer.pb.c +++ b/src/core/proto/grpc/lb/v0/load_balancer.pb.c @@ -39,112 +39,81 @@ #error Regenerate this file with the current version of nanopb generator. #endif + + const pb_field_t grpc_lb_v0_Duration_fields[3] = { - PB_FIELD(1, INT64, OPTIONAL, STATIC, FIRST, grpc_lb_v0_Duration, seconds, - seconds, 0), - PB_FIELD(2, INT32, OPTIONAL, STATIC, OTHER, grpc_lb_v0_Duration, nanos, - seconds, 0), - PB_LAST_FIELD}; + PB_FIELD( 1, INT64 , OPTIONAL, STATIC , FIRST, grpc_lb_v0_Duration, seconds, seconds, 0), + PB_FIELD( 2, INT32 , OPTIONAL, STATIC , OTHER, grpc_lb_v0_Duration, nanos, seconds, 0), + PB_LAST_FIELD +}; const pb_field_t grpc_lb_v0_LoadBalanceRequest_fields[3] = { - PB_FIELD(1, MESSAGE, OPTIONAL, STATIC, FIRST, grpc_lb_v0_LoadBalanceRequest, - initial_request, initial_request, - &grpc_lb_v0_InitialLoadBalanceRequest_fields), - PB_FIELD(2, MESSAGE, OPTIONAL, STATIC, OTHER, grpc_lb_v0_LoadBalanceRequest, - client_stats, initial_request, &grpc_lb_v0_ClientStats_fields), - PB_LAST_FIELD}; + PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, grpc_lb_v0_LoadBalanceRequest, initial_request, initial_request, &grpc_lb_v0_InitialLoadBalanceRequest_fields), + PB_FIELD( 2, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v0_LoadBalanceRequest, client_stats, initial_request, &grpc_lb_v0_ClientStats_fields), + PB_LAST_FIELD +}; const pb_field_t grpc_lb_v0_InitialLoadBalanceRequest_fields[2] = { - PB_FIELD(1, STRING, OPTIONAL, STATIC, FIRST, - grpc_lb_v0_InitialLoadBalanceRequest, name, name, 0), - PB_LAST_FIELD}; + PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v0_InitialLoadBalanceRequest, name, name, 0), + PB_LAST_FIELD +}; const pb_field_t grpc_lb_v0_ClientStats_fields[4] = { - PB_FIELD(1, INT64, OPTIONAL, STATIC, FIRST, grpc_lb_v0_ClientStats, - total_requests, total_requests, 0), - PB_FIELD(2, INT64, OPTIONAL, STATIC, OTHER, grpc_lb_v0_ClientStats, - client_rpc_errors, total_requests, 0), - PB_FIELD(3, INT64, OPTIONAL, STATIC, OTHER, grpc_lb_v0_ClientStats, - dropped_requests, client_rpc_errors, 0), - PB_LAST_FIELD}; + PB_FIELD( 1, INT64 , OPTIONAL, STATIC , FIRST, grpc_lb_v0_ClientStats, total_requests, total_requests, 0), + PB_FIELD( 2, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v0_ClientStats, client_rpc_errors, total_requests, 0), + PB_FIELD( 3, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v0_ClientStats, dropped_requests, client_rpc_errors, 0), + PB_LAST_FIELD +}; const pb_field_t grpc_lb_v0_LoadBalanceResponse_fields[3] = { - PB_FIELD(1, MESSAGE, OPTIONAL, STATIC, FIRST, - grpc_lb_v0_LoadBalanceResponse, initial_response, initial_response, - &grpc_lb_v0_InitialLoadBalanceResponse_fields), - PB_FIELD(2, MESSAGE, OPTIONAL, STATIC, OTHER, - grpc_lb_v0_LoadBalanceResponse, server_list, initial_response, - &grpc_lb_v0_ServerList_fields), - PB_LAST_FIELD}; + PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, grpc_lb_v0_LoadBalanceResponse, initial_response, initial_response, &grpc_lb_v0_InitialLoadBalanceResponse_fields), + PB_FIELD( 2, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v0_LoadBalanceResponse, server_list, initial_response, &grpc_lb_v0_ServerList_fields), + PB_LAST_FIELD +}; const pb_field_t grpc_lb_v0_InitialLoadBalanceResponse_fields[4] = { - PB_FIELD(1, STRING, OPTIONAL, STATIC, FIRST, - grpc_lb_v0_InitialLoadBalanceResponse, client_config, - client_config, 0), - PB_FIELD(2, STRING, OPTIONAL, STATIC, OTHER, - grpc_lb_v0_InitialLoadBalanceResponse, load_balancer_delegate, - client_config, 0), - PB_FIELD(3, MESSAGE, OPTIONAL, STATIC, OTHER, - grpc_lb_v0_InitialLoadBalanceResponse, - client_stats_report_interval, load_balancer_delegate, - &grpc_lb_v0_Duration_fields), - PB_LAST_FIELD}; + PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v0_InitialLoadBalanceResponse, client_config, client_config, 0), + PB_FIELD( 2, STRING , OPTIONAL, STATIC , OTHER, grpc_lb_v0_InitialLoadBalanceResponse, load_balancer_delegate, client_config, 0), + PB_FIELD( 3, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v0_Duration_fields), + PB_LAST_FIELD +}; const pb_field_t grpc_lb_v0_ServerList_fields[3] = { - PB_FIELD(1, MESSAGE, REPEATED, CALLBACK, FIRST, grpc_lb_v0_ServerList, - servers, servers, &grpc_lb_v0_Server_fields), - PB_FIELD(3, MESSAGE, OPTIONAL, STATIC, OTHER, grpc_lb_v0_ServerList, - expiration_interval, servers, &grpc_lb_v0_Duration_fields), - PB_LAST_FIELD}; + PB_FIELD( 1, MESSAGE , REPEATED, CALLBACK, FIRST, grpc_lb_v0_ServerList, servers, servers, &grpc_lb_v0_Server_fields), + PB_FIELD( 3, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v0_ServerList, expiration_interval, servers, &grpc_lb_v0_Duration_fields), + PB_LAST_FIELD +}; const pb_field_t grpc_lb_v0_Server_fields[5] = { - PB_FIELD(1, STRING, OPTIONAL, STATIC, FIRST, grpc_lb_v0_Server, ip_address, - ip_address, 0), - PB_FIELD(2, INT32, OPTIONAL, STATIC, OTHER, grpc_lb_v0_Server, port, - ip_address, 0), - PB_FIELD(3, BYTES, OPTIONAL, STATIC, OTHER, grpc_lb_v0_Server, - load_balance_token, port, 0), - PB_FIELD(4, BOOL, OPTIONAL, STATIC, OTHER, grpc_lb_v0_Server, drop_request, - load_balance_token, 0), - PB_LAST_FIELD}; + PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v0_Server, ip_address, ip_address, 0), + PB_FIELD( 2, INT32 , OPTIONAL, STATIC , OTHER, grpc_lb_v0_Server, port, ip_address, 0), + PB_FIELD( 3, BYTES , OPTIONAL, STATIC , OTHER, grpc_lb_v0_Server, load_balance_token, port, 0), + PB_FIELD( 4, BOOL , OPTIONAL, STATIC , OTHER, grpc_lb_v0_Server, drop_request, load_balance_token, 0), + PB_LAST_FIELD +}; + /* Check that field information fits in pb_field_t */ #if !defined(PB_FIELD_32BIT) /* If you get an error here, it means that you need to define PB_FIELD_32BIT * compile-time option. You can do that in pb.h or on compiler command line. - * + * * The reason you need to do this is that some of your messages contain tag * numbers or field sizes that are larger than what can fit in 8 or 16 bit * field descriptors. */ -PB_STATIC_ASSERT( - (pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 65536 && - pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 65536 && - pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 65536 && - pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 65536 && - pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, - client_stats_report_interval) < 65536 && - pb_membersize(grpc_lb_v0_ServerList, servers) < 65536 && - pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 65536), - YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server) +PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v0_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server) #endif #if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) /* If you get an error here, it means that you need to define PB_FIELD_16BIT * compile-time option. You can do that in pb.h or on compiler command line. - * + * * The reason you need to do this is that some of your messages contain tag * numbers or field sizes that are larger than what can fit in the default * 8 bit descriptors. */ -PB_STATIC_ASSERT( - (pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 256 && - pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 256 && - pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 256 && - pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 256 && - pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, - client_stats_report_interval) < 256 && - pb_membersize(grpc_lb_v0_ServerList, servers) < 256 && - pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 256), - YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server) +PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v0_ServerList, servers) < 256 && pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server) #endif + + diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.h b/src/core/proto/grpc/lb/v0/load_balancer.pb.h index ac836658399..87037213998 100644 --- a/src/core/proto/grpc/lb/v0/load_balancer.pb.h +++ b/src/core/proto/grpc/lb/v0/load_balancer.pb.h @@ -47,144 +47,104 @@ extern "C" { /* Enum definitions */ /* Struct definitions */ typedef struct _grpc_lb_v0_ClientStats { - bool has_total_requests; - int64_t total_requests; - bool has_client_rpc_errors; - int64_t client_rpc_errors; - bool has_dropped_requests; - int64_t dropped_requests; + bool has_total_requests; + int64_t total_requests; + bool has_client_rpc_errors; + int64_t client_rpc_errors; + bool has_dropped_requests; + int64_t dropped_requests; } grpc_lb_v0_ClientStats; typedef struct _grpc_lb_v0_Duration { - bool has_seconds; - int64_t seconds; - bool has_nanos; - int32_t nanos; + bool has_seconds; + int64_t seconds; + bool has_nanos; + int32_t nanos; } grpc_lb_v0_Duration; typedef struct _grpc_lb_v0_InitialLoadBalanceRequest { - bool has_name; - char name[128]; + bool has_name; + char name[128]; } grpc_lb_v0_InitialLoadBalanceRequest; typedef PB_BYTES_ARRAY_T(64) grpc_lb_v0_Server_load_balance_token_t; typedef struct _grpc_lb_v0_Server { - bool has_ip_address; - char ip_address[46]; - bool has_port; - int32_t port; - bool has_load_balance_token; - grpc_lb_v0_Server_load_balance_token_t load_balance_token; - bool has_drop_request; - bool drop_request; + bool has_ip_address; + char ip_address[46]; + bool has_port; + int32_t port; + bool has_load_balance_token; + grpc_lb_v0_Server_load_balance_token_t load_balance_token; + bool has_drop_request; + bool drop_request; } grpc_lb_v0_Server; typedef struct _grpc_lb_v0_InitialLoadBalanceResponse { - bool has_client_config; - char client_config[64]; - bool has_load_balancer_delegate; - char load_balancer_delegate[64]; - bool has_client_stats_report_interval; - grpc_lb_v0_Duration client_stats_report_interval; + bool has_client_config; + char client_config[64]; + bool has_load_balancer_delegate; + char load_balancer_delegate[64]; + bool has_client_stats_report_interval; + grpc_lb_v0_Duration client_stats_report_interval; } grpc_lb_v0_InitialLoadBalanceResponse; typedef struct _grpc_lb_v0_LoadBalanceRequest { - bool has_initial_request; - grpc_lb_v0_InitialLoadBalanceRequest initial_request; - bool has_client_stats; - grpc_lb_v0_ClientStats client_stats; + bool has_initial_request; + grpc_lb_v0_InitialLoadBalanceRequest initial_request; + bool has_client_stats; + grpc_lb_v0_ClientStats client_stats; } grpc_lb_v0_LoadBalanceRequest; typedef struct _grpc_lb_v0_ServerList { - pb_callback_t servers; - bool has_expiration_interval; - grpc_lb_v0_Duration expiration_interval; + pb_callback_t servers; + bool has_expiration_interval; + grpc_lb_v0_Duration expiration_interval; } grpc_lb_v0_ServerList; typedef struct _grpc_lb_v0_LoadBalanceResponse { - bool has_initial_response; - grpc_lb_v0_InitialLoadBalanceResponse initial_response; - bool has_server_list; - grpc_lb_v0_ServerList server_list; + bool has_initial_response; + grpc_lb_v0_InitialLoadBalanceResponse initial_response; + bool has_server_list; + grpc_lb_v0_ServerList server_list; } grpc_lb_v0_LoadBalanceResponse; /* Default values for struct fields */ /* Initializer values for message structs */ -#define grpc_lb_v0_Duration_init_default \ - { false, 0, false, 0 } -#define grpc_lb_v0_LoadBalanceRequest_init_default \ - { \ - false, grpc_lb_v0_InitialLoadBalanceRequest_init_default, false, \ - grpc_lb_v0_ClientStats_init_default \ - } -#define grpc_lb_v0_InitialLoadBalanceRequest_init_default \ - { false, "" } -#define grpc_lb_v0_ClientStats_init_default \ - { false, 0, false, 0, false, 0 } -#define grpc_lb_v0_LoadBalanceResponse_init_default \ - { \ - false, grpc_lb_v0_InitialLoadBalanceResponse_init_default, false, \ - grpc_lb_v0_ServerList_init_default \ - } -#define grpc_lb_v0_InitialLoadBalanceResponse_init_default \ - { false, "", false, "", false, grpc_lb_v0_Duration_init_default } -#define grpc_lb_v0_ServerList_init_default \ - { \ - { \ - { NULL } \ - , NULL \ - } \ - , false, grpc_lb_v0_Duration_init_default \ - } -#define grpc_lb_v0_Server_init_default \ - { false, "", false, 0, false, {0, {0}}, false, 0 } -#define grpc_lb_v0_Duration_init_zero \ - { false, 0, false, 0 } -#define grpc_lb_v0_LoadBalanceRequest_init_zero \ - { \ - false, grpc_lb_v0_InitialLoadBalanceRequest_init_zero, false, \ - grpc_lb_v0_ClientStats_init_zero \ - } -#define grpc_lb_v0_InitialLoadBalanceRequest_init_zero \ - { false, "" } -#define grpc_lb_v0_ClientStats_init_zero \ - { false, 0, false, 0, false, 0 } -#define grpc_lb_v0_LoadBalanceResponse_init_zero \ - { \ - false, grpc_lb_v0_InitialLoadBalanceResponse_init_zero, false, \ - grpc_lb_v0_ServerList_init_zero \ - } -#define grpc_lb_v0_InitialLoadBalanceResponse_init_zero \ - { false, "", false, "", false, grpc_lb_v0_Duration_init_zero } -#define grpc_lb_v0_ServerList_init_zero \ - { \ - { \ - { NULL } \ - , NULL \ - } \ - , false, grpc_lb_v0_Duration_init_zero \ - } -#define grpc_lb_v0_Server_init_zero \ - { false, "", false, 0, false, {0, {0}}, false, 0 } +#define grpc_lb_v0_Duration_init_default {false, 0, false, 0} +#define grpc_lb_v0_LoadBalanceRequest_init_default {false, grpc_lb_v0_InitialLoadBalanceRequest_init_default, false, grpc_lb_v0_ClientStats_init_default} +#define grpc_lb_v0_InitialLoadBalanceRequest_init_default {false, ""} +#define grpc_lb_v0_ClientStats_init_default {false, 0, false, 0, false, 0} +#define grpc_lb_v0_LoadBalanceResponse_init_default {false, grpc_lb_v0_InitialLoadBalanceResponse_init_default, false, grpc_lb_v0_ServerList_init_default} +#define grpc_lb_v0_InitialLoadBalanceResponse_init_default {false, "", false, "", false, grpc_lb_v0_Duration_init_default} +#define grpc_lb_v0_ServerList_init_default {{{NULL}, NULL}, false, grpc_lb_v0_Duration_init_default} +#define grpc_lb_v0_Server_init_default {false, "", false, 0, false, {0, {0}}, false, 0} +#define grpc_lb_v0_Duration_init_zero {false, 0, false, 0} +#define grpc_lb_v0_LoadBalanceRequest_init_zero {false, grpc_lb_v0_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v0_ClientStats_init_zero} +#define grpc_lb_v0_InitialLoadBalanceRequest_init_zero {false, ""} +#define grpc_lb_v0_ClientStats_init_zero {false, 0, false, 0, false, 0} +#define grpc_lb_v0_LoadBalanceResponse_init_zero {false, grpc_lb_v0_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v0_ServerList_init_zero} +#define grpc_lb_v0_InitialLoadBalanceResponse_init_zero {false, "", false, "", false, grpc_lb_v0_Duration_init_zero} +#define grpc_lb_v0_ServerList_init_zero {{{NULL}, NULL}, false, grpc_lb_v0_Duration_init_zero} +#define grpc_lb_v0_Server_init_zero {false, "", false, 0, false, {0, {0}}, false, 0} /* Field tags (for use in manual encoding/decoding) */ #define grpc_lb_v0_ClientStats_total_requests_tag 1 #define grpc_lb_v0_ClientStats_client_rpc_errors_tag 2 #define grpc_lb_v0_ClientStats_dropped_requests_tag 3 -#define grpc_lb_v0_Duration_seconds_tag 1 -#define grpc_lb_v0_Duration_nanos_tag 2 +#define grpc_lb_v0_Duration_seconds_tag 1 +#define grpc_lb_v0_Duration_nanos_tag 2 #define grpc_lb_v0_InitialLoadBalanceRequest_name_tag 1 -#define grpc_lb_v0_Server_ip_address_tag 1 -#define grpc_lb_v0_Server_port_tag 2 +#define grpc_lb_v0_Server_ip_address_tag 1 +#define grpc_lb_v0_Server_port_tag 2 #define grpc_lb_v0_Server_load_balance_token_tag 3 -#define grpc_lb_v0_Server_drop_request_tag 4 +#define grpc_lb_v0_Server_drop_request_tag 4 #define grpc_lb_v0_InitialLoadBalanceResponse_client_config_tag 1 #define grpc_lb_v0_InitialLoadBalanceResponse_load_balancer_delegate_tag 2 #define grpc_lb_v0_InitialLoadBalanceResponse_client_stats_report_interval_tag 3 #define grpc_lb_v0_LoadBalanceRequest_initial_request_tag 1 #define grpc_lb_v0_LoadBalanceRequest_client_stats_tag 2 -#define grpc_lb_v0_ServerList_servers_tag 1 +#define grpc_lb_v0_ServerList_servers_tag 1 #define grpc_lb_v0_ServerList_expiration_interval_tag 3 #define grpc_lb_v0_LoadBalanceResponse_initial_response_tag 1 #define grpc_lb_v0_LoadBalanceResponse_server_list_tag 2 @@ -200,17 +160,18 @@ extern const pb_field_t grpc_lb_v0_ServerList_fields[3]; extern const pb_field_t grpc_lb_v0_Server_fields[5]; /* Maximum encoded size of messages (where known) */ -#define grpc_lb_v0_Duration_size 22 -#define grpc_lb_v0_LoadBalanceRequest_size 169 +#define grpc_lb_v0_Duration_size 22 +#define grpc_lb_v0_LoadBalanceRequest_size 169 #define grpc_lb_v0_InitialLoadBalanceRequest_size 131 -#define grpc_lb_v0_ClientStats_size 33 +#define grpc_lb_v0_ClientStats_size 33 #define grpc_lb_v0_InitialLoadBalanceResponse_size 156 -#define grpc_lb_v0_Server_size 127 +#define grpc_lb_v0_Server_size 127 /* Message IDs (where set with "msgid" option) */ #ifdef PB_MSGID -#define LOAD_BALANCER_MESSAGES +#define LOAD_BALANCER_MESSAGES \ + #endif diff --git a/tools/codegen/core/gen_load_balancing_proto.sh b/tools/codegen/core/gen_load_balancing_proto.sh index 114dd9d70d8..6d974ce31b9 100755 --- a/tools/codegen/core/gen_load_balancing_proto.sh +++ b/tools/codegen/core/gen_load_balancing_proto.sh @@ -70,9 +70,6 @@ read -r -d '' COPYRIGHT <<'EOF' EOF -# build clang-format docker image -docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format - CURRENT_YEAR=$(date +%Y) COPYRIGHT_FILE=$(mktemp) echo "${COPYRIGHT//$CURRENT_YEAR}" > $COPYRIGHT_FILE @@ -106,6 +103,15 @@ if [[ ! -f "${EXPECTED_OPTIONS_FILE_PATH}" ]]; then exit 4 fi +readonly VENV_DIR=$(mktemp -d) +readonly VENV_NAME="nanopb-$(date '+%Y%m%d_%H%M%S_%N')" +pushd $VENV_DIR +virtualenv $VENV_NAME +. $VENV_NAME/bin/activate +popd + +pip install protobuf==3.0.0b2 + pushd "$(dirname $1)" > /dev/null protoc \ @@ -124,10 +130,7 @@ mv -v $TMPFILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.c" cat $COPYRIGHT_FILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.h" > $TMPFILE mv -v $TMPFILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.h" -readonly MOUNTPOINT='/protos' -docker run --rm=true -v ${HOST_GIT_ROOT}/gens/src/proto/grpc/lb/v0:$MOUNTPOINT \ - -t grpc_clang_format \ - clang-format-3.6 -style="{BasedOnStyle: Google, Language: Cpp}" \ - -i $MOUNTPOINT/load_balancer.pb.c $MOUNTPOINT/load_balancer.pb.h +deactivate +rm -rf $VENV_DIR popd > /dev/null diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 78d3a734e96..3b01140996b 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -33,22 +33,6 @@ set -ex apt-get install -y autoconf automake libtool curl python-virtualenv readonly NANOPB_TMP_OUTPUT="${LOCAL_GIT_ROOT}/gens/src/proto/grpc/lb/v0" -readonly VENV_DIR=$(mktemp -d) -# create a virtualenv for nanopb's compiler -pushd $VENV_DIR -readonly VENV_NAME="nanopb-$(date '+%Y%m%d_%H%M%S_%N')" -virtualenv $VENV_NAME -. $VENV_NAME/bin/activate -popd - -# install proto3 -pip install protobuf==3.0.0b2 - -# change to root directory -cd $(dirname $0)/../.. - -# build clang-format docker image -docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format # install protoc version 3 pushd third_party/protobuf diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh index 86ba8b2e90b..af0b22a07f9 100755 --- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh @@ -44,7 +44,7 @@ for dir in $DIRS do for glob in $GLOB do - files="$files `find /local-code/$dir -name $glob -and -not -name *.generated.*`" + files="$files `find /local-code/$dir -name $glob -and -not -name *.generated.* -and -not -name *.pb.h -and -not -name *.pb.c`" done done From ddcc39267089a4886c420af8776e572f24714cbb Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 12 Feb 2016 09:19:05 -0800 Subject: [PATCH 006/236] Fix script --- tools/distrib/check_nanopb_output.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 3b01140996b..9cb7581d87f 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -67,4 +67,3 @@ if [ $? != 0 ]; then echo "Outputs differ: $NANOPB_TMP_OUTPUT vs src/core/proto/grpc/lb/v0" exit 1 fi -deactivate From ccdea1900fdad3d507617c8b1b639c7f5914d06b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 16 Feb 2016 08:06:46 -0800 Subject: [PATCH 007/236] Separate timer checking from pollsets --- BUILD | 3 - build.yaml | 1 - gRPC.podspec | 2 - grpc.gemspec | 1 - package.json | 1 - src/core/iomgr/iocp_windows.c | 2 +- src/core/iomgr/iomgr.c | 2 +- src/core/iomgr/pollset_posix.c | 11 ---- src/core/iomgr/pollset_windows.c | 4 -- src/core/iomgr/timer.c | 1 - src/core/iomgr/timer.h | 20 ++++++ src/core/iomgr/timer_internal.h | 61 ------------------- src/core/surface/completion_queue.c | 28 ++++++++- test/core/iomgr/timer_list_test.c | 1 - tools/doxygen/Doxyfile.core.internal | 1 - tools/run_tests/sources_and_headers.json | 4 -- vsprojects/vcxproj/grpc/grpc.vcxproj | 1 - vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 3 - .../grpc_unsecure/grpc_unsecure.vcxproj | 1 - .../grpc_unsecure.vcxproj.filters | 3 - 20 files changed, 48 insertions(+), 103 deletions(-) delete mode 100644 src/core/iomgr/timer_internal.h diff --git a/BUILD b/BUILD index 72d5caa8d43..24e3e540661 100644 --- a/BUILD +++ b/BUILD @@ -231,7 +231,6 @@ cc_library( "src/core/iomgr/time_averaged_stats.h", "src/core/iomgr/timer.h", "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", "src/core/iomgr/udp_server.h", "src/core/iomgr/wakeup_fd_pipe.h", "src/core/iomgr/wakeup_fd_posix.h", @@ -533,7 +532,6 @@ cc_library( "src/core/iomgr/time_averaged_stats.h", "src/core/iomgr/timer.h", "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", "src/core/iomgr/udp_server.h", "src/core/iomgr/wakeup_fd_pipe.h", "src/core/iomgr/wakeup_fd_posix.h", @@ -1490,7 +1488,6 @@ objc_library( "src/core/iomgr/time_averaged_stats.h", "src/core/iomgr/timer.h", "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", "src/core/iomgr/udp_server.h", "src/core/iomgr/wakeup_fd_pipe.h", "src/core/iomgr/wakeup_fd_posix.h", diff --git a/build.yaml b/build.yaml index 7f33ef3f0e5..f8fc4883832 100644 --- a/build.yaml +++ b/build.yaml @@ -307,7 +307,6 @@ filegroups: - src/core/iomgr/time_averaged_stats.h - src/core/iomgr/timer.h - src/core/iomgr/timer_heap.h - - src/core/iomgr/timer_internal.h - src/core/iomgr/udp_server.h - src/core/iomgr/wakeup_fd_pipe.h - src/core/iomgr/wakeup_fd_posix.h diff --git a/gRPC.podspec b/gRPC.podspec index 5b4d24e4820..13c303a8c7b 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -235,7 +235,6 @@ Pod::Spec.new do |s| 'src/core/iomgr/time_averaged_stats.h', 'src/core/iomgr/timer.h', 'src/core/iomgr/timer_heap.h', - 'src/core/iomgr/timer_internal.h', 'src/core/iomgr/udp_server.h', 'src/core/iomgr/wakeup_fd_pipe.h', 'src/core/iomgr/wakeup_fd_posix.h', @@ -541,7 +540,6 @@ Pod::Spec.new do |s| 'src/core/iomgr/time_averaged_stats.h', 'src/core/iomgr/timer.h', 'src/core/iomgr/timer_heap.h', - 'src/core/iomgr/timer_internal.h', 'src/core/iomgr/udp_server.h', 'src/core/iomgr/wakeup_fd_pipe.h', 'src/core/iomgr/wakeup_fd_posix.h', diff --git a/grpc.gemspec b/grpc.gemspec index 32fe4932a9f..4485b440d6a 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -231,7 +231,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/iomgr/time_averaged_stats.h ) s.files += %w( src/core/iomgr/timer.h ) s.files += %w( src/core/iomgr/timer_heap.h ) - s.files += %w( src/core/iomgr/timer_internal.h ) s.files += %w( src/core/iomgr/udp_server.h ) s.files += %w( src/core/iomgr/wakeup_fd_pipe.h ) s.files += %w( src/core/iomgr/wakeup_fd_posix.h ) diff --git a/package.json b/package.json index 8cbfb290559..3042c91680e 100644 --- a/package.json +++ b/package.json @@ -176,7 +176,6 @@ "src/core/iomgr/time_averaged_stats.h", "src/core/iomgr/timer.h", "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", "src/core/iomgr/udp_server.h", "src/core/iomgr/wakeup_fd_pipe.h", "src/core/iomgr/wakeup_fd_posix.h", diff --git a/src/core/iomgr/iocp_windows.c b/src/core/iomgr/iocp_windows.c index 759340e00ef..807729708ea 100644 --- a/src/core/iomgr/iocp_windows.c +++ b/src/core/iomgr/iocp_windows.c @@ -42,7 +42,7 @@ #include #include -#include "src/core/iomgr/timer_internal.h" +#include "src/core/iomgr/timer.h" #include "src/core/iomgr/iocp_windows.h" #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/socket_windows.h" diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c index 212ce5534dd..3283b586b06 100644 --- a/src/core/iomgr/iomgr.c +++ b/src/core/iomgr/iomgr.c @@ -43,7 +43,7 @@ #include #include "src/core/iomgr/iomgr_internal.h" -#include "src/core/iomgr/timer_internal.h" +#include "src/core/iomgr/timer.h" #include "src/core/support/string.h" static gpr_mu g_mu; diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 19ee6650f00..1063727248a 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -42,7 +42,6 @@ #include #include -#include "src/core/iomgr/timer_internal.h" #include "src/core/iomgr/fd_posix.h" #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/socket_utils_posix.h" @@ -274,16 +273,6 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); goto done; } - /* Check alarms - these are a global resource so we just ping - each time through on every pollset. - May update deadline to ensure timely wakeups. - TODO(ctiller): can this work be localized? */ - if (grpc_timer_check(exec_ctx, now, &deadline)) { - GPR_TIMER_MARK("grpc_pollset_work.alarm_triggered", 0); - gpr_mu_unlock(&pollset->mu); - locked = 0; - goto done; - } /* If we're shutting down then we don't execute any extended work */ if (pollset->shutting_down) { GPR_TIMER_MARK("grpc_pollset_work.shutting_down", 0); diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index 02c66783631..35a956b27fd 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -38,7 +38,6 @@ #include #include -#include "src/core/iomgr/timer_internal.h" #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/iocp_windows.h" #include "src/core/iomgr/pollset.h" @@ -136,9 +135,6 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, worker->kicked = 0; worker->pollset = pollset; gpr_cv_init(&worker->cv); - if (grpc_timer_check(exec_ctx, now, &deadline)) { - goto done; - } if (!pollset->kicked_without_pollers && !pollset->shutting_down) { if (g_active_poller == NULL) { grpc_pollset_worker *next_worker; diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index a33d8f63a05..5e7fadb7906 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -34,7 +34,6 @@ #include "src/core/iomgr/timer.h" #include "src/core/iomgr/timer_heap.h" -#include "src/core/iomgr/timer_internal.h" #include "src/core/iomgr/time_averaged_stats.h" #include #include diff --git a/src/core/iomgr/timer.h b/src/core/iomgr/timer.h index 720c9d5ab94..2f74b6e5d3d 100644 --- a/src/core/iomgr/timer.h +++ b/src/core/iomgr/timer.h @@ -86,4 +86,24 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, Requires: cancel() must happen after add() on a given timer */ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer); +/* iomgr internal api for dealing with timers */ + +/* Check for timers to be run, and run them. + Return non zero if timer callbacks were executed. + Drops drop_mu if it is non-null before executing callbacks. + If next is non-null, TRY to update *next with the next running timer + IF that timer occurs before *next current value. + *next is never guaranteed to be updated on any given execution; however, + with high probability at least one thread in the system will see an update + at any time slice. */ + +int grpc_timer_check(grpc_exec_ctx* exec_ctx, gpr_timespec now, + gpr_timespec* next); +void grpc_timer_list_init(gpr_timespec now); +void grpc_timer_list_shutdown(grpc_exec_ctx* exec_ctx); + +/* the following must be implemented by each iomgr implementation */ + +void grpc_kick_poller(void); + #endif /* GRPC_INTERNAL_CORE_IOMGR_TIMER_H */ diff --git a/src/core/iomgr/timer_internal.h b/src/core/iomgr/timer_internal.h deleted file mode 100644 index f182e737646..00000000000 --- a/src/core/iomgr/timer_internal.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef GRPC_INTERNAL_CORE_IOMGR_TIMER_INTERNAL_H -#define GRPC_INTERNAL_CORE_IOMGR_TIMER_INTERNAL_H - -#include "src/core/iomgr/exec_ctx.h" -#include -#include - -/* iomgr internal api for dealing with timers */ - -/* Check for timers to be run, and run them. - Return non zero if timer callbacks were executed. - Drops drop_mu if it is non-null before executing callbacks. - If next is non-null, TRY to update *next with the next running timer - IF that timer occurs before *next current value. - *next is never guaranteed to be updated on any given execution; however, - with high probability at least one thread in the system will see an update - at any time slice. */ - -int grpc_timer_check(grpc_exec_ctx* exec_ctx, gpr_timespec now, - gpr_timespec* next); -void grpc_timer_list_init(gpr_timespec now); -void grpc_timer_list_shutdown(grpc_exec_ctx* exec_ctx); - -/* the following must be implemented by each iomgr implementation */ - -void grpc_kick_poller(void); - -#endif /* GRPC_INTERNAL_CORE_IOMGR_TIMER_INTERNAL_H */ diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index 75298eb795c..6597c83cdca 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -323,7 +323,19 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, break; } first_loop = 0; - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, deadline); + /* Check alarms - these are a global resource so we just ping + each time through on every pollset. + May update deadline to ensure timely wakeups. + TODO(ctiller): can this work be localized? */ + gpr_timespec iteration_deadline = deadline; + if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { + GPR_TIMER_MARK("alarm_triggered", 0); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + grpc_exec_ctx_flush(&exec_ctx); + gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + continue; + } + grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, iteration_deadline); } GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); GRPC_CQ_INTERNAL_UNREF(cc, "next"); @@ -427,7 +439,19 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, break; } first_loop = 0; - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, deadline); + /* Check alarms - these are a global resource so we just ping + each time through on every pollset. + May update deadline to ensure timely wakeups. + TODO(ctiller): can this work be localized? */ + gpr_timespec iteration_deadline = deadline; + if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { + GPR_TIMER_MARK("alarm_triggered", 0); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + grpc_exec_ctx_flush(&exec_ctx); + gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + continue; + } + grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, iteration_deadline); del_plucker(cc, tag, &worker); } done: diff --git a/test/core/iomgr/timer_list_test.c b/test/core/iomgr/timer_list_test.c index 15de87c5a1a..487527fbf5d 100644 --- a/test/core/iomgr/timer_list_test.c +++ b/test/core/iomgr/timer_list_test.c @@ -35,7 +35,6 @@ #include -#include "src/core/iomgr/timer_internal.h" #include #include "test/core/util/test_config.h" diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index b6268432335..ffc40dfc19b 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -850,7 +850,6 @@ src/core/iomgr/tcp_windows.h \ src/core/iomgr/time_averaged_stats.h \ src/core/iomgr/timer.h \ src/core/iomgr/timer_heap.h \ -src/core/iomgr/timer_internal.h \ src/core/iomgr/udp_server.h \ src/core/iomgr/wakeup_fd_pipe.h \ src/core/iomgr/wakeup_fd_posix.h \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 6538ddc37e1..fdba0417ca6 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -3032,7 +3032,6 @@ "src/core/iomgr/time_averaged_stats.h", "src/core/iomgr/timer.h", "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", "src/core/iomgr/udp_server.h", "src/core/iomgr/wakeup_fd_pipe.h", "src/core/iomgr/wakeup_fd_posix.h", @@ -3251,7 +3250,6 @@ "src/core/iomgr/timer.h", "src/core/iomgr/timer_heap.c", "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", "src/core/iomgr/udp_server.c", "src/core/iomgr/udp_server.h", "src/core/iomgr/wakeup_fd_eventfd.c", @@ -3557,7 +3555,6 @@ "src/core/iomgr/time_averaged_stats.h", "src/core/iomgr/timer.h", "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", "src/core/iomgr/udp_server.h", "src/core/iomgr/wakeup_fd_pipe.h", "src/core/iomgr/wakeup_fd_posix.h", @@ -3760,7 +3757,6 @@ "src/core/iomgr/timer.h", "src/core/iomgr/timer_heap.c", "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", "src/core/iomgr/udp_server.c", "src/core/iomgr/udp_server.h", "src/core/iomgr/wakeup_fd_eventfd.c", diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 76975322be8..8e8f29aee99 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -359,7 +359,6 @@ - diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 4660572f979..e55c6673cea 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -734,9 +734,6 @@ src\core\iomgr - - src\core\iomgr - src\core\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 541000af404..af89435885c 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -335,7 +335,6 @@ - diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 48814f997e1..809ea59c5f4 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -629,9 +629,6 @@ src\core\iomgr - - src\core\iomgr - src\core\iomgr From eb62c943389d4b34e8a246a8fe63783aed885a01 Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 17 Feb 2016 15:37:25 -0800 Subject: [PATCH 008/236] Add a way to override channel arguments for server creation --- include/grpc++/server.h | 6 ++++-- src/cpp/server/server.cc | 14 ++++++-------- src/cpp/server/server_builder.cc | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/grpc++/server.h b/include/grpc++/server.h index 2a71073a7e2..c177805236b 100644 --- a/include/grpc++/server.h +++ b/include/grpc++/server.h @@ -79,6 +79,8 @@ class Server GRPC_FINAL : public ServerInterface, private GrpcLibrary { class GlobalCallbacks { public: virtual ~GlobalCallbacks() {} + /// Called before server is created. + virtual void UpdateArguments(ChannelArguments* args) {} /// Called before application callback for each synchronous server request virtual void PreSynchronousRequest(ServerContext* context) = 0; /// Called after application callback for each synchronous server request @@ -108,7 +110,7 @@ class Server GRPC_FINAL : public ServerInterface, private GrpcLibrary { /// \param max_message_size Maximum message length that the channel can /// receive. Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned, - int max_message_size, const ChannelArguments& args); + int max_message_size, ChannelArguments* args); /// Register a service. This call does not take ownership of the service. /// The service must exist for the lifetime of the Server instance. @@ -177,7 +179,7 @@ class Server GRPC_FINAL : public ServerInterface, private GrpcLibrary { bool has_generic_service_; // Pointer to the c grpc server. - grpc_server* const server_; + grpc_server* server_; ThreadPoolInterface* thread_pool_; // Whether the thread pool is created and owned by the server. diff --git a/src/cpp/server/server.cc b/src/cpp/server/server.cc index 0d31140924b..6d31a608c80 100644 --- a/src/cpp/server/server.cc +++ b/src/cpp/server/server.cc @@ -272,27 +272,25 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag { grpc_completion_queue* cq_; }; -static grpc_server* CreateServer(const ChannelArguments& args) { - grpc_channel_args channel_args; - args.SetChannelArgs(&channel_args); - return grpc_server_create(&channel_args, nullptr); -} - static internal::GrpcLibraryInitializer g_gli_initializer; Server::Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned, - int max_message_size, const ChannelArguments& args) + int max_message_size, ChannelArguments* args) : max_message_size_(max_message_size), started_(false), shutdown_(false), num_running_cb_(0), sync_methods_(new std::list), has_generic_service_(false), - server_(CreateServer(args)), + server_(nullptr), thread_pool_(thread_pool), thread_pool_owned_(thread_pool_owned) { g_gli_initializer.summon(); gpr_once_init(&g_once_init_callbacks, InitGlobalCallbacks); global_callbacks_ = g_callbacks; + global_callbacks_->UpdateArguments(args); + grpc_channel_args channel_args; + args->SetChannelArgs(&channel_args); + server_ = grpc_server_create(&channel_args, nullptr); grpc_server_register_completion_queue(server_, cq_.cq(), nullptr); } diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc index c54cf6474f1..134e5f1d5ff 100644 --- a/src/cpp/server/server_builder.cc +++ b/src/cpp/server/server_builder.cc @@ -103,7 +103,7 @@ std::unique_ptr ServerBuilder::BuildAndStart() { args.SetInt(GRPC_COMPRESSION_ALGORITHM_STATE_ARG, compression_options_.enabled_algorithms_bitset); std::unique_ptr server( - new Server(thread_pool.release(), true, max_message_size_, args)); + new Server(thread_pool.release(), true, max_message_size_, &args)); for (auto cq = cqs_.begin(); cq != cqs_.end(); ++cq) { grpc_server_register_completion_queue(server->server_, (*cq)->cq(), nullptr); From 5e07d76a86c42e4e4e5d103b7579e364339af4d0 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 17 Feb 2016 17:10:41 -0800 Subject: [PATCH 009/236] Made Alarm's constructor a template for deadline --- include/grpc++/alarm.h | 8 +++++++- src/cpp/common/alarm.cc | 10 +++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/grpc++/alarm.h b/include/grpc++/alarm.h index 9979c34e4f1..66904deb48b 100644 --- a/include/grpc++/alarm.h +++ b/include/grpc++/alarm.h @@ -39,6 +39,8 @@ #include #include #include +#include +#include struct grpc_alarm; @@ -54,7 +56,11 @@ class Alarm : private GrpcLibrary { /// Once the alarm expires (at \a deadline) or it's cancelled (see \a Cancel), /// an event with tag \a tag will be added to \a cq. If the alarm expired, the /// event's success bit will be true, false otherwise (ie, upon cancellation). - Alarm(CompletionQueue* cq, gpr_timespec deadline, void* tag); + template + Alarm(CompletionQueue* cq, const T& deadline, void* tag) + : tag_(tag), + alarm_(grpc_alarm_create(cq->cq(), TimePoint(deadline).raw_time(), + static_cast(&tag_))) {} /// Destroy the given completion queue alarm, cancelling it in the process. ~Alarm(); diff --git a/src/cpp/common/alarm.cc b/src/cpp/common/alarm.cc index a2896887682..8af17597ef0 100644 --- a/src/cpp/common/alarm.cc +++ b/src/cpp/common/alarm.cc @@ -31,21 +31,17 @@ */ #include -#include #include -#include namespace grpc { static internal::GrpcLibraryInitializer g_gli_initializer; -Alarm::Alarm(CompletionQueue* cq, gpr_timespec deadline, void* tag) - : tag_(tag), - alarm_(grpc_alarm_create(cq->cq(), deadline, static_cast(&tag_))) { + +Alarm::~Alarm() { g_gli_initializer.summon(); + grpc_alarm_destroy(alarm_); } -Alarm::~Alarm() { grpc_alarm_destroy(alarm_); } - void Alarm::Cancel() { grpc_alarm_cancel(alarm_); } } // namespace grpc From 311445fd32956e9383823400c82ce3fcd71b2b31 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 18 Feb 2016 07:31:39 -0800 Subject: [PATCH 010/236] Fix tcp_client_posix_test --- src/core/iomgr/timer.c | 4 ++-- src/core/iomgr/timer.h | 8 ++++---- src/core/surface/completion_queue.c | 6 ++++-- test/core/iomgr/tcp_client_posix_test.c | 26 +++++++++++++++++-------- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 5e7fadb7906..8379fffad02 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -335,8 +335,8 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, return (int)n; } -int grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, - gpr_timespec *next) { +bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, + gpr_timespec *next) { GPR_ASSERT(now.clock_type == g_clock_type); return run_some_expired_timers( exec_ctx, now, next, diff --git a/src/core/iomgr/timer.h b/src/core/iomgr/timer.h index 2f74b6e5d3d..906255ddfb9 100644 --- a/src/core/iomgr/timer.h +++ b/src/core/iomgr/timer.h @@ -89,7 +89,7 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer); /* iomgr internal api for dealing with timers */ /* Check for timers to be run, and run them. - Return non zero if timer callbacks were executed. + Return true if timer callbacks were executed. Drops drop_mu if it is non-null before executing callbacks. If next is non-null, TRY to update *next with the next running timer IF that timer occurs before *next current value. @@ -97,10 +97,10 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer); with high probability at least one thread in the system will see an update at any time slice. */ -int grpc_timer_check(grpc_exec_ctx* exec_ctx, gpr_timespec now, - gpr_timespec* next); +bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, + gpr_timespec *next); void grpc_timer_list_init(gpr_timespec now); -void grpc_timer_list_shutdown(grpc_exec_ctx* exec_ctx); +void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx); /* the following must be implemented by each iomgr implementation */ diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index 6597c83cdca..de295ab9410 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -335,7 +335,8 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); continue; } - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, iteration_deadline); + grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, + iteration_deadline); } GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); GRPC_CQ_INTERNAL_UNREF(cc, "next"); @@ -451,7 +452,8 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); continue; } - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, iteration_deadline); + grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, + iteration_deadline); del_plucker(cc, tag, &worker); } done: diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 9725d8a3b64..b57478059f0 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -45,6 +45,7 @@ #include "src/core/iomgr/iomgr.h" #include "src/core/iomgr/socket_utils_posix.h" +#include "src/core/iomgr/timer.h" #include "test/core/util/test_config.h" static grpc_pollset_set g_pollset_set; @@ -125,11 +126,13 @@ void test_succeeds(void) { gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5)); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); - grpc_exec_ctx_finish(&exec_ctx); + grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + + grpc_exec_ctx_finish(&exec_ctx); } void test_fails(void) { @@ -159,14 +162,18 @@ void test_fails(void) { /* wait for the connection callback to finish */ while (g_connections_complete == connections_complete_before) { grpc_pollset_worker worker; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, - gpr_now(GPR_CLOCK_MONOTONIC), test_deadline()); + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec polling_deadline = test_deadline(); + if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, now, polling_deadline); + } gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); - grpc_exec_ctx_finish(&exec_ctx); + grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_exec_ctx_finish(&exec_ctx); } void test_times_out(void) { @@ -243,15 +250,18 @@ void test_times_out(void) { GPR_ASSERT(g_connections_complete == connections_complete_before + is_after_deadline); } - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, - gpr_now(GPR_CLOCK_MONOTONIC), - GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10)); + gpr_timespec polling_deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10); + if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, now, polling_deadline); + } gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); - grpc_exec_ctx_finish(&exec_ctx); + grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_exec_ctx_finish(&exec_ctx); + close(svr_fd); for (i = 0; i < NUM_CLIENT_CONNECTS; ++i) { close(client_fd[i]); From d0a8ae126625856aa84ba151fc5584b60097a65e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 18 Feb 2016 08:01:19 -0800 Subject: [PATCH 011/236] Move worker into pollset --- src/core/iomgr/pollset.h | 2 +- src/core/iomgr/pollset_posix.c | 38 ++++++++++--------- .../security/google_default_credentials.c | 2 +- src/core/surface/completion_queue.c | 12 +++--- test/core/httpcli/httpcli_test.c | 4 +- test/core/httpcli/httpscli_test.c | 4 +- test/core/iomgr/endpoint_tests.c | 2 +- test/core/iomgr/fd_posix_test.c | 8 ++-- test/core/iomgr/tcp_client_posix_test.c | 6 +-- test/core/iomgr/tcp_posix_test.c | 12 +++--- test/core/iomgr/tcp_server_posix_test.c | 2 +- test/core/security/oauth2_utils.c | 2 +- test/core/util/port_posix.c | 4 +- test/core/util/test_tcp_server.c | 2 +- 14 files changed, 52 insertions(+), 48 deletions(-) diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index c6b0214dea2..0c0efad7600 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -83,7 +83,7 @@ void grpc_pollset_destroy(grpc_pollset *pollset); pollset lock */ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_pollset_worker *worker, gpr_timespec now, + grpc_pollset_worker **worker, gpr_timespec now, gpr_timespec deadline); /* Break one polling thread out of polling work for this pollset. diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 1063727248a..ee7e9f48f4c 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -246,8 +246,11 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { } void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_pollset_worker *worker, gpr_timespec now, + grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { + grpc_pollset_worker worker; + *worker_hdl = &worker; + /* pollset->mu already held */ int added_worker = 0; int locked = 1; @@ -255,16 +258,16 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, int keep_polling = 0; GPR_TIMER_BEGIN("grpc_pollset_work", 0); /* this must happen before we (potentially) drop pollset->mu */ - worker->next = worker->prev = NULL; - worker->reevaluate_polling_on_wakeup = 0; + worker.next = worker.prev = NULL; + worker.reevaluate_polling_on_wakeup = 0; if (pollset->local_wakeup_cache != NULL) { - worker->wakeup_fd = pollset->local_wakeup_cache; - pollset->local_wakeup_cache = worker->wakeup_fd->next; + worker.wakeup_fd = pollset->local_wakeup_cache; + pollset->local_wakeup_cache = worker.wakeup_fd->next; } else { - worker->wakeup_fd = gpr_malloc(sizeof(*worker->wakeup_fd)); - grpc_wakeup_fd_init(&worker->wakeup_fd->fd); + worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd)); + grpc_wakeup_fd_init(&worker.wakeup_fd->fd); } - worker->kicked_specifically = 0; + worker.kicked_specifically = 0; /* If there's work waiting for the pollset to be idle, and the pollset is idle, then do that work */ if (!grpc_pollset_has_workers(pollset) && @@ -293,13 +296,13 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, keep_polling = 0; if (!pollset->kicked_without_pollers) { if (!added_worker) { - push_front_worker(pollset, worker); + push_front_worker(pollset, &worker); added_worker = 1; - gpr_tls_set(&g_current_thread_worker, (intptr_t)worker); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); } gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset); GPR_TIMER_BEGIN("maybe_work_and_unlock", 0); - pollset->vtable->maybe_work_and_unlock(exec_ctx, pollset, worker, + pollset->vtable->maybe_work_and_unlock(exec_ctx, pollset, &worker, deadline, now); GPR_TIMER_END("maybe_work_and_unlock", 0); locked = 0; @@ -321,10 +324,10 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* If we're forced to re-evaluate polling (via grpc_pollset_kick with GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force a loop */ - if (worker->reevaluate_polling_on_wakeup) { - worker->reevaluate_polling_on_wakeup = 0; + if (worker.reevaluate_polling_on_wakeup) { + worker.reevaluate_polling_on_wakeup = 0; pollset->kicked_without_pollers = 0; - if (queued_work || worker->kicked_specifically) { + if (queued_work || worker.kicked_specifically) { /* If there's queued work on the list, then set the deadline to be immediate so we get back out of the polling loop quickly */ deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); @@ -333,12 +336,12 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } if (added_worker) { - remove_worker(pollset, worker); + remove_worker(pollset, &worker); gpr_tls_set(&g_current_thread_worker, 0); } /* release wakeup fd to the local pool */ - worker->wakeup_fd->next = pollset->local_wakeup_cache; - pollset->local_wakeup_cache = worker->wakeup_fd; + worker.wakeup_fd->next = pollset->local_wakeup_cache; + pollset->local_wakeup_cache = worker.wakeup_fd; /* check shutdown conditions */ if (pollset->shutting_down) { if (grpc_pollset_has_workers(pollset)) { @@ -360,6 +363,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_mu_lock(&pollset->mu); } } + *worker_hdl = NULL; GPR_TIMER_END("grpc_pollset_work", 0); } diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c index f3ac14568a6..cc9c9582986 100644 --- a/src/core/security/google_default_credentials.c +++ b/src/core/security/google_default_credentials.c @@ -122,7 +122,7 @@ static int is_stack_running_on_compute_engine(void) { called once for the lifetime of the process by the default credentials. */ gpr_mu_lock(GRPC_POLLSET_MU(&detector.pollset)); while (!detector.is_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &detector.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index de295ab9410..376feb1bbe8 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -50,7 +50,7 @@ #include typedef struct { - grpc_pollset_worker *worker; + grpc_pollset_worker **worker; void *tag; } plucker; @@ -252,7 +252,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, pluck_worker = NULL; for (i = 0; i < cc->num_pluckers; i++) { if (cc->pluckers[i].tag == tag) { - pluck_worker = cc->pluckers[i].worker; + pluck_worker = *cc->pluckers[i].worker; break; } } @@ -275,7 +275,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_timespec deadline, void *reserved) { grpc_event ret; - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; int first_loop = 1; gpr_timespec now; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -348,7 +348,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, } static int add_plucker(grpc_completion_queue *cc, void *tag, - grpc_pollset_worker *worker) { + grpc_pollset_worker **worker) { if (cc->num_pluckers == GRPC_MAX_COMPLETION_QUEUE_PLUCKERS) { return 0; } @@ -359,7 +359,7 @@ static int add_plucker(grpc_completion_queue *cc, void *tag, } static void del_plucker(grpc_completion_queue *cc, void *tag, - grpc_pollset_worker *worker) { + grpc_pollset_worker **worker) { int i; for (i = 0; i < cc->num_pluckers; i++) { if (cc->pluckers[i].tag == tag && cc->pluckers[i].worker == worker) { @@ -376,7 +376,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, grpc_event ret; grpc_cq_completion *c; grpc_cq_completion *prev; - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; gpr_timespec now; int first_loop = 1; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; diff --git a/test/core/httpcli/httpcli_test.c b/test/core/httpcli/httpcli_test.c index 651ef1fa3b8..e954a920b04 100644 --- a/test/core/httpcli/httpcli_test.c +++ b/test/core/httpcli/httpcli_test.c @@ -89,7 +89,7 @@ static void test_get(int port) { on_finish, (void *)42); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!g_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); @@ -120,7 +120,7 @@ static void test_post(int port) { n_seconds_time(15), on_finish, (void *)42); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!g_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); diff --git a/test/core/httpcli/httpscli_test.c b/test/core/httpcli/httpscli_test.c index db41be17e73..9f31eae2782 100644 --- a/test/core/httpcli/httpscli_test.c +++ b/test/core/httpcli/httpscli_test.c @@ -90,7 +90,7 @@ static void test_get(int port) { on_finish, (void *)42); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!g_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); @@ -122,7 +122,7 @@ static void test_post(int port) { n_seconds_time(15), on_finish, (void *)42); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!g_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c index 1b6a78da9ac..6ba67df3b10 100644 --- a/test/core/iomgr/endpoint_tests.c +++ b/test/core/iomgr/endpoint_tests.c @@ -234,7 +234,7 @@ static void read_and_write_test(grpc_endpoint_test_config config, gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); while (!state.read_done || !state.write_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; GPR_ASSERT(gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0); grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 347a86af108..07761b6576d 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -252,7 +252,7 @@ static void server_wait_and_shutdown(server *sv) { gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!sv->done) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); @@ -366,7 +366,7 @@ static void client_start(grpc_exec_ctx *exec_ctx, client *cl, int port) { static void client_wait_and_shutdown(client *cl) { gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!cl->done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), @@ -467,7 +467,7 @@ static void test_grpc_fd_change(void) { /* And now wait for it to run. */ gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (a.cb_that_ran == NULL) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); @@ -491,7 +491,7 @@ static void test_grpc_fd_change(void) { gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (b.cb_that_ran == NULL) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index b57478059f0..cdd5b096fbc 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -121,7 +121,7 @@ void test_succeeds(void) { gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (g_connections_complete == connections_complete_before) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5)); @@ -161,7 +161,7 @@ void test_fails(void) { /* wait for the connection callback to finish */ while (g_connections_complete == connections_complete_before) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); gpr_timespec polling_deadline = test_deadline(); if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { @@ -228,7 +228,7 @@ void test_times_out(void) { /* Make sure the event doesn't trigger early */ gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); for (;;) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; gpr_timespec now = gpr_now(connect_deadline.clock_type); gpr_timespec continue_verifying_time = gpr_time_from_seconds(5, GPR_TIMESPAN); diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index d290c6bc3ab..20b0b9e13b7 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -190,7 +190,7 @@ static void read_test(size_t num_bytes, size_t slice_size) { gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (state.read_bytes < state.target_read_bytes) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); @@ -236,7 +236,7 @@ static void large_read_test(size_t slice_size) { gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (state.read_bytes < state.target_read_bytes) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); @@ -303,7 +303,7 @@ void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { GPR_ASSERT(fcntl(fd, F_SETFL, flags & ~O_NONBLOCK) == 0); for (;;) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), @@ -365,7 +365,7 @@ static void write_test(size_t num_bytes, size_t slice_size) { drain_socket_blocking(sv[0], num_bytes, num_bytes); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); for (;;) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; if (state.write_done) { break; } @@ -425,7 +425,7 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (state.read_bytes < state.target_read_bytes) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); @@ -439,7 +439,7 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { grpc_tcp_destroy_and_release_fd(&exec_ctx, ep, &fd, &fd_released_cb); gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!fd_released_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); } diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 272d97bfcbd..aaab0fc359e 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -183,7 +183,7 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, gpr_log(GPR_DEBUG, "wait"); while (g_nconnects == nconnects_before && gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index 55ac31e62ca..4dd595df956 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -95,7 +95,7 @@ char *grpc_test_fetch_oauth2_token_with_credentials( gpr_mu_lock(GRPC_POLLSET_MU(&request.pollset)); while (!request.is_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &request.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c index 4b31f810e59..ad21fe0f536 100644 --- a/test/core/util/port_posix.c +++ b/test/core/util/port_posix.c @@ -114,7 +114,7 @@ static void free_port_using_server(char *server, int port) { &pr); gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); while (!pr.done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); @@ -280,7 +280,7 @@ static int pick_port_using_server(char *server) { grpc_exec_ctx_finish(&exec_ctx); gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); while (pr.port == -1) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index 66470c02886..e99d5dcffdd 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -85,7 +85,7 @@ void test_tcp_server_start(test_tcp_server *server, int port) { } void test_tcp_server_poll(test_tcp_server *server, int seconds) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; gpr_timespec deadline = gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(seconds, GPR_TIMESPAN)); From cb6c7f66d3b08eab81c3d51de4e489fe24c63e4c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 18 Feb 2016 08:02:45 -0800 Subject: [PATCH 012/236] Fix test --- test/core/iomgr/timer_list_test.c | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/test/core/iomgr/timer_list_test.c b/test/core/iomgr/timer_list_test.c index 487527fbf5d..7a21fdd5c10 100644 --- a/test/core/iomgr/timer_list_test.c +++ b/test/core/iomgr/timer_list_test.c @@ -71,20 +71,19 @@ static void add_test(void) { } /* collect timers. Only the first batch should be ready. */ - GPR_ASSERT(10 == grpc_timer_check(&exec_ctx, - gpr_time_add(start, gpr_time_from_millis( - 500, GPR_TIMESPAN)), - NULL)); + GPR_ASSERT(grpc_timer_check( + &exec_ctx, gpr_time_add(start, gpr_time_from_millis(500, GPR_TIMESPAN)), + NULL)); grpc_exec_ctx_finish(&exec_ctx); for (i = 0; i < 20; i++) { GPR_ASSERT(cb_called[i][1] == (i < 10)); GPR_ASSERT(cb_called[i][0] == 0); } - GPR_ASSERT(0 == grpc_timer_check(&exec_ctx, - gpr_time_add(start, gpr_time_from_millis( - 600, GPR_TIMESPAN)), - NULL)); + GPR_ASSERT(!grpc_timer_check( + &exec_ctx, + gpr_time_add(start, gpr_time_from_millis(600, GPR_TIMESPAN)), + NULL)); grpc_exec_ctx_finish(&exec_ctx); for (i = 0; i < 30; i++) { GPR_ASSERT(cb_called[i][1] == (i < 10)); @@ -92,20 +91,19 @@ static void add_test(void) { } /* collect the rest of the timers */ - GPR_ASSERT(10 == grpc_timer_check( - &exec_ctx, gpr_time_add(start, gpr_time_from_millis( - 1500, GPR_TIMESPAN)), - NULL)); + GPR_ASSERT(grpc_timer_check( + &exec_ctx, gpr_time_add(start, gpr_time_from_millis(1500, GPR_TIMESPAN)), + NULL)); grpc_exec_ctx_finish(&exec_ctx); for (i = 0; i < 30; i++) { GPR_ASSERT(cb_called[i][1] == (i < 20)); GPR_ASSERT(cb_called[i][0] == 0); } - GPR_ASSERT(0 == grpc_timer_check(&exec_ctx, - gpr_time_add(start, gpr_time_from_millis( - 1600, GPR_TIMESPAN)), - NULL)); + GPR_ASSERT(!grpc_timer_check( + &exec_ctx, + gpr_time_add(start, gpr_time_from_millis(1600, GPR_TIMESPAN)), + NULL)); for (i = 0; i < 30; i++) { GPR_ASSERT(cb_called[i][1] == (i < 20)); GPR_ASSERT(cb_called[i][0] == 0); From f290e30d087855f7104857d294851b9e7d35c11c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 18 Feb 2016 08:04:36 -0800 Subject: [PATCH 013/236] Finish porting posix --- test/core/end2end/fixtures/h2_uchannel.c | 2 +- test/core/iomgr/tcp_server_posix_test.c | 2 +- test/core/iomgr/workqueue_test.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/core/end2end/fixtures/h2_uchannel.c b/test/core/end2end/fixtures/h2_uchannel.c index dbdd3524ed0..33055561cb3 100644 --- a/test/core/end2end/fixtures/h2_uchannel.c +++ b/test/core/end2end/fixtures/h2_uchannel.c @@ -264,7 +264,7 @@ static grpc_connected_subchannel *connect_subchannel(grpc_subchannel *c) { grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(GRPC_POLLSET_MU(&pollset)); while (g_state != GRPC_CHANNEL_READY) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index aaab0fc359e..43180b16988 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -183,7 +183,7 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, gpr_log(GPR_DEBUG, "wait"); while (g_nconnects == nconnects_before && gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) { - grpc_pollset_worker *worker = NULL; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); diff --git a/test/core/iomgr/workqueue_test.c b/test/core/iomgr/workqueue_test.c index 500170b542c..a0249152566 100644 --- a/test/core/iomgr/workqueue_test.c +++ b/test/core/iomgr/workqueue_test.c @@ -63,7 +63,7 @@ static void test_add_closure(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_workqueue *wq = grpc_workqueue_create(&exec_ctx); gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5); - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_closure_init(&c, must_succeed, &done); grpc_workqueue_push(wq, &c, 1); @@ -87,7 +87,7 @@ static void test_flush(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_workqueue *wq = grpc_workqueue_create(&exec_ctx); gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5); - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_closure_init(&c, must_succeed, &done); grpc_exec_ctx_enqueue(&exec_ctx, &c, true, NULL); From bd479284dc55d8e61590c182f25b10af02ff2592 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 18 Feb 2016 08:07:09 -0800 Subject: [PATCH 014/236] Update windows --- src/core/iomgr/pollset_windows.c | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index 35a956b27fd..8f35a465091 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -125,22 +125,25 @@ void grpc_pollset_reset(grpc_pollset *pollset) { } void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_pollset_worker *worker, gpr_timespec now, + grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { + grpc_pollset_worker worker; + *worker_hdl = &worker; + int added_worker = 0; - worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = - worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = - worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next = - worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = NULL; - worker->kicked = 0; - worker->pollset = pollset; - gpr_cv_init(&worker->cv); + worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = + worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = + worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next = + worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = NULL; + worker.kicked = 0; + worker.pollset = pollset; + gpr_cv_init(&worker.cv); if (!pollset->kicked_without_pollers && !pollset->shutting_down) { if (g_active_poller == NULL) { grpc_pollset_worker *next_worker; /* become poller */ pollset->is_iocp_worker = 1; - g_active_poller = worker; + g_active_poller = &worker; gpr_mu_unlock(&grpc_polling_mu); grpc_iocp_work(exec_ctx, deadline); grpc_exec_ctx_flush(exec_ctx); @@ -167,12 +170,12 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, goto done; } push_front_worker(&g_global_root_worker, GRPC_POLLSET_WORKER_LINK_GLOBAL, - worker); + &worker); push_front_worker(&pollset->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET, - worker); + &worker); added_worker = 1; - while (!worker->kicked) { - if (gpr_cv_wait(&worker->cv, &grpc_polling_mu, deadline)) { + while (!worker.kicked) { + if (gpr_cv_wait(&worker.cv, &grpc_polling_mu, deadline)) { break; } } @@ -186,10 +189,11 @@ done: gpr_mu_lock(&grpc_polling_mu); } if (added_worker) { - remove_worker(worker, GRPC_POLLSET_WORKER_LINK_GLOBAL); - remove_worker(worker, GRPC_POLLSET_WORKER_LINK_POLLSET); + remove_worker(&worker, GRPC_POLLSET_WORKER_LINK_GLOBAL); + remove_worker(&worker, GRPC_POLLSET_WORKER_LINK_POLLSET); } gpr_cv_destroy(&worker->cv); + *worker_hdl = NULL; } void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { From 3633ce48a9ec540c9608d2a7e773d4e1113bb1e1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 18 Feb 2016 08:08:53 -0800 Subject: [PATCH 015/236] Fix tools --- test/core/security/print_google_default_creds_token.c | 2 +- test/core/security/verify_jwt.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index 50fe61c9969..fcbbe334f45 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -100,7 +100,7 @@ int main(int argc, char **argv) { gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); while (!sync.is_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index 378a37f26c5..c9071fa08b0 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -111,7 +111,7 @@ int main(int argc, char **argv) { gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); while (!sync.is_done) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); From f315bc198009eff04a53e48cec9b9506c85a2417 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 18 Feb 2016 08:25:15 -0800 Subject: [PATCH 016/236] Fix copyrights --- src/core/iomgr/iomgr.c | 2 +- src/core/iomgr/timer.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c index 3283b586b06..04580150f3a 100644 --- a/src/core/iomgr/iomgr.c +++ b/src/core/iomgr/iomgr.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/timer.h b/src/core/iomgr/timer.h index 906255ddfb9..9ad1e92f42e 100644 --- a/src/core/iomgr/timer.h +++ b/src/core/iomgr/timer.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 89d517c86a7ce072fec93d435b27f5a7903a7f1b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 18 Feb 2016 08:27:29 -0800 Subject: [PATCH 017/236] Fix copyrights --- src/core/iomgr/pollset.h | 2 +- test/core/security/print_google_default_creds_token.c | 2 +- test/core/security/verify_jwt.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index 0c0efad7600..6585326f81a 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index fcbbe334f45..6043bf54204 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index c9071fa08b0..5070cf0492c 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 64e7875510c0a92ff613608325ea3de0767e0a72 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 18 Feb 2016 13:38:32 -0800 Subject: [PATCH 018/236] Should have a test for 0 and negative alarms to make sure that those make it on to the CQ like any other. --- test/cpp/common/alarm_cpp_test.cc | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/cpp/common/alarm_cpp_test.cc b/test/cpp/common/alarm_cpp_test.cc index 4745ef14ec7..b81f85a8898 100644 --- a/test/cpp/common/alarm_cpp_test.cc +++ b/test/cpp/common/alarm_cpp_test.cc @@ -55,6 +55,36 @@ TEST(AlarmTest, RegularExpiry) { EXPECT_EQ(junk, output_tag); } +TEST(AlarmTest, ZeroExpiry) { + CompletionQueue cq; + void* junk = reinterpret_cast(1618033); + Alarm alarm(&cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(0), junk); + + void* output_tag; + bool ok; + const CompletionQueue::NextStatus status = cq.AsyncNext( + (void**)&output_tag, &ok, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); + + EXPECT_EQ(status, CompletionQueue::GOT_EVENT); + EXPECT_TRUE(ok); + EXPECT_EQ(junk, output_tag); +} + +TEST(AlarmTest, NegativeExpiry) { + CompletionQueue cq; + void* junk = reinterpret_cast(1618033); + Alarm alarm(&cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(-1), junk); + + void* output_tag; + bool ok; + const CompletionQueue::NextStatus status = cq.AsyncNext( + (void**)&output_tag, &ok, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); + + EXPECT_EQ(status, CompletionQueue::GOT_EVENT); + EXPECT_TRUE(ok); + EXPECT_EQ(junk, output_tag); +} + TEST(AlarmTest, Cancellation) { CompletionQueue cq; void* junk = reinterpret_cast(1618033); From 1cbb5673179677810bd6a5632b6faae0a9b6cef4 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 18 Feb 2016 14:27:28 -0800 Subject: [PATCH 019/236] minor tweaks to C# qpswoker --- src/csharp/Grpc.IntegrationTesting/Control.cs | 419 +++++++++++++++--- .../Grpc.IntegrationTesting/QpsWorker.cs | 13 +- .../RunnerClientServerTest.cs | 4 +- .../Grpc.IntegrationTesting/ServerRunners.cs | 2 +- .../Grpc.IntegrationTesting/Services.cs | 7 +- .../Grpc.IntegrationTesting/ServicesGrpc.cs | 71 ++- .../WorkerServiceImpl.cs | 18 + 7 files changed, 468 insertions(+), 66 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/Control.cs b/src/csharp/Grpc.IntegrationTesting/Control.cs index b90243c2bd5..291bc753978 100644 --- a/src/csharp/Grpc.IntegrationTesting/Control.cs +++ b/src/csharp/Grpc.IntegrationTesting/Control.cs @@ -38,7 +38,7 @@ namespace Grpc.Testing { "LmdycGMudGVzdGluZy5EZXRlcm1pbmlzdGljUGFyYW1zSAASLAoGcGFyZXRv", "GAUgASgLMhouZ3JwYy50ZXN0aW5nLlBhcmV0b1BhcmFtc0gAQgYKBGxvYWQi", "QwoOU2VjdXJpdHlQYXJhbXMSEwoLdXNlX3Rlc3RfY2EYASABKAgSHAoUc2Vy", - "dmVyX2hvc3Rfb3ZlcnJpZGUYAiABKAkirwMKDENsaWVudENvbmZpZxIWCg5z", + "dmVyX2hvc3Rfb3ZlcnJpZGUYAiABKAki1gMKDENsaWVudENvbmZpZxIWCg5z", "ZXJ2ZXJfdGFyZ2V0cxgBIAMoCRItCgtjbGllbnRfdHlwZRgCIAEoDjIYLmdy", "cGMudGVzdGluZy5DbGllbnRUeXBlEjUKD3NlY3VyaXR5X3BhcmFtcxgDIAEo", "CzIcLmdycGMudGVzdGluZy5TZWN1cml0eVBhcmFtcxIkChxvdXRzdGFuZGlu", @@ -48,24 +48,27 @@ namespace Grpc.Testing { "GAogASgLMhguZ3JwYy50ZXN0aW5nLkxvYWRQYXJhbXMSMwoOcGF5bG9hZF9j", "b25maWcYCyABKAsyGy5ncnBjLnRlc3RpbmcuUGF5bG9hZENvbmZpZxI3ChBo", "aXN0b2dyYW1fcGFyYW1zGAwgASgLMh0uZ3JwYy50ZXN0aW5nLkhpc3RvZ3Jh", - "bVBhcmFtcyI4CgxDbGllbnRTdGF0dXMSKAoFc3RhdHMYASABKAsyGS5ncnBj", - "LnRlc3RpbmcuQ2xpZW50U3RhdHMiFQoETWFyaxINCgVyZXNldBgBIAEoCCJo", - "CgpDbGllbnRBcmdzEisKBXNldHVwGAEgASgLMhouZ3JwYy50ZXN0aW5nLkNs", - "aWVudENvbmZpZ0gAEiIKBG1hcmsYAiABKAsyEi5ncnBjLnRlc3RpbmcuTWFy", - "a0gAQgkKB2FyZ3R5cGUi9wEKDFNlcnZlckNvbmZpZxItCgtzZXJ2ZXJfdHlw", - "ZRgBIAEoDjIYLmdycGMudGVzdGluZy5TZXJ2ZXJUeXBlEjUKD3NlY3VyaXR5", - "X3BhcmFtcxgCIAEoCzIcLmdycGMudGVzdGluZy5TZWN1cml0eVBhcmFtcxIM", - "CgRob3N0GAMgASgJEgwKBHBvcnQYBCABKAUSHAoUYXN5bmNfc2VydmVyX3Ro", - "cmVhZHMYByABKAUSEgoKY29yZV9saW1pdBgIIAEoBRIzCg5wYXlsb2FkX2Nv", - "bmZpZxgJIAEoCzIbLmdycGMudGVzdGluZy5QYXlsb2FkQ29uZmlnImgKClNl", - "cnZlckFyZ3MSKwoFc2V0dXAYASABKAsyGi5ncnBjLnRlc3RpbmcuU2VydmVy", - "Q29uZmlnSAASIgoEbWFyaxgCIAEoCzISLmdycGMudGVzdGluZy5NYXJrSABC", - "CQoHYXJndHlwZSJVCgxTZXJ2ZXJTdGF0dXMSKAoFc3RhdHMYASABKAsyGS5n", - "cnBjLnRlc3RpbmcuU2VydmVyU3RhdHMSDAoEcG9ydBgCIAEoBRINCgVjb3Jl", - "cxgDIAEoBSovCgpDbGllbnRUeXBlEg8KC1NZTkNfQ0xJRU5UEAASEAoMQVNZ", - "TkNfQ0xJRU5UEAEqLwoKU2VydmVyVHlwZRIPCgtTWU5DX1NFUlZFUhAAEhAK", - "DEFTWU5DX1NFUlZFUhABKiMKB1JwY1R5cGUSCQoFVU5BUlkQABINCglTVFJF", - "QU1JTkcQAWIGcHJvdG8z")); + "bVBhcmFtcxIRCgljb3JlX2xpc3QYDSADKAUSEgoKY29yZV9saW1pdBgOIAEo", + "BSI4CgxDbGllbnRTdGF0dXMSKAoFc3RhdHMYASABKAsyGS5ncnBjLnRlc3Rp", + "bmcuQ2xpZW50U3RhdHMiFQoETWFyaxINCgVyZXNldBgBIAEoCCJoCgpDbGll", + "bnRBcmdzEisKBXNldHVwGAEgASgLMhouZ3JwYy50ZXN0aW5nLkNsaWVudENv", + "bmZpZ0gAEiIKBG1hcmsYAiABKAsyEi5ncnBjLnRlc3RpbmcuTWFya0gAQgkK", + "B2FyZ3R5cGUi/AEKDFNlcnZlckNvbmZpZxItCgtzZXJ2ZXJfdHlwZRgBIAEo", + "DjIYLmdycGMudGVzdGluZy5TZXJ2ZXJUeXBlEjUKD3NlY3VyaXR5X3BhcmFt", + "cxgCIAEoCzIcLmdycGMudGVzdGluZy5TZWN1cml0eVBhcmFtcxIMCgRwb3J0", + "GAQgASgFEhwKFGFzeW5jX3NlcnZlcl90aHJlYWRzGAcgASgFEhIKCmNvcmVf", + "bGltaXQYCCABKAUSMwoOcGF5bG9hZF9jb25maWcYCSABKAsyGy5ncnBjLnRl", + "c3RpbmcuUGF5bG9hZENvbmZpZxIRCgljb3JlX2xpc3QYCiADKAUiaAoKU2Vy", + "dmVyQXJncxIrCgVzZXR1cBgBIAEoCzIaLmdycGMudGVzdGluZy5TZXJ2ZXJD", + "b25maWdIABIiCgRtYXJrGAIgASgLMhIuZ3JwYy50ZXN0aW5nLk1hcmtIAEIJ", + "Cgdhcmd0eXBlIlUKDFNlcnZlclN0YXR1cxIoCgVzdGF0cxgBIAEoCzIZLmdy", + "cGMudGVzdGluZy5TZXJ2ZXJTdGF0cxIMCgRwb3J0GAIgASgFEg0KBWNvcmVz", + "GAMgASgFIg0KC0NvcmVSZXF1ZXN0Ih0KDENvcmVSZXNwb25zZRINCgVjb3Jl", + "cxgBIAEoBSIGCgRWb2lkKi8KCkNsaWVudFR5cGUSDwoLU1lOQ19DTElFTlQQ", + "ABIQCgxBU1lOQ19DTElFTlQQASpJCgpTZXJ2ZXJUeXBlEg8KC1NZTkNfU0VS", + "VkVSEAASEAoMQVNZTkNfU0VSVkVSEAESGAoUQVNZTkNfR0VORVJJQ19TRVJW", + "RVIQAiojCgdScGNUeXBlEgkKBVVOQVJZEAASDQoJU1RSRUFNSU5HEAFiBnBy", + "b3RvMw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { global::Grpc.Testing.PayloadsReflection.Descriptor, global::Grpc.Testing.StatsReflection.Descriptor, }, new pbr::GeneratedCodeInfo(new[] {typeof(global::Grpc.Testing.ClientType), typeof(global::Grpc.Testing.ServerType), typeof(global::Grpc.Testing.RpcType), }, new pbr::GeneratedCodeInfo[] { @@ -76,13 +79,16 @@ namespace Grpc.Testing { new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClosedLoopParams), global::Grpc.Testing.ClosedLoopParams.Parser, null, null, null, null), new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.LoadParams), global::Grpc.Testing.LoadParams.Parser, new[]{ "ClosedLoop", "Poisson", "Uniform", "Determ", "Pareto" }, new[]{ "Load" }, null, null), new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.SecurityParams), global::Grpc.Testing.SecurityParams.Parser, new[]{ "UseTestCa", "ServerHostOverride" }, null, null, null), - new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClientConfig), global::Grpc.Testing.ClientConfig.Parser, new[]{ "ServerTargets", "ClientType", "SecurityParams", "OutstandingRpcsPerChannel", "ClientChannels", "AsyncClientThreads", "RpcType", "LoadParams", "PayloadConfig", "HistogramParams" }, null, null, null), + new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClientConfig), global::Grpc.Testing.ClientConfig.Parser, new[]{ "ServerTargets", "ClientType", "SecurityParams", "OutstandingRpcsPerChannel", "ClientChannels", "AsyncClientThreads", "RpcType", "LoadParams", "PayloadConfig", "HistogramParams", "CoreList", "CoreLimit" }, null, null, null), new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClientStatus), global::Grpc.Testing.ClientStatus.Parser, new[]{ "Stats" }, null, null, null), new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.Mark), global::Grpc.Testing.Mark.Parser, new[]{ "Reset" }, null, null, null), new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ClientArgs), global::Grpc.Testing.ClientArgs.Parser, new[]{ "Setup", "Mark" }, new[]{ "Argtype" }, null, null), - new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ServerConfig), global::Grpc.Testing.ServerConfig.Parser, new[]{ "ServerType", "SecurityParams", "Host", "Port", "AsyncServerThreads", "CoreLimit", "PayloadConfig" }, null, null, null), + new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ServerConfig), global::Grpc.Testing.ServerConfig.Parser, new[]{ "ServerType", "SecurityParams", "Port", "AsyncServerThreads", "CoreLimit", "PayloadConfig", "CoreList" }, null, null, null), new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ServerArgs), global::Grpc.Testing.ServerArgs.Parser, new[]{ "Setup", "Mark" }, new[]{ "Argtype" }, null, null), - new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ServerStatus), global::Grpc.Testing.ServerStatus.Parser, new[]{ "Stats", "Port", "Cores" }, null, null, null) + new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.ServerStatus), global::Grpc.Testing.ServerStatus.Parser, new[]{ "Stats", "Port", "Cores" }, null, null, null), + new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.CoreRequest), global::Grpc.Testing.CoreRequest.Parser, null, null, null, null), + new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.CoreResponse), global::Grpc.Testing.CoreResponse.Parser, new[]{ "Cores" }, null, null, null), + new pbr::GeneratedCodeInfo(typeof(global::Grpc.Testing.Void), global::Grpc.Testing.Void.Parser, null, null, null, null) })); } #endregion @@ -97,6 +103,7 @@ namespace Grpc.Testing { public enum ServerType { SYNC_SERVER = 0, ASYNC_SERVER = 1, + ASYNC_GENERIC_SERVER = 2, } public enum RpcType { @@ -1097,6 +1104,8 @@ namespace Grpc.Testing { LoadParams = other.loadParams_ != null ? other.LoadParams.Clone() : null; PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; HistogramParams = other.histogramParams_ != null ? other.HistogramParams.Clone() : null; + coreList_ = other.coreList_.Clone(); + coreLimit_ = other.coreLimit_; } public ClientConfig Clone() { @@ -1219,6 +1228,28 @@ namespace Grpc.Testing { } } + /// Field number for the "core_list" field. + public const int CoreListFieldNumber = 13; + private static readonly pb::FieldCodec _repeated_coreList_codec + = pb::FieldCodec.ForInt32(106); + private readonly pbc::RepeatedField coreList_ = new pbc::RepeatedField(); + /// + /// Specify the cores we should run the client on, if desired + /// + public pbc::RepeatedField CoreList { + get { return coreList_; } + } + + /// Field number for the "core_limit" field. + public const int CoreLimitFieldNumber = 14; + private int coreLimit_; + public int CoreLimit { + get { return coreLimit_; } + set { + coreLimit_ = value; + } + } + public override bool Equals(object other) { return Equals(other as ClientConfig); } @@ -1240,6 +1271,8 @@ namespace Grpc.Testing { if (!object.Equals(LoadParams, other.LoadParams)) return false; if (!object.Equals(PayloadConfig, other.PayloadConfig)) return false; if (!object.Equals(HistogramParams, other.HistogramParams)) return false; + if(!coreList_.Equals(other.coreList_)) return false; + if (CoreLimit != other.CoreLimit) return false; return true; } @@ -1255,6 +1288,8 @@ namespace Grpc.Testing { if (loadParams_ != null) hash ^= LoadParams.GetHashCode(); if (payloadConfig_ != null) hash ^= PayloadConfig.GetHashCode(); if (histogramParams_ != null) hash ^= HistogramParams.GetHashCode(); + hash ^= coreList_.GetHashCode(); + if (CoreLimit != 0) hash ^= CoreLimit.GetHashCode(); return hash; } @@ -1300,6 +1335,11 @@ namespace Grpc.Testing { output.WriteRawTag(98); output.WriteMessage(HistogramParams); } + coreList_.WriteTo(output, _repeated_coreList_codec); + if (CoreLimit != 0) { + output.WriteRawTag(112); + output.WriteInt32(CoreLimit); + } } public int CalculateSize() { @@ -1332,6 +1372,10 @@ namespace Grpc.Testing { if (histogramParams_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(HistogramParams); } + size += coreList_.CalculateSize(_repeated_coreList_codec); + if (CoreLimit != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(CoreLimit); + } return size; } @@ -1379,6 +1423,10 @@ namespace Grpc.Testing { } HistogramParams.MergeFrom(other.HistogramParams); } + coreList_.Add(other.coreList_); + if (other.CoreLimit != 0) { + CoreLimit = other.CoreLimit; + } } public void MergeFrom(pb::CodedInputStream input) { @@ -1440,6 +1488,15 @@ namespace Grpc.Testing { input.ReadMessage(histogramParams_); break; } + case 106: + case 104: { + coreList_.AddEntriesFrom(input, _repeated_coreList_codec); + break; + } + case 112: { + CoreLimit = input.ReadInt32(); + break; + } } } } @@ -1855,11 +1912,11 @@ namespace Grpc.Testing { public ServerConfig(ServerConfig other) : this() { serverType_ = other.serverType_; SecurityParams = other.securityParams_ != null ? other.SecurityParams.Clone() : null; - host_ = other.host_; port_ = other.port_; asyncServerThreads_ = other.asyncServerThreads_; coreLimit_ = other.coreLimit_; PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; + coreList_ = other.coreList_.Clone(); } public ServerConfig Clone() { @@ -1886,19 +1943,6 @@ namespace Grpc.Testing { } } - /// Field number for the "host" field. - public const int HostFieldNumber = 3; - private string host_ = ""; - /// - /// Host on which to listen. - /// - public string Host { - get { return host_; } - set { - host_ = pb::Preconditions.CheckNotNull(value, "value"); - } - } - /// Field number for the "port" field. public const int PortFieldNumber = 4; private int port_; @@ -1929,7 +1973,7 @@ namespace Grpc.Testing { public const int CoreLimitFieldNumber = 8; private int coreLimit_; /// - /// restrict core usage, currently unused + /// Specify the number of cores to limit server to, if desired /// public int CoreLimit { get { return coreLimit_; } @@ -1941,6 +1985,9 @@ namespace Grpc.Testing { /// Field number for the "payload_config" field. public const int PayloadConfigFieldNumber = 9; private global::Grpc.Testing.PayloadConfig payloadConfig_; + /// + /// payload config, used in generic server + /// public global::Grpc.Testing.PayloadConfig PayloadConfig { get { return payloadConfig_; } set { @@ -1948,6 +1995,18 @@ namespace Grpc.Testing { } } + /// Field number for the "core_list" field. + public const int CoreListFieldNumber = 10; + private static readonly pb::FieldCodec _repeated_coreList_codec + = pb::FieldCodec.ForInt32(82); + private readonly pbc::RepeatedField coreList_ = new pbc::RepeatedField(); + /// + /// Specify the cores we should run the server on, if desired + /// + public pbc::RepeatedField CoreList { + get { return coreList_; } + } + public override bool Equals(object other) { return Equals(other as ServerConfig); } @@ -1961,11 +2020,11 @@ namespace Grpc.Testing { } if (ServerType != other.ServerType) return false; if (!object.Equals(SecurityParams, other.SecurityParams)) return false; - if (Host != other.Host) return false; if (Port != other.Port) return false; if (AsyncServerThreads != other.AsyncServerThreads) return false; if (CoreLimit != other.CoreLimit) return false; if (!object.Equals(PayloadConfig, other.PayloadConfig)) return false; + if(!coreList_.Equals(other.coreList_)) return false; return true; } @@ -1973,11 +2032,11 @@ namespace Grpc.Testing { int hash = 1; if (ServerType != global::Grpc.Testing.ServerType.SYNC_SERVER) hash ^= ServerType.GetHashCode(); if (securityParams_ != null) hash ^= SecurityParams.GetHashCode(); - if (Host.Length != 0) hash ^= Host.GetHashCode(); if (Port != 0) hash ^= Port.GetHashCode(); if (AsyncServerThreads != 0) hash ^= AsyncServerThreads.GetHashCode(); if (CoreLimit != 0) hash ^= CoreLimit.GetHashCode(); if (payloadConfig_ != null) hash ^= PayloadConfig.GetHashCode(); + hash ^= coreList_.GetHashCode(); return hash; } @@ -1994,10 +2053,6 @@ namespace Grpc.Testing { output.WriteRawTag(18); output.WriteMessage(SecurityParams); } - if (Host.Length != 0) { - output.WriteRawTag(26); - output.WriteString(Host); - } if (Port != 0) { output.WriteRawTag(32); output.WriteInt32(Port); @@ -2014,6 +2069,7 @@ namespace Grpc.Testing { output.WriteRawTag(74); output.WriteMessage(PayloadConfig); } + coreList_.WriteTo(output, _repeated_coreList_codec); } public int CalculateSize() { @@ -2024,9 +2080,6 @@ namespace Grpc.Testing { if (securityParams_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(SecurityParams); } - if (Host.Length != 0) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(Host); - } if (Port != 0) { size += 1 + pb::CodedOutputStream.ComputeInt32Size(Port); } @@ -2039,6 +2092,7 @@ namespace Grpc.Testing { if (payloadConfig_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(PayloadConfig); } + size += coreList_.CalculateSize(_repeated_coreList_codec); return size; } @@ -2055,9 +2109,6 @@ namespace Grpc.Testing { } SecurityParams.MergeFrom(other.SecurityParams); } - if (other.Host.Length != 0) { - Host = other.Host; - } if (other.Port != 0) { Port = other.Port; } @@ -2073,6 +2124,7 @@ namespace Grpc.Testing { } PayloadConfig.MergeFrom(other.PayloadConfig); } + coreList_.Add(other.coreList_); } public void MergeFrom(pb::CodedInputStream input) { @@ -2093,10 +2145,6 @@ namespace Grpc.Testing { input.ReadMessage(securityParams_); break; } - case 26: { - Host = input.ReadString(); - break; - } case 32: { Port = input.ReadInt32(); break; @@ -2116,6 +2164,11 @@ namespace Grpc.Testing { input.ReadMessage(payloadConfig_); break; } + case 82: + case 80: { + coreList_.AddEntriesFrom(input, _repeated_coreList_codec); + break; + } } } } @@ -2347,7 +2400,7 @@ namespace Grpc.Testing { public const int CoresFieldNumber = 3; private int cores_; /// - /// Number of cores on the server. See gpr_cpu_num_cores. + /// Number of cores available to the server /// public int Cores { get { return cores_; } @@ -2460,6 +2513,264 @@ namespace Grpc.Testing { } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + public sealed partial class CoreRequest : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new CoreRequest()); + public static pb::MessageParser Parser { get { return _parser; } } + + public static pbr::MessageDescriptor Descriptor { + get { return global::Grpc.Testing.ControlReflection.Descriptor.MessageTypes[14]; } + } + + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + public CoreRequest() { + OnConstruction(); + } + + partial void OnConstruction(); + + public CoreRequest(CoreRequest other) : this() { + } + + public CoreRequest Clone() { + return new CoreRequest(this); + } + + public override bool Equals(object other) { + return Equals(other as CoreRequest); + } + + public bool Equals(CoreRequest other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + return true; + } + + public override int GetHashCode() { + int hash = 1; + return hash; + } + + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + public void WriteTo(pb::CodedOutputStream output) { + } + + public int CalculateSize() { + int size = 0; + return size; + } + + public void MergeFrom(CoreRequest other) { + if (other == null) { + return; + } + } + + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + input.SkipLastField(); + break; + } + } + } + + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + public sealed partial class CoreResponse : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new CoreResponse()); + public static pb::MessageParser Parser { get { return _parser; } } + + public static pbr::MessageDescriptor Descriptor { + get { return global::Grpc.Testing.ControlReflection.Descriptor.MessageTypes[15]; } + } + + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + public CoreResponse() { + OnConstruction(); + } + + partial void OnConstruction(); + + public CoreResponse(CoreResponse other) : this() { + cores_ = other.cores_; + } + + public CoreResponse Clone() { + return new CoreResponse(this); + } + + /// Field number for the "cores" field. + public const int CoresFieldNumber = 1; + private int cores_; + /// + /// Number of cores available on the server + /// + public int Cores { + get { return cores_; } + set { + cores_ = value; + } + } + + public override bool Equals(object other) { + return Equals(other as CoreResponse); + } + + public bool Equals(CoreResponse other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Cores != other.Cores) return false; + return true; + } + + public override int GetHashCode() { + int hash = 1; + if (Cores != 0) hash ^= Cores.GetHashCode(); + return hash; + } + + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + public void WriteTo(pb::CodedOutputStream output) { + if (Cores != 0) { + output.WriteRawTag(8); + output.WriteInt32(Cores); + } + } + + public int CalculateSize() { + int size = 0; + if (Cores != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(Cores); + } + return size; + } + + public void MergeFrom(CoreResponse other) { + if (other == null) { + return; + } + if (other.Cores != 0) { + Cores = other.Cores; + } + } + + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + input.SkipLastField(); + break; + case 8: { + Cores = input.ReadInt32(); + break; + } + } + } + } + + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + public sealed partial class Void : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new Void()); + public static pb::MessageParser Parser { get { return _parser; } } + + public static pbr::MessageDescriptor Descriptor { + get { return global::Grpc.Testing.ControlReflection.Descriptor.MessageTypes[16]; } + } + + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + public Void() { + OnConstruction(); + } + + partial void OnConstruction(); + + public Void(Void other) : this() { + } + + public Void Clone() { + return new Void(this); + } + + public override bool Equals(object other) { + return Equals(other as Void); + } + + public bool Equals(Void other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + return true; + } + + public override int GetHashCode() { + int hash = 1; + return hash; + } + + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + public void WriteTo(pb::CodedOutputStream output) { + } + + public int CalculateSize() { + int size = 0; + return size; + } + + public void MergeFrom(Void other) { + if (other == null) { + return; + } + } + + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + input.SkipLastField(); + break; + } + } + } + + } + #endregion } diff --git a/src/csharp/Grpc.IntegrationTesting/QpsWorker.cs b/src/csharp/Grpc.IntegrationTesting/QpsWorker.cs index 686b4843453..a7c9fa894de 100644 --- a/src/csharp/Grpc.IntegrationTesting/QpsWorker.cs +++ b/src/csharp/Grpc.IntegrationTesting/QpsWorker.cs @@ -85,24 +85,27 @@ namespace Grpc.IntegrationTesting } var workerServer = new QpsWorker(options); - workerServer.Run(); + workerServer.RunAsync().Wait(); } - private void Run() + private async Task RunAsync() { string host = "0.0.0.0"; int port = options.DriverPort; + var tcs = new TaskCompletionSource(); + var workerServiceImpl = new WorkerServiceImpl(() => { Task.Run(() => tcs.SetResult(null)); }); + var server = new Server { - Services = { WorkerService.BindService(new WorkerServiceImpl()) }, + Services = { WorkerService.BindService(workerServiceImpl) }, Ports = { new ServerPort(host, options.DriverPort, ServerCredentials.Insecure )} }; int boundPort = server.Ports.Single().BoundPort; Console.WriteLine("Running qps worker server on " + string.Format("{0}:{1}", host, boundPort)); server.Start(); - - server.ShutdownTask.Wait(); + await tcs.Task; + await server.ShutdownAsync(); } } } diff --git a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs index 3dd91b79485..e87f3b7dc1b 100644 --- a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs @@ -48,7 +48,6 @@ namespace Grpc.IntegrationTesting /// public class RunnerClientServerTest { - const string Host = "localhost"; IServerRunner serverRunner; [TestFixtureSetUp] @@ -57,7 +56,6 @@ namespace Grpc.IntegrationTesting var serverConfig = new ServerConfig { ServerType = ServerType.ASYNC_SERVER, - Host = Host, PayloadConfig = new PayloadConfig { SimpleParams = new SimpleProtoParams @@ -83,7 +81,7 @@ namespace Grpc.IntegrationTesting { var config = new ClientConfig { - ServerTargets = { string.Format("{0}:{1}", Host, serverRunner.BoundPort) }, + ServerTargets = { string.Format("{0}:{1}", "localhost", serverRunner.BoundPort) }, RpcType = RpcType.UNARY, LoadParams = new LoadParams { ClosedLoop = new ClosedLoopParams() }, PayloadConfig = new PayloadConfig diff --git a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs index e8be7758cee..d401fa82d21 100644 --- a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs +++ b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs @@ -65,7 +65,7 @@ namespace Grpc.IntegrationTesting var server = new Server { Services = { BenchmarkService.BindService(new BenchmarkServiceImpl(responseSize)) }, - Ports = { new ServerPort(config.Host, config.Port, credentials) } + Ports = { new ServerPort("[::]", config.Port, credentials) } }; server.Start(); diff --git a/src/csharp/Grpc.IntegrationTesting/Services.cs b/src/csharp/Grpc.IntegrationTesting/Services.cs index 04a092ccd79..a8475c18172 100644 --- a/src/csharp/Grpc.IntegrationTesting/Services.cs +++ b/src/csharp/Grpc.IntegrationTesting/Services.cs @@ -29,11 +29,14 @@ namespace Grpc.Testing { "QmVuY2htYXJrU2VydmljZRJGCglVbmFyeUNhbGwSGy5ncnBjLnRlc3Rpbmcu", "U2ltcGxlUmVxdWVzdBocLmdycGMudGVzdGluZy5TaW1wbGVSZXNwb25zZRJO", "Cg1TdHJlYW1pbmdDYWxsEhsuZ3JwYy50ZXN0aW5nLlNpbXBsZVJlcXVlc3Qa", - "HC5ncnBjLnRlc3RpbmcuU2ltcGxlUmVzcG9uc2UoATABMp0BCg1Xb3JrZXJT", + "HC5ncnBjLnRlc3RpbmcuU2ltcGxlUmVzcG9uc2UoATABMpcCCg1Xb3JrZXJT", "ZXJ2aWNlEkUKCVJ1blNlcnZlchIYLmdycGMudGVzdGluZy5TZXJ2ZXJBcmdz", "GhouZ3JwYy50ZXN0aW5nLlNlcnZlclN0YXR1cygBMAESRQoJUnVuQ2xpZW50", "EhguZ3JwYy50ZXN0aW5nLkNsaWVudEFyZ3MaGi5ncnBjLnRlc3RpbmcuQ2xp", - "ZW50U3RhdHVzKAEwAWIGcHJvdG8z")); + "ZW50U3RhdHVzKAEwARJCCglDb3JlQ291bnQSGS5ncnBjLnRlc3RpbmcuQ29y", + "ZVJlcXVlc3QaGi5ncnBjLnRlc3RpbmcuQ29yZVJlc3BvbnNlEjQKClF1aXRX", + "b3JrZXISEi5ncnBjLnRlc3RpbmcuVm9pZBoSLmdycGMudGVzdGluZy5Wb2lk", + "YgZwcm90bzM=")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { global::Grpc.Testing.MessagesReflection.Descriptor, global::Grpc.Testing.ControlReflection.Descriptor, }, new pbr::GeneratedCodeInfo(null, null)); diff --git a/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs b/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs index dd30afb427f..996439afbf1 100644 --- a/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/ServicesGrpc.cs @@ -114,6 +114,9 @@ namespace Grpc.Testing { static readonly Marshaller __Marshaller_ServerStatus = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ServerStatus.Parser.ParseFrom); static readonly Marshaller __Marshaller_ClientArgs = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ClientArgs.Parser.ParseFrom); static readonly Marshaller __Marshaller_ClientStatus = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ClientStatus.Parser.ParseFrom); + static readonly Marshaller __Marshaller_CoreRequest = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.CoreRequest.Parser.ParseFrom); + static readonly Marshaller __Marshaller_CoreResponse = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.CoreResponse.Parser.ParseFrom); + static readonly Marshaller __Marshaller_Void = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.Void.Parser.ParseFrom); static readonly Method __Method_RunServer = new Method( MethodType.DuplexStreaming, @@ -129,6 +132,20 @@ namespace Grpc.Testing { __Marshaller_ClientArgs, __Marshaller_ClientStatus); + static readonly Method __Method_CoreCount = new Method( + MethodType.Unary, + __ServiceName, + "CoreCount", + __Marshaller_CoreRequest, + __Marshaller_CoreResponse); + + static readonly Method __Method_QuitWorker = new Method( + MethodType.Unary, + __ServiceName, + "QuitWorker", + __Marshaller_Void, + __Marshaller_Void); + // service descriptor public static global::Google.Protobuf.Reflection.ServiceDescriptor Descriptor { @@ -142,6 +159,14 @@ namespace Grpc.Testing { AsyncDuplexStreamingCall RunServer(CallOptions options); AsyncDuplexStreamingCall RunClient(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)); AsyncDuplexStreamingCall RunClient(CallOptions options); + global::Grpc.Testing.CoreResponse CoreCount(global::Grpc.Testing.CoreRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)); + global::Grpc.Testing.CoreResponse CoreCount(global::Grpc.Testing.CoreRequest request, CallOptions options); + AsyncUnaryCall CoreCountAsync(global::Grpc.Testing.CoreRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)); + AsyncUnaryCall CoreCountAsync(global::Grpc.Testing.CoreRequest request, CallOptions options); + global::Grpc.Testing.Void QuitWorker(global::Grpc.Testing.Void request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)); + global::Grpc.Testing.Void QuitWorker(global::Grpc.Testing.Void request, CallOptions options); + AsyncUnaryCall QuitWorkerAsync(global::Grpc.Testing.Void request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)); + AsyncUnaryCall QuitWorkerAsync(global::Grpc.Testing.Void request, CallOptions options); } // server-side interface @@ -149,6 +174,8 @@ namespace Grpc.Testing { { Task RunServer(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); Task RunClient(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context); + Task CoreCount(global::Grpc.Testing.CoreRequest request, ServerCallContext context); + Task QuitWorker(global::Grpc.Testing.Void request, ServerCallContext context); } // client stub @@ -177,6 +204,46 @@ namespace Grpc.Testing { var call = CreateCall(__Method_RunClient, options); return Calls.AsyncDuplexStreamingCall(call); } + public global::Grpc.Testing.CoreResponse CoreCount(global::Grpc.Testing.CoreRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + { + var call = CreateCall(__Method_CoreCount, new CallOptions(headers, deadline, cancellationToken)); + return Calls.BlockingUnaryCall(call, request); + } + public global::Grpc.Testing.CoreResponse CoreCount(global::Grpc.Testing.CoreRequest request, CallOptions options) + { + var call = CreateCall(__Method_CoreCount, options); + return Calls.BlockingUnaryCall(call, request); + } + public AsyncUnaryCall CoreCountAsync(global::Grpc.Testing.CoreRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + { + var call = CreateCall(__Method_CoreCount, new CallOptions(headers, deadline, cancellationToken)); + return Calls.AsyncUnaryCall(call, request); + } + public AsyncUnaryCall CoreCountAsync(global::Grpc.Testing.CoreRequest request, CallOptions options) + { + var call = CreateCall(__Method_CoreCount, options); + return Calls.AsyncUnaryCall(call, request); + } + public global::Grpc.Testing.Void QuitWorker(global::Grpc.Testing.Void request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + { + var call = CreateCall(__Method_QuitWorker, new CallOptions(headers, deadline, cancellationToken)); + return Calls.BlockingUnaryCall(call, request); + } + public global::Grpc.Testing.Void QuitWorker(global::Grpc.Testing.Void request, CallOptions options) + { + var call = CreateCall(__Method_QuitWorker, options); + return Calls.BlockingUnaryCall(call, request); + } + public AsyncUnaryCall QuitWorkerAsync(global::Grpc.Testing.Void request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + { + var call = CreateCall(__Method_QuitWorker, new CallOptions(headers, deadline, cancellationToken)); + return Calls.AsyncUnaryCall(call, request); + } + public AsyncUnaryCall QuitWorkerAsync(global::Grpc.Testing.Void request, CallOptions options) + { + var call = CreateCall(__Method_QuitWorker, options); + return Calls.AsyncUnaryCall(call, request); + } } // creates service definition that can be registered with a server @@ -184,7 +251,9 @@ namespace Grpc.Testing { { return ServerServiceDefinition.CreateBuilder(__ServiceName) .AddMethod(__Method_RunServer, serviceImpl.RunServer) - .AddMethod(__Method_RunClient, serviceImpl.RunClient).Build(); + .AddMethod(__Method_RunClient, serviceImpl.RunClient) + .AddMethod(__Method_CoreCount, serviceImpl.CoreCount) + .AddMethod(__Method_QuitWorker, serviceImpl.QuitWorker).Build(); } // creates a new client diff --git a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs index bb2918bf463..4edd7027540 100644 --- a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs +++ b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs @@ -47,6 +47,13 @@ namespace Grpc.Testing /// public class WorkerServiceImpl : WorkerService.IWorkerService { + readonly Action stopRequestHandler; + + public WorkerServiceImpl(Action stopRequestHandler) + { + this.stopRequestHandler = Grpc.Core.Utils.Preconditions.CheckNotNull(stopRequestHandler); + } + public async Task RunServer(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) { Grpc.Core.Utils.Preconditions.CheckState(await requestStream.MoveNext()); @@ -92,5 +99,16 @@ namespace Grpc.Testing } await runner.StopAsync(); } + + public Task CoreCount(CoreRequest request, ServerCallContext context) + { + return Task.FromResult(new CoreResponse { Cores = Environment.ProcessorCount }); + } + + public Task QuitWorker(Void request, ServerCallContext context) + { + stopRequestHandler(); + return Task.FromResult(new Void()); + } } } From 13b63a0711bd32728fb45ee4352cc3a28b31c3df Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 18 Feb 2016 14:29:32 -0800 Subject: [PATCH 020/236] fix copyrights --- src/csharp/Grpc.IntegrationTesting/QpsWorker.cs | 2 +- src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs | 2 +- src/csharp/Grpc.IntegrationTesting/ServerRunners.cs | 2 +- src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/QpsWorker.cs b/src/csharp/Grpc.IntegrationTesting/QpsWorker.cs index a7c9fa894de..e407792c4b5 100644 --- a/src/csharp/Grpc.IntegrationTesting/QpsWorker.cs +++ b/src/csharp/Grpc.IntegrationTesting/QpsWorker.cs @@ -1,6 +1,6 @@ #region Copyright notice and license -// Copyright 2015, Google Inc. +// Copyright 2015-2016, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without diff --git a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs index e87f3b7dc1b..06d5ee93d88 100644 --- a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs @@ -1,6 +1,6 @@ #region Copyright notice and license -// Copyright 2015, Google Inc. +// Copyright 2015-2016, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without diff --git a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs index d401fa82d21..7d6f1a51794 100644 --- a/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs +++ b/src/csharp/Grpc.IntegrationTesting/ServerRunners.cs @@ -1,6 +1,6 @@ #region Copyright notice and license -// Copyright 2015, Google Inc. +// Copyright 2015-2016, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without diff --git a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs index 4edd7027540..62d03edbdde 100644 --- a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs +++ b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs @@ -1,6 +1,6 @@ #region Copyright notice and license -// Copyright 2015, Google Inc. +// Copyright 2015-2016, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without From 9656eca5102c6ee99f7cff227226a8c927eda722 Mon Sep 17 00:00:00 2001 From: Test User Date: Thu, 18 Feb 2016 14:47:22 -0800 Subject: [PATCH 021/236] Added tests for code coverage --- src/objective-c/tests/GRPCClientTests.m | 45 ++++++++++++++++--- .../tests/InteropTestsLocalCleartext.m | 2 +- src/objective-c/tests/InteropTestsLocalSSL.m | 11 +++++ src/objective-c/tests/RxLibraryUnitTests.m | 23 ++++++++++ 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 00c4b8830d3..1a829939556 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -41,11 +41,11 @@ #import #import #import - +#import static NSString * const kHostAddress = @"localhost:5050"; static NSString * const kPackage = @"grpc.testing"; static NSString * const kService = @"TestService"; - +static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; static ProtoMethod *kInexistentMethod; static ProtoMethod *kEmptyCallMethod; static ProtoMethod *kUnaryCallMethod; @@ -127,8 +127,8 @@ static ProtoMethod *kUnaryCallMethod; XCTFail(@"Received unexpected response: %@", value); } completionHandler:^(NSError *errorOrNil) { XCTAssertNotNil(errorOrNil, @"Finished without error!"); - // TODO(jcanizales): The server should return code 12 UNIMPLEMENTED, not 5 NOT FOUND. - XCTAssertEqual(errorOrNil.code, 5, @"Finished with unexpected error: %@", errorOrNil); + // + XCTAssertEqual(errorOrNil.code, 12, @"Finished with unexpected error: %@", errorOrNil); [expectation fulfill]; }]; @@ -200,7 +200,7 @@ static ProtoMethod *kUnaryCallMethod; request.fillOauthScope = YES; GRXWriter *requestsWriter = [GRXWriter writerWithValue:[request data]]; - GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress + GRPCCall *call = [[GRPCCall alloc] initWithHost:kRemoteSSLHost path:kUnaryCallMethod.HTTPPath requestsWriter:requestsWriter]; @@ -257,4 +257,39 @@ static ProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:8 handler:nil]; } +- (void)testExceptions { + // Try to set userAgentPrefix for host that is nil. This should cause + // an exception. + @try { + [GRPCCall setUserAgentPrefix:@"Foo" forHost:nil]; + XCTFail(@"Did not receive an exception when host is nil"); + } @catch(NSException *theException) { + NSLog(@"Received exception as expected: %@", theException.name); + } + + // Try to set parameters to nil for GRPCCall. This should cause an exception + @try { + GRPCCall *call = [[GRPCCall alloc] initWithHost:nil + path:nil + requestsWriter:nil]; + XCTFail(@"Did not receive an exception when parameters are nil"); + } @catch(NSException *theException) { + NSLog(@"Received exception as expected: %@", theException.name); + } + + + // Set state to Finished by force + GRXWriter *requestsWriter = [GRXWriter emptyWriter]; + [requestsWriter finishWithError:nil]; + @try { + GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress + path:kUnaryCallMethod.HTTPPath + requestsWriter:requestsWriter]; + XCTFail(@"Did not receive an exception when GRXWriter has incorrect state."); + } @catch(NSException *theException) { + NSLog(@"Received exception as expected: %@", theException.name); + } + +} + @end diff --git a/src/objective-c/tests/InteropTestsLocalCleartext.m b/src/objective-c/tests/InteropTestsLocalCleartext.m index 56927a8af6d..f4542851104 100644 --- a/src/objective-c/tests/InteropTestsLocalCleartext.m +++ b/src/objective-c/tests/InteropTestsLocalCleartext.m @@ -49,7 +49,7 @@ static NSString * const kLocalCleartextHost = @"localhost:5050"; - (void)setUp { // Register test server as non-SSL. - [GRPCCall useInsecureConnectionsForHost:kLocalCleartextHost]; + [GRPCCall useInsecureConnectionsForHost:nil]; [super setUp]; } diff --git a/src/objective-c/tests/InteropTestsLocalSSL.m b/src/objective-c/tests/InteropTestsLocalSSL.m index 9d7afefbfe3..f0f4b1d71f0 100644 --- a/src/objective-c/tests/InteropTestsLocalSSL.m +++ b/src/objective-c/tests/InteropTestsLocalSSL.m @@ -57,4 +57,15 @@ static NSString * const kLocalSSLHost = @"localhost:5051"; [super setUp]; } +- (void)testExceptions { + // Try to set userAgentPrefix for host that is nil. This should cause + // an exception. + @try { + [GRPCCall useTestCertsPath:nil testName:nil forHost:nil]; + XCTFail(@"Did not receive an exception when parameters are nil"); + } @catch(NSException *theException) { + NSLog(@"Received exception as expected: %@", theException.name); + } +} + @end diff --git a/src/objective-c/tests/RxLibraryUnitTests.m b/src/objective-c/tests/RxLibraryUnitTests.m index a67a4c6cd93..ba79191dc48 100644 --- a/src/objective-c/tests/RxLibraryUnitTests.m +++ b/src/objective-c/tests/RxLibraryUnitTests.m @@ -137,4 +137,27 @@ XCTAssertEqualObjects(handler.errorOrNil, anyError); } +- (void)testBufferedPipeFinishWriteWhilePaused { + // Given: + CapturingSingleValueHandler *handler = [CapturingSingleValueHandler handler]; + id writeable = [GRXWriteable writeableWithSingleHandler:handler.block]; + id anyValue = @7; + + // If: + GRXBufferedPipe *pipe = [GRXBufferedPipe pipe]; + // Write something, then finish + [pipe writeValue:anyValue]; + [pipe writesFinishedWithError:nil]; + // then start the writeable + [pipe startWithWriteable:writeable]; + + // Then: + XCTAssertEqual(handler.timesCalled, 1); + XCTAssertEqualObjects(handler.value, anyValue); + XCTAssertEqualObjects(handler.errorOrNil, nil); +} + +- (void)testBufferedPipeSetState { +} + @end From 9c5f0b160576ac33147b7386ffe15b01a54eb4dc Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Thu, 18 Feb 2016 15:00:07 -0800 Subject: [PATCH 022/236] Move Node JUnit reports.xml to repo root --- tools/run_tests/run_node.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/run_node.sh index 178584ae8ed..fb84b79e7fc 100755 --- a/tools/run_tests/run_node.sh +++ b/tools/run_tests/run_node.sh @@ -44,6 +44,7 @@ root=`pwd` test_directory='src/node/test' timeout=8000 + if [ "$CONFIG" = "gcov" ] then ./node_modules/.bin/istanbul cover --dir reports/node_coverage \ @@ -58,7 +59,7 @@ then echo '' > \ ../reports/node_coverage/index.html else - JUNIT_REPORT_PATH=src/node/reports.xml JUNIT_REPORT_STACK=1 \ + JUNIT_REPORT_PATH=reports.xml JUNIT_REPORT_STACK=1 \ ./node_modules/.bin/mocha --timeout $timeout \ --reporter mocha-jenkins-reporter $test_directory fi From 894bd37434c4e1a73d90ee3a2edf84e8f8fe392f Mon Sep 17 00:00:00 2001 From: makdharma Date: Thu, 18 Feb 2016 15:31:41 -0800 Subject: [PATCH 023/236] Updated travis to use Xcode 7.2 and SDK 9.2 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d2d1c8ba639..004d44f3a55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: objective-c -osx_image: xcode7.1 +osx_image: xcode7.2 env: global: - CONFIG=opt @@ -27,6 +27,6 @@ xcode_scheme: - InteropTestsLocalCleartext # TODO(jcanizales): Investigate why they time out: # - InteropTestsRemote -xcode_sdk: iphonesimulator9.1 +xcode_sdk: iphonesimulator9.2 notifications: email: false From b46672602965727de647a720b6cc3f3dca436a11 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Thu, 18 Feb 2016 15:35:56 -0800 Subject: [PATCH 024/236] Extract reports.xml files from docker images in reports.zip --- tools/jenkins/build_docker_and_run_tests.sh | 5 ----- tools/jenkins/docker_run_tests.sh | 1 + tools/run_tests/run_node.sh | 3 +-- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh index e2ac7518f09..0ddf261032c 100755 --- a/tools/jenkins/build_docker_and_run_tests.sh +++ b/tools/jenkins/build_docker_and_run_tests.sh @@ -82,11 +82,6 @@ docker run \ $DOCKER_IMAGE_NAME \ bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || DOCKER_FAILED="true" -if [ "$XML_REPORT" != "" ] -then - docker cp "$CONTAINER_NAME:/var/local/git/grpc/$XML_REPORT" $git_root || true -fi - docker cp "$CONTAINER_NAME:/var/local/git/grpc/reports.zip" $git_root || true unzip -o $git_root/reports.zip -d $git_root || true rm -f reports.zip diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 282b8573511..578bda170ae 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -60,5 +60,6 @@ echo '' >> index.html cd .. zip -r reports.zip reports +find . -name reports.xml | xargs zip reports.xml exit $exit_code diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/run_node.sh index fb84b79e7fc..178584ae8ed 100755 --- a/tools/run_tests/run_node.sh +++ b/tools/run_tests/run_node.sh @@ -44,7 +44,6 @@ root=`pwd` test_directory='src/node/test' timeout=8000 - if [ "$CONFIG" = "gcov" ] then ./node_modules/.bin/istanbul cover --dir reports/node_coverage \ @@ -59,7 +58,7 @@ then echo '' > \ ../reports/node_coverage/index.html else - JUNIT_REPORT_PATH=reports.xml JUNIT_REPORT_STACK=1 \ + JUNIT_REPORT_PATH=src/node/reports.xml JUNIT_REPORT_STACK=1 \ ./node_modules/.bin/mocha --timeout $timeout \ --reporter mocha-jenkins-reporter $test_directory fi From 9ddd46587827cd0498b67a3972ec9c263ba393ab Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Thu, 18 Feb 2016 15:39:18 -0800 Subject: [PATCH 025/236] Fixed zip file name --- tools/jenkins/docker_run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 578bda170ae..2e40f8fd65a 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -60,6 +60,6 @@ echo '' >> index.html cd .. zip -r reports.zip reports -find . -name reports.xml | xargs zip reports.xml +find . -name reports.xml | xargs zip reports.zip exit $exit_code From 3b28872210fc2019eba52f310e824b1aa864f8f5 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 19 Feb 2016 00:28:28 -0800 Subject: [PATCH 026/236] 1. Adjust tsan/asan/msan slowdowns according to documentation tsan documentation says 2-20x, so set it at 20x asan documentation says 1.2-2.7x, so set it at 3x msan documentation says 2-4x, so set it at 4x This is now much less optimistic than before 2. Reactive tsan tests for qps_test 3. Set CPU load for qps_openloop_test 4. Divide qps_openloop_test Poisson rate by the slowdown factor of the configuration --- Makefile | 8 ++++---- build.yaml | 11 +++++------ test/cpp/qps/qps_openloop_test.cc | 5 +++-- tools/run_tests/configs.json | 8 ++++---- tools/run_tests/tests.json | 6 ++---- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 6c7febdabc3..445e6bcdf33 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ LD_asan-noleaks = clang LDXX_asan-noleaks = clang++ CPPFLAGS_asan-noleaks = -O0 -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-noleaks = -fsanitize=address -DEFINES_asan-noleaks += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=1.5 +DEFINES_asan-noleaks += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=3 VALID_CONFIG_ubsan = 1 REQUIRE_CUSTOM_LIBRARIES_ubsan = 1 @@ -177,7 +177,7 @@ LD_asan = clang LDXX_asan = clang++ CPPFLAGS_asan = -O0 -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan = -fsanitize=address -DEFINES_asan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=1.5 +DEFINES_asan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=3 VALID_CONFIG_tsan = 1 REQUIRE_CUSTOM_LIBRARIES_tsan = 1 @@ -187,7 +187,7 @@ LD_tsan = clang LDXX_tsan = clang++ CPPFLAGS_tsan = -O0 -fsanitize=thread -fno-omit-frame-pointer -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_tsan = -fsanitize=thread -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) -DEFINES_tsan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=2 +DEFINES_tsan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20 VALID_CONFIG_msan = 1 REQUIRE_CUSTOM_LIBRARIES_msan = 1 @@ -198,7 +198,7 @@ LDXX_msan = clang++ CPPFLAGS_msan = -O0 -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) DEFINES_msan = NDEBUG -DEFINES_msan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=2 +DEFINES_msan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=4 VALID_CONFIG_mutrace = 1 CC_mutrace = $(DEFAULT_CC) diff --git a/build.yaml b/build.yaml index b639b5d21e6..e60300cbc52 100644 --- a/build.yaml +++ b/build.yaml @@ -2310,6 +2310,7 @@ targets: - linux - posix - name: qps_openloop_test + cpu_cost: 10 build: test language: c++ src: @@ -2342,8 +2343,6 @@ targets: - gpr_test_util - gpr - grpc++_test_config - exclude_configs: - - tsan platforms: - mac - linux @@ -2631,7 +2630,7 @@ configs: test_environ: ASAN_OPTIONS: detect_leaks=1:color=always LSAN_OPTIONS: suppressions=tools/lsan_suppressions.txt:report_objects=1 - timeout_multiplier: 1.5 + timeout_multiplier: 3 asan-noleaks: CC: clang CPPFLAGS: -O0 -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument @@ -2643,7 +2642,7 @@ configs: compile_the_world: true test_environ: ASAN_OPTIONS: detect_leaks=0:color=always - timeout_multiplier: 1.5 + timeout_multiplier: 3 basicprof: CPPFLAGS: -O2 -DGRPC_BASIC_PROFILER -DGRPC_TIMERS_RDTSC DEFINES: NDEBUG @@ -2682,7 +2681,7 @@ configs: -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) LDXX: clang++ compile_the_world: true - timeout_multiplier: 2 + timeout_multiplier: 4 mutrace: CPPFLAGS: -O0 DEFINES: _DEBUG DEBUG @@ -2704,7 +2703,7 @@ configs: compile_the_world: true test_environ: TSAN_OPTIONS: suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 - timeout_multiplier: 2 + timeout_multiplier: 20 ubsan: CC: clang CPPFLAGS: -O1 -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument diff --git a/test/cpp/qps/qps_openloop_test.cc b/test/cpp/qps/qps_openloop_test.cc index 0ac41d9f963..90f21733ee2 100644 --- a/test/cpp/qps/qps_openloop_test.cc +++ b/test/cpp/qps/qps_openloop_test.cc @@ -35,6 +35,7 @@ #include +#include "test/core/util/test_config.h" #include "test/cpp/qps/driver.h" #include "test/cpp/qps/report.h" #include "test/cpp/util/benchmark_config.h" @@ -55,11 +56,11 @@ static void RunQPS() { client_config.set_async_client_threads(8); client_config.set_rpc_type(STREAMING); client_config.mutable_load_params()->mutable_poisson()->set_offered_load( - 1000.0); + 1000.0/g_fixture_slowdown_factor); ServerConfig server_config; server_config.set_server_type(ASYNC_SERVER); - server_config.set_async_server_threads(4); + server_config.set_async_server_threads(8); const auto result = RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2); diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json index 9d7b8a3c725..aacca409f7c 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/configs.json @@ -18,7 +18,7 @@ "environ": { "ASAN_OPTIONS": "detect_leaks=0:color=always" }, - "timeout_multiplier": 1.5 + "timeout_multiplier": 3 }, { "config": "ubsan", @@ -48,18 +48,18 @@ "ASAN_OPTIONS": "detect_leaks=1:color=always", "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" }, - "timeout_multiplier": 1.5 + "timeout_multiplier": 3 }, { "config": "tsan", "environ": { "TSAN_OPTIONS": "suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" }, - "timeout_multiplier": 2 + "timeout_multiplier": 20 }, { "config": "msan", - "timeout_multiplier": 2 + "timeout_multiplier": 4 }, { "config": "mutrace" diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index ea9c129101d..6f98a2c179f 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -2104,7 +2104,7 @@ "mac", "posix" ], - "cpu_cost": 1.0, + "cpu_cost": 10, "exclude_configs": [], "flaky": false, "language": "c++", @@ -2123,9 +2123,7 @@ "posix" ], "cpu_cost": 10, - "exclude_configs": [ - "tsan" - ], + "exclude_configs": [], "flaky": false, "language": "c++", "name": "qps_test", From 2d435b9178b04a38168fc9d54d0a6dbb235baa93 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 19 Feb 2016 00:42:08 -0800 Subject: [PATCH 027/236] Fix the slowdown factor --- test/cpp/qps/qps_openloop_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/qps_openloop_test.cc b/test/cpp/qps/qps_openloop_test.cc index 90f21733ee2..27f266b32be 100644 --- a/test/cpp/qps/qps_openloop_test.cc +++ b/test/cpp/qps/qps_openloop_test.cc @@ -56,7 +56,7 @@ static void RunQPS() { client_config.set_async_client_threads(8); client_config.set_rpc_type(STREAMING); client_config.mutable_load_params()->mutable_poisson()->set_offered_load( - 1000.0/g_fixture_slowdown_factor); + 1000.0 / GRPC_TEST_SLOWDOWN_FACTOR); ServerConfig server_config; server_config.set_server_type(ASYNC_SERVER); From e046f712c5ed1723972af41c1ff402a33d538e29 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 18 Feb 2016 16:06:10 -0800 Subject: [PATCH 028/236] build protoc artifacts on mac and linux --- .../grpc_artifact_protoc/Dockerfile | 70 +++++++++++++++++++ tools/run_tests/artifact_targets.py | 50 ++++++++++++- tools/run_tests/build_artifact_protoc.sh | 41 +++++++++++ 3 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 tools/dockerfile/grpc_artifact_protoc/Dockerfile create mode 100755 tools/run_tests/build_artifact_protoc.sh diff --git a/tools/dockerfile/grpc_artifact_protoc/Dockerfile b/tools/dockerfile/grpc_artifact_protoc/Dockerfile new file mode 100644 index 00000000000..0e405655718 --- /dev/null +++ b/tools/dockerfile/grpc_artifact_protoc/Dockerfile @@ -0,0 +1,70 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Docker file for building protoc and gRPC protoc plugin artifacts. +# forked from https://github.com/google/protobuf/blob/master/protoc-artifacts/Dockerfile + +FROM centos:6.6 + +RUN yum install -y git \ + tar \ + wget \ + make \ + autoconf \ + curl-devel \ + unzip \ + automake \ + libtool \ + glibc-static.i686 \ + glibc-devel \ + glibc-devel.i686 + +# Install Java 8 +RUN wget -q --no-cookies --no-check-certificate \ + --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u45-b14/jdk-8u45-linux-x64.tar.gz" \ + -O - | tar xz -C /var/local +ENV JAVA_HOME /var/local/jdk1.8.0_45 +ENV PATH $JAVA_HOME/bin:$PATH + +# Install Maven +RUN wget -q http://apache.cs.utah.edu/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz -O - | \ + tar xz -C /var/local +ENV PATH /var/local/apache-maven-3.3.3/bin:$PATH + +# Install GCC 4.7 to support -static-libstdc++ +RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d +RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' +RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" +RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo +RUN yum install -y devtoolset-1.1 \ + devtoolset-1.1-libstdc++-devel \ + devtoolset-1.1-libstdc++-devel.i686 + +# Start in devtoolset environment that uses GCC 4.7 +CMD ["scl", "enable", "devtoolset-1.1", "bash"] \ No newline at end of file diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 9cd02c5e432..3933564273d 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -79,6 +79,12 @@ def macos_arch_env(arch): raise Exception('Unsupported arch') return {'CFLAGS': arch_arg, 'LDFLAGS': arch_arg} +_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' + +_ARCH_FLAG_MAP = { + 'x86': '-m32', + 'x64': '-m64' +} class PythonArtifact: """Builds Python artifacts.""" @@ -190,6 +196,7 @@ class CSharpExtArtifact: def __str__(self): return self.name + node_gyp_arch_map = { 'x86': 'ia32', 'x64': 'x64' @@ -226,6 +233,43 @@ class NodeExtArtifact: self.gyp_arch]) +class ProtocArtifact: + """Builds protoc and protoc-plugin artifacts""" + + def __init__(self, platform, arch): + self.name = 'protoc_%s_%s' % (platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'protoc', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if self.platform != 'windows': + cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] + ldflags = ' -static-libgcc -static-libstdc++ -s %s' % _ARCH_FLAG_MAP[self.arch] + environ={'CONFIG': 'opt', + 'CXXFLAGS': cxxflags, + 'LDFLAGS': ldflags, + 'PROTOBUF_LDFLAGS_EXTRA': ldflags} + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/dockerfile/grpc_artifact_protoc', + 'tools/run_tests/build_artifact_protoc.sh', + environ=environ) + else: + environ['CXXFLAGS'] += ' %s' % _MACOS_COMPAT_FLAG + return create_jobspec(self.name, + ['tools/run_tests/build_artifact_protoc.sh'], + environ=environ) + else: + raise Exception('Not yet supported') + + def __str__(self): + return self.name + + def targets(): """Gets list of supported targets""" return ([Cls(platform, arch) @@ -237,4 +281,8 @@ def targets(): PythonArtifact('macos', 'x64'), RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), - RubyArtifact('macos', 'x64')]) + RubyArtifact('macos', 'x64'), + ProtocArtifact('linux', 'x86'), + ProtocArtifact('linux', 'x64'), + ProtocArtifact('macos', 'x86'), + ProtocArtifact('macos', 'x64')]) diff --git a/tools/run_tests/build_artifact_protoc.sh b/tools/run_tests/build_artifact_protoc.sh new file mode 100755 index 00000000000..161d3a84d6e --- /dev/null +++ b/tools/run_tests/build_artifact_protoc.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Use devtoolset environment that has GCC 4.7 before set -ex +source scl_source enable devtoolset-1.1 + +set -ex + +cd $(dirname $0)/../.. + +make plugins + +mkdir -p artifacts +cp bins/opt/protobuf/protoc bins/opt/*_plugin artifacts/ From 2146fe8f88a06669e29a0cef2474e4063e7efe53 Mon Sep 17 00:00:00 2001 From: vjpai Date: Fri, 19 Feb 2016 10:05:57 -0800 Subject: [PATCH 029/236] Tune down multiplier for tsan to 5 --- Makefile | 2 +- build.yaml | 2 +- tools/run_tests/configs.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 445e6bcdf33..cf7af69ddca 100644 --- a/Makefile +++ b/Makefile @@ -187,7 +187,7 @@ LD_tsan = clang LDXX_tsan = clang++ CPPFLAGS_tsan = -O0 -fsanitize=thread -fno-omit-frame-pointer -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_tsan = -fsanitize=thread -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) -DEFINES_tsan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20 +DEFINES_tsan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=5 VALID_CONFIG_msan = 1 REQUIRE_CUSTOM_LIBRARIES_msan = 1 diff --git a/build.yaml b/build.yaml index e60300cbc52..a8646a2f37a 100644 --- a/build.yaml +++ b/build.yaml @@ -2703,7 +2703,7 @@ configs: compile_the_world: true test_environ: TSAN_OPTIONS: suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 - timeout_multiplier: 20 + timeout_multiplier: 5 ubsan: CC: clang CPPFLAGS: -O1 -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json index aacca409f7c..cbb8ec57b61 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/configs.json @@ -55,7 +55,7 @@ "environ": { "TSAN_OPTIONS": "suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" }, - "timeout_multiplier": 20 + "timeout_multiplier": 5 }, { "config": "msan", From aea13f1c8519d7a0e3aea3837fd9fc18e68ee9b4 Mon Sep 17 00:00:00 2001 From: yang-g Date: Fri, 19 Feb 2016 11:05:28 -0800 Subject: [PATCH 030/236] global replace health check proto v1alpha to v1 --- doc/health-checking.md | 4 +-- .../Grpc.HealthCheck/Grpc.HealthCheck.nuspec | 2 +- src/csharp/Grpc.HealthCheck/HealthGrpc.cs | 30 +++++++++---------- src/csharp/generate_proto_csharp.sh | 2 +- src/node/health_check/health.js | 6 ++-- .../grpc/health/{v1alpha => v1}/health.proto | 7 ++--- .../grpc/health/{v1alpha => v1}/__init__.py | 2 +- .../grpc/health/{v1alpha => v1}/health.proto | 4 +-- .../grpc/health/{v1alpha => v1}/health.py | 6 ++-- src/ruby/.rubocop.yml | 2 +- src/ruby/pb/README.md | 4 +-- src/ruby/pb/generate_proto_ruby.sh | 4 +-- src/ruby/pb/grpc/health/checker.rb | 8 ++--- src/ruby/pb/grpc/health/v1/health.rb | 29 ++++++++++++++++++ .../health/{v1alpha => v1}/health_services.rb | 8 ++--- src/ruby/pb/grpc/health/v1alpha/health.rb | 29 ------------------ src/ruby/spec/pb/health/checker_spec.rb | 16 +++++----- 17 files changed, 81 insertions(+), 82 deletions(-) rename src/proto/grpc/health/{v1alpha => v1}/health.proto (93%) rename src/python/grpcio_health_checking/grpc/health/{v1alpha => v1}/__init__.py (97%) rename src/python/grpcio_health_checking/grpc/health/{v1alpha => v1}/health.proto (96%) rename src/python/grpcio_health_checking/grpc/health/{v1alpha => v1}/health.py (96%) create mode 100644 src/ruby/pb/grpc/health/v1/health.rb rename src/ruby/pb/grpc/health/{v1alpha => v1}/health_services.rb (71%) delete mode 100644 src/ruby/pb/grpc/health/v1alpha/health.rb diff --git a/doc/health-checking.md b/doc/health-checking.md index 0b3f9c6a034..92512e942bd 100644 --- a/doc/health-checking.md +++ b/doc/health-checking.md @@ -26,7 +26,7 @@ The server should export a service defined in the following proto: ``` syntax = "proto3"; -package grpc.health.v1alpha; +package grpc.health.v1; message HealthCheckRequest { string service = 1; @@ -49,7 +49,7 @@ service Health { A client can query the server’s health status by calling the `Check` method, and a deadline should be set on the rpc. The client can optionally set the service name it wants to query for health status. The suggested format of service name -is `package_names.ServiceName`, such as `grpc.health.v1alpha.Health`. +is `package_names.ServiceName`, such as `grpc.health.v1.Health`. The server should register all the services manually and set the individual status, including an empty service name and its status. For each diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.nuspec b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.nuspec index 66386288df1..7b3b391009e 100644 --- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.nuspec +++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.nuspec @@ -4,7 +4,7 @@ Grpc.HealthCheck gRPC C# Healthchecking Implementation of gRPC health service - Example implementation of grpc.health.v1alpha service that can be used for health-checking. + Example implementation of grpc.health.v1 service that can be used for health-checking. $version$ Google Inc. grpc-packages diff --git a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs index 882edd56270..68320eb5c2e 100644 --- a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs +++ b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs @@ -7,15 +7,15 @@ using System.Threading; using System.Threading.Tasks; using Grpc.Core; -namespace Grpc.Health.V1Alpha { +namespace Grpc.Health.V1 { public static class Health { - static readonly string __ServiceName = "grpc.health.v1alpha.Health"; + static readonly string __ServiceName = "grpc.health.v1.Health"; - static readonly Marshaller __Marshaller_HealthCheckRequest = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Health.V1Alpha.HealthCheckRequest.Parser.ParseFrom); - static readonly Marshaller __Marshaller_HealthCheckResponse = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Health.V1Alpha.HealthCheckResponse.Parser.ParseFrom); + static readonly Marshaller __Marshaller_HealthCheckRequest = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Health.V1.HealthCheckRequest.Parser.ParseFrom); + static readonly Marshaller __Marshaller_HealthCheckResponse = Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Health.V1.HealthCheckResponse.Parser.ParseFrom); - static readonly Method __Method_Check = new Method( + static readonly Method __Method_Check = new Method( MethodType.Unary, __ServiceName, "Check", @@ -25,22 +25,22 @@ namespace Grpc.Health.V1Alpha { // service descriptor public static global::Google.Protobuf.Reflection.ServiceDescriptor Descriptor { - get { return global::Grpc.Health.V1Alpha.HealthReflection.Descriptor.Services[0]; } + get { return global::Grpc.Health.V1.HealthReflection.Descriptor.Services[0]; } } // client interface public interface IHealthClient { - global::Grpc.Health.V1Alpha.HealthCheckResponse Check(global::Grpc.Health.V1Alpha.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)); - global::Grpc.Health.V1Alpha.HealthCheckResponse Check(global::Grpc.Health.V1Alpha.HealthCheckRequest request, CallOptions options); - AsyncUnaryCall CheckAsync(global::Grpc.Health.V1Alpha.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)); - AsyncUnaryCall CheckAsync(global::Grpc.Health.V1Alpha.HealthCheckRequest request, CallOptions options); + global::Grpc.Health.V1.HealthCheckResponse Check(global::Grpc.Health.V1.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)); + global::Grpc.Health.V1.HealthCheckResponse Check(global::Grpc.Health.V1.HealthCheckRequest request, CallOptions options); + AsyncUnaryCall CheckAsync(global::Grpc.Health.V1.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)); + AsyncUnaryCall CheckAsync(global::Grpc.Health.V1.HealthCheckRequest request, CallOptions options); } // server-side interface public interface IHealth { - Task Check(global::Grpc.Health.V1Alpha.HealthCheckRequest request, ServerCallContext context); + Task Check(global::Grpc.Health.V1.HealthCheckRequest request, ServerCallContext context); } // client stub @@ -49,22 +49,22 @@ namespace Grpc.Health.V1Alpha { public HealthClient(Channel channel) : base(channel) { } - public global::Grpc.Health.V1Alpha.HealthCheckResponse Check(global::Grpc.Health.V1Alpha.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + public global::Grpc.Health.V1.HealthCheckResponse Check(global::Grpc.Health.V1.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) { var call = CreateCall(__Method_Check, new CallOptions(headers, deadline, cancellationToken)); return Calls.BlockingUnaryCall(call, request); } - public global::Grpc.Health.V1Alpha.HealthCheckResponse Check(global::Grpc.Health.V1Alpha.HealthCheckRequest request, CallOptions options) + public global::Grpc.Health.V1.HealthCheckResponse Check(global::Grpc.Health.V1.HealthCheckRequest request, CallOptions options) { var call = CreateCall(__Method_Check, options); return Calls.BlockingUnaryCall(call, request); } - public AsyncUnaryCall CheckAsync(global::Grpc.Health.V1Alpha.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) + public AsyncUnaryCall CheckAsync(global::Grpc.Health.V1.HealthCheckRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken)) { var call = CreateCall(__Method_Check, new CallOptions(headers, deadline, cancellationToken)); return Calls.AsyncUnaryCall(call, request); } - public AsyncUnaryCall CheckAsync(global::Grpc.Health.V1Alpha.HealthCheckRequest request, CallOptions options) + public AsyncUnaryCall CheckAsync(global::Grpc.Health.V1.HealthCheckRequest request, CallOptions options) { var call = CreateCall(__Method_Check, options); return Calls.AsyncUnaryCall(call, request); diff --git a/src/csharp/generate_proto_csharp.sh b/src/csharp/generate_proto_csharp.sh index 0261a458af7..23e0540253a 100755 --- a/src/csharp/generate_proto_csharp.sh +++ b/src/csharp/generate_proto_csharp.sh @@ -42,7 +42,7 @@ $PROTOC --plugin=$PLUGIN --csharp_out=$EXAMPLES_DIR --grpc_out=$EXAMPLES_DIR \ -I src/proto/math src/proto/math/math.proto $PROTOC --plugin=$PLUGIN --csharp_out=$HEALTHCHECK_DIR --grpc_out=$HEALTHCHECK_DIR \ - -I src/proto/grpc/health/v1alpha src/proto/grpc/health/v1alpha/health.proto + -I src/proto/grpc/health/v1 src/proto/grpc/health/v1/health.proto $PROTOC --plugin=$PLUGIN --csharp_out=$TESTING_DIR --grpc_out=$TESTING_DIR \ -I . src/proto/grpc/testing/{control,empty,messages,payloads,services,stats,test}.proto diff --git a/src/node/health_check/health.js b/src/node/health_check/health.js index 1a2c0366875..6ab41571831 100644 --- a/src/node/health_check/health.js +++ b/src/node/health_check/health.js @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,9 +38,9 @@ var grpc = require('../'); var _ = require('lodash'); var health_proto = grpc.load(__dirname + - '/../../proto/grpc/health/v1alpha/health.proto'); + '/../../proto/grpc/health/v1/health.proto'); -var HealthClient = health_proto.grpc.health.v1alpha.Health; +var HealthClient = health_proto.grpc.health.v1.Health; function HealthImplementation(statusMap) { this.statusMap = _.clone(statusMap); diff --git a/src/proto/grpc/health/v1alpha/health.proto b/src/proto/grpc/health/v1/health.proto similarity index 93% rename from src/proto/grpc/health/v1alpha/health.proto rename to src/proto/grpc/health/v1/health.proto index 05f837dd990..6e27606d1b6 100644 --- a/src/proto/grpc/health/v1alpha/health.proto +++ b/src/proto/grpc/health/v1/health.proto @@ -29,12 +29,11 @@ syntax = "proto3"; -package grpc.health.v1alpha; -option csharp_namespace = "Grpc.Health.V1Alpha"; +package grpc.health.v1; +option csharp_namespace = "Grpc.Health.V1"; message HealthCheckRequest { - string host = 1; - string service = 2; + string service = 1; } message HealthCheckResponse { diff --git a/src/python/grpcio_health_checking/grpc/health/v1alpha/__init__.py b/src/python/grpcio_health_checking/grpc/health/v1/__init__.py similarity index 97% rename from src/python/grpcio_health_checking/grpc/health/v1alpha/__init__.py rename to src/python/grpcio_health_checking/grpc/health/v1/__init__.py index 70865191060..13aac79160b 100644 --- a/src/python/grpcio_health_checking/grpc/health/v1alpha/__init__.py +++ b/src/python/grpcio_health_checking/grpc/health/v1/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/src/python/grpcio_health_checking/grpc/health/v1alpha/health.proto b/src/python/grpcio_health_checking/grpc/health/v1/health.proto similarity index 96% rename from src/python/grpcio_health_checking/grpc/health/v1alpha/health.proto rename to src/python/grpcio_health_checking/grpc/health/v1/health.proto index 57f4aaa9c08..de10719b6ce 100644 --- a/src/python/grpcio_health_checking/grpc/health/v1alpha/health.proto +++ b/src/python/grpcio_health_checking/grpc/health/v1/health.proto @@ -1,4 +1,4 @@ -// Copyright 2015, Google Inc. +// Copyright 2015-2016, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without @@ -29,7 +29,7 @@ syntax = "proto3"; -package grpc.health.v1alpha; +package grpc.health.v1; message HealthCheckRequest { string service = 1; diff --git a/src/python/grpcio_health_checking/grpc/health/v1alpha/health.py b/src/python/grpcio_health_checking/grpc/health/v1/health.py similarity index 96% rename from src/python/grpcio_health_checking/grpc/health/v1alpha/health.py rename to src/python/grpcio_health_checking/grpc/health/v1/health.py index 9dfcd962f06..60cbd644330 100644 --- a/src/python/grpcio_health_checking/grpc/health/v1alpha/health.py +++ b/src/python/grpcio_health_checking/grpc/health/v1/health.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -33,7 +33,7 @@ import abc import enum import threading -from grpc.health.v1alpha import health_pb2 +from grpc.health.v1 import health_pb2 @enum.unique @@ -64,7 +64,7 @@ class _HealthServicer(health_pb2.EarlyAdopterHealthServicer): def set(service, status): if not isinstance(status, HealthStatus): - raise TypeError('expected grpc.health.v1alpha.health.HealthStatus ' + raise TypeError('expected grpc.health.v1.health.HealthStatus ' 'for argument `status` but got {}'.format(status)) with self._server_status_lock: self._server_status[service] = status diff --git a/src/ruby/.rubocop.yml b/src/ruby/.rubocop.yml index dd57ab60828..ff5cf8db831 100644 --- a/src/ruby/.rubocop.yml +++ b/src/ruby/.rubocop.yml @@ -7,7 +7,7 @@ AllCops: - 'bin/apis/**/*' - 'bin/math.rb' - 'bin/math_services.rb' - - 'pb/grpc/health/v1alpha/*' + - 'pb/grpc/health/v1/*' - 'pb/test/**/*' Metrics/CyclomaticComplexity: diff --git a/src/ruby/pb/README.md b/src/ruby/pb/README.md index e04aef185ca..d9e30bbc854 100644 --- a/src/ruby/pb/README.md +++ b/src/ruby/pb/README.md @@ -11,7 +11,7 @@ The code is is generated using the protoc (> 3.0.0.alpha.1) and the grpc_ruby_plugin. These must be installed to regenerate the IDL defined classes, but that's not necessary just to use them. -health_check/v1alpha +health_check/v1 -------------------- This package defines the surface of a simple health check service that gRPC @@ -20,7 +20,7 @@ re-generate the surface. ```bash $ # (from this directory) -$ protoc -I ../../proto ../../proto/grpc/health/v1alpha/health.proto \ +$ protoc -I ../../proto ../../proto/grpc/health/v1/health.proto \ --grpc_out=. \ --ruby_out=. \ --plugin=protoc-gen-grpc=`which grpc_ruby_plugin` diff --git a/src/ruby/pb/generate_proto_ruby.sh b/src/ruby/pb/generate_proto_ruby.sh index 576b1c08d30..86c082099d9 100755 --- a/src/ruby/pb/generate_proto_ruby.sh +++ b/src/ruby/pb/generate_proto_ruby.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -35,7 +35,7 @@ cd $(dirname $0)/../../.. PROTOC=bins/opt/protobuf/protoc PLUGIN=protoc-gen-grpc=bins/opt/grpc_ruby_plugin -$PROTOC -I src/proto src/proto/grpc/health/v1alpha/health.proto \ +$PROTOC -I src/proto src/proto/grpc/health/v1/health.proto \ --grpc_out=src/ruby/pb \ --ruby_out=src/ruby/pb \ --plugin=$PLUGIN diff --git a/src/ruby/pb/grpc/health/checker.rb b/src/ruby/pb/grpc/health/checker.rb index 8c692e74f90..0af4c5b0a8d 100644 --- a/src/ruby/pb/grpc/health/checker.rb +++ b/src/ruby/pb/grpc/health/checker.rb @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,7 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. require 'grpc' -require 'grpc/health/v1alpha/health_services' +require 'grpc/health/v1/health_services' require 'thread' module Grpc @@ -36,9 +36,9 @@ module Grpc # service. module Health # Checker is implementation of the schema-specified health checking service. - class Checker < V1alpha::Health::Service + class Checker < V1::Health::Service StatusCodes = GRPC::Core::StatusCodes - HealthCheckResponse = V1alpha::HealthCheckResponse + HealthCheckResponse = V1::HealthCheckResponse # Initializes the statuses of participating services def initialize diff --git a/src/ruby/pb/grpc/health/v1/health.rb b/src/ruby/pb/grpc/health/v1/health.rb new file mode 100644 index 00000000000..ca27afb8e1c --- /dev/null +++ b/src/ruby/pb/grpc/health/v1/health.rb @@ -0,0 +1,29 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: grpc/health/v1/health.proto + +require 'google/protobuf' + +Google::Protobuf::DescriptorPool.generated_pool.build do + add_message "grpc.health.v1.HealthCheckRequest" do + optional :host, :string, 1 + optional :service, :string, 2 + end + add_message "grpc.health.v1.HealthCheckResponse" do + optional :status, :enum, 1, "grpc.health.v1.HealthCheckResponse.ServingStatus" + end + add_enum "grpc.health.v1.HealthCheckResponse.ServingStatus" do + value :UNKNOWN, 0 + value :SERVING, 1 + value :NOT_SERVING, 2 + end +end + +module Grpc + module Health + module V1 + HealthCheckRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.health.v1.HealthCheckRequest").msgclass + HealthCheckResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.health.v1.HealthCheckResponse").msgclass + HealthCheckResponse::ServingStatus = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.health.v1.HealthCheckResponse.ServingStatus").enummodule + end + end +end diff --git a/src/ruby/pb/grpc/health/v1alpha/health_services.rb b/src/ruby/pb/grpc/health/v1/health_services.rb similarity index 71% rename from src/ruby/pb/grpc/health/v1alpha/health_services.rb rename to src/ruby/pb/grpc/health/v1/health_services.rb index d5cba2e9ec7..cb79b20437f 100644 --- a/src/ruby/pb/grpc/health/v1alpha/health_services.rb +++ b/src/ruby/pb/grpc/health/v1/health_services.rb @@ -1,12 +1,12 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! -# Source: grpc/health/v1alpha/health.proto for package 'grpc.health.v1alpha' +# Source: grpc/health/v1/health.proto for package 'grpc.health.v1' require 'grpc' -require 'grpc/health/v1alpha/health' +require 'grpc/health/v1/health' module Grpc module Health - module V1alpha + module V1 module Health # TODO: add proto service documentation here @@ -16,7 +16,7 @@ module Grpc self.marshal_class_method = :encode self.unmarshal_class_method = :decode - self.service_name = 'grpc.health.v1alpha.Health' + self.service_name = 'grpc.health.v1.Health' rpc :Check, HealthCheckRequest, HealthCheckResponse end diff --git a/src/ruby/pb/grpc/health/v1alpha/health.rb b/src/ruby/pb/grpc/health/v1alpha/health.rb deleted file mode 100644 index 9c04298ea54..00000000000 --- a/src/ruby/pb/grpc/health/v1alpha/health.rb +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: grpc/health/v1alpha/health.proto - -require 'google/protobuf' - -Google::Protobuf::DescriptorPool.generated_pool.build do - add_message "grpc.health.v1alpha.HealthCheckRequest" do - optional :host, :string, 1 - optional :service, :string, 2 - end - add_message "grpc.health.v1alpha.HealthCheckResponse" do - optional :status, :enum, 1, "grpc.health.v1alpha.HealthCheckResponse.ServingStatus" - end - add_enum "grpc.health.v1alpha.HealthCheckResponse.ServingStatus" do - value :UNKNOWN, 0 - value :SERVING, 1 - value :NOT_SERVING, 2 - end -end - -module Grpc - module Health - module V1alpha - HealthCheckRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.health.v1alpha.HealthCheckRequest").msgclass - HealthCheckResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.health.v1alpha.HealthCheckResponse").msgclass - HealthCheckResponse::ServingStatus = Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.health.v1alpha.HealthCheckResponse.ServingStatus").enummodule - end - end -end diff --git a/src/ruby/spec/pb/health/checker_spec.rb b/src/ruby/spec/pb/health/checker_spec.rb index c1decd822a7..fe4e5269bb0 100644 --- a/src/ruby/spec/pb/health/checker_spec.rb +++ b/src/ruby/spec/pb/health/checker_spec.rb @@ -28,7 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. require 'grpc' -require 'grpc/health/v1alpha/health' +require 'grpc/health/v1/health' require 'grpc/health/checker' require 'open3' require 'tmpdir' @@ -43,7 +43,7 @@ describe 'Health protobuf code generation' do skip 'protoc || grpc_ruby_plugin missing, cannot verify health code-gen' else it 'should already be loaded indirectly i.e, used by the other specs' do - expect(require('grpc/health/v1alpha/health_services')).to be(false) + expect(require('grpc/health/v1/health_services')).to be(false) end it 'should have the same content as created by code generation' do @@ -52,7 +52,7 @@ describe 'Health protobuf code generation' do # Get the current content service_path = File.join(root_dir, 'ruby', 'pb', 'grpc', - 'health', 'v1alpha', 'health_services.rb') + 'health', 'v1', 'health_services.rb') want = nil File.open(service_path) { |f| want = f.read } @@ -61,12 +61,12 @@ describe 'Health protobuf code generation' do plugin = plugin.strip got = nil Dir.mktmpdir do |tmp_dir| - gen_out = File.join(tmp_dir, 'grpc', 'health', 'v1alpha', + gen_out = File.join(tmp_dir, 'grpc', 'health', 'v1', 'health_services.rb') pid = spawn( 'protoc', '-I.', - 'grpc/health/v1alpha/health.proto', + 'grpc/health/v1/health.proto', "--grpc_out=#{tmp_dir}", "--plugin=protoc-gen-grpc=#{plugin}", chdir: pb_dir) @@ -81,9 +81,9 @@ end describe Grpc::Health::Checker do StatusCodes = GRPC::Core::StatusCodes - ServingStatus = Grpc::Health::V1alpha::HealthCheckResponse::ServingStatus - HCResp = Grpc::Health::V1alpha::HealthCheckResponse - HCReq = Grpc::Health::V1alpha::HealthCheckRequest + ServingStatus = Grpc::Health::V1::HealthCheckResponse::ServingStatus + HCResp = Grpc::Health::V1::HealthCheckResponse + HCReq = Grpc::Health::V1::HealthCheckRequest success_tests = [ { From 3165c09ce5f5930b605961441724fe090705d4e0 Mon Sep 17 00:00:00 2001 From: yang-g Date: Fri, 19 Feb 2016 12:36:40 -0800 Subject: [PATCH 031/236] regenerate csharp and ruby service from proto --- src/csharp/Grpc.HealthCheck/Health.cs | 72 ++++++++------------------- src/ruby/pb/grpc/health/v1/health.rb | 3 +- 2 files changed, 23 insertions(+), 52 deletions(-) diff --git a/src/csharp/Grpc.HealthCheck/Health.cs b/src/csharp/Grpc.HealthCheck/Health.cs index 56673f1adf6..d0d0c0b5196 100644 --- a/src/csharp/Grpc.HealthCheck/Health.cs +++ b/src/csharp/Grpc.HealthCheck/Health.cs @@ -7,7 +7,7 @@ using pb = global::Google.Protobuf; using pbc = global::Google.Protobuf.Collections; using pbr = global::Google.Protobuf.Reflection; using scg = global::System.Collections.Generic; -namespace Grpc.Health.V1Alpha { +namespace Grpc.Health.V1 { /// Holder for reflection information generated from health.proto [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] @@ -23,20 +23,19 @@ namespace Grpc.Health.V1Alpha { static HealthReflection() { byte[] descriptorData = global::System.Convert.FromBase64String( string.Concat( - "CgxoZWFsdGgucHJvdG8SE2dycGMuaGVhbHRoLnYxYWxwaGEiMwoSSGVhbHRo", - "Q2hlY2tSZXF1ZXN0EgwKBGhvc3QYASABKAkSDwoHc2VydmljZRgCIAEoCSKZ", - "AQoTSGVhbHRoQ2hlY2tSZXNwb25zZRJGCgZzdGF0dXMYASABKA4yNi5ncnBj", - "LmhlYWx0aC52MWFscGhhLkhlYWx0aENoZWNrUmVzcG9uc2UuU2VydmluZ1N0", - "YXR1cyI6Cg1TZXJ2aW5nU3RhdHVzEgsKB1VOS05PV04QABILCgdTRVJWSU5H", - "EAESDwoLTk9UX1NFUlZJTkcQAjJkCgZIZWFsdGgSWgoFQ2hlY2sSJy5ncnBj", - "LmhlYWx0aC52MWFscGhhLkhlYWx0aENoZWNrUmVxdWVzdBooLmdycGMuaGVh", - "bHRoLnYxYWxwaGEuSGVhbHRoQ2hlY2tSZXNwb25zZUIWqgITR3JwYy5IZWFs", - "dGguVjFBbHBoYWIGcHJvdG8z")); + "CgxoZWFsdGgucHJvdG8SDmdycGMuaGVhbHRoLnYxIiUKEkhlYWx0aENoZWNr", + "UmVxdWVzdBIPCgdzZXJ2aWNlGAEgASgJIpQBChNIZWFsdGhDaGVja1Jlc3Bv", + "bnNlEkEKBnN0YXR1cxgBIAEoDjIxLmdycGMuaGVhbHRoLnYxLkhlYWx0aENo", + "ZWNrUmVzcG9uc2UuU2VydmluZ1N0YXR1cyI6Cg1TZXJ2aW5nU3RhdHVzEgsK", + "B1VOS05PV04QABILCgdTRVJWSU5HEAESDwoLTk9UX1NFUlZJTkcQAjJaCgZI", + "ZWFsdGgSUAoFQ2hlY2sSIi5ncnBjLmhlYWx0aC52MS5IZWFsdGhDaGVja1Jl", + "cXVlc3QaIy5ncnBjLmhlYWx0aC52MS5IZWFsdGhDaGVja1Jlc3BvbnNlQhGq", + "Ag5HcnBjLkhlYWx0aC5WMWIGcHJvdG8z")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedCodeInfo(null, new pbr::GeneratedCodeInfo[] { - new pbr::GeneratedCodeInfo(typeof(global::Grpc.Health.V1Alpha.HealthCheckRequest), global::Grpc.Health.V1Alpha.HealthCheckRequest.Parser, new[]{ "Host", "Service" }, null, null, null), - new pbr::GeneratedCodeInfo(typeof(global::Grpc.Health.V1Alpha.HealthCheckResponse), global::Grpc.Health.V1Alpha.HealthCheckResponse.Parser, new[]{ "Status" }, null, new[]{ typeof(global::Grpc.Health.V1Alpha.HealthCheckResponse.Types.ServingStatus) }, null) + new pbr::GeneratedCodeInfo(typeof(global::Grpc.Health.V1.HealthCheckRequest), global::Grpc.Health.V1.HealthCheckRequest.Parser, new[]{ "Service" }, null, null, null), + new pbr::GeneratedCodeInfo(typeof(global::Grpc.Health.V1.HealthCheckResponse), global::Grpc.Health.V1.HealthCheckResponse.Parser, new[]{ "Status" }, null, new[]{ typeof(global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus) }, null) })); } #endregion @@ -49,7 +48,7 @@ namespace Grpc.Health.V1Alpha { public static pb::MessageParser Parser { get { return _parser; } } public static pbr::MessageDescriptor Descriptor { - get { return global::Grpc.Health.V1Alpha.HealthReflection.Descriptor.MessageTypes[0]; } + get { return global::Grpc.Health.V1.HealthReflection.Descriptor.MessageTypes[0]; } } pbr::MessageDescriptor pb::IMessage.Descriptor { @@ -63,7 +62,6 @@ namespace Grpc.Health.V1Alpha { partial void OnConstruction(); public HealthCheckRequest(HealthCheckRequest other) : this() { - host_ = other.host_; service_ = other.service_; } @@ -71,18 +69,8 @@ namespace Grpc.Health.V1Alpha { return new HealthCheckRequest(this); } - /// Field number for the "host" field. - public const int HostFieldNumber = 1; - private string host_ = ""; - public string Host { - get { return host_; } - set { - host_ = pb::Preconditions.CheckNotNull(value, "value"); - } - } - /// Field number for the "service" field. - public const int ServiceFieldNumber = 2; + public const int ServiceFieldNumber = 1; private string service_ = ""; public string Service { get { return service_; } @@ -102,14 +90,12 @@ namespace Grpc.Health.V1Alpha { if (ReferenceEquals(other, this)) { return true; } - if (Host != other.Host) return false; if (Service != other.Service) return false; return true; } public override int GetHashCode() { int hash = 1; - if (Host.Length != 0) hash ^= Host.GetHashCode(); if (Service.Length != 0) hash ^= Service.GetHashCode(); return hash; } @@ -119,21 +105,14 @@ namespace Grpc.Health.V1Alpha { } public void WriteTo(pb::CodedOutputStream output) { - if (Host.Length != 0) { - output.WriteRawTag(10); - output.WriteString(Host); - } if (Service.Length != 0) { - output.WriteRawTag(18); + output.WriteRawTag(10); output.WriteString(Service); } } public int CalculateSize() { int size = 0; - if (Host.Length != 0) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(Host); - } if (Service.Length != 0) { size += 1 + pb::CodedOutputStream.ComputeStringSize(Service); } @@ -144,9 +123,6 @@ namespace Grpc.Health.V1Alpha { if (other == null) { return; } - if (other.Host.Length != 0) { - Host = other.Host; - } if (other.Service.Length != 0) { Service = other.Service; } @@ -160,10 +136,6 @@ namespace Grpc.Health.V1Alpha { input.SkipLastField(); break; case 10: { - Host = input.ReadString(); - break; - } - case 18: { Service = input.ReadString(); break; } @@ -179,7 +151,7 @@ namespace Grpc.Health.V1Alpha { public static pb::MessageParser Parser { get { return _parser; } } public static pbr::MessageDescriptor Descriptor { - get { return global::Grpc.Health.V1Alpha.HealthReflection.Descriptor.MessageTypes[1]; } + get { return global::Grpc.Health.V1.HealthReflection.Descriptor.MessageTypes[1]; } } pbr::MessageDescriptor pb::IMessage.Descriptor { @@ -202,8 +174,8 @@ namespace Grpc.Health.V1Alpha { /// Field number for the "status" field. public const int StatusFieldNumber = 1; - private global::Grpc.Health.V1Alpha.HealthCheckResponse.Types.ServingStatus status_ = global::Grpc.Health.V1Alpha.HealthCheckResponse.Types.ServingStatus.UNKNOWN; - public global::Grpc.Health.V1Alpha.HealthCheckResponse.Types.ServingStatus Status { + private global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus status_ = global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN; + public global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus Status { get { return status_; } set { status_ = value; @@ -227,7 +199,7 @@ namespace Grpc.Health.V1Alpha { public override int GetHashCode() { int hash = 1; - if (Status != global::Grpc.Health.V1Alpha.HealthCheckResponse.Types.ServingStatus.UNKNOWN) hash ^= Status.GetHashCode(); + if (Status != global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN) hash ^= Status.GetHashCode(); return hash; } @@ -236,7 +208,7 @@ namespace Grpc.Health.V1Alpha { } public void WriteTo(pb::CodedOutputStream output) { - if (Status != global::Grpc.Health.V1Alpha.HealthCheckResponse.Types.ServingStatus.UNKNOWN) { + if (Status != global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN) { output.WriteRawTag(8); output.WriteEnum((int) Status); } @@ -244,7 +216,7 @@ namespace Grpc.Health.V1Alpha { public int CalculateSize() { int size = 0; - if (Status != global::Grpc.Health.V1Alpha.HealthCheckResponse.Types.ServingStatus.UNKNOWN) { + if (Status != global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN) { size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) Status); } return size; @@ -254,7 +226,7 @@ namespace Grpc.Health.V1Alpha { if (other == null) { return; } - if (other.Status != global::Grpc.Health.V1Alpha.HealthCheckResponse.Types.ServingStatus.UNKNOWN) { + if (other.Status != global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus.UNKNOWN) { Status = other.Status; } } @@ -267,7 +239,7 @@ namespace Grpc.Health.V1Alpha { input.SkipLastField(); break; case 8: { - status_ = (global::Grpc.Health.V1Alpha.HealthCheckResponse.Types.ServingStatus) input.ReadEnum(); + status_ = (global::Grpc.Health.V1.HealthCheckResponse.Types.ServingStatus) input.ReadEnum(); break; } } diff --git a/src/ruby/pb/grpc/health/v1/health.rb b/src/ruby/pb/grpc/health/v1/health.rb index ca27afb8e1c..aa87a93918b 100644 --- a/src/ruby/pb/grpc/health/v1/health.rb +++ b/src/ruby/pb/grpc/health/v1/health.rb @@ -5,8 +5,7 @@ require 'google/protobuf' Google::Protobuf::DescriptorPool.generated_pool.build do add_message "grpc.health.v1.HealthCheckRequest" do - optional :host, :string, 1 - optional :service, :string, 2 + optional :service, :string, 1 end add_message "grpc.health.v1.HealthCheckResponse" do optional :status, :enum, 1, "grpc.health.v1.HealthCheckResponse.ServingStatus" From e1711624e1ce76545af09947c89ffd761a413051 Mon Sep 17 00:00:00 2001 From: yang-g Date: Fri, 19 Feb 2016 13:06:37 -0800 Subject: [PATCH 032/236] V1Alpha as well --- .../Grpc.HealthCheck.Tests/HealthClientServerTest.cs | 8 ++++---- .../Grpc.HealthCheck.Tests/HealthServiceImplTest.cs | 4 ++-- src/csharp/Grpc.HealthCheck/HealthServiceImpl.cs | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs b/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs index a8a76c74922..7956f33524d 100644 --- a/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs +++ b/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs @@ -36,7 +36,7 @@ using System.Text; using System.Threading.Tasks; using Grpc.Core; -using Grpc.Health.V1Alpha; +using Grpc.Health.V1; using NUnit.Framework; namespace Grpc.HealthCheck.Tests @@ -49,7 +49,7 @@ namespace Grpc.HealthCheck.Tests const string Host = "localhost"; Server server; Channel channel; - Grpc.Health.V1Alpha.Health.IHealthClient client; + Grpc.Health.V1.Health.IHealthClient client; Grpc.HealthCheck.HealthServiceImpl serviceImpl; [TestFixtureSetUp] @@ -59,13 +59,13 @@ namespace Grpc.HealthCheck.Tests server = new Server { - Services = { Grpc.Health.V1Alpha.Health.BindService(serviceImpl) }, + Services = { Grpc.Health.V1.Health.BindService(serviceImpl) }, Ports = { { Host, ServerPort.PickUnused, ServerCredentials.Insecure } } }; server.Start(); channel = new Channel(Host, server.Ports.Single().BoundPort, ChannelCredentials.Insecure); - client = Grpc.Health.V1Alpha.Health.NewClient(channel); + client = Grpc.Health.V1.Health.NewClient(channel); } [TestFixtureTearDown] diff --git a/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs b/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs index 2097c0dc8cf..b86d511b163 100644 --- a/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs +++ b/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs @@ -1,5 +1,5 @@ #region Copyright notice and license -// Copyright 2015, Google Inc. +// Copyright 2015-2016, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without @@ -36,7 +36,7 @@ using System.Text; using System.Threading.Tasks; using Grpc.Core; -using Grpc.Health.V1Alpha; +using Grpc.Health.V1; using NUnit.Framework; namespace Grpc.HealthCheck.Tests diff --git a/src/csharp/Grpc.HealthCheck/HealthServiceImpl.cs b/src/csharp/Grpc.HealthCheck/HealthServiceImpl.cs index e2ad1a834bf..e93b9b96dc5 100644 --- a/src/csharp/Grpc.HealthCheck/HealthServiceImpl.cs +++ b/src/csharp/Grpc.HealthCheck/HealthServiceImpl.cs @@ -37,7 +37,7 @@ using System.Threading.Tasks; using Grpc.Core; using Grpc.Core.Utils; -using Grpc.Health.V1Alpha; +using Grpc.Health.V1; namespace Grpc.HealthCheck { @@ -48,10 +48,10 @@ namespace Grpc.HealthCheck /// /// var serviceImpl = new HealthServiceImpl(); /// server = new Server(); - /// server.AddServiceDefinition(Grpc.Health.V1Alpha.Health.BindService(serviceImpl)); + /// server.AddServiceDefinition(Grpc.Health.V1.Health.BindService(serviceImpl)); /// /// - public class HealthServiceImpl : Grpc.Health.V1Alpha.Health.IHealth + public class HealthServiceImpl : Grpc.Health.V1.Health.IHealth { private readonly object myLock = new object(); private readonly Dictionary statusMap = From e7b7d86fde3475978c3f7d4dfaabaf549152cd02 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 19 Feb 2016 09:49:35 -0800 Subject: [PATCH 033/236] fix protoc artifact build on mac --- tools/run_tests/artifact_targets.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 3933564273d..5d678a561f9 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -248,7 +248,9 @@ class ProtocArtifact: def build_jobspec(self): if self.platform != 'windows': cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] - ldflags = ' -static-libgcc -static-libstdc++ -s %s' % _ARCH_FLAG_MAP[self.arch] + ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] + if self.platform != 'macos': + ldflags += ' -static-libgcc -static-libstdc++ -s' environ={'CONFIG': 'opt', 'CXXFLAGS': cxxflags, 'LDFLAGS': ldflags, @@ -259,7 +261,7 @@ class ProtocArtifact: 'tools/run_tests/build_artifact_protoc.sh', environ=environ) else: - environ['CXXFLAGS'] += ' %s' % _MACOS_COMPAT_FLAG + environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG return create_jobspec(self.name, ['tools/run_tests/build_artifact_protoc.sh'], environ=environ) From a4598b4c47694b67cad94ccdb9385ec20940cf4c Mon Sep 17 00:00:00 2001 From: yang-g Date: Fri, 19 Feb 2016 13:30:23 -0800 Subject: [PATCH 034/236] fix impl and test about host --- .../HealthClientServerTest.cs | 6 +-- .../HealthServiceImplTest.cs | 44 +++++++++---------- .../Grpc.HealthCheck/HealthServiceImpl.cs | 40 +++++------------ 3 files changed, 34 insertions(+), 56 deletions(-) diff --git a/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs b/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs index 7956f33524d..c3fac05324c 100644 --- a/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs +++ b/src/csharp/Grpc.HealthCheck.Tests/HealthClientServerTest.cs @@ -79,16 +79,16 @@ namespace Grpc.HealthCheck.Tests [Test] public void ServiceIsRunning() { - serviceImpl.SetStatus("", "", HealthCheckResponse.Types.ServingStatus.SERVING); + serviceImpl.SetStatus("", HealthCheckResponse.Types.ServingStatus.SERVING); - var response = client.Check(new HealthCheckRequest { Host = "", Service = "" }); + var response = client.Check(new HealthCheckRequest { Service = "" }); Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.SERVING, response.Status); } [Test] public void ServiceDoesntExist() { - Assert.Throws(Is.TypeOf(typeof(RpcException)).And.Property("Status").Property("StatusCode").EqualTo(StatusCode.NotFound), () => client.Check(new HealthCheckRequest { Host = "", Service = "nonexistent.service" })); + Assert.Throws(Is.TypeOf(typeof(RpcException)).And.Property("Status").Property("StatusCode").EqualTo(StatusCode.NotFound), () => client.Check(new HealthCheckRequest { Service = "nonexistent.service" })); } // TODO(jtattermusch): add test with timeout once timeouts are supported diff --git a/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs b/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs index b86d511b163..47e4b7c2a70 100644 --- a/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs +++ b/src/csharp/Grpc.HealthCheck.Tests/HealthServiceImplTest.cs @@ -50,58 +50,56 @@ namespace Grpc.HealthCheck.Tests public void SetStatus() { var impl = new HealthServiceImpl(); - impl.SetStatus("", "", HealthCheckResponse.Types.ServingStatus.SERVING); - Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.SERVING, GetStatusHelper(impl, "", "")); + impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.SERVING); + Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.SERVING, GetStatusHelper(impl, "")); - impl.SetStatus("", "", HealthCheckResponse.Types.ServingStatus.NOT_SERVING); - Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.NOT_SERVING, GetStatusHelper(impl, "", "")); + impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.NOT_SERVING); + Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.NOT_SERVING, GetStatusHelper(impl, "")); - impl.SetStatus("virtual-host", "", HealthCheckResponse.Types.ServingStatus.UNKNOWN); - Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.UNKNOWN, GetStatusHelper(impl, "virtual-host", "")); + impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.UNKNOWN); + Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.UNKNOWN, GetStatusHelper(impl, "")); - impl.SetStatus("virtual-host", "grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.SERVING); - Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.SERVING, GetStatusHelper(impl, "virtual-host", "grpc.test.TestService")); + impl.SetStatus("grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.SERVING); + Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.SERVING, GetStatusHelper(impl, "grpc.test.TestService")); } [Test] public void ClearStatus() { var impl = new HealthServiceImpl(); - impl.SetStatus("", "", HealthCheckResponse.Types.ServingStatus.SERVING); - impl.SetStatus("virtual-host", "", HealthCheckResponse.Types.ServingStatus.UNKNOWN); + impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.SERVING); + impl.SetStatus("grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.UNKNOWN); - impl.ClearStatus("", ""); + impl.ClearStatus(""); - Assert.Throws(Is.TypeOf(typeof(RpcException)).And.Property("Status").Property("StatusCode").EqualTo(StatusCode.NotFound), () => GetStatusHelper(impl, "", "")); - Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.UNKNOWN, GetStatusHelper(impl, "virtual-host", "")); + Assert.Throws(Is.TypeOf(typeof(RpcException)).And.Property("Status").Property("StatusCode").EqualTo(StatusCode.NotFound), () => GetStatusHelper(impl, "")); + Assert.AreEqual(HealthCheckResponse.Types.ServingStatus.UNKNOWN, GetStatusHelper(impl, "grpc.test.TestService")); } [Test] public void ClearAll() { var impl = new HealthServiceImpl(); - impl.SetStatus("", "", HealthCheckResponse.Types.ServingStatus.SERVING); - impl.SetStatus("virtual-host", "", HealthCheckResponse.Types.ServingStatus.UNKNOWN); + impl.SetStatus("", HealthCheckResponse.Types.ServingStatus.SERVING); + impl.SetStatus("grpc.test.TestService", HealthCheckResponse.Types.ServingStatus.UNKNOWN); impl.ClearAll(); - Assert.Throws(typeof(RpcException), () => GetStatusHelper(impl, "", "")); - Assert.Throws(typeof(RpcException), () => GetStatusHelper(impl, "virtual-host", "")); + Assert.Throws(typeof(RpcException), () => GetStatusHelper(impl, "")); + Assert.Throws(typeof(RpcException), () => GetStatusHelper(impl, "grpc.test.TestService")); } [Test] public void NullsRejected() { var impl = new HealthServiceImpl(); - Assert.Throws(typeof(ArgumentNullException), () => impl.SetStatus(null, "", HealthCheckResponse.Types.ServingStatus.SERVING)); - Assert.Throws(typeof(ArgumentNullException), () => impl.SetStatus("", null, HealthCheckResponse.Types.ServingStatus.SERVING)); + Assert.Throws(typeof(ArgumentNullException), () => impl.SetStatus(null, HealthCheckResponse.Types.ServingStatus.SERVING)); - Assert.Throws(typeof(ArgumentNullException), () => impl.ClearStatus(null, "")); - Assert.Throws(typeof(ArgumentNullException), () => impl.ClearStatus("", null)); + Assert.Throws(typeof(ArgumentNullException), () => impl.ClearStatus(null)); } - private static HealthCheckResponse.Types.ServingStatus GetStatusHelper(HealthServiceImpl impl, string host, string service) + private static HealthCheckResponse.Types.ServingStatus GetStatusHelper(HealthServiceImpl impl, string service) { - return impl.Check(new HealthCheckRequest { Host = host, Service = service }, null).Result.Status; + return impl.Check(new HealthCheckRequest { Service = service }, null).Result.Status; } } } diff --git a/src/csharp/Grpc.HealthCheck/HealthServiceImpl.cs b/src/csharp/Grpc.HealthCheck/HealthServiceImpl.cs index e93b9b96dc5..21482b302b4 100644 --- a/src/csharp/Grpc.HealthCheck/HealthServiceImpl.cs +++ b/src/csharp/Grpc.HealthCheck/HealthServiceImpl.cs @@ -54,38 +54,36 @@ namespace Grpc.HealthCheck public class HealthServiceImpl : Grpc.Health.V1.Health.IHealth { private readonly object myLock = new object(); - private readonly Dictionary statusMap = - new Dictionary(); + private readonly Dictionary statusMap = + new Dictionary(); /// - /// Sets the health status for given host and service. + /// Sets the health status for given service. /// - /// The host. Cannot be null. /// The service. Cannot be null. /// the health status - public void SetStatus(string host, string service, HealthCheckResponse.Types.ServingStatus status) + public void SetStatus(string service, HealthCheckResponse.Types.ServingStatus status) { lock (myLock) { - statusMap[CreateKey(host, service)] = status; + statusMap[service] = status; } } /// - /// Clears health status for given host and service. + /// Clears health status for given service. /// - /// The host. Cannot be null. /// The service. Cannot be null. - public void ClearStatus(string host, string service) + public void ClearStatus(string service) { lock (myLock) { - statusMap.Remove(CreateKey(host, service)); + statusMap.Remove(service); } } /// - /// Clears statuses for all hosts and services. + /// Clears statuses for all services. /// public void ClearAll() { @@ -105,11 +103,10 @@ namespace Grpc.HealthCheck { lock (myLock) { - var host = request.Host; var service = request.Service; HealthCheckResponse.Types.ServingStatus status; - if (!statusMap.TryGetValue(CreateKey(host, service), out status)) + if (!statusMap.TryGetValue(service, out status)) { // TODO(jtattermusch): returning specific status from server handler is not supported yet. throw new RpcException(new Status(StatusCode.NotFound, "")); @@ -117,22 +114,5 @@ namespace Grpc.HealthCheck return Task.FromResult(new HealthCheckResponse { Status = status }); } } - - private static Key CreateKey(string host, string service) - { - return new Key(host, service); - } - - private struct Key - { - public Key(string host, string service) - { - this.Host = GrpcPreconditions.CheckNotNull(host); - this.Service = GrpcPreconditions.CheckNotNull(service); - } - - readonly string Host; - readonly string Service; - } } } From 07591a5dfcc04266410084a249b7dda53e2a9d43 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 19 Feb 2016 10:28:24 -0800 Subject: [PATCH 035/236] fix centos6.6 docker build with overlay --- .../grpc_artifact_protoc/Dockerfile | 21 +++++++------------ tools/run_tests/artifact_targets.py | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tools/dockerfile/grpc_artifact_protoc/Dockerfile b/tools/dockerfile/grpc_artifact_protoc/Dockerfile index 0e405655718..74fc96c00ed 100644 --- a/tools/dockerfile/grpc_artifact_protoc/Dockerfile +++ b/tools/dockerfile/grpc_artifact_protoc/Dockerfile @@ -45,26 +45,21 @@ RUN yum install -y git \ glibc-devel \ glibc-devel.i686 -# Install Java 8 -RUN wget -q --no-cookies --no-check-certificate \ - --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u45-b14/jdk-8u45-linux-x64.tar.gz" \ - -O - | tar xz -C /var/local -ENV JAVA_HOME /var/local/jdk1.8.0_45 -ENV PATH $JAVA_HOME/bin:$PATH - -# Install Maven -RUN wget -q http://apache.cs.utah.edu/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz -O - | \ - tar xz -C /var/local -ENV PATH /var/local/apache-maven-3.3.3/bin:$PATH - # Install GCC 4.7 to support -static-libstdc++ RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo + +# Make yum install twice to overcome docker issue when using overlay storage driver. + +# We'll get and "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error caused by +# docker issue when using overlay storage driver, but all the stuff we need +# will be installed, so for now we just ignore the error. +# https://github.com/docker/docker/issues/10180 RUN yum install -y devtoolset-1.1 \ devtoolset-1.1-libstdc++-devel \ - devtoolset-1.1-libstdc++-devel.i686 + devtoolset-1.1-libstdc++-devel.i686 || true # Start in devtoolset environment that uses GCC 4.7 CMD ["scl", "enable", "devtoolset-1.1", "bash"] \ No newline at end of file diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 5d678a561f9..cf056ec9293 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -250,7 +250,7 @@ class ProtocArtifact: cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] if self.platform != 'macos': - ldflags += ' -static-libgcc -static-libstdc++ -s' + ldflags += ' -static-libgcc -static-libstdc++ -s' environ={'CONFIG': 'opt', 'CXXFLAGS': cxxflags, 'LDFLAGS': ldflags, From f4fc61cf680dc1efdd029ce8c33f72cc14380866 Mon Sep 17 00:00:00 2001 From: yang-g Date: Fri, 19 Feb 2016 13:48:03 -0800 Subject: [PATCH 036/236] fix ruby impl and test --- src/ruby/pb/grpc/health/checker.rb | 14 ++++----- src/ruby/spec/pb/health/checker_spec.rb | 42 +++++++++---------------- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/src/ruby/pb/grpc/health/checker.rb b/src/ruby/pb/grpc/health/checker.rb index 0af4c5b0a8d..9f1ee65c419 100644 --- a/src/ruby/pb/grpc/health/checker.rb +++ b/src/ruby/pb/grpc/health/checker.rb @@ -50,20 +50,20 @@ module Grpc def check(req, _call) status = nil @status_mutex.synchronize do - status = @statuses["#{req.host}/#{req.service}"] + status = @statuses["#{req.service}"] end fail GRPC::BadStatus, StatusCodes::NOT_FOUND if status.nil? HealthCheckResponse.new(status: status) end - # Adds the health status for a given host and service. - def add_status(host, service, status) - @status_mutex.synchronize { @statuses["#{host}/#{service}"] = status } + # Adds the health status for a given service. + def add_status(service, status) + @status_mutex.synchronize { @statuses["#{service}"] = status } end - # Clears the status for the given host or service. - def clear_status(host, service) - @status_mutex.synchronize { @statuses.delete("#{host}/#{service}") } + # Clears the status for the given service. + def clear_status(service) + @status_mutex.synchronize { @statuses.delete("#{service}") } end # Clears alls the statuses. diff --git a/src/ruby/spec/pb/health/checker_spec.rb b/src/ruby/spec/pb/health/checker_spec.rb index fe4e5269bb0..9bb79bb4cae 100644 --- a/src/ruby/spec/pb/health/checker_spec.rb +++ b/src/ruby/spec/pb/health/checker_spec.rb @@ -87,21 +87,11 @@ describe Grpc::Health::Checker do success_tests = [ { - desc: 'neither host or service are specified', - host: '', + desc: 'the service is not specified', service: '' }, { - desc: 'only the host is specified', - host: 'test-fake-host', - service: '' - }, { - desc: 'the host and service are specified', - host: 'test-fake-host', + desc: 'the service is specified', service: 'fake-service-1' - }, { - desc: 'only the service is specified', - host: '', - service: 'fake-service-2' } ] @@ -114,9 +104,8 @@ describe Grpc::Health::Checker do context 'method `add_status` and `check`' do success_tests.each do |t| it "should succeed when #{t[:desc]}" do - subject.add_status(t[:host], t[:service], ServingStatus::NOT_SERVING) - got = subject.check(HCReq.new(host: t[:host], service: t[:service]), - nil) + subject.add_status(t[:service], ServingStatus::NOT_SERVING) + got = subject.check(HCReq.new(service: t[:service]), nil) want = HCResp.new(status: ServingStatus::NOT_SERVING) expect(got).to eq(want) end @@ -127,7 +116,7 @@ describe Grpc::Health::Checker do success_tests.each do |t| it "should fail with NOT_FOUND when #{t[:desc]}" do blk = proc do - subject.check(HCReq.new(host: t[:host], service: t[:service]), nil) + subject.check(HCReq.new(service: t[:service]), nil) end expected_msg = /#{StatusCodes::NOT_FOUND}/ expect(&blk).to raise_error GRPC::BadStatus, expected_msg @@ -138,16 +127,14 @@ describe Grpc::Health::Checker do context 'method `clear_status`' do success_tests.each do |t| it "should fail after clearing status when #{t[:desc]}" do - subject.add_status(t[:host], t[:service], ServingStatus::NOT_SERVING) - got = subject.check(HCReq.new(host: t[:host], service: t[:service]), - nil) + subject.add_status(t[:service], ServingStatus::NOT_SERVING) + got = subject.check(HCReq.new(service: t[:service]), nil) want = HCResp.new(status: ServingStatus::NOT_SERVING) expect(got).to eq(want) - subject.clear_status(t[:host], t[:service]) + subject.clear_status(t[:service]) blk = proc do - subject.check(HCReq.new(host: t[:host], service: t[:service]), - nil) + subject.check(HCReq.new(service: t[:service]), nil) end expected_msg = /#{StatusCodes::NOT_FOUND}/ expect(&blk).to raise_error GRPC::BadStatus, expected_msg @@ -158,9 +145,8 @@ describe Grpc::Health::Checker do context 'method `clear_all`' do it 'should return NOT_FOUND after being invoked' do success_tests.each do |t| - subject.add_status(t[:host], t[:service], ServingStatus::NOT_SERVING) - got = subject.check(HCReq.new(host: t[:host], service: t[:service]), - nil) + subject.add_status(t[:service], ServingStatus::NOT_SERVING) + got = subject.check(HCReq.new(service: t[:service]), nil) want = HCResp.new(status: ServingStatus::NOT_SERVING) expect(got).to eq(want) end @@ -169,7 +155,7 @@ describe Grpc::Health::Checker do success_tests.each do |t| blk = proc do - subject.check(HCReq.new(host: t[:host], service: t[:service]), nil) + subject.check(HCReq.new(service: t[:service]), nil) end expected_msg = /#{StatusCodes::NOT_FOUND}/ expect(&blk).to raise_error GRPC::BadStatus, expected_msg @@ -203,7 +189,7 @@ describe Grpc::Health::Checker do it 'should receive the correct status', server: true do @srv.handle(subject) - subject.add_status('', '', ServingStatus::NOT_SERVING) + subject.add_status('', ServingStatus::NOT_SERVING) t = Thread.new { @srv.run } @srv.wait_till_running @@ -221,7 +207,7 @@ describe Grpc::Health::Checker do @srv.wait_till_running blk = proc do stub = CheckerStub.new(@host, :this_channel_is_insecure, **@client_opts) - stub.check(HCReq.new(host: 'unknown', service: 'unknown')) + stub.check(HCReq.new(service: 'unknown')) end expected_msg = /#{StatusCodes::NOT_FOUND}/ expect(&blk).to raise_error GRPC::BadStatus, expected_msg From 2e8710414f34c7ff5d50f45f35fb98df9bc96ee0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 19 Feb 2016 13:57:13 -0800 Subject: [PATCH 037/236] Remove extraneous comment. --- tools/dockerfile/grpc_artifact_protoc/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/dockerfile/grpc_artifact_protoc/Dockerfile b/tools/dockerfile/grpc_artifact_protoc/Dockerfile index 74fc96c00ed..1bbc6e021bc 100644 --- a/tools/dockerfile/grpc_artifact_protoc/Dockerfile +++ b/tools/dockerfile/grpc_artifact_protoc/Dockerfile @@ -51,8 +51,6 @@ RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo -# Make yum install twice to overcome docker issue when using overlay storage driver. - # We'll get and "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error caused by # docker issue when using overlay storage driver, but all the stuff we need # will be installed, so for now we just ignore the error. @@ -62,4 +60,4 @@ RUN yum install -y devtoolset-1.1 \ devtoolset-1.1-libstdc++-devel.i686 || true # Start in devtoolset environment that uses GCC 4.7 -CMD ["scl", "enable", "devtoolset-1.1", "bash"] \ No newline at end of file +CMD ["scl", "enable", "devtoolset-1.1", "bash"] From a8be91b3153fc75a425718799130172357507dd3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 19 Feb 2016 16:22:44 -0800 Subject: [PATCH 038/236] Provide an interface firewall between pollset and its implementations Starting to allow for >1 implementation of pollset within a binary. Do so without requiring an extra allocation for completion queues (which we could not tolerate). --- src/core/iomgr/fd_posix.c | 6 +- src/core/iomgr/pollset.h | 15 ++- .../iomgr/pollset_multipoller_with_epoll.c | 9 +- .../pollset_multipoller_with_poll_posix.c | 12 ++- src/core/iomgr/pollset_posix.c | 45 ++++----- src/core/iomgr/pollset_posix.h | 14 +-- src/core/iomgr/pollset_set_posix.c | 1 + src/core/iomgr/pollset_set_posix.h | 1 - src/core/iomgr/pollset_windows.h | 2 - src/core/iomgr/tcp_posix.c | 5 +- src/core/iomgr/workqueue_posix.c | 1 + src/core/iomgr/workqueue_posix.h | 2 + .../security/google_default_credentials.c | 23 +++-- src/core/surface/completion_queue.c | 84 ++++++++-------- .../set_initial_connect_string_test.c | 2 +- .../core/end2end/fixtures/h2_full+poll+pipe.c | 16 +-- test/core/end2end/fixtures/h2_full+poll.c | 14 +-- test/core/end2end/fixtures/h2_ssl+poll.c | 12 ++- test/core/end2end/fixtures/h2_uchannel.c | 39 ++++---- test/core/end2end/fixtures/h2_uds+poll.c | 16 +-- test/core/httpcli/httpcli_test.c | 46 +++++---- test/core/httpcli/httpscli_test.c | 45 +++++---- test/core/iomgr/endpoint_pair_test.c | 21 ++-- test/core/iomgr/endpoint_tests.c | 18 ++-- test/core/iomgr/endpoint_tests.h | 2 +- test/core/iomgr/fd_posix_test.c | 91 +++++++++-------- test/core/iomgr/tcp_client_posix_test.c | 64 ++++++------ test/core/iomgr/tcp_posix_test.c | 97 ++++++++++--------- test/core/iomgr/tcp_server_posix_test.c | 52 +++++----- test/core/iomgr/workqueue_test.c | 41 ++++---- test/core/security/oauth2_utils.c | 25 ++--- .../print_google_default_creds_token.c | 32 +++--- test/core/security/secure_endpoint_test.c | 28 +++--- test/core/security/verify_jwt.c | 31 +++--- test/core/util/port_posix.c | 57 ++++++----- test/core/util/test_tcp_server.c | 19 ++-- test/core/util/test_tcp_server.h | 4 +- 37 files changed, 552 insertions(+), 440 deletions(-) diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c index 85eadd754b9..812ff0992e3 100644 --- a/src/core/iomgr/fd_posix.c +++ b/src/core/iomgr/fd_posix.c @@ -46,6 +46,8 @@ #include #include +#include "src/core/iomgr/pollset_posix.h" + #define CLOSURE_NOT_READY ((grpc_closure *)0) #define CLOSURE_READY ((grpc_closure *)1) @@ -175,11 +177,11 @@ int grpc_fd_is_orphaned(grpc_fd *fd) { } static void pollset_kick_locked(grpc_fd_watcher *watcher) { - gpr_mu_lock(GRPC_POLLSET_MU(watcher->pollset)); + gpr_mu_lock(watcher->pollset->mu); GPR_ASSERT(watcher->worker); grpc_pollset_kick_ext(watcher->pollset, watcher->worker, GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); - gpr_mu_unlock(GRPC_POLLSET_MU(watcher->pollset)); + gpr_mu_unlock(watcher->pollset->mu); } static void maybe_wake_one_watcher_locked(grpc_fd *fd) { diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index 0c0efad7600..dfbd4a40ec4 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -35,8 +35,11 @@ #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_H #include +#include #include +#include "src/core/iomgr/exec_ctx.h" + #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) /* A grpc_pollset is a set of file descriptors that a higher level item is @@ -46,15 +49,11 @@ - a completion queue might keep a pollset with an entry for each transport that is servicing a call that it's tracking */ -#ifdef GPR_POSIX_SOCKET -#include "src/core/iomgr/pollset_posix.h" -#endif - -#ifdef GPR_WIN32 -#include "src/core/iomgr/pollset_windows.h" -#endif +typedef struct grpc_pollset grpc_pollset; +typedef struct grpc_pollset_worker grpc_pollset_worker; -void grpc_pollset_init(grpc_pollset *pollset); +size_t grpc_pollset_size(void); +void grpc_pollset_init(grpc_pollset *pollset, gpr_mu *mu); /* Begin shutting down the pollset, and call closure when done. * GRPC_POLLSET_MU(pollset) must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index 4acae2bb712..e1af2b52410 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -45,6 +45,7 @@ #include #include #include "src/core/iomgr/fd_posix.h" +#include "src/core/iomgr/pollset_posix.h" #include "src/core/profiling/timers.h" #include "src/core/support/block_annotate.h" @@ -148,7 +149,7 @@ static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg, finally_add_fd(exec_ctx, da->pollset, da->fd); } - gpr_mu_lock(&da->pollset->mu); + gpr_mu_lock(da->pollset->mu); da->pollset->in_flight_cbs--; if (da->pollset->shutting_down) { /* We don't care about this pollset anymore. */ @@ -157,7 +158,7 @@ static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg, grpc_exec_ctx_enqueue(exec_ctx, da->pollset->shutdown_done, true, NULL); } } - gpr_mu_unlock(&da->pollset->mu); + gpr_mu_unlock(da->pollset->mu); GRPC_FD_UNREF(da->fd, "delayed_add"); @@ -169,7 +170,7 @@ static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int and_unlock_pollset) { if (and_unlock_pollset) { - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); finally_add_fd(exec_ctx, pollset, fd); } else { delayed_add *da = gpr_malloc(sizeof(*da)); @@ -201,7 +202,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock( * here. */ - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); timeout_ms = grpc_poll_deadline_to_millis_timeout(deadline, now); diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 809f8f39daa..348d3391048 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -42,13 +42,15 @@ #include #include -#include "src/core/iomgr/fd_posix.h" -#include "src/core/iomgr/iomgr_internal.h" -#include "src/core/support/block_annotate.h" #include #include #include +#include "src/core/iomgr/fd_posix.h" +#include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/support/block_annotate.h" + typedef struct { /* all polled fds */ size_t fd_count; @@ -78,7 +80,7 @@ static void multipoll_with_poll_pollset_add_fd(grpc_exec_ctx *exec_ctx, GRPC_FD_REF(fd, "multipoller"); exit: if (and_unlock_pollset) { - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); } } @@ -130,7 +132,7 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( } h->del_count = 0; h->fd_count = fd_count; - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); for (i = 2; i < pfd_count; i++) { pfds[i].events = (short)grpc_fd_begin_poll(watchers[i].fd, pollset, worker, diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index ee7e9f48f4c..63321638bab 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -42,16 +42,16 @@ #include #include -#include "src/core/iomgr/fd_posix.h" -#include "src/core/iomgr/iomgr_internal.h" -#include "src/core/iomgr/socket_utils_posix.h" -#include "src/core/profiling/timers.h" -#include "src/core/support/block_annotate.h" #include #include #include #include #include +#include "src/core/iomgr/fd_posix.h" +#include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/socket_utils_posix.h" +#include "src/core/profiling/timers.h" +#include "src/core/support/block_annotate.h" GPR_TLS_DECL(g_current_thread_poller); GPR_TLS_DECL(g_current_thread_worker); @@ -97,6 +97,8 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { worker->prev->next = worker->next->prev = worker; } +size_t grpc_pollset_size(void) { return sizeof(grpc_pollset); } + void grpc_pollset_kick_ext(grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) { @@ -186,8 +188,8 @@ void grpc_kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); } static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null); -void grpc_pollset_init(grpc_pollset *pollset) { - gpr_mu_init(&pollset->mu); +void grpc_pollset_init(grpc_pollset *pollset, gpr_mu *mu) { + pollset->mu = mu; pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; pollset->in_flight_cbs = 0; pollset->shutting_down = 0; @@ -204,7 +206,6 @@ void grpc_pollset_destroy(grpc_pollset *pollset) { GPR_ASSERT(!grpc_pollset_has_workers(pollset)); GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); pollset->vtable->destroy(pollset); - gpr_mu_destroy(&pollset->mu); while (pollset->local_wakeup_cache) { grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next; grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd); @@ -227,15 +228,15 @@ void grpc_pollset_reset(grpc_pollset *pollset) { void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { - gpr_mu_lock(&pollset->mu); + gpr_mu_lock(pollset->mu); pollset->vtable->add_fd(exec_ctx, pollset, fd, 1); /* the following (enabled only in debug) will reacquire and then release our lock - meaning that if the unlocking flag passed to add_fd above is not respected, the code will deadlock (in a way that we have a chance of debugging) */ #ifndef NDEBUG - gpr_mu_lock(&pollset->mu); - gpr_mu_unlock(&pollset->mu); + gpr_mu_lock(pollset->mu); + gpr_mu_unlock(pollset->mu); #endif } @@ -284,7 +285,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* Give do_promote priority so we don't starve it out */ if (pollset->in_flight_cbs) { GPR_TIMER_MARK("grpc_pollset_work.in_flight_cbs", 0); - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); locked = 0; goto done; } @@ -318,7 +319,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, done: if (!locked) { queued_work |= grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->mu); + gpr_mu_lock(pollset->mu); locked = 1; } /* If we're forced to re-evaluate polling (via grpc_pollset_kick with @@ -348,19 +349,19 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_kick(pollset, NULL); } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) { pollset->called_shutdown = 1; - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); finish_shutdown(exec_ctx, pollset); grpc_exec_ctx_flush(exec_ctx); /* Continuing to access pollset here is safe -- it is the caller's * responsibility to not destroy when it has outstanding calls to * grpc_pollset_work. * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */ - gpr_mu_lock(&pollset->mu); + gpr_mu_lock(pollset->mu); } else if (!grpc_closure_list_empty(pollset->idle_jobs)) { grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->mu); + gpr_mu_lock(pollset->mu); } } *worker_hdl = NULL; @@ -428,7 +429,7 @@ static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args, * 4. The pollset may be shutting down. */ - gpr_mu_lock(&pollset->mu); + gpr_mu_lock(pollset->mu); /* First we need to ensure that nobody is polling concurrently */ GPR_ASSERT(!grpc_pollset_has_workers(pollset)); @@ -469,7 +470,7 @@ static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args, } } - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); /* Matching ref in basic_pollset_add_fd */ GRPC_FD_UNREF(fd, "basicpoll_add"); @@ -522,7 +523,7 @@ static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, exit: if (and_unlock_pollset) { - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); } } @@ -558,14 +559,14 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx, pfd[2].fd = fd->fd; pfd[2].revents = 0; GRPC_FD_REF(fd, "basicpoll_begin"); - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); pfd[2].events = (short)grpc_fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, &fd_watcher); if (pfd[2].events != 0) { nfds++; } } else { - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(pollset->mu); } /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h index b34bb094268..58158e3d46a 100644 --- a/src/core/iomgr/pollset_posix.h +++ b/src/core/iomgr/pollset_posix.h @@ -37,8 +37,10 @@ #include #include + #include "src/core/iomgr/exec_ctx.h" #include "src/core/iomgr/iomgr.h" +#include "src/core/iomgr/pollset.h" #include "src/core/iomgr/wakeup_fd_posix.h" typedef struct grpc_pollset_vtable grpc_pollset_vtable; @@ -53,21 +55,21 @@ typedef struct grpc_cached_wakeup_fd { struct grpc_cached_wakeup_fd *next; } grpc_cached_wakeup_fd; -typedef struct grpc_pollset_worker { +struct grpc_pollset_worker { grpc_cached_wakeup_fd *wakeup_fd; int reevaluate_polling_on_wakeup; int kicked_specifically; struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; -} grpc_pollset_worker; +}; -typedef struct grpc_pollset { +struct grpc_pollset { /* pollsets under posix can mutate representation as fds are added and removed. For example, we may choose a poll() based implementation on linux for few fds, and an epoll() based implementation for many fds */ const grpc_pollset_vtable *vtable; - gpr_mu mu; + gpr_mu *mu; grpc_pollset_worker root_worker; int in_flight_cbs; int shutting_down; @@ -81,7 +83,7 @@ typedef struct grpc_pollset { } data; /* Local cache of eventfds for workers */ grpc_cached_wakeup_fd *local_wakeup_cache; -} grpc_pollset; +}; struct grpc_pollset_vtable { void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -93,8 +95,6 @@ struct grpc_pollset_vtable { void (*destroy)(grpc_pollset *pollset); }; -#define GRPC_POLLSET_MU(pollset) (&(pollset)->mu) - /* Add an fd to a pollset */ void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, struct grpc_fd *fd); diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/iomgr/pollset_set_posix.c index 4ec92202e30..85a0cadfc7c 100644 --- a/src/core/iomgr/pollset_set_posix.c +++ b/src/core/iomgr/pollset_set_posix.c @@ -41,6 +41,7 @@ #include #include +#include "src/core/iomgr/pollset_posix.h" #include "src/core/iomgr/pollset_set.h" void grpc_pollset_set_init(grpc_pollset_set *pollset_set) { diff --git a/src/core/iomgr/pollset_set_posix.h b/src/core/iomgr/pollset_set_posix.h index 4820a61e4b2..7ce8ec7343f 100644 --- a/src/core/iomgr/pollset_set_posix.h +++ b/src/core/iomgr/pollset_set_posix.h @@ -35,7 +35,6 @@ #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_POSIX_H #include "src/core/iomgr/fd_posix.h" -#include "src/core/iomgr/pollset_posix.h" typedef struct grpc_pollset_set { gpr_mu mu; diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h index 65ba80619b7..dfec5821b35 100644 --- a/src/core/iomgr/pollset_windows.h +++ b/src/core/iomgr/pollset_windows.h @@ -74,6 +74,4 @@ struct grpc_pollset { extern gpr_mu grpc_polling_mu; -#define GRPC_POLLSET_MU(pollset) (&grpc_polling_mu) - #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index 048e9074412..fba35634276 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -40,8 +40,8 @@ #include #include #include -#include #include +#include #include #include @@ -51,9 +51,10 @@ #include #include -#include "src/core/support/string.h" #include "src/core/debug/trace.h" +#include "src/core/iomgr/pollset_posix.h" #include "src/core/profiling/timers.h" +#include "src/core/support/string.h" #ifdef GPR_HAVE_MSG_NOSIGNAL #define SENDMSG_FLAGS MSG_NOSIGNAL diff --git a/src/core/iomgr/workqueue_posix.c b/src/core/iomgr/workqueue_posix.c index da11df67efb..c096dbfb30c 100644 --- a/src/core/iomgr/workqueue_posix.c +++ b/src/core/iomgr/workqueue_posix.c @@ -44,6 +44,7 @@ #include #include "src/core/iomgr/fd_posix.h" +#include "src/core/iomgr/pollset_posix.h" static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, bool success); diff --git a/src/core/iomgr/workqueue_posix.h b/src/core/iomgr/workqueue_posix.h index 589034fe1bb..6f29f4004ce 100644 --- a/src/core/iomgr/workqueue_posix.h +++ b/src/core/iomgr/workqueue_posix.h @@ -34,6 +34,8 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H #define GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H +#include "src/core/iomgr/wakeup_fd_posix.h" + struct grpc_fd; struct grpc_workqueue { diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c index cc9c9582986..25006e16c71 100644 --- a/src/core/security/google_default_credentials.c +++ b/src/core/security/google_default_credentials.c @@ -58,7 +58,7 @@ static gpr_once g_once = GPR_ONCE_INIT; static void init_default_credentials(void) { gpr_mu_init(&g_mu); } typedef struct { - grpc_pollset pollset; + grpc_pollset *pollset; int is_done; int success; } compute_engine_detector; @@ -80,10 +80,10 @@ static void on_compute_engine_detection_http_response( } } } - gpr_mu_lock(GRPC_POLLSET_MU(&detector->pollset)); + gpr_mu_lock(&g_mu); detector->is_done = 1; - grpc_pollset_kick(&detector->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&detector->pollset)); + grpc_pollset_kick(detector->pollset, NULL); + gpr_mu_unlock(&g_mu); } static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool s) { @@ -101,7 +101,8 @@ static int is_stack_running_on_compute_engine(void) { on compute engine. */ gpr_timespec max_detection_delay = gpr_time_from_seconds(1, GPR_TIMESPAN); - grpc_pollset_init(&detector.pollset); + detector.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(detector.pollset, &g_mu); detector.is_done = 0; detector.success = 0; @@ -112,7 +113,7 @@ static int is_stack_running_on_compute_engine(void) { grpc_httpcli_context_init(&context); grpc_httpcli_get( - &exec_ctx, &context, &detector.pollset, &request, + &exec_ctx, &context, detector.pollset, &request, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), max_detection_delay), on_compute_engine_detection_http_response, &detector); @@ -120,20 +121,22 @@ static int is_stack_running_on_compute_engine(void) { /* Block until we get the response. This is not ideal but this should only be called once for the lifetime of the process by the default credentials. */ - gpr_mu_lock(GRPC_POLLSET_MU(&detector.pollset)); + gpr_mu_lock(&g_mu); while (!detector.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &detector.pollset, &worker, + grpc_pollset_work(&exec_ctx, detector.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); } - gpr_mu_unlock(GRPC_POLLSET_MU(&detector.pollset)); + gpr_mu_unlock(&g_mu); grpc_httpcli_context_destroy(&context); grpc_closure_init(&destroy_closure, destroy_pollset, &detector.pollset); - grpc_pollset_shutdown(&exec_ctx, &detector.pollset, &destroy_closure); + grpc_pollset_shutdown(&exec_ctx, detector.pollset, &destroy_closure); grpc_exec_ctx_finish(&exec_ctx); + gpr_free(detector.pollset); + return detector.success; } diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index 376feb1bbe8..d0659c7e522 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -36,18 +36,19 @@ #include #include -#include "src/core/iomgr/timer.h" +#include +#include +#include +#include + #include "src/core/iomgr/pollset.h" +#include "src/core/iomgr/timer.h" +#include "src/core/profiling/timers.h" #include "src/core/support/string.h" #include "src/core/surface/api_trace.h" #include "src/core/surface/call.h" #include "src/core/surface/event_string.h" #include "src/core/surface/surface_trace.h" -#include "src/core/profiling/timers.h" -#include -#include -#include -#include typedef struct { grpc_pollset_worker **worker; @@ -56,6 +57,7 @@ typedef struct { /* Completion queue structure */ struct grpc_completion_queue { + gpr_mu mu; /** completed events */ grpc_cq_completion completed_head; grpc_cq_completion *completed_tail; @@ -63,8 +65,6 @@ struct grpc_completion_queue { gpr_refcount pending_events; /** Once owning_refs drops to zero, we will destroy the cq */ gpr_refcount owning_refs; - /** the set of low level i/o things that concern this cq */ - grpc_pollset pollset; /** 0 initially, 1 once we've begun shutting down */ int shutdown; int shutdown_called; @@ -82,6 +82,8 @@ struct grpc_completion_queue { grpc_completion_queue *next_free; }; +#define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1)) + static gpr_mu g_freelist_mu; grpc_completion_queue *g_freelist; @@ -94,7 +96,8 @@ void grpc_cq_global_shutdown(void) { gpr_mu_destroy(&g_freelist_mu); while (g_freelist) { grpc_completion_queue *next = g_freelist->next_free; - grpc_pollset_destroy(&g_freelist->pollset); + grpc_pollset_destroy(POLLSET_FROM_CQ(g_freelist)); + gpr_mu_destroy(&g_freelist->mu); #ifndef NDEBUG gpr_free(g_freelist->outstanding_tags); #endif @@ -124,8 +127,9 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) { if (g_freelist == NULL) { gpr_mu_unlock(&g_freelist_mu); - cc = gpr_malloc(sizeof(grpc_completion_queue)); - grpc_pollset_init(&cc->pollset); + cc = gpr_malloc(sizeof(grpc_completion_queue) + grpc_pollset_size()); + gpr_mu_init(&cc->mu); + grpc_pollset_init(POLLSET_FROM_CQ(cc), &cc->mu); #ifndef NDEBUG cc->outstanding_tags = NULL; cc->outstanding_tag_capacity = 0; @@ -184,7 +188,7 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { #endif if (gpr_unref(&cc->owning_refs)) { GPR_ASSERT(cc->completed_head.next == (uintptr_t)&cc->completed_head); - grpc_pollset_reset(&cc->pollset); + grpc_pollset_reset(POLLSET_FROM_CQ(cc)); gpr_mu_lock(&g_freelist_mu); cc->next_free = g_freelist; g_freelist = cc; @@ -194,7 +198,7 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) { #ifndef NDEBUG - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(&cc->mu); GPR_ASSERT(!cc->shutdown_called); if (cc->outstanding_tag_count == cc->outstanding_tag_capacity) { cc->outstanding_tag_capacity = GPR_MAX(4, 2 * cc->outstanding_tag_capacity); @@ -203,7 +207,7 @@ void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) { cc->outstanding_tag_capacity); } cc->outstanding_tags[cc->outstanding_tag_count++] = tag; - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); #endif gpr_ref(&cc->pending_events); } @@ -231,7 +235,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, storage->next = ((uintptr_t)&cc->completed_head) | ((uintptr_t)(success != 0)); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(&cc->mu); #ifndef NDEBUG for (i = 0; i < (int)cc->outstanding_tag_count; i++) { if (cc->outstanding_tags[i] == tag) { @@ -256,8 +260,8 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, break; } } - grpc_pollset_kick(&cc->pollset, pluck_worker); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker); + gpr_mu_unlock(&cc->mu); } else { cc->completed_tail->next = ((uintptr_t)storage) | (1u & (uintptr_t)cc->completed_tail->next); @@ -265,8 +269,9 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, GPR_ASSERT(!cc->shutdown); GPR_ASSERT(cc->shutdown_called); cc->shutdown = 1; - grpc_pollset_shutdown(exec_ctx, &cc->pollset, &cc->pollset_shutdown_done); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + grpc_pollset_shutdown(exec_ctx, POLLSET_FROM_CQ(cc), + &cc->pollset_shutdown_done); + gpr_mu_unlock(&cc->mu); } GPR_TIMER_END("grpc_cq_end_op", 0); @@ -294,7 +299,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); GRPC_CQ_INTERNAL_REF(cc, "next"); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(&cc->mu); for (;;) { if (cc->completed_tail != &cc->completed_head) { grpc_cq_completion *c = (grpc_cq_completion *)cc->completed_head.next; @@ -302,7 +307,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, if (c == cc->completed_tail) { cc->completed_tail = &cc->completed_head; } - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); ret.type = GRPC_OP_COMPLETE; ret.success = c->next & 1u; ret.tag = c->tag; @@ -310,14 +315,14 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, break; } if (cc->shutdown) { - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_SHUTDOWN; break; } now = gpr_now(GPR_CLOCK_MONOTONIC); if (!first_loop && gpr_time_cmp(now, deadline) >= 0) { - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_TIMEOUT; break; @@ -330,12 +335,12 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_timespec iteration_deadline = deadline; if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { GPR_TIMER_MARK("alarm_triggered", 0); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(&cc->mu); continue; } - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, + grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, iteration_deadline); } GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); @@ -395,7 +400,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); GRPC_CQ_INTERNAL_REF(cc, "pluck"); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(&cc->mu); for (;;) { prev = &cc->completed_head; while ((c = (grpc_cq_completion *)(prev->next & ~(uintptr_t)1)) != @@ -405,7 +410,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, if (c == cc->completed_tail) { cc->completed_tail = prev; } - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); ret.type = GRPC_OP_COMPLETE; ret.success = c->next & 1u; ret.tag = c->tag; @@ -415,7 +420,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, prev = c; } if (cc->shutdown) { - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_SHUTDOWN; break; @@ -425,7 +430,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, "Too many outstanding grpc_completion_queue_pluck calls: maximum " "is %d", GRPC_MAX_COMPLETION_QUEUE_PLUCKERS); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); memset(&ret, 0, sizeof(ret)); /* TODO(ctiller): should we use a different result here */ ret.type = GRPC_QUEUE_TIMEOUT; @@ -434,7 +439,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, now = gpr_now(GPR_CLOCK_MONOTONIC); if (!first_loop && gpr_time_cmp(now, deadline) >= 0) { del_plucker(cc, tag, &worker); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_TIMEOUT; break; @@ -447,12 +452,12 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, gpr_timespec iteration_deadline = deadline; if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { GPR_TIMER_MARK("alarm_triggered", 0); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(&cc->mu); continue; } - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, + grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, iteration_deadline); del_plucker(cc, tag, &worker); } @@ -472,9 +477,9 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_completion_queue_shutdown", 0); GRPC_API_TRACE("grpc_completion_queue_shutdown(cc=%p)", 1, (cc)); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(&cc->mu); if (cc->shutdown_called) { - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); GPR_TIMER_END("grpc_completion_queue_shutdown", 0); return; } @@ -482,9 +487,10 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { if (gpr_unref(&cc->pending_events)) { GPR_ASSERT(!cc->shutdown); cc->shutdown = 1; - grpc_pollset_shutdown(&exec_ctx, &cc->pollset, &cc->pollset_shutdown_done); + grpc_pollset_shutdown(&exec_ctx, POLLSET_FROM_CQ(cc), + &cc->pollset_shutdown_done); } - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(&cc->mu); grpc_exec_ctx_finish(&exec_ctx); GPR_TIMER_END("grpc_completion_queue_shutdown", 0); } @@ -498,7 +504,7 @@ void grpc_completion_queue_destroy(grpc_completion_queue *cc) { } grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) { - return &cc->pollset; + return POLLSET_FROM_CQ(cc); } void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; } diff --git a/test/core/client_config/set_initial_connect_string_test.c b/test/core/client_config/set_initial_connect_string_test.c index bcd1f261235..3cf267fb3bd 100644 --- a/test/core/client_config/set_initial_connect_string_test.c +++ b/test/core/client_config/set_initial_connect_string_test.c @@ -85,7 +85,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_slice_buffer_init(&state.incoming_buffer); gpr_slice_buffer_init(&state.temp_incoming_buffer); state.tcp = tcp; - grpc_endpoint_add_to_pollset(exec_ctx, tcp, &server->pollset); + grpc_endpoint_add_to_pollset(exec_ctx, tcp, server->pollset); grpc_endpoint_read(exec_ctx, tcp, &state.temp_incoming_buffer, &on_read); } diff --git a/test/core/end2end/fixtures/h2_full+poll+pipe.c b/test/core/end2end/fixtures/h2_full+poll+pipe.c index d475a7bb552..50df34fcc68 100644 --- a/test/core/end2end/fixtures/h2_full+poll+pipe.c +++ b/test/core/end2end/fixtures/h2_full+poll+pipe.c @@ -35,21 +35,23 @@ #include -#include "src/core/channel/client_channel.h" -#include "src/core/channel/connected_channel.h" -#include "src/core/channel/http_server_filter.h" -#include "src/core/surface/channel.h" -#include "src/core/surface/server.h" -#include "src/core/transport/chttp2_transport.h" #include #include #include #include #include #include + +#include "src/core/channel/client_channel.h" +#include "src/core/channel/connected_channel.h" +#include "src/core/channel/http_server_filter.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/iomgr/wakeup_fd_posix.h" +#include "src/core/surface/channel.h" +#include "src/core/surface/server.h" +#include "src/core/transport/chttp2_transport.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" -#include "src/core/iomgr/wakeup_fd_posix.h" typedef struct fullstack_fixture_data { char *localaddr; diff --git a/test/core/end2end/fixtures/h2_full+poll.c b/test/core/end2end/fixtures/h2_full+poll.c index 3f5e6096f63..76d6c990666 100644 --- a/test/core/end2end/fixtures/h2_full+poll.c +++ b/test/core/end2end/fixtures/h2_full+poll.c @@ -35,18 +35,20 @@ #include -#include "src/core/channel/client_channel.h" -#include "src/core/channel/connected_channel.h" -#include "src/core/channel/http_server_filter.h" -#include "src/core/surface/channel.h" -#include "src/core/surface/server.h" -#include "src/core/transport/chttp2_transport.h" #include #include #include #include #include #include + +#include "src/core/channel/client_channel.h" +#include "src/core/channel/connected_channel.h" +#include "src/core/channel/http_server_filter.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/surface/channel.h" +#include "src/core/surface/server.h" +#include "src/core/transport/chttp2_transport.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/end2end/fixtures/h2_ssl+poll.c b/test/core/end2end/fixtures/h2_ssl+poll.c index 614654ed524..5085880a176 100644 --- a/test/core/end2end/fixtures/h2_ssl+poll.c +++ b/test/core/end2end/fixtures/h2_ssl+poll.c @@ -36,17 +36,19 @@ #include #include +#include +#include +#include + #include "src/core/channel/channel_args.h" +#include "src/core/iomgr/pollset_posix.h" #include "src/core/security/credentials.h" #include "src/core/support/env.h" #include "src/core/support/file.h" #include "src/core/support/string.h" -#include -#include -#include -#include "test/core/util/test_config.h" -#include "test/core/util/port.h" #include "test/core/end2end/data/ssl_test_data.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" typedef struct fullstack_secure_fixture_data { char *localaddr; diff --git a/test/core/end2end/fixtures/h2_uchannel.c b/test/core/end2end/fixtures/h2_uchannel.c index 33055561cb3..f363b60cba2 100644 --- a/test/core/end2end/fixtures/h2_uchannel.c +++ b/test/core/end2end/fixtures/h2_uchannel.c @@ -35,6 +35,13 @@ #include +#include +#include +#include +#include +#include +#include +#include #include "src/core/channel/channel_args.h" #include "src/core/channel/client_channel.h" #include "src/core/channel/client_uchannel.h" @@ -46,13 +53,6 @@ #include "src/core/surface/channel.h" #include "src/core/surface/server.h" #include "src/core/transport/chttp2_transport.h" -#include -#include -#include -#include -#include -#include -#include #include "test/core/util/port.h" #include "test/core/util/test_config.h" @@ -253,30 +253,33 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *arg, bool success) { } static grpc_connected_subchannel *connect_subchannel(grpc_subchannel *c) { - grpc_pollset pollset; + gpr_mu mu; + gpr_mu_init(&mu); + grpc_pollset *pollset = gpr_malloc(grpc_pollset_size()); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_pollset_init(&pollset); + grpc_pollset_init(pollset, &mu); grpc_pollset_set_init(&g_interested_parties); - grpc_pollset_set_add_pollset(&exec_ctx, &g_interested_parties, &pollset); + grpc_pollset_set_add_pollset(&exec_ctx, &g_interested_parties, pollset); grpc_subchannel_notify_on_state_change(&exec_ctx, c, &g_interested_parties, &g_state, grpc_closure_create(state_changed, c)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&pollset)); + gpr_mu_lock(&mu); while (g_state != GRPC_CHANNEL_READY) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &pollset, &worker, - gpr_now(GPR_CLOCK_MONOTONIC), + grpc_pollset_work(&exec_ctx, pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); - gpr_mu_unlock(GRPC_POLLSET_MU(&pollset)); + gpr_mu_unlock(&mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&pollset)); + gpr_mu_lock(&mu); } - grpc_pollset_shutdown(&exec_ctx, &pollset, - grpc_closure_create(destroy_pollset, &pollset)); + grpc_pollset_shutdown(&exec_ctx, pollset, + grpc_closure_create(destroy_pollset, pollset)); grpc_pollset_set_destroy(&g_interested_parties); - gpr_mu_unlock(GRPC_POLLSET_MU(&pollset)); + gpr_mu_unlock(&mu); grpc_exec_ctx_finish(&exec_ctx); + gpr_free(pollset); + gpr_mu_destroy(&mu); return grpc_subchannel_get_connected_subchannel(c); } diff --git a/test/core/end2end/fixtures/h2_uds+poll.c b/test/core/end2end/fixtures/h2_uds+poll.c index 155017c8875..47106edde52 100644 --- a/test/core/end2end/fixtures/h2_uds+poll.c +++ b/test/core/end2end/fixtures/h2_uds+poll.c @@ -37,13 +37,6 @@ #include #include -#include "src/core/channel/client_channel.h" -#include "src/core/channel/connected_channel.h" -#include "src/core/channel/http_server_filter.h" -#include "src/core/support/string.h" -#include "src/core/surface/channel.h" -#include "src/core/surface/server.h" -#include "src/core/transport/chttp2_transport.h" #include #include #include @@ -51,6 +44,15 @@ #include #include #include + +#include "src/core/channel/client_channel.h" +#include "src/core/channel/connected_channel.h" +#include "src/core/channel/http_server_filter.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/support/string.h" +#include "src/core/surface/channel.h" +#include "src/core/surface/server.h" +#include "src/core/transport/chttp2_transport.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/httpcli/httpcli_test.c b/test/core/httpcli/httpcli_test.c index e954a920b04..cfa18292149 100644 --- a/test/core/httpcli/httpcli_test.c +++ b/test/core/httpcli/httpcli_test.c @@ -36,18 +36,19 @@ #include #include -#include "src/core/iomgr/iomgr.h" #include #include #include #include #include +#include "src/core/iomgr/iomgr.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" static int g_done = 0; static grpc_httpcli_context g_context; -static grpc_pollset g_pollset; +static gpr_mu g_mu; +static grpc_pollset *g_pollset; static gpr_timespec n_seconds_time(int seconds) { return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(seconds); @@ -63,10 +64,10 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, GPR_ASSERT(response->status == 200); GPR_ASSERT(response->body_length == strlen(expect)); GPR_ASSERT(0 == memcmp(expect, response->body, response->body_length)); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); g_done = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(&g_mu); } static void test_get(int port) { @@ -85,18 +86,18 @@ static void test_get(int port) { req.path = "/get"; req.handshaker = &grpc_httpcli_plaintext; - grpc_httpcli_get(&exec_ctx, &g_context, &g_pollset, &req, n_seconds_time(15), + grpc_httpcli_get(&exec_ctx, &g_context, g_pollset, &req, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); gpr_free(host); } @@ -116,18 +117,18 @@ static void test_post(int port) { req.path = "/post"; req.handshaker = &grpc_httpcli_plaintext; - grpc_httpcli_post(&exec_ctx, &g_context, &g_pollset, &req, "hello", 5, + grpc_httpcli_post(&exec_ctx, &g_context, g_pollset, &req, "hello", 5, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); gpr_free(host); } @@ -175,17 +176,22 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); grpc_httpcli_context_init(&g_context); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&g_mu); + grpc_pollset_init(g_pollset, &g_mu); test_get(port); test_post(port); grpc_httpcli_context_destroy(&g_context); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_mu_destroy(&g_mu); + gpr_free(g_pollset); + gpr_subprocess_destroy(server); return 0; diff --git a/test/core/httpcli/httpscli_test.c b/test/core/httpcli/httpscli_test.c index 9f31eae2782..b12f9472b15 100644 --- a/test/core/httpcli/httpscli_test.c +++ b/test/core/httpcli/httpscli_test.c @@ -36,18 +36,19 @@ #include #include -#include "src/core/iomgr/iomgr.h" #include #include #include #include #include +#include "src/core/iomgr/iomgr.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" static int g_done = 0; static grpc_httpcli_context g_context; -static grpc_pollset g_pollset; +static gpr_mu g_mu; +static grpc_pollset *g_pollset; static gpr_timespec n_seconds_time(int seconds) { return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(seconds); @@ -63,10 +64,10 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, GPR_ASSERT(response->status == 200); GPR_ASSERT(response->body_length == strlen(expect)); GPR_ASSERT(0 == memcmp(expect, response->body, response->body_length)); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); g_done = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(&g_mu); } static void test_get(int port) { @@ -86,18 +87,18 @@ static void test_get(int port) { req.path = "/get"; req.handshaker = &grpc_httpcli_ssl; - grpc_httpcli_get(&exec_ctx, &g_context, &g_pollset, &req, n_seconds_time(15), + grpc_httpcli_get(&exec_ctx, &g_context, g_pollset, &req, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); gpr_free(host); } @@ -118,18 +119,18 @@ static void test_post(int port) { req.path = "/post"; req.handshaker = &grpc_httpcli_ssl; - grpc_httpcli_post(&exec_ctx, &g_context, &g_pollset, &req, "hello", 5, + grpc_httpcli_post(&exec_ctx, &g_context, g_pollset, &req, "hello", 5, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); gpr_free(host); } @@ -178,17 +179,21 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); grpc_httpcli_context_init(&g_context); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); test_get(port); test_post(port); grpc_httpcli_context_destroy(&g_context); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); + gpr_mu_destroy(&g_mu); + gpr_subprocess_destroy(server); return 0; diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c index 7e266ebfb99..c02e4149359 100644 --- a/test/core/iomgr/endpoint_pair_test.c +++ b/test/core/iomgr/endpoint_pair_test.c @@ -39,10 +39,11 @@ #include #include #include "src/core/iomgr/endpoint_pair.h" -#include "test/core/util/test_config.h" #include "test/core/iomgr/endpoint_tests.h" +#include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu g_mu; +static grpc_pollset *g_pollset; static void clean_up(void) {} @@ -54,8 +55,8 @@ static grpc_endpoint_test_fixture create_fixture_endpoint_pair( f.client_ep = p.client; f.server_ep = p.server; - grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, &g_pollset); - grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, g_pollset); grpc_exec_ctx_finish(&exec_ctx); return f; @@ -72,14 +73,18 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) { int main(int argc, char **argv) { grpc_closure destroyed; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_mu_init(&g_mu); grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); - grpc_endpoint_tests(configs[0], &g_pollset); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); + grpc_endpoint_tests(configs[0], g_pollset, &g_mu); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_mu_destroy(&g_mu); + gpr_free(g_pollset); return 0; } diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c index 6ba67df3b10..f689e4ba7fb 100644 --- a/test/core/iomgr/endpoint_tests.c +++ b/test/core/iomgr/endpoint_tests.c @@ -36,8 +36,8 @@ #include #include -#include #include +#include #include #include #include "test/core/util/test_config.h" @@ -58,6 +58,7 @@ */ +static gpr_mu *g_mu; static grpc_pollset *g_pollset; size_t count_slices(gpr_slice *slices, size_t nslices, int *current_data) { @@ -134,10 +135,10 @@ static void read_and_write_test_read_handler(grpc_exec_ctx *exec_ctx, state->incoming.slices, state->incoming.count, &state->current_read_data); if (state->bytes_read == state->target_bytes || !success) { gpr_log(GPR_INFO, "Read handler done"); - gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_lock(g_mu); state->read_done = 1 + success; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_unlock(g_mu); } else if (success) { grpc_endpoint_read(exec_ctx, state->read_ep, &state->incoming, &state->done_read); @@ -169,10 +170,10 @@ static void read_and_write_test_write_handler(grpc_exec_ctx *exec_ctx, } gpr_log(GPR_INFO, "Write handler done"); - gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_lock(g_mu); state->write_done = 1 + success; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_unlock(g_mu); } /* Do both reading and writing using the grpc_endpoint API. @@ -232,14 +233,14 @@ static void read_and_write_test(grpc_endpoint_test_config config, } grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_lock(g_mu); while (!state.read_done || !state.write_done) { grpc_pollset_worker *worker = NULL; GPR_ASSERT(gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0); grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); } - gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); end_test(config); @@ -251,9 +252,10 @@ static void read_and_write_test(grpc_endpoint_test_config config, } void grpc_endpoint_tests(grpc_endpoint_test_config config, - grpc_pollset *pollset) { + grpc_pollset *pollset, gpr_mu *mu) { size_t i; g_pollset = pollset; + g_mu = mu; read_and_write_test(config, 10000000, 100000, 8192, 0); read_and_write_test(config, 1000000, 100000, 1, 0); read_and_write_test(config, 100000000, 100000, 1, 1); diff --git a/test/core/iomgr/endpoint_tests.h b/test/core/iomgr/endpoint_tests.h index 700f854891e..e1a231ea238 100644 --- a/test/core/iomgr/endpoint_tests.h +++ b/test/core/iomgr/endpoint_tests.h @@ -53,6 +53,6 @@ struct grpc_endpoint_test_config { }; void grpc_endpoint_tests(grpc_endpoint_test_config config, - grpc_pollset *pollset); + grpc_pollset *pollset, gpr_mu *mu); #endif /* GRPC_TEST_CORE_IOMGR_ENDPOINT_TESTS_H */ diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 07761b6576d..d199e0c272a 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -49,9 +49,12 @@ #include #include #include + +#include "src/core/iomgr/pollset_posix.h" #include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu g_mu; +static grpc_pollset *g_pollset; /* buffer size used to send and receive data. 1024 is the minimal value to set TCP send and receive buffer. */ @@ -179,10 +182,10 @@ static void listen_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg /*server */, grpc_fd_orphan(exec_ctx, sv->em_fd, NULL, NULL, "b"); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); sv->done = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(&g_mu); } /* Called when a new TCP connection request arrives in the listening port. */ @@ -209,7 +212,7 @@ static void listen_cb(grpc_exec_ctx *exec_ctx, void *arg, /*=sv_arg*/ se = gpr_malloc(sizeof(*se)); se->sv = sv; se->em_fd = grpc_fd_create(fd, "listener"); - grpc_pollset_add_fd(exec_ctx, &g_pollset, se->em_fd); + grpc_pollset_add_fd(exec_ctx, g_pollset, se->em_fd); se->session_read_closure.cb = session_read_cb; se->session_read_closure.cb_arg = se; grpc_fd_notify_on_read(exec_ctx, se->em_fd, &se->session_read_closure); @@ -238,7 +241,7 @@ static int server_start(grpc_exec_ctx *exec_ctx, server *sv) { GPR_ASSERT(listen(fd, MAX_NUM_FD) == 0); sv->em_fd = grpc_fd_create(fd, "server"); - grpc_pollset_add_fd(exec_ctx, &g_pollset, sv->em_fd); + grpc_pollset_add_fd(exec_ctx, g_pollset, sv->em_fd); /* Register to be interested in reading from listen_fd. */ sv->listen_closure.cb = listen_cb; sv->listen_closure.cb_arg = sv; @@ -249,18 +252,18 @@ static int server_start(grpc_exec_ctx *exec_ctx, server *sv) { /* Wait and shutdown a sever. */ static void server_wait_and_shutdown(server *sv) { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (!sv->done) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); } /* ===An upload client to test notify_on_write=== */ @@ -296,7 +299,7 @@ static void client_session_shutdown_cb(grpc_exec_ctx *exec_ctx, client *cl = arg; grpc_fd_orphan(exec_ctx, cl->em_fd, NULL, NULL, "c"); cl->done = 1; - grpc_pollset_kick(&g_pollset, NULL); + grpc_pollset_kick(g_pollset, NULL); } /* Write as much as possible, then register notify_on_write. */ @@ -307,9 +310,9 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ ssize_t write_once = 0; if (!success) { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); client_session_shutdown_cb(exec_ctx, arg, 1); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); return; } @@ -319,7 +322,7 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ } while (write_once > 0); if (errno == EAGAIN) { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); if (cl->client_write_cnt < CLIENT_TOTAL_WRITE_CNT) { cl->write_closure.cb = client_session_write; cl->write_closure.cb_arg = cl; @@ -328,7 +331,7 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ } else { client_session_shutdown_cb(exec_ctx, arg, 1); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); } else { gpr_log(GPR_ERROR, "unknown errno %s", strerror(errno)); abort(); @@ -357,25 +360,25 @@ static void client_start(grpc_exec_ctx *exec_ctx, client *cl, int port) { } cl->em_fd = grpc_fd_create(fd, "client"); - grpc_pollset_add_fd(exec_ctx, &g_pollset, cl->em_fd); + grpc_pollset_add_fd(exec_ctx, g_pollset, cl->em_fd); client_session_write(exec_ctx, cl, 1); } /* Wait for the signal to shutdown a client. */ static void client_wait_and_shutdown(client *cl) { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (!cl->done) { grpc_pollset_worker *worker = NULL; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); } /* Test grpc_fd. Start an upload server and client, upload a stream of @@ -410,20 +413,20 @@ static void first_read_callback(grpc_exec_ctx *exec_ctx, void *arg /* fd_change_data */, bool success) { fd_change_data *fdc = arg; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); fdc->cb_that_ran = first_read_callback; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(&g_mu); } static void second_read_callback(grpc_exec_ctx *exec_ctx, void *arg /* fd_change_data */, bool success) { fd_change_data *fdc = arg; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); fdc->cb_that_ran = second_read_callback; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(&g_mu); } /* Test that changing the callback we use for notify_on_read actually works. @@ -456,7 +459,7 @@ static void test_grpc_fd_change(void) { GPR_ASSERT(fcntl(sv[1], F_SETFL, flags | O_NONBLOCK) == 0); em_fd = grpc_fd_create(sv[0], "test_grpc_fd_change"); - grpc_pollset_add_fd(&exec_ctx, &g_pollset, em_fd); + grpc_pollset_add_fd(&exec_ctx, g_pollset, em_fd); /* Register the first callback, then make its FD readable */ grpc_fd_notify_on_read(&exec_ctx, em_fd, &first_closure); @@ -465,18 +468,18 @@ static void test_grpc_fd_change(void) { GPR_ASSERT(result == 1); /* And now wait for it to run. */ - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (a.cb_that_ran == NULL) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } GPR_ASSERT(a.cb_that_ran == first_read_callback); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); /* And drain the socket so we can generate a new read edge */ result = read(sv[0], &data, 1); @@ -489,19 +492,19 @@ static void test_grpc_fd_change(void) { result = write(sv[1], &data, 1); GPR_ASSERT(result == 1); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (b.cb_that_ran == NULL) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } /* Except now we verify that second_read_callback ran instead */ GPR_ASSERT(b.cb_that_ran == second_read_callback); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_fd_orphan(&exec_ctx, em_fd, NULL, NULL, "d"); grpc_exec_ctx_finish(&exec_ctx); @@ -519,12 +522,16 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_iomgr_init(); - grpc_pollset_init(&g_pollset); + gpr_mu_init(&g_mu); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); test_grpc_fd(); test_grpc_fd_change(); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); + gpr_free(g_pollset); + gpr_mu_destroy(&g_mu); grpc_iomgr_shutdown(); return 0; } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index cdd5b096fbc..58d6d2cb56e 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -40,6 +40,7 @@ #include #include +#include #include #include @@ -49,7 +50,8 @@ #include "test/core/util/test_config.h" static grpc_pollset_set g_pollset_set; -static grpc_pollset g_pollset; +static gpr_mu g_mu; +static grpc_pollset *g_pollset; static int g_connections_complete = 0; static grpc_endpoint *g_connecting = NULL; @@ -58,10 +60,10 @@ static gpr_timespec test_deadline(void) { } static void finish_connection() { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); g_connections_complete++; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(&g_mu); } static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, bool success) { @@ -99,9 +101,9 @@ void test_succeeds(void) { GPR_ASSERT(0 == bind(svr_fd, (struct sockaddr *)&addr, addr_len)); GPR_ASSERT(0 == listen(svr_fd, 1)); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); connections_complete_before = g_connections_complete; - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); /* connect to it */ GPR_ASSERT(getsockname(svr_fd, (struct sockaddr *)&addr, &addr_len) == 0); @@ -118,19 +120,19 @@ void test_succeeds(void) { GPR_ASSERT(r >= 0); close(r); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (g_connections_complete == connections_complete_before) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); } @@ -147,9 +149,9 @@ void test_fails(void) { memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); connections_complete_before = g_connections_complete; - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); /* connect to a broken address */ grpc_closure_init(&done, must_fail, NULL); @@ -157,7 +159,7 @@ void test_fails(void) { (struct sockaddr *)&addr, addr_len, gpr_inf_future(GPR_CLOCK_REALTIME)); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); /* wait for the connection callback to finish */ while (g_connections_complete == connections_complete_before) { @@ -165,14 +167,14 @@ void test_fails(void) { gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); gpr_timespec polling_deadline = test_deadline(); if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, now, polling_deadline); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, now, polling_deadline); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); } @@ -217,16 +219,16 @@ void test_times_out(void) { connect_deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); connections_complete_before = g_connections_complete; - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_closure_init(&done, must_fail, NULL); grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, &g_pollset_set, (struct sockaddr *)&addr, addr_len, connect_deadline); /* Make sure the event doesn't trigger early */ - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); for (;;) { grpc_pollset_worker *worker = NULL; gpr_timespec now = gpr_now(connect_deadline.clock_type); @@ -252,13 +254,13 @@ void test_times_out(void) { } gpr_timespec polling_deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10); if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, now, polling_deadline); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, now, polling_deadline); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); @@ -278,17 +280,21 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); grpc_pollset_set_init(&g_pollset_set); - grpc_pollset_init(&g_pollset); - grpc_pollset_set_add_pollset(&exec_ctx, &g_pollset_set, &g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&g_mu); + grpc_pollset_init(g_pollset, &g_mu); + grpc_pollset_set_add_pollset(&exec_ctx, &g_pollset_set, g_pollset); grpc_exec_ctx_finish(&exec_ctx); test_succeeds(); gpr_log(GPR_ERROR, "End of first test"); test_fails(); test_times_out(); grpc_pollset_set_destroy(&g_pollset_set); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); + gpr_mu_destroy(&g_mu); return 0; } diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index 20b0b9e13b7..d6758f8fe1b 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -36,8 +36,8 @@ #include #include #include -#include #include +#include #include #include @@ -45,10 +45,11 @@ #include #include #include -#include "test/core/util/test_config.h" #include "test/core/iomgr/endpoint_tests.h" +#include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu g_mu; +static grpc_pollset *g_pollset; /* General test notes: @@ -145,7 +146,7 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, bool success) { GPR_ASSERT(success); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); current_data = state->read_bytes % 256; read_bytes = count_slices(state->incoming.slices, state->incoming.count, ¤t_data); @@ -153,10 +154,10 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, bool success) { gpr_log(GPR_INFO, "Read %d bytes of %d", read_bytes, state->target_read_bytes); if (state->read_bytes >= state->target_read_bytes) { - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); } else { grpc_endpoint_read(exec_ctx, state->ep, &state->incoming, &state->read_cb); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); } } @@ -175,7 +176,7 @@ static void read_test(size_t num_bytes, size_t slice_size) { create_sockets(sv); ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), slice_size, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); gpr_log(GPR_INFO, "Wrote %d bytes", written_bytes); @@ -188,17 +189,17 @@ static void read_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); gpr_slice_buffer_destroy(&state.incoming); grpc_endpoint_destroy(&exec_ctx, ep); @@ -221,7 +222,7 @@ static void large_read_test(size_t slice_size) { ep = grpc_tcp_create(grpc_fd_create(sv[1], "large_read_test"), slice_size, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket(sv[0]); gpr_log(GPR_INFO, "Wrote %d bytes", written_bytes); @@ -234,17 +235,17 @@ static void large_read_test(size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); gpr_slice_buffer_destroy(&state.incoming); grpc_endpoint_destroy(&exec_ctx, ep); @@ -283,11 +284,11 @@ static void write_done(grpc_exec_ctx *exec_ctx, void *user_data /* write_socket_state */, bool success) { struct write_socket_state *state = (struct write_socket_state *)user_data; gpr_log(GPR_INFO, "Write done callback called"); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); gpr_log(GPR_INFO, "Signalling write done"); state->write_done = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(&g_mu); } void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { @@ -304,11 +305,11 @@ void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { for (;;) { grpc_pollset_worker *worker = NULL; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + gpr_mu_lock(&g_mu); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); do { bytes_read = @@ -350,7 +351,7 @@ static void write_test(size_t num_bytes, size_t slice_size) { ep = grpc_tcp_create(grpc_fd_create(sv[1], "write_test"), GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); state.ep = ep; state.write_done = 0; @@ -363,19 +364,19 @@ static void write_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_write(&exec_ctx, ep, &outgoing, &write_done_closure); drain_socket_blocking(sv[0], num_bytes, num_bytes); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); for (;;) { grpc_pollset_worker *worker = NULL; if (state.write_done) { break; } - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); gpr_slice_buffer_destroy(&outgoing); grpc_endpoint_destroy(&exec_ctx, ep); @@ -386,7 +387,7 @@ static void write_test(size_t num_bytes, size_t slice_size) { void on_fd_released(grpc_exec_ctx *exec_ctx, void *arg, bool success) { int *done = arg; *done = 1; - grpc_pollset_kick(&g_pollset, NULL); + grpc_pollset_kick(g_pollset, NULL); } /* Do a read_test, then release fd and try to read/write again. Verify that @@ -410,7 +411,7 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), slice_size, "test"); GPR_ASSERT(grpc_tcp_fd(ep) == sv[1] && sv[1] >= 0); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); gpr_log(GPR_INFO, "Wrote %d bytes", written_bytes); @@ -423,27 +424,27 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); gpr_slice_buffer_destroy(&state.incoming); grpc_tcp_destroy_and_release_fd(&exec_ctx, ep, &fd, &fd_released_cb); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); while (!fd_released_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); GPR_ASSERT(fd_released_done == 1); GPR_ASSERT(fd == sv[1]); grpc_exec_ctx_finish(&exec_ctx); @@ -491,8 +492,8 @@ static grpc_endpoint_test_fixture create_fixture_tcp_socketpair( slice_size, "test"); f.server_ep = grpc_tcp_create(grpc_fd_create(sv[1], "fixture:server"), slice_size, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, &g_pollset); - grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, g_pollset); grpc_exec_ctx_finish(&exec_ctx); @@ -512,13 +513,17 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&g_mu); + grpc_pollset_init(g_pollset, &g_mu); run_tests(); - grpc_endpoint_tests(configs[0], &g_pollset); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_endpoint_tests(configs[0], g_pollset, &g_mu); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); + gpr_mu_destroy(&g_mu); return 0; } diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 43180b16988..ed627734bfa 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -32,24 +32,28 @@ */ #include "src/core/iomgr/tcp_server.h" -#include "src/core/iomgr/iomgr.h" -#include "src/core/iomgr/sockaddr_utils.h" + +#include +#include +#include +#include +#include + #include +#include #include #include #include + +#include "src/core/iomgr/iomgr.h" +#include "src/core/iomgr/sockaddr_utils.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" -#include -#include -#include -#include -#include - #define LOG_TEST(x) gpr_log(GPR_INFO, "%s", #x) -static grpc_pollset g_pollset; +static gpr_mu g_mu; +static grpc_pollset *g_pollset; static int g_nconnects = 0; typedef struct on_connect_result { @@ -113,11 +117,11 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_endpoint_shutdown(exec_ctx, tcp); grpc_endpoint_destroy(exec_ctx, tcp); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); on_connect_result_set(&g_result, acceptor); g_nconnects++; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(&g_mu); } static void test_no_op(void) { @@ -174,7 +178,7 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, int clifd = socket(remote->sa_family, SOCK_STREAM, 0); int nconnects_before; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); nconnects_before = g_nconnects; on_connect_result_init(&g_result); GPR_ASSERT(clifd >= 0); @@ -184,18 +188,18 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, while (g_nconnects == nconnects_before && gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(exec_ctx, &g_pollset, &worker, + grpc_pollset_work(exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); } gpr_log(GPR_DEBUG, "wait done"); GPR_ASSERT(g_nconnects == nconnects_before + 1); close(clifd); *result = g_result; - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(&g_mu); } /* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for @@ -210,7 +214,6 @@ static void test_connect(unsigned n) { unsigned svr1_fd_count; int svr1_port; grpc_tcp_server *s = grpc_tcp_server_create(NULL); - grpc_pollset *pollsets[1]; unsigned i; server_weak_ref weak_ref; server_weak_ref_init(&weak_ref); @@ -259,8 +262,7 @@ static void test_connect(unsigned n) { } } - pollsets[0] = &g_pollset; - grpc_tcp_server_start(&exec_ctx, s, pollsets, 1, on_connect, NULL); + grpc_tcp_server_start(&exec_ctx, s, &g_pollset, 1, on_connect, NULL); for (i = 0; i < n; i++) { on_connect_result result; @@ -312,7 +314,9 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&g_mu); + grpc_pollset_init(g_pollset, &g_mu); test_no_op(); test_no_op_with_start(); @@ -321,9 +325,11 @@ int main(int argc, char **argv) { test_connect(1); test_connect(10); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); + gpr_mu_destroy(&g_mu); return 0; } diff --git a/test/core/iomgr/workqueue_test.c b/test/core/iomgr/workqueue_test.c index a0249152566..f557042e39b 100644 --- a/test/core/iomgr/workqueue_test.c +++ b/test/core/iomgr/workqueue_test.c @@ -34,18 +34,20 @@ #include "src/core/iomgr/workqueue.h" #include +#include #include #include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu g_mu; +static grpc_pollset *g_pollset; static void must_succeed(grpc_exec_ctx *exec_ctx, void *p, bool success) { GPR_ASSERT(success == 1); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); *(int *)p = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(&g_mu); } static void test_ref_unref(void) { @@ -67,13 +69,13 @@ static void test_add_closure(void) { grpc_closure_init(&c, must_succeed, &done); grpc_workqueue_push(wq, &c, 1); - grpc_workqueue_add_to_pollset(&exec_ctx, wq, &g_pollset); + grpc_workqueue_add_to_pollset(&exec_ctx, wq, g_pollset); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); GPR_ASSERT(!done); - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, - gpr_now(deadline.clock_type), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(deadline.clock_type), + deadline); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(done); @@ -92,13 +94,13 @@ static void test_flush(void) { grpc_exec_ctx_enqueue(&exec_ctx, &c, true, NULL); grpc_workqueue_flush(&exec_ctx, wq); - grpc_workqueue_add_to_pollset(&exec_ctx, wq, &g_pollset); + grpc_workqueue_add_to_pollset(&exec_ctx, wq, g_pollset); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(&g_mu); GPR_ASSERT(!done); - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, - gpr_now(deadline.clock_type), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(deadline.clock_type), + deadline); + gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(done); @@ -115,15 +117,20 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); + gpr_mu_init(&g_mu); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); test_ref_unref(); test_add_closure(); test_flush(); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + + gpr_free(g_pollset); + gpr_mu_destroy(&g_mu); return 0; } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index 4dd595df956..3280eac801c 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -45,7 +45,8 @@ #include "src/core/security/credentials.h" typedef struct { - grpc_pollset pollset; + gpr_mu mu; + grpc_pollset *pollset; int is_done; char *token; } oauth2_request; @@ -66,11 +67,11 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data, GPR_SLICE_LENGTH(token_slice)); token[GPR_SLICE_LENGTH(token_slice)] = '\0'; } - gpr_mu_lock(GRPC_POLLSET_MU(&request->pollset)); + gpr_mu_lock(&request->mu); request->is_done = 1; request->token = token; - grpc_pollset_kick(&request->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&request->pollset)); + grpc_pollset_kick(request->pollset, NULL); + gpr_mu_unlock(&request->mu); } static void do_nothing(grpc_exec_ctx *exec_ctx, void *unused, bool success) {} @@ -82,28 +83,30 @@ char *grpc_test_fetch_oauth2_token_with_credentials( grpc_closure do_nothing_closure; grpc_auth_metadata_context null_ctx = {"", "", NULL, NULL}; - grpc_pollset_init(&request.pollset); + request.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(request.pollset, &request.mu); request.is_done = 0; grpc_closure_init(&do_nothing_closure, do_nothing, NULL); - grpc_call_credentials_get_request_metadata(&exec_ctx, creds, &request.pollset, + grpc_call_credentials_get_request_metadata(&exec_ctx, creds, request.pollset, null_ctx, on_oauth2_response, &request); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&request.pollset)); + gpr_mu_lock(&request.mu); while (!request.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &request.pollset, &worker, + grpc_pollset_work(&exec_ctx, request.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); } - gpr_mu_unlock(GRPC_POLLSET_MU(&request.pollset)); + gpr_mu_unlock(&request.mu); - grpc_pollset_shutdown(&exec_ctx, &request.pollset, &do_nothing_closure); + grpc_pollset_shutdown(&exec_ctx, request.pollset, &do_nothing_closure); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_destroy(&request.pollset); + grpc_pollset_destroy(request.pollset); + gpr_free(request.pollset); return request.token; } diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index fcbbe334f45..cd4e990f45e 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -34,8 +34,6 @@ #include #include -#include "src/core/security/credentials.h" -#include "src/core/support/string.h" #include #include #include @@ -44,8 +42,12 @@ #include #include +#include "src/core/security/credentials.h" +#include "src/core/support/string.h" + typedef struct { - grpc_pollset pollset; + gpr_mu mu; + grpc_pollset *pollset; int is_done; } synchronizer; @@ -62,10 +64,10 @@ static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *user_data, printf("\nGot token: %s\n\n", token); gpr_free(token); } - gpr_mu_lock(GRPC_POLLSET_MU(&sync->pollset)); + gpr_mu_lock(&sync->mu); sync->is_done = 1; - grpc_pollset_kick(&sync->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&sync->pollset)); + grpc_pollset_kick(sync->pollset, NULL); + gpr_mu_unlock(&sync->mu); } int main(int argc, char **argv) { @@ -91,26 +93,30 @@ int main(int argc, char **argv) { goto end; } - grpc_pollset_init(&sync.pollset); + sync.pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&sync.mu); + grpc_pollset_init(sync.pollset, &sync.mu); sync.is_done = 0; grpc_call_credentials_get_request_metadata( &exec_ctx, ((grpc_composite_channel_credentials *)creds)->call_creds, - &sync.pollset, context, on_metadata_response, &sync); + sync.pollset, context, on_metadata_response, &sync); - gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_lock(&sync.mu); while (!sync.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &sync.pollset, &worker, + grpc_pollset_work(&exec_ctx, sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_unlock(&sync.mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_lock(&sync.mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_unlock(&sync.mu); grpc_channel_credentials_release(creds); + gpr_free(sync.pollset); + gpr_mu_destroy(&sync.mu); end: gpr_cmdline_destroy(cl); diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index fb4bd30e2df..61543ada43e 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -36,16 +36,17 @@ #include #include -#include "src/core/security/secure_endpoint.h" -#include "src/core/iomgr/endpoint_pair.h" -#include "src/core/iomgr/iomgr.h" #include #include #include -#include "test/core/util/test_config.h" +#include "src/core/iomgr/endpoint_pair.h" +#include "src/core/iomgr/iomgr.h" +#include "src/core/security/secure_endpoint.h" #include "src/core/tsi/fake_transport_security.h" +#include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu g_mu; +static grpc_pollset *g_pollset; static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( size_t slice_size, gpr_slice *leftover_slices, size_t leftover_nslices) { @@ -56,8 +57,8 @@ static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( grpc_endpoint_pair tcp; tcp = grpc_iomgr_create_endpoint_pair("fixture", slice_size); - grpc_endpoint_add_to_pollset(&exec_ctx, tcp.client, &g_pollset); - grpc_endpoint_add_to_pollset(&exec_ctx, tcp.server, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, tcp.client, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, tcp.server, g_pollset); if (leftover_nslices == 0) { f.client_ep = @@ -181,13 +182,18 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); - grpc_endpoint_tests(configs[0], &g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&g_mu); + grpc_pollset_init(g_pollset, &g_mu); + grpc_endpoint_tests(configs[0], g_pollset, &g_mu); test_leftover(configs[1], 1); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); + gpr_mu_destroy(&g_mu); + return 0; } diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index c9071fa08b0..f22b076171c 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -34,7 +34,6 @@ #include #include -#include "src/core/security/jwt_verifier.h" #include #include #include @@ -43,8 +42,11 @@ #include #include +#include "src/core/security/jwt_verifier.h" + typedef struct { - grpc_pollset pollset; + grpc_pollset *pollset; + gpr_mu mu; int is_done; int success; } synchronizer; @@ -77,10 +79,10 @@ static void on_jwt_verification_done(void *user_data, grpc_jwt_verifier_status_to_string(status)); } - gpr_mu_lock(GRPC_POLLSET_MU(&sync->pollset)); + gpr_mu_lock(&sync->mu); sync->is_done = 1; - grpc_pollset_kick(&sync->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&sync->pollset)); + grpc_pollset_kick(sync->pollset, NULL); + gpr_mu_unlock(&sync->mu); } int main(int argc, char **argv) { @@ -103,23 +105,28 @@ int main(int argc, char **argv) { grpc_init(); - grpc_pollset_init(&sync.pollset); + sync.pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&sync.mu); + grpc_pollset_init(sync.pollset, &sync.mu); sync.is_done = 0; - grpc_jwt_verifier_verify(&exec_ctx, verifier, &sync.pollset, jwt, aud, + grpc_jwt_verifier_verify(&exec_ctx, verifier, sync.pollset, jwt, aud, on_jwt_verification_done, &sync); - gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_lock(&sync.mu); while (!sync.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &sync.pollset, &worker, + grpc_pollset_work(&exec_ctx, sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_unlock(&sync.mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_lock(&sync.mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_unlock(&sync.mu); + + gpr_mu_destroy(&sync.mu); + gpr_free(sync.pollset); grpc_jwt_verifier_destroy(verifier); gpr_cmdline_destroy(cl); diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c index ad21fe0f536..a04471706ee 100644 --- a/test/core/util/port_posix.c +++ b/test/core/util/port_posix.c @@ -69,7 +69,8 @@ static int has_port_been_chosen(int port) { } typedef struct freereq { - grpc_pollset pollset; + gpr_mu mu; + grpc_pollset *pollset; int done; } freereq; @@ -82,10 +83,10 @@ static void destroy_pollset_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, const grpc_httpcli_response *response) { freereq *pr = arg; - gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); + gpr_mu_lock(&pr->mu); pr->done = 1; - grpc_pollset_kick(&pr->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); + grpc_pollset_kick(pr->pollset, NULL); + gpr_mu_unlock(&pr->mu); } static void free_port_using_server(char *server, int port) { @@ -100,31 +101,36 @@ static void free_port_using_server(char *server, int port) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); - grpc_pollset_init(&pr.pollset); + + pr.pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&pr.mu); + grpc_pollset_init(pr.pollset, &pr.mu); grpc_closure_init(&shutdown_closure, destroy_pollset_and_shutdown, - &pr.pollset); + pr.pollset); req.host = server; gpr_asprintf(&path, "/drop/%d", port); req.path = path; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, &pr.pollset, &req, + grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), freed_port_from_server, &pr); - gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_lock(&pr.mu); while (!pr.done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, + grpc_pollset_work(&exec_ctx, pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); } - gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_unlock(&pr.mu); grpc_httpcli_context_destroy(&context); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_shutdown(&exec_ctx, &pr.pollset, &shutdown_closure); + grpc_pollset_shutdown(&exec_ctx, pr.pollset, &shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); + gpr_free(pr.pollset); + gpr_mu_destroy(&pr.mu); gpr_free(path); } @@ -202,7 +208,8 @@ static int is_port_available(int *port, int is_tcp) { } typedef struct portreq { - grpc_pollset pollset; + gpr_mu mu; + grpc_pollset *pollset; int port; int retries; char *server; @@ -234,7 +241,7 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->retries++; req.host = pr->server; req.path = "/get"; - grpc_httpcli_get(exec_ctx, pr->ctx, &pr->pollset, &req, + grpc_httpcli_get(exec_ctx, pr->ctx, pr->pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, pr); return; @@ -246,10 +253,10 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, port = port * 10 + response->body[i] - '0'; } GPR_ASSERT(port > 1024); - gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); + gpr_mu_lock(&pr->mu); pr->port = port; - grpc_pollset_kick(&pr->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); + grpc_pollset_kick(pr->pollset, NULL); + gpr_mu_unlock(&pr->mu); } static int pick_port_using_server(char *server) { @@ -263,9 +270,11 @@ static int pick_port_using_server(char *server) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); - grpc_pollset_init(&pr.pollset); + pr.pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&pr.mu); + grpc_pollset_init(pr.pollset, &pr.mu); grpc_closure_init(&shutdown_closure, destroy_pollset_and_shutdown, - &pr.pollset); + pr.pollset); pr.port = -1; pr.server = server; pr.ctx = &context; @@ -274,22 +283,24 @@ static int pick_port_using_server(char *server) { req.path = "/get"; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, &pr.pollset, &req, + grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, &pr); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_lock(&pr.mu); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, + grpc_pollset_work(&exec_ctx, pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); } - gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_unlock(&pr.mu); grpc_httpcli_context_destroy(&context); - grpc_pollset_shutdown(&exec_ctx, &pr.pollset, &shutdown_closure); + grpc_pollset_shutdown(&exec_ctx, pr.pollset, &shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); + gpr_free(pr.pollset); + gpr_mu_destroy(&pr.mu); return pr.port; } diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index e99d5dcffdd..8f1db4e5015 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -57,8 +57,9 @@ void test_tcp_server_init(test_tcp_server *server, server->tcp_server = NULL; grpc_closure_init(&server->shutdown_complete, on_server_destroyed, server); server->shutdown = 0; - grpc_pollset_init(&server->pollset); - server->pollsets[0] = &server->pollset; + server->pollset = gpr_malloc(grpc_pollset_size()); + gpr_mu_init(&server->mu); + grpc_pollset_init(server->pollset, &server->mu); server->on_connect = on_connect; server->cb_data = user_data; } @@ -77,7 +78,7 @@ void test_tcp_server_start(test_tcp_server *server, int port) { grpc_tcp_server_add_port(server->tcp_server, &addr, sizeof(addr)); GPR_ASSERT(port_added == port); - grpc_tcp_server_start(&exec_ctx, server->tcp_server, server->pollsets, 1, + grpc_tcp_server_start(&exec_ctx, server->tcp_server, &server->pollset, 1, server->on_connect, server->cb_data); gpr_log(GPR_INFO, "test tcp server listening on 0.0.0.0:%d", port); @@ -90,10 +91,10 @@ void test_tcp_server_poll(test_tcp_server *server, int seconds) { gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(seconds, GPR_TIMESPAN)); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - gpr_mu_lock(GRPC_POLLSET_MU(&server->pollset)); - grpc_pollset_work(&exec_ctx, &server->pollset, &worker, + gpr_mu_lock(&server->mu); + grpc_pollset_work(&exec_ctx, server->pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&server->pollset)); + gpr_mu_unlock(&server->mu); grpc_exec_ctx_finish(&exec_ctx); } @@ -111,8 +112,10 @@ void test_tcp_server_destroy(test_tcp_server *server) { gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), shutdown_deadline) < 0) { test_tcp_server_poll(server, 1); } - grpc_pollset_shutdown(&exec_ctx, &server->pollset, &do_nothing_cb); + grpc_pollset_shutdown(&exec_ctx, server->pollset, &do_nothing_cb); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_destroy(&server->pollset); + grpc_pollset_destroy(server->pollset); + gpr_free(server->pollset); + gpr_mu_destroy(&server->mu); grpc_shutdown(); } diff --git a/test/core/util/test_tcp_server.h b/test/core/util/test_tcp_server.h index 51119cf6c80..ef9dd007c77 100644 --- a/test/core/util/test_tcp_server.h +++ b/test/core/util/test_tcp_server.h @@ -41,8 +41,8 @@ typedef struct test_tcp_server { grpc_tcp_server *tcp_server; grpc_closure shutdown_complete; int shutdown; - grpc_pollset pollset; - grpc_pollset *pollsets[1]; + gpr_mu mu; + grpc_pollset *pollset; grpc_tcp_server_cb on_connect; void *cb_data; } test_tcp_server; From c605c62b30ca15c83a7c4e98386062c62de0d36d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 19 Feb 2016 16:24:26 -0800 Subject: [PATCH 039/236] Update copyrights --- src/core/iomgr/iomgr.c | 2 +- src/core/iomgr/pollset.h | 2 +- src/core/iomgr/pollset_set_posix.c | 2 +- src/core/iomgr/pollset_set_posix.h | 2 +- src/core/iomgr/pollset_windows.h | 2 +- src/core/iomgr/timer.h | 2 +- src/core/iomgr/workqueue_posix.h | 2 +- test/core/end2end/fixtures/h2_full+poll+pipe.c | 2 +- test/core/end2end/fixtures/h2_full+poll.c | 2 +- test/core/end2end/fixtures/h2_ssl+poll.c | 2 +- test/core/end2end/fixtures/h2_uds+poll.c | 2 +- test/core/iomgr/endpoint_tests.h | 2 +- test/core/security/print_google_default_creds_token.c | 2 +- test/core/security/verify_jwt.c | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c index 3283b586b06..04580150f3a 100644 --- a/src/core/iomgr/iomgr.c +++ b/src/core/iomgr/iomgr.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index dfbd4a40ec4..39e4ff24408 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/iomgr/pollset_set_posix.c index 85a0cadfc7c..3610e36b586 100644 --- a/src/core/iomgr/pollset_set_posix.c +++ b/src/core/iomgr/pollset_set_posix.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/pollset_set_posix.h b/src/core/iomgr/pollset_set_posix.h index 7ce8ec7343f..5ee83b5dd6a 100644 --- a/src/core/iomgr/pollset_set_posix.h +++ b/src/core/iomgr/pollset_set_posix.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h index dfec5821b35..c2f13fdfa8b 100644 --- a/src/core/iomgr/pollset_windows.h +++ b/src/core/iomgr/pollset_windows.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/timer.h b/src/core/iomgr/timer.h index 906255ddfb9..9ad1e92f42e 100644 --- a/src/core/iomgr/timer.h +++ b/src/core/iomgr/timer.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/workqueue_posix.h b/src/core/iomgr/workqueue_posix.h index 6f29f4004ce..68f195ee0d2 100644 --- a/src/core/iomgr/workqueue_posix.h +++ b/src/core/iomgr/workqueue_posix.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/fixtures/h2_full+poll+pipe.c b/test/core/end2end/fixtures/h2_full+poll+pipe.c index 50df34fcc68..682598fbe2d 100644 --- a/test/core/end2end/fixtures/h2_full+poll+pipe.c +++ b/test/core/end2end/fixtures/h2_full+poll+pipe.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/fixtures/h2_full+poll.c b/test/core/end2end/fixtures/h2_full+poll.c index 76d6c990666..5a0b2ef4953 100644 --- a/test/core/end2end/fixtures/h2_full+poll.c +++ b/test/core/end2end/fixtures/h2_full+poll.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/fixtures/h2_ssl+poll.c b/test/core/end2end/fixtures/h2_ssl+poll.c index 5085880a176..9138b84376f 100644 --- a/test/core/end2end/fixtures/h2_ssl+poll.c +++ b/test/core/end2end/fixtures/h2_ssl+poll.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/fixtures/h2_uds+poll.c b/test/core/end2end/fixtures/h2_uds+poll.c index 47106edde52..c3a855ff883 100644 --- a/test/core/end2end/fixtures/h2_uds+poll.c +++ b/test/core/end2end/fixtures/h2_uds+poll.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/iomgr/endpoint_tests.h b/test/core/iomgr/endpoint_tests.h index e1a231ea238..8ea47e345cf 100644 --- a/test/core/iomgr/endpoint_tests.h +++ b/test/core/iomgr/endpoint_tests.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index cd4e990f45e..2145a79ff97 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index f22b076171c..c3cf6bb7399 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 23a329838588eb3dc7bcfee365007c5194288912 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 21 Feb 2016 22:43:21 -0800 Subject: [PATCH 040/236] Fix plucking problem --- src/core/surface/completion_queue.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index de295ab9410..0a80680f02e 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -333,10 +333,10 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); - continue; + } else { + grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, + iteration_deadline); } - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, - iteration_deadline); } GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); GRPC_CQ_INTERNAL_UNREF(cc, "next"); @@ -450,10 +450,10 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); - continue; + } else { + grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, + iteration_deadline); } - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, - iteration_deadline); del_plucker(cc, tag, &worker); } done: From 6d15982ce70cee38e0d26e00d52191a4d8c156ac Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 19 Feb 2016 16:03:22 -0800 Subject: [PATCH 041/236] building protoc artifacts on windows --- tools/run_tests/artifact_targets.py | 15 ++++---- tools/run_tests/build_artifact_protoc.bat | 45 +++++++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 tools/run_tests/build_artifact_protoc.bat diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index cf056ec9293..2a9f248d92e 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -266,7 +266,12 @@ class ProtocArtifact: ['tools/run_tests/build_artifact_protoc.sh'], environ=environ) else: - raise Exception('Not yet supported') + generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' + vcplatform = 'x64' if self.arch == 'x64' else 'Win32' + return create_jobspec(self.name, + ['tools\\run_tests\\build_artifact_protoc.bat'], + environ={'generator': generator, + 'Platform': vcplatform}) def __str__(self): return self.name @@ -275,7 +280,7 @@ class ProtocArtifact: def targets(): """Gets list of supported targets""" return ([Cls(platform, arch) - for Cls in (CSharpExtArtifact, NodeExtArtifact) + for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64')] + [PythonArtifact('linux', 'x86'), @@ -283,8 +288,4 @@ def targets(): PythonArtifact('macos', 'x64'), RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), - RubyArtifact('macos', 'x64'), - ProtocArtifact('linux', 'x86'), - ProtocArtifact('linux', 'x64'), - ProtocArtifact('macos', 'x86'), - ProtocArtifact('macos', 'x64')]) + RubyArtifact('macos', 'x64')]) diff --git a/tools/run_tests/build_artifact_protoc.bat b/tools/run_tests/build_artifact_protoc.bat new file mode 100644 index 00000000000..1fa7f6046ae --- /dev/null +++ b/tools/run_tests/build_artifact_protoc.bat @@ -0,0 +1,45 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +mkdir artifacts + +setlocal +cd third_party/protobuf/cmake +cmake -G "%generator%" || goto :error +endlocal + +call vsprojects/build_plugins.bat || goto :error + +xcopy /Y third_party\protobuf\cmake\Release\protoc.exe artifacts\ || goto :error +xcopy /Y vsprojects\Release\*_plugin.exe artifacts\ || goto :error + +goto :EOF + +:error +exit /b 1 \ No newline at end of file From 4ad14db9960d62b57abc84723e58cd51b18f492b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 19 Feb 2016 16:33:46 -0800 Subject: [PATCH 042/236] download and extract gmock --- tools/run_tests/build_artifact_protoc.bat | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/build_artifact_protoc.bat b/tools/run_tests/build_artifact_protoc.bat index 1fa7f6046ae..e1dc032188f 100644 --- a/tools/run_tests/build_artifact_protoc.bat +++ b/tools/run_tests/build_artifact_protoc.bat @@ -30,14 +30,20 @@ mkdir artifacts setlocal -cd third_party/protobuf/cmake +cd third_party/protobuf + +powershell -Command "Invoke-WebRequest https://googlemock.googlecode.com/files/gmock-1.7.0.zip -OutFile gmock.zip" +powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('gmock.zip', '.');" +rename gmock-1.7.0 gmock + +cd cmake cmake -G "%generator%" || goto :error endlocal call vsprojects/build_plugins.bat || goto :error xcopy /Y third_party\protobuf\cmake\Release\protoc.exe artifacts\ || goto :error -xcopy /Y vsprojects\Release\*_plugin.exe artifacts\ || goto :error +xcopy /Y vsprojects\Release\*_plugin.exe artifacts\ || xcopy /Y vsprojects\x64\Release\*_plugin.exe artifacts\ || goto :error goto :EOF From 6611dde2615be4853bc3cc3b8405ba67aa9dc2dd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 22 Feb 2016 08:48:02 -0800 Subject: [PATCH 043/236] enable 64bit windows compilation of protoc plugins --- build.yaml | 2 ++ templates/vsprojects/protoc.props.template | 3 +++ 2 files changed, 5 insertions(+) diff --git a/build.yaml b/build.yaml index b639b5d21e6..761e7d537d8 100644 --- a/build.yaml +++ b/build.yaml @@ -776,6 +776,8 @@ libs: - gpr_codegen secure: false vs_project_guid: '{B6E81D84-2ACB-41B8-8781-493A944C7817}' + vs_props: + - protoc - name: interop_client_helper build: private language: c++ diff --git a/templates/vsprojects/protoc.props.template b/templates/vsprojects/protoc.props.template index a869005daec..ced6028a4be 100644 --- a/templates/vsprojects/protoc.props.template +++ b/templates/vsprojects/protoc.props.template @@ -4,6 +4,9 @@ + + 4244;4267;%(DisableSpecificWarnings) + libprotoc.lib;%(AdditionalDependencies) $(SolutionDir)\..\third_party\protobuf\cmake\$(Configuration);%(AdditionalLibraryDirectories) From a0d7ea6410d4006a23d472ad461ab0039846715d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 22 Feb 2016 08:49:14 -0800 Subject: [PATCH 044/236] generated projects --- vsprojects/protoc.props | 2 +- .../vcxproj/grpc_plugin_support/grpc_plugin_support.vcxproj | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/vsprojects/protoc.props b/vsprojects/protoc.props index ecaf248446e..1bdc07193bc 100644 --- a/vsprojects/protoc.props +++ b/vsprojects/protoc.props @@ -1 +1 @@ - libprotoc.lib;%(AdditionalDependencies) $(SolutionDir)\..\third_party\protobuf\cmake\$(Configuration);%(AdditionalLibraryDirectories) \ No newline at end of file + 4244;4267;%(DisableSpecificWarnings) libprotoc.lib;%(AdditionalDependencies) $(SolutionDir)\..\third_party\protobuf\cmake\$(Configuration);%(AdditionalLibraryDirectories) \ No newline at end of file diff --git a/vsprojects/vcxproj/grpc_plugin_support/grpc_plugin_support.vcxproj b/vsprojects/vcxproj/grpc_plugin_support/grpc_plugin_support.vcxproj index 89183902d74..a76c883903a 100644 --- a/vsprojects/vcxproj/grpc_plugin_support/grpc_plugin_support.vcxproj +++ b/vsprojects/vcxproj/grpc_plugin_support/grpc_plugin_support.vcxproj @@ -53,6 +53,7 @@ + From ad0df7bf1fa9a2ad6302016cfbe0b86793f810c5 Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 22 Feb 2016 10:00:20 -0800 Subject: [PATCH 045/236] Discard the read buffer on stream error --- src/core/transport/chttp2_transport.c | 5 +++++ test/cpp/end2end/async_end2end_test.cc | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 617d98875c3..c3efc36cc5d 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -1019,6 +1019,11 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx, stream_global->recv_initial_metadata_ready = NULL; } if (stream_global->recv_message_ready != NULL) { + while (stream_global->seen_error && + (bs = grpc_chttp2_incoming_frame_queue_pop( + &stream_global->incoming_frames)) != NULL) { + grpc_byte_stream_destroy(exec_ctx, bs); + } if (stream_global->incoming_frames.head != NULL) { *stream_global->recv_message = grpc_chttp2_incoming_frame_queue_pop( &stream_global->incoming_frames); diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index a15cbd7ee2e..9ca3bf98f85 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -989,6 +989,10 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { if (server_try_cancel == CANCEL_AFTER_PROCESSING) { ServerTryCancel(&srv_ctx); + + // Client reads may fail bacause it is notified that the stream is + // cancelled. + ignore_cq_result = true; } // Client attemts to read the three messages from the server From 276e32d0fbbfab490fd26dcbfb4e65c3c87f31ae Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 22 Feb 2016 13:15:30 -0800 Subject: [PATCH 046/236] Fix race between add_writing_stalled and destroy stream --- src/core/transport/chttp2/internal.h | 9 +++++++-- src/core/transport/chttp2/stream_lists.c | 22 ++++++++++++++++++---- src/core/transport/chttp2/writing.c | 10 +++++----- src/core/transport/chttp2_transport.c | 2 +- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h index 0e1e2c42650..d76d31be23f 100644 --- a/src/core/transport/chttp2/internal.h +++ b/src/core/transport/chttp2/internal.h @@ -485,7 +485,8 @@ struct grpc_chttp2_stream { /** Someone is unlocking the transport mutex: check to see if writes are required, and schedule them if so */ -int grpc_chttp2_unlocking_check_writes(grpc_chttp2_transport_global *global, +int grpc_chttp2_unlocking_check_writes(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *global, grpc_chttp2_transport_writing *writing, int is_parsing); void grpc_chttp2_perform_writes( @@ -568,8 +569,12 @@ void grpc_chttp2_list_add_writing_stalled_by_transport( grpc_chttp2_transport_writing *transport_writing, grpc_chttp2_stream_writing *stream_writing); void grpc_chttp2_list_flush_writing_stalled_by_transport( - grpc_chttp2_transport_writing *transport_writing, bool is_window_available); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing, + bool is_window_available); +void grpc_chttp2_list_add_stalled_by_transport( + grpc_chttp2_transport_writing *transport_writing, + grpc_chttp2_stream_writing *stream_writing); int grpc_chttp2_list_pop_stalled_by_transport( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global); diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/transport/chttp2/stream_lists.c index 2f31a47cb3d..b284c788183 100644 --- a/src/core/transport/chttp2/stream_lists.c +++ b/src/core/transport/chttp2/stream_lists.c @@ -316,13 +316,16 @@ int grpc_chttp2_list_pop_check_read_ops( void grpc_chttp2_list_add_writing_stalled_by_transport( grpc_chttp2_transport_writing *transport_writing, grpc_chttp2_stream_writing *stream_writing) { - stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), - STREAM_FROM_WRITING(stream_writing), + grpc_chttp2_stream *stream = STREAM_FROM_WRITING(stream_writing); + if (!stream->included[GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT]) { + GRPC_CHTTP2_STREAM_REF(&stream->global, "chttp2_writing_stalled"); + } + stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), stream, GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT); } void grpc_chttp2_list_flush_writing_stalled_by_transport( - grpc_chttp2_transport_writing *transport_writing, + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing, bool is_window_available) { grpc_chttp2_stream *stream; grpc_chttp2_transport *transport = TRANSPORT_FROM_WRITING(transport_writing); @@ -331,11 +334,22 @@ void grpc_chttp2_list_flush_writing_stalled_by_transport( if (is_window_available) { grpc_chttp2_list_add_writable_stream(&transport->global, &stream->global); } else { - stream_list_add(transport, stream, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); + grpc_chttp2_list_add_stalled_by_transport(transport_writing, + &stream->writing); } + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &stream->global, + "chttp2_writing_stalled"); } } +void grpc_chttp2_list_add_stalled_by_transport( + grpc_chttp2_transport_writing *transport_writing, + grpc_chttp2_stream_writing *stream_writing) { + stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), + STREAM_FROM_WRITING(stream_writing), + GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); +} + int grpc_chttp2_list_pop_stalled_by_transport( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global) { diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c index cafecf10465..356fd8174a7 100644 --- a/src/core/transport/chttp2/writing.c +++ b/src/core/transport/chttp2/writing.c @@ -44,7 +44,7 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing); int grpc_chttp2_unlocking_check_writes( - grpc_chttp2_transport_global *transport_global, + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, grpc_chttp2_transport_writing *transport_writing, int is_parsing) { grpc_chttp2_stream_global *stream_global; grpc_chttp2_stream_writing *stream_writing; @@ -76,8 +76,8 @@ int grpc_chttp2_unlocking_check_writes( GRPC_CHTTP2_FLOW_MOVE_TRANSPORT("write", transport_writing, outgoing_window, transport_global, outgoing_window); bool is_window_available = transport_writing->outgoing_window > 0; - grpc_chttp2_list_flush_writing_stalled_by_transport(transport_writing, - is_window_available); + grpc_chttp2_list_flush_writing_stalled_by_transport( + exec_ctx, transport_writing, is_window_available); /* for each grpc_chttp2_stream that's become writable, frame it's data (according to available window sizes) and add to the output buffer */ @@ -133,8 +133,8 @@ int grpc_chttp2_unlocking_check_writes( GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); } } else { - grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing, - stream_writing); + grpc_chttp2_list_add_stalled_by_transport(transport_writing, + stream_writing); } } if (stream_global->send_trailing_metadata) { diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 617d98875c3..89e64876af7 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -598,7 +598,7 @@ static void lock(grpc_chttp2_transport *t) { gpr_mu_lock(&t->mu); } static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { GPR_TIMER_BEGIN("unlock", 0); if (!t->writing_active && !t->closed && - grpc_chttp2_unlocking_check_writes(&t->global, &t->writing, + grpc_chttp2_unlocking_check_writes(exec_ctx, &t->global, &t->writing, t->parsing_active)) { t->writing_active = 1; REF_TRANSPORT(t, "writing"); From 732a875fe82c80a735cc9ca6a430f962811188ab Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 22 Feb 2016 15:59:19 -0800 Subject: [PATCH 047/236] Add a sanity test for name aliasing --- build.yaml | 61 +- src/core/census/{log.c => mlog.c} | 2 +- src/core/census/{log.h => mlog.h} | 0 .../{algorithm.c => compression_algorithm.c} | 0 src/core/security/{base64.c => b64.c} | 2 +- src/core/security/{base64.h => b64.h} | 0 .../security/google_default_credentials.c | 2 +- src/core/security/json_token.c | 2 +- src/core/security/jwt_verifier.c | 2 +- src/core/security/security_connector.c | 14 +- src/core/support/{file.c => load_file.c} | 2 +- src/core/support/{file.h => load_file.h} | 0 src/core/support/tmpfile.h | 0 .../support/{file_posix.c => tmpfile_posix.c} | 2 +- .../support/{file_win32.c => tmpfile_win32.c} | 2 +- .../sources_and_headers.json.template | 47 +- test/core/bad_ssl/gen_build_yaml.py | 4 +- .../bad_ssl/{server.c => server_common.c} | 2 +- .../bad_ssl/{server.h => server_common.h} | 0 test/core/bad_ssl/servers/alpn.c | 2 +- test/core/bad_ssl/servers/cert.c | 4 +- test/core/census/{log_test.c => mlog_test.c} | 2 +- test/core/end2end/fixtures/h2_ssl+poll.c | 13 +- test/core/end2end/fixtures/h2_ssl.c | 13 +- test/core/end2end/fixtures/h2_ssl_proxy.c | 11 +- test/core/end2end/gen_build_yaml.py | 6 +- ...{channel_connectivity.c => connectivity.c} | 0 .../end2end/tests/{channel_ping.c => ping.c} | 0 .../tests/{metadata.c => simple_metadata.c} | 0 .../security/{base64_test.c => b64_test.c} | 2 +- test/core/security/create_jwt.c | 2 +- test/core/security/credentials_test.c | 17 +- test/core/security/fetch_oauth2.c | 2 +- test/core/security/json_token_test.c | 7 +- test/core/security/jwt_verifier_test.c | 2 +- test/core/security/security_connector_test.c | 15 +- .../support/{file_test.c => load_file_test.c} | 2 +- .../cpp/interop/{server.cc => server_main.cc} | 0 test/cpp/qps/client.h | 4 +- test/cpp/qps/client_async.cc | 28 +- test/cpp/qps/client_sync.cc | 2 +- test/cpp/qps/server.h | 2 +- test/cpp/qps/server_sync.cc | 12 +- test/cpp/qps/{timer.cc => usage_timer.cc} | 6 +- test/cpp/qps/{timer.h => usage_timer.h} | 6 +- .../sanity/check_sources_and_headers.py | 54 +- tools/run_tests/sources_and_headers.json | 5970 +++++++++++------ 47 files changed, 4174 insertions(+), 2154 deletions(-) rename src/core/census/{log.c => mlog.c} (99%) rename src/core/census/{log.h => mlog.h} (100%) rename src/core/compression/{algorithm.c => compression_algorithm.c} (100%) rename src/core/security/{base64.c => b64.c} (99%) rename src/core/security/{base64.h => b64.h} (100%) rename src/core/support/{file.c => load_file.c} (98%) rename src/core/support/{file.h => load_file.h} (100%) create mode 100644 src/core/support/tmpfile.h rename src/core/support/{file_posix.c => tmpfile_posix.c} (98%) rename src/core/support/{file_win32.c => tmpfile_win32.c} (98%) rename test/core/bad_ssl/{server.c => server_common.c} (98%) rename test/core/bad_ssl/{server.h => server_common.h} (100%) rename test/core/census/{log_test.c => mlog_test.c} (99%) rename test/core/end2end/tests/{channel_connectivity.c => connectivity.c} (100%) rename test/core/end2end/tests/{channel_ping.c => ping.c} (100%) rename test/core/end2end/tests/{metadata.c => simple_metadata.c} (100%) rename test/core/security/{base64_test.c => b64_test.c} (99%) rename test/core/support/{file_test.c => load_file_test.c} (99%) rename test/cpp/interop/{server.cc => server_main.cc} (100%) rename test/cpp/qps/{timer.cc => usage_timer.cc} (98%) rename test/cpp/qps/{timer.h => usage_timer.h} (95%) diff --git a/build.yaml b/build.yaml index cb6cb568448..9a20f2a55ab 100644 --- a/build.yaml +++ b/build.yaml @@ -14,12 +14,12 @@ filegroups: - include/grpc/census.h headers: - src/core/census/aggregation.h - - src/core/census/log.h + - src/core/census/mlog.h - src/core/census/rpc_metric_id.h src: - src/core/census/context.c - src/core/census/initialize.c - - src/core/census/log.c + - src/core/census/mlog.c - src/core/census/operation.c - src/core/census/placeholders.c - src/core/census/tracing.c @@ -57,13 +57,14 @@ filegroups: - src/core/profiling/timers.h - src/core/support/block_annotate.h - src/core/support/env.h - - src/core/support/file.h + - src/core/support/load_file.h - src/core/support/murmur_hash.h - src/core/support/stack_lockfree.h - src/core/support/string.h - src/core/support/string_win32.h - src/core/support/thd_internal.h - src/core/support/time_precise.h + - src/core/support/tmpfile.h src: - src/core/profiling/basic_timers.c - src/core/profiling/stap_timers.c @@ -77,11 +78,9 @@ filegroups: - src/core/support/env_linux.c - src/core/support/env_posix.c - src/core/support/env_win32.c - - src/core/support/file.c - - src/core/support/file_posix.c - - src/core/support/file_win32.c - src/core/support/histogram.c - src/core/support/host_port.c + - src/core/support/load_file.c - src/core/support/log.c - src/core/support/log_android.c - src/core/support/log_linux.c @@ -107,6 +106,8 @@ filegroups: - src/core/support/time_precise.c - src/core/support/time_win32.c - src/core/support/tls_pthread.c + - src/core/support/tmpfile_posix.c + - src/core/support/tmpfile_win32.c - src/core/support/wrap_memcpy.c - name: gpr_codegen public_headers: @@ -388,7 +389,7 @@ filegroups: - src/core/client_config/subchannel_factory.c - src/core/client_config/subchannel_index.c - src/core/client_config/uri_parser.c - - src/core/compression/algorithm.c + - src/core/compression/compression_algorithm.c - src/core/compression/message_compress.c - src/core/debug/trace.c - src/core/httpcli/format_request.c @@ -498,7 +499,7 @@ filegroups: - name: grpc_secure headers: - src/core/security/auth_filters.h - - src/core/security/base64.h + - src/core/security/b64.h - src/core/security/credentials.h - src/core/security/handshake.h - src/core/security/json_token.h @@ -513,7 +514,7 @@ filegroups: - src/core/tsi/transport_security_interface.h src: - src/core/httpcli/httpcli_security_connector.c - - src/core/security/base64.c + - src/core/security/b64.c - src/core/security/client_auth_filter.c - src/core/security/credentials.c - src/core/security/credentials_metadata.c @@ -852,7 +853,7 @@ libs: - src/proto/grpc/testing/empty.proto - src/proto/grpc/testing/messages.proto - src/proto/grpc/testing/test.proto - - test/cpp/interop/server.cc + - test/cpp/interop/server_main.cc deps: - interop_server_helper - grpc++_test_util @@ -876,7 +877,7 @@ libs: - test/cpp/qps/report.h - test/cpp/qps/server.h - test/cpp/qps/stats.h - - test/cpp/qps/timer.h + - test/cpp/qps/usage_timer.h - test/cpp/util/benchmark_config.h src: - src/proto/grpc/testing/messages.proto @@ -894,7 +895,7 @@ libs: - test/cpp/qps/report.cc - test/cpp/qps/server_async.cc - test/cpp/qps/server_sync.cc - - test/cpp/qps/timer.cc + - test/cpp/qps/usage_timer.cc - test/cpp/util/benchmark_config.cc deps: - grpc_test_util @@ -978,16 +979,6 @@ targets: - grpc - gpr_test_util - gpr -- name: census_log_test - build: test - language: c - src: - - test/core/census/log_test.c - deps: - - grpc_test_util - - grpc - - gpr_test_util - - gpr - name: channel_create_test build: test language: c @@ -1209,27 +1200,27 @@ targets: deps: - gpr_test_util - gpr -- name: gpr_file_test +- name: gpr_histogram_test build: test language: c src: - - test/core/support/file_test.c + - test/core/support/histogram_test.c deps: - gpr_test_util - gpr -- name: gpr_histogram_test +- name: gpr_host_port_test build: test language: c src: - - test/core/support/histogram_test.c + - test/core/support/host_port_test.c deps: - gpr_test_util - gpr -- name: gpr_host_port_test +- name: gpr_load_file_test build: test language: c src: - - test/core/support/host_port_test.c + - test/core/support/load_file_test.c deps: - gpr_test_util - gpr @@ -1326,11 +1317,11 @@ targets: - grpc - gpr_test_util - gpr -- name: grpc_base64_test +- name: grpc_b64_test build: test language: c src: - - test/core/security/base64_test.c + - test/core/security/b64_test.c deps: - grpc_test_util - grpc @@ -1642,6 +1633,16 @@ targets: - grpc - gpr_test_util - gpr +- name: mlog_test + build: test + language: c + src: + - test/core/census/mlog_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: multiple_server_queues_test build: test language: c diff --git a/src/core/census/log.c b/src/core/census/mlog.c similarity index 99% rename from src/core/census/log.c rename to src/core/census/mlog.c index 91b26941b83..a2cc46d3f26 100644 --- a/src/core/census/log.c +++ b/src/core/census/mlog.c @@ -88,7 +88,7 @@ // include the name of the structure, which will be passed as the first // argument. E.g. cl_block_initialize() will initialize a cl_block. -#include "src/core/census/log.h" +#include "src/core/census/mlog.h" #include #include #include diff --git a/src/core/census/log.h b/src/core/census/mlog.h similarity index 100% rename from src/core/census/log.h rename to src/core/census/mlog.h diff --git a/src/core/compression/algorithm.c b/src/core/compression/compression_algorithm.c similarity index 100% rename from src/core/compression/algorithm.c rename to src/core/compression/compression_algorithm.c diff --git a/src/core/security/base64.c b/src/core/security/b64.c similarity index 99% rename from src/core/security/base64.c rename to src/core/security/b64.c index 8367c160c36..c40b528e2f5 100644 --- a/src/core/security/base64.c +++ b/src/core/security/b64.c @@ -31,7 +31,7 @@ * */ -#include "src/core/security/base64.h" +#include "src/core/security/b64.h" #include #include diff --git a/src/core/security/base64.h b/src/core/security/b64.h similarity index 100% rename from src/core/security/base64.h rename to src/core/security/b64.h diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c index f3ac14568a6..bd4b0e462ad 100644 --- a/src/core/security/google_default_credentials.c +++ b/src/core/security/google_default_credentials.c @@ -41,7 +41,7 @@ #include "src/core/httpcli/httpcli.h" #include "src/core/support/env.h" -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include "src/core/surface/api_trace.h" /* -- Constants. -- */ diff --git a/src/core/security/json_token.c b/src/core/security/json_token.c index 762f02989ac..372e5bfc5ab 100644 --- a/src/core/security/json_token.c +++ b/src/core/security/json_token.c @@ -39,7 +39,7 @@ #include #include -#include "src/core/security/base64.h" +#include "src/core/security/b64.h" #include "src/core/support/string.h" #include diff --git a/src/core/security/jwt_verifier.c b/src/core/security/jwt_verifier.c index 042a117f5d7..928c6c148de 100644 --- a/src/core/security/jwt_verifier.c +++ b/src/core/security/jwt_verifier.c @@ -37,7 +37,7 @@ #include #include "src/core/httpcli/httpcli.h" -#include "src/core/security/base64.h" +#include "src/core/security/b64.h" #include "src/core/tsi/ssl_types.h" #include diff --git a/src/core/security/security_connector.c b/src/core/security/security_connector.c index b46205323bf..51a99b4492d 100644 --- a/src/core/security/security_connector.c +++ b/src/core/security/security_connector.c @@ -35,20 +35,20 @@ #include +#include +#include +#include +#include +#include + #include "src/core/security/credentials.h" #include "src/core/security/handshake.h" #include "src/core/security/secure_endpoint.h" #include "src/core/security/security_context.h" #include "src/core/support/env.h" -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include "src/core/support/string.h" #include "src/core/transport/chttp2/alpn.h" - -#include -#include -#include -#include -#include #include "src/core/tsi/fake_transport_security.h" #include "src/core/tsi/ssl_transport_security.h" diff --git a/src/core/support/file.c b/src/core/support/load_file.c similarity index 98% rename from src/core/support/file.c rename to src/core/support/load_file.c index 8c673dbcc66..2ea08a50664 100644 --- a/src/core/support/file.c +++ b/src/core/support/load_file.c @@ -31,7 +31,7 @@ * */ -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include #include diff --git a/src/core/support/file.h b/src/core/support/load_file.h similarity index 100% rename from src/core/support/file.h rename to src/core/support/load_file.h diff --git a/src/core/support/tmpfile.h b/src/core/support/tmpfile.h new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/core/support/file_posix.c b/src/core/support/tmpfile_posix.c similarity index 98% rename from src/core/support/file_posix.c rename to src/core/support/tmpfile_posix.c index c11c07148ab..70ecfc1b335 100644 --- a/src/core/support/file_posix.c +++ b/src/core/support/tmpfile_posix.c @@ -35,7 +35,7 @@ #ifdef GPR_POSIX_FILE -#include "src/core/support/file.h" +#include "src/core/support/tmpfile.h" #include #include diff --git a/src/core/support/file_win32.c b/src/core/support/tmpfile_win32.c similarity index 98% rename from src/core/support/file_win32.c rename to src/core/support/tmpfile_win32.c index 355744f79a5..834e2684b26 100644 --- a/src/core/support/file_win32.c +++ b/src/core/support/tmpfile_win32.c @@ -44,8 +44,8 @@ #include #include -#include "src/core/support/file.h" #include "src/core/support/string_win32.h" +#include "src/core/support/tmpfile.h" FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) { FILE *result = NULL; diff --git a/templates/tools/run_tests/sources_and_headers.json.template b/templates/tools/run_tests/sources_and_headers.json.template index 04802772af5..cf424edd0b2 100644 --- a/templates/tools/run_tests/sources_and_headers.json.template +++ b/templates/tools/run_tests/sources_and_headers.json.template @@ -3,34 +3,41 @@ <%! import json import os - + def proto_headers(src): - out = [] - for f in src: - name, ext = os.path.splitext(f) - if ext == '.proto': - out.extend(fmt % name for fmt in ['%s.grpc.pb.h', '%s.pb.h']) - return out - + out = [] + for f in src: + name, ext = os.path.splitext(f) + if ext == '.proto': + out.extend(fmt % name for fmt in ['%s.grpc.pb.h', '%s.pb.h']) + return out + def no_protos(src): - out = [] - for f in src: - if os.path.splitext(f)[1] != '.proto': - out.append(f) - return out + out = [] + for f in src: + if os.path.splitext(f)[1] != '.proto': + out.append(f) + return out + + def all_targets(targets, libs): + for tgt in targets: + yield ('target', tgt) + for tgt in libs: + yield ('lib', tgt) %> - + ${json.dumps([{"name": tgt.name, + "type": typ, "language": tgt.language, + "third_party": tgt.boringssl or tgt.zlib, "src": sorted( - no_protos(tgt.src) + - tgt.get('public_headers', []) + + no_protos(tgt.src) + + tgt.get('public_headers', []) + tgt.get('headers', [])), "headers": sorted( - tgt.get('public_headers', []) + - tgt.get('headers', []) + + tgt.get('public_headers', []) + + tgt.get('headers', []) + proto_headers(tgt.src)), "deps": sorted(tgt.get('deps', []))} - for tgt in (targets + libs) - if not tgt.boringssl and not tgt.zlib], + for typ, tgt in all_targets(targets, libs)], sort_keys=True, indent=2)} diff --git a/test/core/bad_ssl/gen_build_yaml.py b/test/core/bad_ssl/gen_build_yaml.py index cc097a8fdf6..e2a3febe5d9 100755 --- a/test/core/bad_ssl/gen_build_yaml.py +++ b/test/core/bad_ssl/gen_build_yaml.py @@ -52,8 +52,8 @@ def main(): 'name': 'bad_ssl_test_server', 'build': 'private', 'language': 'c', - 'src': ['test/core/bad_ssl/server.c'], - 'headers': ['test/core/bad_ssl/server.h'], + 'src': ['test/core/bad_ssl/server_common.c'], + 'headers': ['test/core/bad_ssl/server_common.h'], 'vs_proj_dir': 'test', 'platforms': ['linux', 'posix', 'mac'], 'deps': [ diff --git a/test/core/bad_ssl/server.c b/test/core/bad_ssl/server_common.c similarity index 98% rename from test/core/bad_ssl/server.c rename to test/core/bad_ssl/server_common.c index 6113d364c9d..14b1892c2e9 100644 --- a/test/core/bad_ssl/server.c +++ b/test/core/bad_ssl/server_common.c @@ -35,7 +35,7 @@ #include #include -#include "test/core/bad_ssl/server.h" +#include "test/core/bad_ssl/server_common.h" #include "test/core/util/test_config.h" /* Common server implementation details for all servers in servers/. diff --git a/test/core/bad_ssl/server.h b/test/core/bad_ssl/server_common.h similarity index 100% rename from test/core/bad_ssl/server.h rename to test/core/bad_ssl/server_common.h diff --git a/test/core/bad_ssl/servers/alpn.c b/test/core/bad_ssl/servers/alpn.c index 7d70690e52d..dd0eb793d47 100644 --- a/test/core/bad_ssl/servers/alpn.c +++ b/test/core/bad_ssl/servers/alpn.c @@ -39,7 +39,7 @@ #include #include "src/core/transport/chttp2/alpn.h" -#include "test/core/bad_ssl/server.h" +#include "test/core/bad_ssl/server_common.h" #include "test/core/end2end/data/ssl_test_data.h" /* This test starts a server that is configured to advertise (via alpn and npn) diff --git a/test/core/bad_ssl/servers/cert.c b/test/core/bad_ssl/servers/cert.c index d67a6ca1d4a..d262ce16d81 100644 --- a/test/core/bad_ssl/servers/cert.c +++ b/test/core/bad_ssl/servers/cert.c @@ -38,9 +38,9 @@ #include #include -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" -#include "test/core/bad_ssl/server.h" +#include "test/core/bad_ssl/server_common.h" #include "test/core/end2end/data/ssl_test_data.h" /* This server will present an untrusted cert to the connecting client, diff --git a/test/core/census/log_test.c b/test/core/census/mlog_test.c similarity index 99% rename from test/core/census/log_test.c rename to test/core/census/mlog_test.c index b68ca115045..5b6c5946ab5 100644 --- a/test/core/census/log_test.c +++ b/test/core/census/mlog_test.c @@ -31,7 +31,7 @@ * */ -#include "src/core/census/log.h" +#include "src/core/census/mlog.h" #include #include #include diff --git a/test/core/end2end/fixtures/h2_ssl+poll.c b/test/core/end2end/fixtures/h2_ssl+poll.c index 614654ed524..0d489d62e2d 100644 --- a/test/core/end2end/fixtures/h2_ssl+poll.c +++ b/test/core/end2end/fixtures/h2_ssl+poll.c @@ -36,17 +36,18 @@ #include #include +#include +#include +#include + #include "src/core/channel/channel_args.h" #include "src/core/security/credentials.h" #include "src/core/support/env.h" -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include "src/core/support/string.h" -#include -#include -#include -#include "test/core/util/test_config.h" -#include "test/core/util/port.h" #include "test/core/end2end/data/ssl_test_data.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" typedef struct fullstack_secure_fixture_data { char *localaddr; diff --git a/test/core/end2end/fixtures/h2_ssl.c b/test/core/end2end/fixtures/h2_ssl.c index 5c63dfb6aad..187399cad1b 100644 --- a/test/core/end2end/fixtures/h2_ssl.c +++ b/test/core/end2end/fixtures/h2_ssl.c @@ -36,17 +36,18 @@ #include #include +#include +#include +#include + #include "src/core/channel/channel_args.h" #include "src/core/security/credentials.h" #include "src/core/support/env.h" -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include "src/core/support/string.h" -#include -#include -#include -#include "test/core/util/test_config.h" -#include "test/core/util/port.h" #include "test/core/end2end/data/ssl_test_data.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" typedef struct fullstack_secure_fixture_data { char *localaddr; diff --git a/test/core/end2end/fixtures/h2_ssl_proxy.c b/test/core/end2end/fixtures/h2_ssl_proxy.c index a93bd83a1f1..dc758061f82 100644 --- a/test/core/end2end/fixtures/h2_ssl_proxy.c +++ b/test/core/end2end/fixtures/h2_ssl_proxy.c @@ -36,18 +36,19 @@ #include #include +#include +#include +#include + #include "src/core/channel/channel_args.h" #include "src/core/security/credentials.h" #include "src/core/support/env.h" -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include "src/core/support/string.h" -#include -#include -#include #include "test/core/end2end/data/ssl_test_data.h" #include "test/core/end2end/fixtures/proxy.h" -#include "test/core/util/test_config.h" #include "test/core/util/port.h" +#include "test/core/util/test_config.h" typedef struct fullstack_secure_fixture_data { grpc_end2end_proxy *proxy; diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index f24dbe72cf5..4dfafcea243 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -94,9 +94,8 @@ END2END_TESTS = { 'cancel_before_invoke': default_test_options._replace(cpu_cost=LOWCPU), 'cancel_in_a_vacuum': default_test_options._replace(cpu_cost=LOWCPU), 'cancel_with_status': default_test_options._replace(cpu_cost=LOWCPU), - 'channel_connectivity': connectivity_test_options._replace(proxyable=False, cpu_cost=LOWCPU), - 'channel_ping': connectivity_test_options._replace(proxyable=False), 'compressed_payload': default_test_options._replace(proxyable=False, cpu_cost=LOWCPU), + 'connectivity': connectivity_test_options._replace(proxyable=False, cpu_cost=LOWCPU), 'default_host': default_test_options._replace(needs_fullstack=True, needs_dns=True), 'disappearing_server': connectivity_test_options, @@ -109,11 +108,11 @@ END2END_TESTS = { 'large_metadata': default_test_options, 'max_concurrent_streams': default_test_options._replace(proxyable=False), 'max_message_length': default_test_options._replace(cpu_cost=LOWCPU), - 'metadata': default_test_options, 'negative_deadline': default_test_options, 'no_op': default_test_options, 'payload': default_test_options._replace(cpu_cost=LOWCPU), 'ping_pong_streaming': default_test_options, + 'ping': connectivity_test_options._replace(proxyable=False), 'registered_call': default_test_options, 'request_with_flags': default_test_options._replace(proxyable=False), 'request_with_payload': default_test_options, @@ -121,6 +120,7 @@ END2END_TESTS = { 'shutdown_finishes_calls': default_test_options, 'shutdown_finishes_tags': default_test_options, 'simple_delayed_request': connectivity_test_options._replace(cpu_cost=LOWCPU), + 'simple_metadata': default_test_options, 'simple_request': default_test_options, 'trailing_metadata': default_test_options, } diff --git a/test/core/end2end/tests/channel_connectivity.c b/test/core/end2end/tests/connectivity.c similarity index 100% rename from test/core/end2end/tests/channel_connectivity.c rename to test/core/end2end/tests/connectivity.c diff --git a/test/core/end2end/tests/channel_ping.c b/test/core/end2end/tests/ping.c similarity index 100% rename from test/core/end2end/tests/channel_ping.c rename to test/core/end2end/tests/ping.c diff --git a/test/core/end2end/tests/metadata.c b/test/core/end2end/tests/simple_metadata.c similarity index 100% rename from test/core/end2end/tests/metadata.c rename to test/core/end2end/tests/simple_metadata.c diff --git a/test/core/security/base64_test.c b/test/core/security/b64_test.c similarity index 99% rename from test/core/security/base64_test.c rename to test/core/security/b64_test.c index e656d4c947f..a45a189457e 100644 --- a/test/core/security/base64_test.c +++ b/test/core/security/b64_test.c @@ -31,7 +31,7 @@ * */ -#include "src/core/security/base64.h" +#include "src/core/security/b64.h" #include diff --git a/test/core/security/create_jwt.c b/test/core/security/create_jwt.c index 237dc9aa3e2..4c0cf436eeb 100644 --- a/test/core/security/create_jwt.c +++ b/test/core/security/create_jwt.c @@ -36,7 +36,7 @@ #include "src/core/security/credentials.h" #include "src/core/security/json_token.h" -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include #include diff --git a/test/core/security/credentials_test.c b/test/core/security/credentials_test.c index 8a210bb3c37..2c3d4cd0d6a 100644 --- a/test/core/security/credentials_test.c +++ b/test/core/security/credentials_test.c @@ -32,25 +32,24 @@ */ #include + #include "src/core/security/credentials.h" +#include #include #include -#include "src/core/httpcli/httpcli.h" -#include "src/core/security/json_token.h" -#include "src/core/support/env.h" -#include "src/core/support/file.h" -#include "src/core/support/string.h" - -#include "test/core/util/test_config.h" - #include #include #include #include -#include +#include "src/core/httpcli/httpcli.h" +#include "src/core/security/json_token.h" +#include "src/core/support/env.h" +#include "src/core/support/load_file.h" +#include "src/core/support/string.h" +#include "test/core/util/test_config.h" /* -- Mock channel credentials. -- */ diff --git a/test/core/security/fetch_oauth2.c b/test/core/security/fetch_oauth2.c index ee1178cbddf..a9e1720a28d 100644 --- a/test/core/security/fetch_oauth2.c +++ b/test/core/security/fetch_oauth2.c @@ -43,7 +43,7 @@ #include #include "src/core/security/credentials.h" -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include "test/core/security/oauth2_utils.h" static grpc_call_credentials *create_refresh_token_creds( diff --git a/test/core/security/json_token_test.c b/test/core/security/json_token_test.c index 7c01a9ce5c9..4d80c16fb98 100644 --- a/test/core/security/json_token_test.c +++ b/test/core/security/json_token_test.c @@ -33,16 +33,17 @@ #include "src/core/security/json_token.h" +#include #include -#include "src/core/security/base64.h" #include #include #include #include -#include "test/core/util/test_config.h" + #include "src/core/json/json.h" -#include +#include "src/core/security/b64.h" +#include "test/core/util/test_config.h" /* This JSON key was generated with the GCE console and revoked immediately. The identifiers have been changed as well. diff --git a/test/core/security/jwt_verifier_test.c b/test/core/security/jwt_verifier_test.c index f396398cefb..f6ec9e12ef3 100644 --- a/test/core/security/jwt_verifier_test.c +++ b/test/core/security/jwt_verifier_test.c @@ -36,7 +36,7 @@ #include #include "src/core/httpcli/httpcli.h" -#include "src/core/security/base64.h" +#include "src/core/security/b64.h" #include "src/core/security/json_token.h" #include "test/core/util/test_config.h" diff --git a/test/core/security/security_connector_test.c b/test/core/security/security_connector_test.c index ee5435f01dd..edb45b66804 100644 --- a/test/core/security/security_connector_test.c +++ b/test/core/security/security_connector_test.c @@ -34,22 +34,21 @@ #include #include +#include +#include +#include +#include +#include + #include "src/core/security/security_connector.h" #include "src/core/security/security_context.h" #include "src/core/support/env.h" -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include "src/core/support/string.h" #include "src/core/tsi/ssl_transport_security.h" #include "src/core/tsi/transport_security.h" #include "test/core/util/test_config.h" -#include - -#include -#include -#include -#include - static int check_transport_security_type(const grpc_auth_context *ctx) { grpc_auth_property_iterator it = grpc_auth_context_find_properties_by_name( ctx, GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME); diff --git a/test/core/support/file_test.c b/test/core/support/load_file_test.c similarity index 99% rename from test/core/support/file_test.c rename to test/core/support/load_file_test.c index 330b2173ef9..59d6ba8912e 100644 --- a/test/core/support/file_test.c +++ b/test/core/support/load_file_test.c @@ -38,7 +38,7 @@ #include #include -#include "src/core/support/file.h" +#include "src/core/support/load_file.h" #include "src/core/support/string.h" #include "test/core/util/test_config.h" diff --git a/test/cpp/interop/server.cc b/test/cpp/interop/server_main.cc similarity index 100% rename from test/cpp/interop/server.cc rename to test/cpp/interop/server_main.cc diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index c94a523fa10..aecbae95f9e 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -46,10 +46,10 @@ #include "src/proto/grpc/testing/payloads.grpc.pb.h" #include "src/proto/grpc/testing/services.grpc.pb.h" -#include "test/cpp/qps/limit_cores.h" #include "test/cpp/qps/histogram.h" #include "test/cpp/qps/interarrival.h" -#include "test/cpp/qps/timer.h" +#include "test/cpp/qps/limit_cores.h" +#include "test/cpp/qps/usage_timer.h" #include "test/cpp/util/create_test_channel.h" namespace grpc { diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 9e8767d1033..f8c1fa3b62b 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -54,7 +54,7 @@ #include "src/proto/grpc/testing/services.grpc.pb.h" #include "test/cpp/qps/client.h" -#include "test/cpp/qps/timer.h" +#include "test/cpp/qps/usage_timer.h" #include "test/cpp/util/create_test_channel.h" namespace grpc { @@ -84,7 +84,8 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function< std::unique_ptr>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*)> start_req, + CompletionQueue*)> + start_req, std::function on_done) : context_(), stub_(stub), @@ -141,7 +142,8 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function next_issue_; std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*)> start_req_; + CompletionQueue*)> + start_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -164,7 +166,8 @@ class AsyncClient : public ClientImpl { AsyncClient(const ClientConfig& config, std::function next_issue, - const RequestType&)> setup_ctx, + const RequestType&)> + setup_ctx, std::function(std::shared_ptr)> create_stub) : ClientImpl(config, create_stub), @@ -277,7 +280,8 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { std::function>( BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, - void*)> start_req, + void*)> + start_req, std::function on_done) : context_(), stub_(stub), @@ -360,10 +364,10 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { State next_state_; std::function callback_; std::function next_issue_; - std::function< - std::unique_ptr>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, - void*)> start_req_; + std::function>( + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, void*)> + start_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -405,7 +409,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function next_issue, std::function( grpc::GenericStub*, grpc::ClientContext*, - const grpc::string& method_name, CompletionQueue*, void*)> start_req, + const grpc::string& method_name, CompletionQueue*, void*)> + start_req, std::function on_done) : context_(), stub_(stub), @@ -493,7 +498,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function next_issue_; std::function( grpc::GenericStub*, grpc::ClientContext*, const grpc::string&, - CompletionQueue*, void*)> start_req_; + CompletionQueue*, void*)> + start_req_; grpc::Status status_; double start_; std::unique_ptr stream_; diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc index edfc246a256..e39768b4981 100644 --- a/test/cpp/qps/client_sync.cc +++ b/test/cpp/qps/client_sync.cc @@ -58,7 +58,7 @@ #include "test/cpp/qps/client.h" #include "test/cpp/qps/histogram.h" #include "test/cpp/qps/interarrival.h" -#include "test/cpp/qps/timer.h" +#include "test/cpp/qps/usage_timer.h" namespace grpc { namespace testing { diff --git a/test/cpp/qps/server.h b/test/cpp/qps/server.h index 94a6f8acfab..3227347e3ff 100644 --- a/test/cpp/qps/server.h +++ b/test/cpp/qps/server.h @@ -43,7 +43,7 @@ #include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/port.h" #include "test/cpp/qps/limit_cores.h" -#include "test/cpp/qps/timer.h" +#include "test/cpp/qps/usage_timer.h" namespace grpc { namespace testing { diff --git a/test/cpp/qps/server_sync.cc b/test/cpp/qps/server_sync.cc index 4b778820d07..b7682f57635 100644 --- a/test/cpp/qps/server_sync.cc +++ b/test/cpp/qps/server_sync.cc @@ -34,18 +34,18 @@ #include #include +#include +#include +#include +#include #include #include #include #include -#include -#include -#include -#include -#include "test/cpp/qps/server.h" -#include "test/cpp/qps/timer.h" #include "src/proto/grpc/testing/services.grpc.pb.h" +#include "test/cpp/qps/server.h" +#include "test/cpp/qps/usage_timer.h" namespace grpc { namespace testing { diff --git a/test/cpp/qps/timer.cc b/test/cpp/qps/usage_timer.cc similarity index 98% rename from test/cpp/qps/timer.cc rename to test/cpp/qps/usage_timer.cc index 3ec7f49f832..5a913d498fa 100644 --- a/test/cpp/qps/timer.cc +++ b/test/cpp/qps/usage_timer.cc @@ -31,11 +31,11 @@ * */ -#include "test/cpp/qps/timer.h" +#include "test/cpp/qps/usage_timer.h" -#include -#include #include +#include +#include Timer::Timer() : start_(Sample()) {} diff --git a/test/cpp/qps/timer.h b/test/cpp/qps/usage_timer.h similarity index 95% rename from test/cpp/qps/timer.h rename to test/cpp/qps/usage_timer.h index d1aee1a9d19..704d4b02a2e 100644 --- a/test/cpp/qps/timer.h +++ b/test/cpp/qps/usage_timer.h @@ -31,10 +31,10 @@ * */ -#ifndef TEST_QPS_TIMER_H -#define TEST_QPS_TIMER_H +#ifndef TEST_QPS_USAGE_TIMER_H +#define TEST_QPS_USAGE_TIMER_H -class Timer { +class UsageTimer { public: Timer(); diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index 3974af0032b..44dc49bb06f 100755 --- a/tools/run_tests/sanity/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -59,25 +59,43 @@ def target_has_header(target, name): return True return False +def produces_object(name): + return os.path.splitext(name)[1] in ['.c', '.cc'] + +obj_producer_to_source = {'c': {}, 'c++': {}, 'csharp': {}} + errors = 0 for target in js: - for fn in target['src']: - with open(os.path.join(root, fn)) as f: - src = f.read().splitlines() - for line in src: - m = re_inc1.match(line) - if m: - if not target_has_header(target, m.group(1)): - print ( - 'target %s (%s) does not name header %s as a dependency' % ( - target['name'], fn, m.group(1))) - errors += 1 - m = re_inc2.match(line) - if m: - if not target_has_header(target, 'include/' + m.group(1)): - print ( - 'target %s (%s) does not name header %s as a dependency' % ( - target['name'], fn, m.group(1))) - errors += 1 + if not target['third_party']: + for fn in target['src']: + with open(os.path.join(root, fn)) as f: + src = f.read().splitlines() + for line in src: + m = re_inc1.match(line) + if m: + if not target_has_header(target, m.group(1)): + print ( + 'target %s (%s) does not name header %s as a dependency' % ( + target['name'], fn, m.group(1))) + errors += 1 + m = re_inc2.match(line) + if m: + if not target_has_header(target, 'include/' + m.group(1)): + print ( + 'target %s (%s) does not name header %s as a dependency' % ( + target['name'], fn, m.group(1))) + errors += 1 + if target['type'] == 'lib': + for fn in target['src']: + language = target['language'] + if produces_object(fn): + obj_base = os.path.splitext(os.path.basename(fn))[0] + if obj_base in obj_producer_to_source[language]: + if obj_producer_to_source[language][obj_base] != fn: + print ( + 'target %s (%s) produces an aliased object file with %s' % ( + target['name'], fn, obj_producer_to_source[language][obj_base])) + else: + obj_producer_to_source[language][obj_base] = fn assert errors == 0 diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 3cdd7b453cb..a1bdd5e6e9b 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -13,7 +13,9 @@ "name": "alarm_test", "src": [ "test/core/surface/alarm_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -27,7 +29,9 @@ "name": "algorithm_test", "src": [ "test/core/compression/algorithm_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -39,7 +43,9 @@ "name": "alloc_test", "src": [ "test/core/support/alloc_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -53,7 +59,9 @@ "name": "alpn_test", "src": [ "test/core/transport/chttp2/alpn_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -65,7 +73,9 @@ "name": "bin_encoder_test", "src": [ "test/core/transport/chttp2/bin_encoder_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -79,21 +89,9 @@ "name": "census_context_test", "src": [ "test/core/census/context_test.c" - ] - }, - { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" ], - "headers": [], - "language": "c", - "name": "census_log_test", - "src": [ - "test/core/census/log_test.c" - ] + "third_party": false, + "type": "target" }, { "deps": [ @@ -107,7 +105,9 @@ "name": "channel_create_test", "src": [ "test/core/surface/channel_create_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -121,7 +121,9 @@ "name": "chttp2_hpack_encoder_test", "src": [ "test/core/transport/chttp2/hpack_encoder_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -135,7 +137,9 @@ "name": "chttp2_status_conversion_test", "src": [ "test/core/transport/chttp2/status_conversion_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -149,7 +153,9 @@ "name": "chttp2_stream_map_test", "src": [ "test/core/transport/chttp2/stream_map_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -163,7 +169,9 @@ "name": "chttp2_varint_test", "src": [ "test/core/transport/chttp2/varint_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -177,7 +185,9 @@ "name": "compression_test", "src": [ "test/core/compression/compression_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -191,7 +201,9 @@ "name": "dns_resolver_test", "src": [ "test/core/client_config/resolvers/dns_resolver_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -205,7 +217,9 @@ "name": "dualstack_socket_test", "src": [ "test/core/end2end/dualstack_socket_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -219,7 +233,9 @@ "name": "endpoint_pair_test", "src": [ "test/core/iomgr/endpoint_pair_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -233,7 +249,9 @@ "name": "fd_conservation_posix_test", "src": [ "test/core/iomgr/fd_conservation_posix_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -247,7 +265,9 @@ "name": "fd_posix_test", "src": [ "test/core/iomgr/fd_posix_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -261,7 +281,9 @@ "name": "fling_client", "src": [ "test/core/fling/client.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -275,7 +297,9 @@ "name": "fling_server", "src": [ "test/core/fling/server.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -289,7 +313,9 @@ "name": "fling_stream_test", "src": [ "test/core/fling/fling_stream_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -303,7 +329,9 @@ "name": "fling_test", "src": [ "test/core/fling/fling_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -315,7 +343,9 @@ "name": "gen_hpack_tables", "src": [ "tools/codegen/core/gen_hpack_tables.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [], @@ -324,7 +354,9 @@ "name": "gen_legal_metadata_characters", "src": [ "tools/codegen/core/gen_legal_metadata_characters.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -336,7 +368,9 @@ "name": "gpr_avl_test", "src": [ "test/core/support/avl_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -348,7 +382,9 @@ "name": "gpr_cmdline_test", "src": [ "test/core/support/cmdline_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -360,7 +396,9 @@ "name": "gpr_cpu_test", "src": [ "test/core/support/cpu_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -372,7 +410,9 @@ "name": "gpr_env_test", "src": [ "test/core/support/env_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -381,10 +421,12 @@ ], "headers": [], "language": "c", - "name": "gpr_file_test", + "name": "gpr_histogram_test", "src": [ - "test/core/support/file_test.c" - ] + "test/core/support/histogram_test.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -393,10 +435,12 @@ ], "headers": [], "language": "c", - "name": "gpr_histogram_test", + "name": "gpr_host_port_test", "src": [ - "test/core/support/histogram_test.c" - ] + "test/core/support/host_port_test.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -405,10 +449,12 @@ ], "headers": [], "language": "c", - "name": "gpr_host_port_test", + "name": "gpr_load_file_test", "src": [ - "test/core/support/host_port_test.c" - ] + "test/core/support/load_file_test.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -420,7 +466,9 @@ "name": "gpr_log_test", "src": [ "test/core/support/log_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -432,7 +480,9 @@ "name": "gpr_slice_buffer_test", "src": [ "test/core/support/slice_buffer_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -444,7 +494,9 @@ "name": "gpr_slice_test", "src": [ "test/core/support/slice_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -456,7 +508,9 @@ "name": "gpr_stack_lockfree_test", "src": [ "test/core/support/stack_lockfree_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -468,7 +522,9 @@ "name": "gpr_string_test", "src": [ "test/core/support/string_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -480,7 +536,9 @@ "name": "gpr_sync_test", "src": [ "test/core/support/sync_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -492,7 +550,9 @@ "name": "gpr_thd_test", "src": [ "test/core/support/thd_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -504,7 +564,9 @@ "name": "gpr_time_test", "src": [ "test/core/support/time_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -516,7 +578,9 @@ "name": "gpr_tls_test", "src": [ "test/core/support/tls_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -528,7 +592,9 @@ "name": "gpr_useful_test", "src": [ "test/core/support/useful_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -542,7 +608,9 @@ "name": "grpc_auth_context_test", "src": [ "test/core/security/auth_context_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -553,10 +621,12 @@ ], "headers": [], "language": "c", - "name": "grpc_base64_test", + "name": "grpc_b64_test", "src": [ - "test/core/security/base64_test.c" - ] + "test/core/security/b64_test.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -570,7 +640,9 @@ "name": "grpc_byte_buffer_reader_test", "src": [ "test/core/surface/byte_buffer_reader_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -584,7 +656,9 @@ "name": "grpc_channel_args_test", "src": [ "test/core/channel/channel_args_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -598,7 +672,9 @@ "name": "grpc_channel_stack_test", "src": [ "test/core/channel/channel_stack_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -612,7 +688,9 @@ "name": "grpc_completion_queue_test", "src": [ "test/core/surface/completion_queue_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -626,7 +704,9 @@ "name": "grpc_create_jwt", "src": [ "test/core/security/create_jwt.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -640,7 +720,9 @@ "name": "grpc_credentials_test", "src": [ "test/core/security/credentials_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -654,7 +736,9 @@ "name": "grpc_fetch_oauth2", "src": [ "test/core/security/fetch_oauth2.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -668,7 +752,9 @@ "name": "grpc_invalid_channel_args_test", "src": [ "test/core/surface/invalid_channel_args_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -682,7 +768,9 @@ "name": "grpc_json_token_test", "src": [ "test/core/security/json_token_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -696,7 +784,9 @@ "name": "grpc_jwt_verifier_test", "src": [ "test/core/security/jwt_verifier_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -710,7 +800,9 @@ "name": "grpc_print_google_default_creds_token", "src": [ "test/core/security/print_google_default_creds_token.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -724,7 +816,9 @@ "name": "grpc_security_connector_test", "src": [ "test/core/security/security_connector_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -738,7 +832,9 @@ "name": "grpc_verify_jwt", "src": [ "test/core/security/verify_jwt.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -752,7 +848,9 @@ "name": "hpack_parser_test", "src": [ "test/core/transport/chttp2/hpack_parser_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -766,7 +864,9 @@ "name": "hpack_table_test", "src": [ "test/core/transport/chttp2/hpack_table_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -780,7 +880,9 @@ "name": "httpcli_format_request_test", "src": [ "test/core/httpcli/format_request_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -794,7 +896,9 @@ "name": "httpcli_parser_test", "src": [ "test/core/httpcli/parser_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -808,7 +912,9 @@ "name": "httpcli_test", "src": [ "test/core/httpcli/httpcli_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -822,7 +928,9 @@ "name": "httpscli_test", "src": [ "test/core/httpcli/httpscli_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -836,7 +944,9 @@ "name": "init_test", "src": [ "test/core/surface/init_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -850,7 +960,9 @@ "name": "invalid_call_argument_test", "src": [ "test/core/end2end/invalid_call_argument_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -862,7 +974,9 @@ "name": "json_rewrite", "src": [ "test/core/json/json_rewrite.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -876,7 +990,9 @@ "name": "json_rewrite_test", "src": [ "test/core/json/json_rewrite_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -890,7 +1006,9 @@ "name": "json_stream_error_test", "src": [ "test/core/json/json_stream_error_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -904,7 +1022,9 @@ "name": "json_test", "src": [ "test/core/json/json_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -918,7 +1038,9 @@ "name": "lame_client_test", "src": [ "test/core/surface/lame_client_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -932,7 +1054,9 @@ "name": "lb_policies_test", "src": [ "test/core/client_config/lb_policies_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -946,7 +1070,9 @@ "name": "low_level_ping_pong_benchmark", "src": [ "test/core/network_benchmarks/low_level_ping_pong.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -960,7 +1086,25 @@ "name": "message_compress_test", "src": [ "test/core/compression/message_compress_test.c" - ] + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "mlog_test", + "src": [ + "test/core/census/mlog_test.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -974,7 +1118,9 @@ "name": "multiple_server_queues_test", "src": [ "test/core/end2end/multiple_server_queues_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -986,7 +1132,9 @@ "name": "murmur_hash_test", "src": [ "test/core/support/murmur_hash_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1000,7 +1148,9 @@ "name": "no_server_test", "src": [ "test/core/end2end/no_server_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1014,7 +1164,9 @@ "name": "resolve_address_test", "src": [ "test/core/iomgr/resolve_address_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1028,7 +1180,9 @@ "name": "secure_channel_create_test", "src": [ "test/core/surface/secure_channel_create_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1042,7 +1196,9 @@ "name": "secure_endpoint_test", "src": [ "test/core/security/secure_endpoint_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1056,7 +1212,9 @@ "name": "server_chttp2_test", "src": [ "test/core/surface/server_chttp2_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1070,7 +1228,9 @@ "name": "server_test", "src": [ "test/core/surface/server_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1085,7 +1245,9 @@ "name": "set_initial_connect_string_test", "src": [ "test/core/client_config/set_initial_connect_string_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1099,7 +1261,9 @@ "name": "sockaddr_resolver_test", "src": [ "test/core/client_config/resolvers/sockaddr_resolver_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1113,7 +1277,9 @@ "name": "sockaddr_utils_test", "src": [ "test/core/iomgr/sockaddr_utils_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1127,7 +1293,9 @@ "name": "socket_utils_test", "src": [ "test/core/iomgr/socket_utils_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1141,7 +1309,9 @@ "name": "tcp_client_posix_test", "src": [ "test/core/iomgr/tcp_client_posix_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1155,7 +1325,9 @@ "name": "tcp_posix_test", "src": [ "test/core/iomgr/tcp_posix_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1169,7 +1341,9 @@ "name": "tcp_server_posix_test", "src": [ "test/core/iomgr/tcp_server_posix_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1183,7 +1357,9 @@ "name": "time_averaged_stats_test", "src": [ "test/core/iomgr/time_averaged_stats_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1197,7 +1373,9 @@ "name": "timeout_encoding_test", "src": [ "test/core/transport/chttp2/timeout_encoding_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1211,7 +1389,9 @@ "name": "timer_heap_test", "src": [ "test/core/iomgr/timer_heap_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1225,7 +1405,9 @@ "name": "timer_list_test", "src": [ "test/core/iomgr/timer_list_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1239,7 +1421,9 @@ "name": "timers_test", "src": [ "test/core/profiling/timers_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1253,7 +1437,9 @@ "name": "transport_connectivity_state_test", "src": [ "test/core/transport/connectivity_state_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1267,7 +1453,9 @@ "name": "transport_metadata_test", "src": [ "test/core/transport/metadata_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1281,7 +1469,9 @@ "name": "transport_security_test", "src": [ "test/core/tsi/transport_security_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1295,7 +1485,9 @@ "name": "udp_server_test", "src": [ "test/core/iomgr/udp_server_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1309,7 +1501,9 @@ "name": "uri_parser_test", "src": [ "test/core/client_config/uri_parser_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1323,7 +1517,9 @@ "name": "workqueue_test", "src": [ "test/core/iomgr/workqueue_test.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1339,7 +1535,9 @@ "name": "alarm_cpp_test", "src": [ "test/cpp/common/alarm_cpp_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1355,7 +1553,9 @@ "name": "async_end2end_test", "src": [ "test/cpp/end2end/async_end2end_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1372,7 +1572,9 @@ "name": "async_streaming_ping_pong_test", "src": [ "test/cpp/qps/async_streaming_ping_pong_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1389,7 +1591,9 @@ "name": "async_unary_ping_pong_test", "src": [ "test/cpp/qps/async_unary_ping_pong_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1405,7 +1609,9 @@ "name": "auth_property_iterator_test", "src": [ "test/cpp/common/auth_property_iterator_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1418,7 +1624,9 @@ "name": "channel_arguments_test", "src": [ "test/cpp/common/channel_arguments_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1434,7 +1642,9 @@ "name": "cli_call_test", "src": [ "test/cpp/util/cli_call_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1450,7 +1660,9 @@ "name": "client_crash_test", "src": [ "test/cpp/end2end/client_crash_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1466,7 +1678,9 @@ "name": "client_crash_test_server", "src": [ "test/cpp/end2end/client_crash_test_server.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1479,7 +1693,9 @@ "name": "credentials_test", "src": [ "test/cpp/client/credentials_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1494,7 +1710,9 @@ "name": "cxx_byte_buffer_test", "src": [ "test/cpp/util/byte_buffer_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1509,7 +1727,9 @@ "name": "cxx_slice_test", "src": [ "test/cpp/util/slice_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1520,7 +1740,9 @@ "name": "cxx_string_ref_test", "src": [ "test/cpp/util/string_ref_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1535,7 +1757,9 @@ "name": "cxx_time_test", "src": [ "test/cpp/util/time_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1551,7 +1775,9 @@ "name": "end2end_test", "src": [ "test/cpp/end2end/end2end_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1568,7 +1794,9 @@ "name": "generic_async_streaming_ping_pong_test", "src": [ "test/cpp/qps/generic_async_streaming_ping_pong_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1584,7 +1812,9 @@ "name": "generic_end2end_test", "src": [ "test/cpp/end2end/generic_end2end_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1601,7 +1831,9 @@ "name": "grpc_cli", "src": [ "test/cpp/util/grpc_cli.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1612,7 +1844,9 @@ "name": "grpc_cpp_plugin", "src": [ "src/compiler/cpp_plugin.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1623,7 +1857,9 @@ "name": "grpc_csharp_plugin", "src": [ "src/compiler/csharp_plugin.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1634,7 +1870,9 @@ "name": "grpc_objective_c_plugin", "src": [ "src/compiler/objective_c_plugin.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1645,7 +1883,9 @@ "name": "grpc_python_plugin", "src": [ "src/compiler/python_plugin.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1656,7 +1896,9 @@ "name": "grpc_ruby_plugin", "src": [ "src/compiler/ruby_plugin.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1672,7 +1914,9 @@ "name": "hybrid_end2end_test", "src": [ "test/cpp/end2end/hybrid_end2end_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1689,7 +1933,9 @@ "headers": [], "language": "c++", "name": "interop_client", - "src": [] + "src": [], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1706,7 +1952,9 @@ "headers": [], "language": "c++", "name": "interop_server", - "src": [] + "src": [], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1720,7 +1968,9 @@ "name": "interop_test", "src": [ "test/cpp/interop/interop_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1739,7 +1989,9 @@ "src": [ "test/cpp/interop/metrics_client.cc", "test/cpp/util/metrics_server.h" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1755,7 +2007,9 @@ "name": "mock_test", "src": [ "test/cpp/end2end/mock_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1773,7 +2027,9 @@ "name": "qps_driver", "src": [ "test/cpp/qps/qps_driver.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1790,7 +2046,9 @@ "name": "qps_interarrival_test", "src": [ "test/cpp/qps/qps_interarrival_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1808,7 +2066,9 @@ "name": "qps_openloop_test", "src": [ "test/cpp/qps/qps_openloop_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1826,7 +2086,9 @@ "name": "qps_test", "src": [ "test/cpp/qps/qps_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1849,7 +2111,9 @@ "test/cpp/qps/client.h", "test/cpp/qps/server.h", "test/cpp/qps/worker.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1873,7 +2137,9 @@ "name": "reconnect_interop_client", "src": [ "test/cpp/interop/reconnect_interop_client.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1899,7 +2165,9 @@ "name": "reconnect_interop_server", "src": [ "test/cpp/interop/reconnect_interop_server.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1915,7 +2183,9 @@ "name": "secure_auth_context_test", "src": [ "test/cpp/common/secure_auth_context_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1932,7 +2202,9 @@ "name": "secure_sync_unary_ping_pong_test", "src": [ "test/cpp/qps/secure_sync_unary_ping_pong_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1948,7 +2220,9 @@ "name": "server_crash_test", "src": [ "test/cpp/end2end/server_crash_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1964,7 +2238,9 @@ "name": "server_crash_test_client", "src": [ "test/cpp/end2end/server_crash_test_client.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1980,7 +2256,9 @@ "name": "shutdown_test", "src": [ "test/cpp/end2end/shutdown_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -1995,7 +2273,9 @@ "name": "status_test", "src": [ "test/cpp/util/status_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -2011,7 +2291,9 @@ "name": "streaming_throughput_test", "src": [ "test/cpp/end2end/streaming_throughput_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -2048,7 +2330,9 @@ "test/cpp/interop/stress_test.cc", "test/cpp/util/metrics_server.cc", "test/cpp/util/metrics_server.h" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -2065,7 +2349,9 @@ "name": "sync_streaming_ping_pong_test", "src": [ "test/cpp/qps/sync_streaming_ping_pong_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -2082,7 +2368,9 @@ "name": "sync_unary_ping_pong_test", "src": [ "test/cpp/qps/sync_unary_ping_pong_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -2098,7 +2386,9 @@ "name": "thread_stress_test", "src": [ "test/cpp/end2end/thread_stress_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -2118,7 +2408,9 @@ "name": "zookeeper_test", "src": [ "test/cpp/end2end/zookeeper_test.cc" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ @@ -2130,522 +2422,481 @@ "name": "public_headers_must_be_c89", "src": [ "test/core/surface/public_headers_must_be_c89.c" - ] + ], + "third_party": false, + "type": "target" }, { "deps": [ - "bad_client_test", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_aes_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "badreq_bad_client_test", - "src": [ - "test/core/bad_client/tests/badreq.c" - ] + "language": "c++", + "name": "boringssl_aes_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "bad_client_test", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_base64_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "connection_prefix_bad_client_test", - "src": [ - "test/core/bad_client/tests/connection_prefix.c" - ] + "language": "c++", + "name": "boringssl_base64_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "bad_client_test", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_bio_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "headers_bad_client_test", - "src": [ - "test/core/bad_client/tests/headers.c" - ] + "language": "c++", + "name": "boringssl_bio_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "bad_client_test", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_bn_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "initial_settings_frame_bad_client_test", - "src": [ - "test/core/bad_client/tests/initial_settings_frame.c" - ] + "language": "c++", + "name": "boringssl_bn_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "bad_client_test", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_bytestring_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "server_registered_method_bad_client_test", - "src": [ - "test/core/bad_client/tests/server_registered_method.c" - ] + "language": "c++", + "name": "boringssl_bytestring_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "bad_client_test", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_aead_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "simple_request_bad_client_test", - "src": [ - "test/core/bad_client/tests/simple_request.c" - ] + "language": "c++", + "name": "boringssl_aead_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "bad_client_test", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_cipher_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "unknown_frame_bad_client_test", - "src": [ - "test/core/bad_client/tests/unknown_frame.c" - ] + "language": "c++", + "name": "boringssl_cipher_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "bad_client_test", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_cmac_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "window_overflow_bad_client_test", - "src": [ - "test/core/bad_client/tests/window_overflow.c" - ] + "language": "c++", + "name": "boringssl_cmac_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "bad_ssl_test_server", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_constant_time_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "bad_ssl_alpn_server", - "src": [ - "test/core/bad_ssl/servers/alpn.c" - ] + "language": "c++", + "name": "boringssl_constant_time_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "bad_ssl_test_server", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_ed25519_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "bad_ssl_cert_server", - "src": [ - "test/core/bad_ssl/servers/cert.c" - ] + "language": "c++", + "name": "boringssl_ed25519_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_test_util", + "boringssl_x25519_test_lib" ], "headers": [], - "language": "c", - "name": "bad_ssl_alpn_test", - "src": [ - "test/core/bad_ssl/bad_ssl_test.c" - ] + "language": "c++", + "name": "boringssl_x25519_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_dh_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "bad_ssl_cert_test", - "src": [ - "test/core/bad_ssl/bad_ssl_test.c" - ] + "language": "c++", + "name": "boringssl_dh_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_digest_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_census_test", - "src": [ - "test/core/end2end/fixtures/h2_census.c" - ] + "language": "c++", + "name": "boringssl_digest_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_dsa_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_compress_test", - "src": [ - "test/core/end2end/fixtures/h2_compress.c" - ] + "language": "c++", + "name": "boringssl_dsa_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_ec_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_fakesec_test", - "src": [ - "test/core/end2end/fixtures/h2_fakesec.c" - ] + "language": "c++", + "name": "boringssl_ec_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_example_mul_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_full_test", - "src": [ - "test/core/end2end/fixtures/h2_full.c" - ] + "language": "c++", + "name": "boringssl_example_mul", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_ecdsa_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_full+pipe_test", - "src": [ - "test/core/end2end/fixtures/h2_full+pipe.c" - ] + "language": "c++", + "name": "boringssl_ecdsa_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_err_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_full+poll_test", - "src": [ - "test/core/end2end/fixtures/h2_full+poll.c" - ] + "language": "c++", + "name": "boringssl_err_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_evp_extra_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_full+poll+pipe_test", - "src": [ - "test/core/end2end/fixtures/h2_full+poll+pipe.c" - ] + "language": "c++", + "name": "boringssl_evp_extra_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_evp_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_oauth2_test", - "src": [ - "test/core/end2end/fixtures/h2_oauth2.c" - ] + "language": "c++", + "name": "boringssl_evp_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_pbkdf_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_proxy_test", - "src": [ - "test/core/end2end/fixtures/h2_proxy.c" - ] + "language": "c++", + "name": "boringssl_pbkdf_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_hkdf_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_sockpair_test", - "src": [ - "test/core/end2end/fixtures/h2_sockpair.c" - ] + "language": "c++", + "name": "boringssl_hkdf_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_hmac_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_sockpair+trace_test", - "src": [ - "test/core/end2end/fixtures/h2_sockpair+trace.c" - ] + "language": "c++", + "name": "boringssl_hmac_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_lhash_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_sockpair_1byte_test", - "src": [ - "test/core/end2end/fixtures/h2_sockpair_1byte.c" - ] + "language": "c++", + "name": "boringssl_lhash_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_gcm_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_ssl_test", - "src": [ - "test/core/end2end/fixtures/h2_ssl.c" - ] + "language": "c++", + "name": "boringssl_gcm_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_pkcs12_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_ssl+poll_test", - "src": [ - "test/core/end2end/fixtures/h2_ssl+poll.c" - ] + "language": "c++", + "name": "boringssl_pkcs12_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_pkcs8_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_ssl_proxy_test", - "src": [ - "test/core/end2end/fixtures/h2_ssl_proxy.c" - ] + "language": "c++", + "name": "boringssl_pkcs8_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_poly1305_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_uchannel_test", - "src": [ - "test/core/end2end/fixtures/h2_uchannel.c" - ] + "language": "c++", + "name": "boringssl_poly1305_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_refcount_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_uds_test", - "src": [ - "test/core/end2end/fixtures/h2_uds.c" - ] + "language": "c++", + "name": "boringssl_refcount_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_certs", - "end2end_tests", - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" + "boringssl", + "boringssl_rsa_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_uds+poll_test", - "src": [ - "test/core/end2end/fixtures/h2_uds+poll.c" - ] + "language": "c++", + "name": "boringssl_rsa_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_test_util", + "boringssl_thread_test_lib" ], "headers": [], - "language": "c", - "name": "h2_census_nosec_test", - "src": [ - "test/core/end2end/fixtures/h2_census.c" - ] + "language": "c++", + "name": "boringssl_thread_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_pkcs7_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_compress_nosec_test", - "src": [ - "test/core/end2end/fixtures/h2_compress.c" - ] + "language": "c++", + "name": "boringssl_pkcs7_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "boringssl", + "boringssl_tab_test_lib", + "boringssl_test_util" ], "headers": [], - "language": "c", - "name": "h2_full_nosec_test", - "src": [ - "test/core/end2end/fixtures/h2_full.c" - ] + "language": "c++", + "name": "boringssl_tab_test", + "src": [], + "third_party": true, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "boringssl", + "boringssl_test_util", + "boringssl_v3name_test_lib" + ], + "headers": [], + "language": "c++", + "name": "boringssl_v3name_test", + "src": [], + "third_party": true, + "type": "target" + }, + { + "deps": [ + "boringssl", + "boringssl_pqueue_test_lib", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_pqueue_test", + "src": [], + "third_party": true, + "type": "target" + }, + { + "deps": [ + "boringssl", + "boringssl_ssl_test_lib", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_ssl_test", + "src": [], + "third_party": true, + "type": "target" + }, + { + "deps": [ + "bad_client_test", "gpr", "gpr_test_util", "grpc_test_util_unsecure", @@ -2653,14 +2904,16 @@ ], "headers": [], "language": "c", - "name": "h2_full+pipe_nosec_test", + "name": "badreq_bad_client_test", "src": [ - "test/core/end2end/fixtures/h2_full+pipe.c" - ] + "test/core/bad_client/tests/badreq.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "bad_client_test", "gpr", "gpr_test_util", "grpc_test_util_unsecure", @@ -2668,14 +2921,16 @@ ], "headers": [], "language": "c", - "name": "h2_full+poll_nosec_test", + "name": "connection_prefix_bad_client_test", "src": [ - "test/core/end2end/fixtures/h2_full+poll.c" - ] + "test/core/bad_client/tests/connection_prefix.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "bad_client_test", "gpr", "gpr_test_util", "grpc_test_util_unsecure", @@ -2683,14 +2938,16 @@ ], "headers": [], "language": "c", - "name": "h2_full+poll+pipe_nosec_test", + "name": "headers_bad_client_test", "src": [ - "test/core/end2end/fixtures/h2_full+poll+pipe.c" - ] + "test/core/bad_client/tests/headers.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "bad_client_test", "gpr", "gpr_test_util", "grpc_test_util_unsecure", @@ -2698,14 +2955,16 @@ ], "headers": [], "language": "c", - "name": "h2_proxy_nosec_test", + "name": "initial_settings_frame_bad_client_test", "src": [ - "test/core/end2end/fixtures/h2_proxy.c" - ] + "test/core/bad_client/tests/initial_settings_frame.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "bad_client_test", "gpr", "gpr_test_util", "grpc_test_util_unsecure", @@ -2713,14 +2972,16 @@ ], "headers": [], "language": "c", - "name": "h2_sockpair_nosec_test", + "name": "server_registered_method_bad_client_test", "src": [ - "test/core/end2end/fixtures/h2_sockpair.c" - ] + "test/core/bad_client/tests/server_registered_method.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "bad_client_test", "gpr", "gpr_test_util", "grpc_test_util_unsecure", @@ -2728,14 +2989,16 @@ ], "headers": [], "language": "c", - "name": "h2_sockpair+trace_nosec_test", + "name": "simple_request_bad_client_test", "src": [ - "test/core/end2end/fixtures/h2_sockpair+trace.c" - ] + "test/core/bad_client/tests/simple_request.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "bad_client_test", "gpr", "gpr_test_util", "grpc_test_util_unsecure", @@ -2743,14 +3006,16 @@ ], "headers": [], "language": "c", - "name": "h2_sockpair_1byte_nosec_test", + "name": "unknown_frame_bad_client_test", "src": [ - "test/core/end2end/fixtures/h2_sockpair_1byte.c" - ] + "test/core/bad_client/tests/unknown_frame.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "bad_client_test", "gpr", "gpr_test_util", "grpc_test_util_unsecure", @@ -2758,897 +3023,1494 @@ ], "headers": [], "language": "c", - "name": "h2_uchannel_nosec_test", + "name": "window_overflow_bad_client_test", "src": [ - "test/core/end2end/fixtures/h2_uchannel.c" - ] + "test/core/bad_client/tests/window_overflow.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "bad_ssl_test_server", "gpr", "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "grpc", + "grpc_test_util" ], "headers": [], "language": "c", - "name": "h2_uds_nosec_test", + "name": "bad_ssl_alpn_server", "src": [ - "test/core/end2end/fixtures/h2_uds.c" - ] + "test/core/bad_ssl/servers/alpn.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "end2end_nosec_tests", + "bad_ssl_test_server", "gpr", "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" + "grpc", + "grpc_test_util" ], "headers": [], "language": "c", - "name": "h2_uds+poll_nosec_test", + "name": "bad_ssl_cert_server", "src": [ - "test/core/end2end/fixtures/h2_uds+poll.c" - ] + "test/core/bad_ssl/servers/cert.c" + ], + "third_party": false, + "type": "target" }, { - "deps": [], - "headers": [ - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/atm_gcc_atomic.h", - "include/grpc/support/atm_gcc_sync.h", - "include/grpc/support/atm_win32.h", - "include/grpc/support/avl.h", - "include/grpc/support/cmdline.h", - "include/grpc/support/cpu.h", - "include/grpc/support/histogram.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/log_win32.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/subprocess.h", - "include/grpc/support/sync.h", - "include/grpc/support/sync_generic.h", - "include/grpc/support/sync_posix.h", - "include/grpc/support/sync_win32.h", - "include/grpc/support/thd.h", - "include/grpc/support/time.h", - "include/grpc/support/tls.h", - "include/grpc/support/tls_gcc.h", - "include/grpc/support/tls_msvc.h", - "include/grpc/support/tls_pthread.h", - "include/grpc/support/useful.h", - "src/core/profiling/timers.h", - "src/core/support/block_annotate.h", - "src/core/support/env.h", - "src/core/support/file.h", - "src/core/support/murmur_hash.h", - "src/core/support/stack_lockfree.h", - "src/core/support/string.h", - "src/core/support/string_win32.h", - "src/core/support/thd_internal.h", - "src/core/support/time_precise.h" + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" ], + "headers": [], "language": "c", - "name": "gpr", + "name": "bad_ssl_alpn_test", "src": [ - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/atm_gcc_atomic.h", - "include/grpc/support/atm_gcc_sync.h", - "include/grpc/support/atm_win32.h", - "include/grpc/support/avl.h", - "include/grpc/support/cmdline.h", - "include/grpc/support/cpu.h", - "include/grpc/support/histogram.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/log_win32.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/subprocess.h", - "include/grpc/support/sync.h", - "include/grpc/support/sync_generic.h", - "include/grpc/support/sync_posix.h", - "include/grpc/support/sync_win32.h", - "include/grpc/support/thd.h", - "include/grpc/support/time.h", - "include/grpc/support/tls.h", - "include/grpc/support/tls_gcc.h", - "include/grpc/support/tls_msvc.h", - "include/grpc/support/tls_pthread.h", - "include/grpc/support/useful.h", - "src/core/profiling/basic_timers.c", - "src/core/profiling/stap_timers.c", - "src/core/profiling/timers.h", - "src/core/support/alloc.c", - "src/core/support/avl.c", - "src/core/support/block_annotate.h", - "src/core/support/cmdline.c", - "src/core/support/cpu_iphone.c", - "src/core/support/cpu_linux.c", - "src/core/support/cpu_posix.c", - "src/core/support/cpu_windows.c", - "src/core/support/env.h", - "src/core/support/env_linux.c", - "src/core/support/env_posix.c", - "src/core/support/env_win32.c", - "src/core/support/file.c", - "src/core/support/file.h", - "src/core/support/file_posix.c", - "src/core/support/file_win32.c", - "src/core/support/histogram.c", - "src/core/support/host_port.c", - "src/core/support/log.c", - "src/core/support/log_android.c", - "src/core/support/log_linux.c", - "src/core/support/log_posix.c", - "src/core/support/log_win32.c", - "src/core/support/murmur_hash.c", - "src/core/support/murmur_hash.h", - "src/core/support/slice.c", - "src/core/support/slice_buffer.c", - "src/core/support/stack_lockfree.c", - "src/core/support/stack_lockfree.h", - "src/core/support/string.c", - "src/core/support/string.h", - "src/core/support/string_posix.c", - "src/core/support/string_win32.c", - "src/core/support/string_win32.h", - "src/core/support/subprocess_posix.c", - "src/core/support/subprocess_windows.c", - "src/core/support/sync.c", - "src/core/support/sync_posix.c", - "src/core/support/sync_win32.c", - "src/core/support/thd.c", - "src/core/support/thd_internal.h", - "src/core/support/thd_posix.c", - "src/core/support/thd_win32.c", - "src/core/support/time.c", - "src/core/support/time_posix.c", - "src/core/support/time_precise.c", - "src/core/support/time_precise.h", - "src/core/support/time_win32.c", - "src/core/support/tls_pthread.c", - "src/core/support/wrap_memcpy.c" - ] + "test/core/bad_ssl/bad_ssl_test.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "gpr" + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" ], - "headers": [ - "test/core/util/test_config.h" + "headers": [], + "language": "c", + "name": "bad_ssl_cert_test", + "src": [ + "test/core/bad_ssl/bad_ssl_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" ], + "headers": [], "language": "c", - "name": "gpr_test_util", + "name": "h2_census_test", "src": [ - "test/core/util/test_config.c", - "test/core/util/test_config.h" - ] + "test/core/end2end/fixtures/h2_census.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "gpr" + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" ], - "headers": [ - "include/grpc/byte_buffer.h", - "include/grpc/byte_buffer_reader.h", - "include/grpc/census.h", - "include/grpc/compression.h", - "include/grpc/grpc.h", - "include/grpc/grpc_security.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/status.h", - "src/core/census/aggregation.h", - "src/core/census/grpc_filter.h", - "src/core/census/log.h", - "src/core/census/rpc_metric_id.h", - "src/core/channel/channel_args.h", - "src/core/channel/channel_stack.h", - "src/core/channel/client_channel.h", - "src/core/channel/client_uchannel.h", - "src/core/channel/compress_filter.h", - "src/core/channel/connected_channel.h", - "src/core/channel/context.h", - "src/core/channel/http_client_filter.h", - "src/core/channel/http_server_filter.h", - "src/core/channel/subchannel_call_holder.h", - "src/core/client_config/client_config.h", - "src/core/client_config/connector.h", - "src/core/client_config/initial_connect_string.h", - "src/core/client_config/lb_policies/pick_first.h", - "src/core/client_config/lb_policies/round_robin.h", - "src/core/client_config/lb_policy.h", - "src/core/client_config/lb_policy_factory.h", - "src/core/client_config/lb_policy_registry.h", - "src/core/client_config/resolver.h", - "src/core/client_config/resolver_factory.h", - "src/core/client_config/resolver_registry.h", - "src/core/client_config/resolvers/dns_resolver.h", - "src/core/client_config/resolvers/sockaddr_resolver.h", - "src/core/client_config/subchannel.h", - "src/core/client_config/subchannel_factory.h", - "src/core/client_config/subchannel_index.h", - "src/core/client_config/uri_parser.h", - "src/core/compression/algorithm_metadata.h", - "src/core/compression/message_compress.h", - "src/core/debug/trace.h", - "src/core/httpcli/format_request.h", - "src/core/httpcli/httpcli.h", - "src/core/httpcli/parser.h", - "src/core/iomgr/closure.h", - "src/core/iomgr/endpoint.h", - "src/core/iomgr/endpoint_pair.h", - "src/core/iomgr/exec_ctx.h", - "src/core/iomgr/executor.h", - "src/core/iomgr/fd_posix.h", - "src/core/iomgr/iocp_windows.h", - "src/core/iomgr/iomgr.h", - "src/core/iomgr/iomgr_internal.h", - "src/core/iomgr/iomgr_posix.h", - "src/core/iomgr/pollset.h", - "src/core/iomgr/pollset_posix.h", - "src/core/iomgr/pollset_set.h", - "src/core/iomgr/pollset_set_posix.h", - "src/core/iomgr/pollset_set_windows.h", - "src/core/iomgr/pollset_windows.h", - "src/core/iomgr/resolve_address.h", - "src/core/iomgr/sockaddr.h", - "src/core/iomgr/sockaddr_posix.h", - "src/core/iomgr/sockaddr_utils.h", - "src/core/iomgr/sockaddr_win32.h", - "src/core/iomgr/socket_utils_posix.h", - "src/core/iomgr/socket_windows.h", - "src/core/iomgr/tcp_client.h", - "src/core/iomgr/tcp_posix.h", - "src/core/iomgr/tcp_server.h", - "src/core/iomgr/tcp_windows.h", - "src/core/iomgr/time_averaged_stats.h", - "src/core/iomgr/timer.h", - "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", - "src/core/iomgr/udp_server.h", - "src/core/iomgr/wakeup_fd_pipe.h", - "src/core/iomgr/wakeup_fd_posix.h", - "src/core/iomgr/workqueue.h", - "src/core/iomgr/workqueue_posix.h", - "src/core/iomgr/workqueue_windows.h", - "src/core/json/json.h", - "src/core/json/json_common.h", - "src/core/json/json_reader.h", - "src/core/json/json_writer.h", - "src/core/security/auth_filters.h", - "src/core/security/base64.h", - "src/core/security/credentials.h", - "src/core/security/handshake.h", - "src/core/security/json_token.h", - "src/core/security/jwt_verifier.h", - "src/core/security/secure_endpoint.h", - "src/core/security/security_connector.h", - "src/core/security/security_context.h", - "src/core/statistics/census_interface.h", - "src/core/statistics/census_rpc_stats.h", - "src/core/surface/api_trace.h", - "src/core/surface/call.h", - "src/core/surface/call_test_only.h", - "src/core/surface/channel.h", - "src/core/surface/completion_queue.h", - "src/core/surface/event_string.h", - "src/core/surface/init.h", - "src/core/surface/server.h", - "src/core/surface/surface_trace.h", - "src/core/transport/byte_stream.h", - "src/core/transport/chttp2/alpn.h", - "src/core/transport/chttp2/bin_encoder.h", - "src/core/transport/chttp2/frame.h", - "src/core/transport/chttp2/frame_data.h", - "src/core/transport/chttp2/frame_goaway.h", - "src/core/transport/chttp2/frame_ping.h", - "src/core/transport/chttp2/frame_rst_stream.h", - "src/core/transport/chttp2/frame_settings.h", - "src/core/transport/chttp2/frame_window_update.h", - "src/core/transport/chttp2/hpack_encoder.h", - "src/core/transport/chttp2/hpack_parser.h", - "src/core/transport/chttp2/hpack_table.h", - "src/core/transport/chttp2/http2_errors.h", - "src/core/transport/chttp2/huffsyms.h", - "src/core/transport/chttp2/incoming_metadata.h", - "src/core/transport/chttp2/internal.h", - "src/core/transport/chttp2/status_conversion.h", - "src/core/transport/chttp2/stream_map.h", - "src/core/transport/chttp2/timeout_encoding.h", - "src/core/transport/chttp2/varint.h", - "src/core/transport/chttp2_transport.h", - "src/core/transport/connectivity_state.h", - "src/core/transport/metadata.h", - "src/core/transport/metadata_batch.h", - "src/core/transport/static_metadata.h", - "src/core/transport/transport.h", - "src/core/transport/transport_impl.h", - "src/core/tsi/fake_transport_security.h", - "src/core/tsi/ssl_transport_security.h", - "src/core/tsi/ssl_types.h", - "src/core/tsi/transport_security.h", - "src/core/tsi/transport_security_interface.h" + "headers": [], + "language": "c", + "name": "h2_compress_test", + "src": [ + "test/core/end2end/fixtures/h2_compress.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" ], + "headers": [], "language": "c", - "name": "grpc", + "name": "h2_fakesec_test", "src": [ - "include/grpc/byte_buffer.h", - "include/grpc/byte_buffer_reader.h", - "include/grpc/census.h", - "include/grpc/compression.h", - "include/grpc/grpc.h", - "include/grpc/grpc_security.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/status.h", - "src/core/census/aggregation.h", - "src/core/census/context.c", - "src/core/census/grpc_context.c", - "src/core/census/grpc_filter.c", - "src/core/census/grpc_filter.h", - "src/core/census/initialize.c", - "src/core/census/log.c", - "src/core/census/log.h", - "src/core/census/operation.c", - "src/core/census/placeholders.c", - "src/core/census/rpc_metric_id.h", - "src/core/census/tracing.c", - "src/core/channel/channel_args.c", - "src/core/channel/channel_args.h", - "src/core/channel/channel_stack.c", - "src/core/channel/channel_stack.h", - "src/core/channel/client_channel.c", - "src/core/channel/client_channel.h", - "src/core/channel/client_uchannel.c", - "src/core/channel/client_uchannel.h", - "src/core/channel/compress_filter.c", - "src/core/channel/compress_filter.h", - "src/core/channel/connected_channel.c", - "src/core/channel/connected_channel.h", - "src/core/channel/context.h", - "src/core/channel/http_client_filter.c", - "src/core/channel/http_client_filter.h", - "src/core/channel/http_server_filter.c", - "src/core/channel/http_server_filter.h", - "src/core/channel/subchannel_call_holder.c", - "src/core/channel/subchannel_call_holder.h", - "src/core/client_config/client_config.c", - "src/core/client_config/client_config.h", - "src/core/client_config/connector.c", - "src/core/client_config/connector.h", - "src/core/client_config/default_initial_connect_string.c", - "src/core/client_config/initial_connect_string.c", - "src/core/client_config/initial_connect_string.h", - "src/core/client_config/lb_policies/pick_first.c", - "src/core/client_config/lb_policies/pick_first.h", - "src/core/client_config/lb_policies/round_robin.c", - "src/core/client_config/lb_policies/round_robin.h", - "src/core/client_config/lb_policy.c", - "src/core/client_config/lb_policy.h", - "src/core/client_config/lb_policy_factory.c", - "src/core/client_config/lb_policy_factory.h", - "src/core/client_config/lb_policy_registry.c", - "src/core/client_config/lb_policy_registry.h", - "src/core/client_config/resolver.c", - "src/core/client_config/resolver.h", - "src/core/client_config/resolver_factory.c", - "src/core/client_config/resolver_factory.h", - "src/core/client_config/resolver_registry.c", - "src/core/client_config/resolver_registry.h", - "src/core/client_config/resolvers/dns_resolver.c", - "src/core/client_config/resolvers/dns_resolver.h", - "src/core/client_config/resolvers/sockaddr_resolver.c", - "src/core/client_config/resolvers/sockaddr_resolver.h", - "src/core/client_config/subchannel.c", - "src/core/client_config/subchannel.h", - "src/core/client_config/subchannel_factory.c", - "src/core/client_config/subchannel_factory.h", - "src/core/client_config/subchannel_index.c", - "src/core/client_config/subchannel_index.h", - "src/core/client_config/uri_parser.c", - "src/core/client_config/uri_parser.h", - "src/core/compression/algorithm.c", - "src/core/compression/algorithm_metadata.h", - "src/core/compression/message_compress.c", - "src/core/compression/message_compress.h", - "src/core/debug/trace.c", - "src/core/debug/trace.h", - "src/core/httpcli/format_request.c", - "src/core/httpcli/format_request.h", - "src/core/httpcli/httpcli.c", - "src/core/httpcli/httpcli.h", - "src/core/httpcli/httpcli_security_connector.c", - "src/core/httpcli/parser.c", - "src/core/httpcli/parser.h", - "src/core/iomgr/closure.c", - "src/core/iomgr/closure.h", - "src/core/iomgr/endpoint.c", - "src/core/iomgr/endpoint.h", - "src/core/iomgr/endpoint_pair.h", - "src/core/iomgr/endpoint_pair_posix.c", - "src/core/iomgr/endpoint_pair_windows.c", - "src/core/iomgr/exec_ctx.c", - "src/core/iomgr/exec_ctx.h", - "src/core/iomgr/executor.c", - "src/core/iomgr/executor.h", - "src/core/iomgr/fd_posix.c", - "src/core/iomgr/fd_posix.h", - "src/core/iomgr/iocp_windows.c", - "src/core/iomgr/iocp_windows.h", - "src/core/iomgr/iomgr.c", - "src/core/iomgr/iomgr.h", - "src/core/iomgr/iomgr_internal.h", - "src/core/iomgr/iomgr_posix.c", - "src/core/iomgr/iomgr_posix.h", - "src/core/iomgr/iomgr_windows.c", - "src/core/iomgr/pollset.h", - "src/core/iomgr/pollset_multipoller_with_epoll.c", - "src/core/iomgr/pollset_multipoller_with_poll_posix.c", - "src/core/iomgr/pollset_posix.c", - "src/core/iomgr/pollset_posix.h", - "src/core/iomgr/pollset_set.h", - "src/core/iomgr/pollset_set_posix.c", - "src/core/iomgr/pollset_set_posix.h", - "src/core/iomgr/pollset_set_windows.c", - "src/core/iomgr/pollset_set_windows.h", - "src/core/iomgr/pollset_windows.c", - "src/core/iomgr/pollset_windows.h", - "src/core/iomgr/resolve_address.h", - "src/core/iomgr/resolve_address_posix.c", - "src/core/iomgr/resolve_address_windows.c", - "src/core/iomgr/sockaddr.h", - "src/core/iomgr/sockaddr_posix.h", - "src/core/iomgr/sockaddr_utils.c", - "src/core/iomgr/sockaddr_utils.h", - "src/core/iomgr/sockaddr_win32.h", - "src/core/iomgr/socket_utils_common_posix.c", - "src/core/iomgr/socket_utils_linux.c", - "src/core/iomgr/socket_utils_posix.c", - "src/core/iomgr/socket_utils_posix.h", - "src/core/iomgr/socket_windows.c", - "src/core/iomgr/socket_windows.h", - "src/core/iomgr/tcp_client.h", - "src/core/iomgr/tcp_client_posix.c", - "src/core/iomgr/tcp_client_windows.c", - "src/core/iomgr/tcp_posix.c", - "src/core/iomgr/tcp_posix.h", - "src/core/iomgr/tcp_server.h", - "src/core/iomgr/tcp_server_posix.c", - "src/core/iomgr/tcp_server_windows.c", - "src/core/iomgr/tcp_windows.c", - "src/core/iomgr/tcp_windows.h", - "src/core/iomgr/time_averaged_stats.c", - "src/core/iomgr/time_averaged_stats.h", - "src/core/iomgr/timer.c", - "src/core/iomgr/timer.h", - "src/core/iomgr/timer_heap.c", - "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", - "src/core/iomgr/udp_server.c", - "src/core/iomgr/udp_server.h", - "src/core/iomgr/wakeup_fd_eventfd.c", - "src/core/iomgr/wakeup_fd_nospecial.c", - "src/core/iomgr/wakeup_fd_pipe.c", - "src/core/iomgr/wakeup_fd_pipe.h", - "src/core/iomgr/wakeup_fd_posix.c", - "src/core/iomgr/wakeup_fd_posix.h", - "src/core/iomgr/workqueue.h", - "src/core/iomgr/workqueue_posix.c", - "src/core/iomgr/workqueue_posix.h", - "src/core/iomgr/workqueue_windows.c", - "src/core/iomgr/workqueue_windows.h", - "src/core/json/json.c", - "src/core/json/json.h", - "src/core/json/json_common.h", - "src/core/json/json_reader.c", - "src/core/json/json_reader.h", - "src/core/json/json_string.c", - "src/core/json/json_writer.c", - "src/core/json/json_writer.h", - "src/core/security/auth_filters.h", - "src/core/security/base64.c", - "src/core/security/base64.h", - "src/core/security/client_auth_filter.c", - "src/core/security/credentials.c", - "src/core/security/credentials.h", - "src/core/security/credentials_metadata.c", - "src/core/security/credentials_posix.c", - "src/core/security/credentials_win32.c", - "src/core/security/google_default_credentials.c", - "src/core/security/handshake.c", - "src/core/security/handshake.h", - "src/core/security/json_token.c", - "src/core/security/json_token.h", - "src/core/security/jwt_verifier.c", - "src/core/security/jwt_verifier.h", - "src/core/security/secure_endpoint.c", - "src/core/security/secure_endpoint.h", - "src/core/security/security_connector.c", - "src/core/security/security_connector.h", - "src/core/security/security_context.c", - "src/core/security/security_context.h", - "src/core/security/server_auth_filter.c", - "src/core/security/server_secure_chttp2.c", - "src/core/statistics/census_interface.h", - "src/core/statistics/census_rpc_stats.h", - "src/core/surface/alarm.c", - "src/core/surface/api_trace.c", - "src/core/surface/api_trace.h", - "src/core/surface/byte_buffer.c", - "src/core/surface/byte_buffer_reader.c", - "src/core/surface/call.c", - "src/core/surface/call.h", - "src/core/surface/call_details.c", - "src/core/surface/call_log_batch.c", - "src/core/surface/call_test_only.h", - "src/core/surface/channel.c", - "src/core/surface/channel.h", - "src/core/surface/channel_connectivity.c", - "src/core/surface/channel_create.c", - "src/core/surface/channel_ping.c", - "src/core/surface/completion_queue.c", - "src/core/surface/completion_queue.h", - "src/core/surface/event_string.c", - "src/core/surface/event_string.h", - "src/core/surface/init.c", - "src/core/surface/init.h", - "src/core/surface/init_secure.c", - "src/core/surface/lame_client.c", - "src/core/surface/metadata_array.c", - "src/core/surface/secure_channel_create.c", - "src/core/surface/server.c", - "src/core/surface/server.h", - "src/core/surface/server_chttp2.c", - "src/core/surface/server_create.c", - "src/core/surface/surface_trace.h", - "src/core/surface/validate_metadata.c", - "src/core/surface/version.c", - "src/core/transport/byte_stream.c", - "src/core/transport/byte_stream.h", - "src/core/transport/chttp2/alpn.c", - "src/core/transport/chttp2/alpn.h", - "src/core/transport/chttp2/bin_encoder.c", - "src/core/transport/chttp2/bin_encoder.h", - "src/core/transport/chttp2/frame.h", - "src/core/transport/chttp2/frame_data.c", - "src/core/transport/chttp2/frame_data.h", - "src/core/transport/chttp2/frame_goaway.c", - "src/core/transport/chttp2/frame_goaway.h", - "src/core/transport/chttp2/frame_ping.c", - "src/core/transport/chttp2/frame_ping.h", - "src/core/transport/chttp2/frame_rst_stream.c", - "src/core/transport/chttp2/frame_rst_stream.h", - "src/core/transport/chttp2/frame_settings.c", - "src/core/transport/chttp2/frame_settings.h", - "src/core/transport/chttp2/frame_window_update.c", - "src/core/transport/chttp2/frame_window_update.h", - "src/core/transport/chttp2/hpack_encoder.c", - "src/core/transport/chttp2/hpack_encoder.h", - "src/core/transport/chttp2/hpack_parser.c", - "src/core/transport/chttp2/hpack_parser.h", - "src/core/transport/chttp2/hpack_table.c", - "src/core/transport/chttp2/hpack_table.h", - "src/core/transport/chttp2/http2_errors.h", - "src/core/transport/chttp2/huffsyms.c", - "src/core/transport/chttp2/huffsyms.h", - "src/core/transport/chttp2/incoming_metadata.c", - "src/core/transport/chttp2/incoming_metadata.h", - "src/core/transport/chttp2/internal.h", - "src/core/transport/chttp2/parsing.c", - "src/core/transport/chttp2/status_conversion.c", - "src/core/transport/chttp2/status_conversion.h", - "src/core/transport/chttp2/stream_lists.c", - "src/core/transport/chttp2/stream_map.c", - "src/core/transport/chttp2/stream_map.h", - "src/core/transport/chttp2/timeout_encoding.c", - "src/core/transport/chttp2/timeout_encoding.h", - "src/core/transport/chttp2/varint.c", - "src/core/transport/chttp2/varint.h", - "src/core/transport/chttp2/writing.c", - "src/core/transport/chttp2_transport.c", - "src/core/transport/chttp2_transport.h", - "src/core/transport/connectivity_state.c", - "src/core/transport/connectivity_state.h", - "src/core/transport/metadata.c", - "src/core/transport/metadata.h", - "src/core/transport/metadata_batch.c", - "src/core/transport/metadata_batch.h", - "src/core/transport/static_metadata.c", - "src/core/transport/static_metadata.h", - "src/core/transport/transport.c", - "src/core/transport/transport.h", - "src/core/transport/transport_impl.h", - "src/core/transport/transport_op_string.c", - "src/core/tsi/fake_transport_security.c", - "src/core/tsi/fake_transport_security.h", - "src/core/tsi/ssl_transport_security.c", - "src/core/tsi/ssl_transport_security.h", - "src/core/tsi/ssl_types.h", - "src/core/tsi/transport_security.c", - "src/core/tsi/transport_security.h", - "src/core/tsi/transport_security_interface.h" - ] + "test/core/end2end/fixtures/h2_fakesec.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ + "end2end_certs", + "end2end_tests", "gpr", - "grpc" + "gpr_test_util", + "grpc", + "grpc_test_util" ], "headers": [], "language": "c", - "name": "grpc_dll", - "src": [] + "name": "h2_full_test", + "src": [ + "test/core/end2end/fixtures/h2_full.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ + "end2end_certs", + "end2end_tests", "gpr", "gpr_test_util", - "grpc" + "grpc", + "grpc_test_util" ], - "headers": [ - "test/core/end2end/cq_verifier.h", - "test/core/end2end/data/ssl_test_data.h", - "test/core/end2end/fixtures/proxy.h", - "test/core/iomgr/endpoint_tests.h", - "test/core/security/oauth2_utils.h", - "test/core/util/grpc_profiler.h", - "test/core/util/parse_hexstring.h", - "test/core/util/port.h", - "test/core/util/slice_splitter.h" + "headers": [], + "language": "c", + "name": "h2_full+pipe_test", + "src": [ + "test/core/end2end/fixtures/h2_full+pipe.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" ], + "headers": [], "language": "c", - "name": "grpc_test_util", + "name": "h2_full+poll_test", "src": [ - "test/core/end2end/cq_verifier.c", - "test/core/end2end/cq_verifier.h", - "test/core/end2end/data/server1_cert.c", - "test/core/end2end/data/server1_key.c", - "test/core/end2end/data/ssl_test_data.h", - "test/core/end2end/data/test_root_cert.c", - "test/core/end2end/fixtures/proxy.c", - "test/core/end2end/fixtures/proxy.h", - "test/core/iomgr/endpoint_tests.c", - "test/core/iomgr/endpoint_tests.h", - "test/core/security/oauth2_utils.c", - "test/core/security/oauth2_utils.h", - "test/core/util/grpc_profiler.c", - "test/core/util/grpc_profiler.h", - "test/core/util/parse_hexstring.c", - "test/core/util/parse_hexstring.h", - "test/core/util/port.h", - "test/core/util/port_posix.c", - "test/core/util/port_windows.c", - "test/core/util/slice_splitter.c", - "test/core/util/slice_splitter.h" - ] + "test/core/end2end/fixtures/h2_full+poll.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ + "end2end_certs", + "end2end_tests", "gpr", "gpr_test_util", - "grpc_unsecure" + "grpc", + "grpc_test_util" ], - "headers": [ - "test/core/end2end/cq_verifier.h", - "test/core/end2end/fixtures/proxy.h", - "test/core/iomgr/endpoint_tests.h", - "test/core/util/grpc_profiler.h", - "test/core/util/parse_hexstring.h", - "test/core/util/port.h", - "test/core/util/slice_splitter.h" + "headers": [], + "language": "c", + "name": "h2_full+poll+pipe_test", + "src": [ + "test/core/end2end/fixtures/h2_full+poll+pipe.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" ], + "headers": [], "language": "c", - "name": "grpc_test_util_unsecure", + "name": "h2_oauth2_test", "src": [ - "test/core/end2end/cq_verifier.c", - "test/core/end2end/cq_verifier.h", - "test/core/end2end/fixtures/proxy.c", - "test/core/end2end/fixtures/proxy.h", - "test/core/iomgr/endpoint_tests.c", - "test/core/iomgr/endpoint_tests.h", - "test/core/util/grpc_profiler.c", - "test/core/util/grpc_profiler.h", - "test/core/util/parse_hexstring.c", - "test/core/util/parse_hexstring.h", - "test/core/util/port.h", - "test/core/util/port_posix.c", - "test/core/util/port_windows.c", - "test/core/util/slice_splitter.c", - "test/core/util/slice_splitter.h" - ] + "test/core/end2end/fixtures/h2_oauth2.c" + ], + "third_party": false, + "type": "target" }, { "deps": [ - "gpr" + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" ], - "headers": [ - "include/grpc/byte_buffer.h", - "include/grpc/byte_buffer_reader.h", - "include/grpc/census.h", - "include/grpc/compression.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/status.h", - "src/core/census/aggregation.h", - "src/core/census/grpc_filter.h", - "src/core/census/log.h", - "src/core/census/rpc_metric_id.h", - "src/core/channel/channel_args.h", - "src/core/channel/channel_stack.h", - "src/core/channel/client_channel.h", - "src/core/channel/client_uchannel.h", - "src/core/channel/compress_filter.h", - "src/core/channel/connected_channel.h", - "src/core/channel/context.h", - "src/core/channel/http_client_filter.h", - "src/core/channel/http_server_filter.h", - "src/core/channel/subchannel_call_holder.h", - "src/core/client_config/client_config.h", - "src/core/client_config/connector.h", - "src/core/client_config/initial_connect_string.h", - "src/core/client_config/lb_policies/pick_first.h", - "src/core/client_config/lb_policies/round_robin.h", - "src/core/client_config/lb_policy.h", - "src/core/client_config/lb_policy_factory.h", - "src/core/client_config/lb_policy_registry.h", - "src/core/client_config/resolver.h", - "src/core/client_config/resolver_factory.h", - "src/core/client_config/resolver_registry.h", - "src/core/client_config/resolvers/dns_resolver.h", - "src/core/client_config/resolvers/sockaddr_resolver.h", - "src/core/client_config/subchannel.h", - "src/core/client_config/subchannel_factory.h", - "src/core/client_config/subchannel_index.h", - "src/core/client_config/uri_parser.h", - "src/core/compression/algorithm_metadata.h", - "src/core/compression/message_compress.h", - "src/core/debug/trace.h", - "src/core/httpcli/format_request.h", - "src/core/httpcli/httpcli.h", - "src/core/httpcli/parser.h", - "src/core/iomgr/closure.h", - "src/core/iomgr/endpoint.h", - "src/core/iomgr/endpoint_pair.h", - "src/core/iomgr/exec_ctx.h", - "src/core/iomgr/executor.h", - "src/core/iomgr/fd_posix.h", - "src/core/iomgr/iocp_windows.h", - "src/core/iomgr/iomgr.h", - "src/core/iomgr/iomgr_internal.h", - "src/core/iomgr/iomgr_posix.h", - "src/core/iomgr/pollset.h", - "src/core/iomgr/pollset_posix.h", - "src/core/iomgr/pollset_set.h", - "src/core/iomgr/pollset_set_posix.h", - "src/core/iomgr/pollset_set_windows.h", - "src/core/iomgr/pollset_windows.h", - "src/core/iomgr/resolve_address.h", - "src/core/iomgr/sockaddr.h", - "src/core/iomgr/sockaddr_posix.h", - "src/core/iomgr/sockaddr_utils.h", - "src/core/iomgr/sockaddr_win32.h", - "src/core/iomgr/socket_utils_posix.h", - "src/core/iomgr/socket_windows.h", - "src/core/iomgr/tcp_client.h", - "src/core/iomgr/tcp_posix.h", - "src/core/iomgr/tcp_server.h", - "src/core/iomgr/tcp_windows.h", - "src/core/iomgr/time_averaged_stats.h", - "src/core/iomgr/timer.h", - "src/core/iomgr/timer_heap.h", - "src/core/iomgr/timer_internal.h", - "src/core/iomgr/udp_server.h", - "src/core/iomgr/wakeup_fd_pipe.h", - "src/core/iomgr/wakeup_fd_posix.h", - "src/core/iomgr/workqueue.h", - "src/core/iomgr/workqueue_posix.h", - "src/core/iomgr/workqueue_windows.h", - "src/core/json/json.h", - "src/core/json/json_common.h", - "src/core/json/json_reader.h", - "src/core/json/json_writer.h", - "src/core/statistics/census_interface.h", - "src/core/statistics/census_rpc_stats.h", - "src/core/surface/api_trace.h", - "src/core/surface/call.h", - "src/core/surface/call_test_only.h", - "src/core/surface/channel.h", - "src/core/surface/completion_queue.h", - "src/core/surface/event_string.h", - "src/core/surface/init.h", - "src/core/surface/server.h", - "src/core/surface/surface_trace.h", - "src/core/transport/byte_stream.h", - "src/core/transport/chttp2/alpn.h", - "src/core/transport/chttp2/bin_encoder.h", - "src/core/transport/chttp2/frame.h", - "src/core/transport/chttp2/frame_data.h", - "src/core/transport/chttp2/frame_goaway.h", - "src/core/transport/chttp2/frame_ping.h", - "src/core/transport/chttp2/frame_rst_stream.h", - "src/core/transport/chttp2/frame_settings.h", - "src/core/transport/chttp2/frame_window_update.h", - "src/core/transport/chttp2/hpack_encoder.h", - "src/core/transport/chttp2/hpack_parser.h", - "src/core/transport/chttp2/hpack_table.h", - "src/core/transport/chttp2/http2_errors.h", - "src/core/transport/chttp2/huffsyms.h", - "src/core/transport/chttp2/incoming_metadata.h", - "src/core/transport/chttp2/internal.h", - "src/core/transport/chttp2/status_conversion.h", - "src/core/transport/chttp2/stream_map.h", - "src/core/transport/chttp2/timeout_encoding.h", - "src/core/transport/chttp2/varint.h", - "src/core/transport/chttp2_transport.h", - "src/core/transport/connectivity_state.h", - "src/core/transport/metadata.h", - "src/core/transport/metadata_batch.h", - "src/core/transport/static_metadata.h", - "src/core/transport/transport.h", - "src/core/transport/transport_impl.h" + "headers": [], + "language": "c", + "name": "h2_proxy_test", + "src": [ + "test/core/end2end/fixtures/h2_proxy.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" ], + "headers": [], "language": "c", - "name": "grpc_unsecure", + "name": "h2_sockpair_test", "src": [ - "include/grpc/byte_buffer.h", - "include/grpc/byte_buffer_reader.h", - "include/grpc/census.h", - "include/grpc/compression.h", - "include/grpc/grpc.h", - "include/grpc/impl/codegen/byte_buffer.h", + "test/core/end2end/fixtures/h2_sockpair.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "h2_sockpair+trace_test", + "src": [ + "test/core/end2end/fixtures/h2_sockpair+trace.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "h2_sockpair_1byte_test", + "src": [ + "test/core/end2end/fixtures/h2_sockpair_1byte.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "h2_ssl_test", + "src": [ + "test/core/end2end/fixtures/h2_ssl.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "h2_ssl+poll_test", + "src": [ + "test/core/end2end/fixtures/h2_ssl+poll.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "h2_ssl_proxy_test", + "src": [ + "test/core/end2end/fixtures/h2_ssl_proxy.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "h2_uchannel_test", + "src": [ + "test/core/end2end/fixtures/h2_uchannel.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "h2_uds_test", + "src": [ + "test/core/end2end/fixtures/h2_uds.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_certs", + "end2end_tests", + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "h2_uds+poll_test", + "src": [ + "test/core/end2end/fixtures/h2_uds+poll.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_census_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_census.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_compress_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_compress.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_full_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_full.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_full+pipe_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_full+pipe.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_full+poll_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_full+poll.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_full+poll+pipe_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_full+poll+pipe.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_proxy_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_proxy.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_sockpair_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_sockpair.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_sockpair+trace_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_sockpair+trace.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_sockpair_1byte_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_sockpair_1byte.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_uchannel_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_uchannel.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_uds_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_uds.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "end2end_nosec_tests", + "gpr", + "gpr_test_util", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "language": "c", + "name": "h2_uds+poll_nosec_test", + "src": [ + "test/core/end2end/fixtures/h2_uds+poll.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [], + "headers": [ + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "include/grpc/support/alloc.h", + "include/grpc/support/atm.h", + "include/grpc/support/atm_gcc_atomic.h", + "include/grpc/support/atm_gcc_sync.h", + "include/grpc/support/atm_win32.h", + "include/grpc/support/avl.h", + "include/grpc/support/cmdline.h", + "include/grpc/support/cpu.h", + "include/grpc/support/histogram.h", + "include/grpc/support/host_port.h", + "include/grpc/support/log.h", + "include/grpc/support/log_win32.h", + "include/grpc/support/port_platform.h", + "include/grpc/support/slice.h", + "include/grpc/support/slice_buffer.h", + "include/grpc/support/string_util.h", + "include/grpc/support/subprocess.h", + "include/grpc/support/sync.h", + "include/grpc/support/sync_generic.h", + "include/grpc/support/sync_posix.h", + "include/grpc/support/sync_win32.h", + "include/grpc/support/thd.h", + "include/grpc/support/time.h", + "include/grpc/support/tls.h", + "include/grpc/support/tls_gcc.h", + "include/grpc/support/tls_msvc.h", + "include/grpc/support/tls_pthread.h", + "include/grpc/support/useful.h", + "src/core/profiling/timers.h", + "src/core/support/block_annotate.h", + "src/core/support/env.h", + "src/core/support/load_file.h", + "src/core/support/murmur_hash.h", + "src/core/support/stack_lockfree.h", + "src/core/support/string.h", + "src/core/support/string_win32.h", + "src/core/support/thd_internal.h", + "src/core/support/time_precise.h", + "src/core/support/tmpfile.h" + ], + "language": "c", + "name": "gpr", + "src": [ + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "include/grpc/support/alloc.h", + "include/grpc/support/atm.h", + "include/grpc/support/atm_gcc_atomic.h", + "include/grpc/support/atm_gcc_sync.h", + "include/grpc/support/atm_win32.h", + "include/grpc/support/avl.h", + "include/grpc/support/cmdline.h", + "include/grpc/support/cpu.h", + "include/grpc/support/histogram.h", + "include/grpc/support/host_port.h", + "include/grpc/support/log.h", + "include/grpc/support/log_win32.h", + "include/grpc/support/port_platform.h", + "include/grpc/support/slice.h", + "include/grpc/support/slice_buffer.h", + "include/grpc/support/string_util.h", + "include/grpc/support/subprocess.h", + "include/grpc/support/sync.h", + "include/grpc/support/sync_generic.h", + "include/grpc/support/sync_posix.h", + "include/grpc/support/sync_win32.h", + "include/grpc/support/thd.h", + "include/grpc/support/time.h", + "include/grpc/support/tls.h", + "include/grpc/support/tls_gcc.h", + "include/grpc/support/tls_msvc.h", + "include/grpc/support/tls_pthread.h", + "include/grpc/support/useful.h", + "src/core/profiling/basic_timers.c", + "src/core/profiling/stap_timers.c", + "src/core/profiling/timers.h", + "src/core/support/alloc.c", + "src/core/support/avl.c", + "src/core/support/block_annotate.h", + "src/core/support/cmdline.c", + "src/core/support/cpu_iphone.c", + "src/core/support/cpu_linux.c", + "src/core/support/cpu_posix.c", + "src/core/support/cpu_windows.c", + "src/core/support/env.h", + "src/core/support/env_linux.c", + "src/core/support/env_posix.c", + "src/core/support/env_win32.c", + "src/core/support/histogram.c", + "src/core/support/host_port.c", + "src/core/support/load_file.c", + "src/core/support/load_file.h", + "src/core/support/log.c", + "src/core/support/log_android.c", + "src/core/support/log_linux.c", + "src/core/support/log_posix.c", + "src/core/support/log_win32.c", + "src/core/support/murmur_hash.c", + "src/core/support/murmur_hash.h", + "src/core/support/slice.c", + "src/core/support/slice_buffer.c", + "src/core/support/stack_lockfree.c", + "src/core/support/stack_lockfree.h", + "src/core/support/string.c", + "src/core/support/string.h", + "src/core/support/string_posix.c", + "src/core/support/string_win32.c", + "src/core/support/string_win32.h", + "src/core/support/subprocess_posix.c", + "src/core/support/subprocess_windows.c", + "src/core/support/sync.c", + "src/core/support/sync_posix.c", + "src/core/support/sync_win32.c", + "src/core/support/thd.c", + "src/core/support/thd_internal.h", + "src/core/support/thd_posix.c", + "src/core/support/thd_win32.c", + "src/core/support/time.c", + "src/core/support/time_posix.c", + "src/core/support/time_precise.c", + "src/core/support/time_precise.h", + "src/core/support/time_win32.c", + "src/core/support/tls_pthread.c", + "src/core/support/tmpfile.h", + "src/core/support/tmpfile_posix.c", + "src/core/support/tmpfile_win32.c", + "src/core/support/wrap_memcpy.c" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr" + ], + "headers": [ + "test/core/util/test_config.h" + ], + "language": "c", + "name": "gpr_test_util", + "src": [ + "test/core/util/test_config.c", + "test/core/util/test_config.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr" + ], + "headers": [ + "include/grpc/byte_buffer.h", + "include/grpc/byte_buffer_reader.h", + "include/grpc/census.h", + "include/grpc/compression.h", + "include/grpc/grpc.h", + "include/grpc/grpc_security.h", + "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/compression_types.h", + "include/grpc/impl/codegen/connectivity_state.h", + "include/grpc/impl/codegen/grpc_types.h", + "include/grpc/impl/codegen/propagation_bits.h", + "include/grpc/impl/codegen/status.h", + "include/grpc/status.h", + "src/core/census/aggregation.h", + "src/core/census/grpc_filter.h", + "src/core/census/mlog.h", + "src/core/census/rpc_metric_id.h", + "src/core/channel/channel_args.h", + "src/core/channel/channel_stack.h", + "src/core/channel/client_channel.h", + "src/core/channel/client_uchannel.h", + "src/core/channel/compress_filter.h", + "src/core/channel/connected_channel.h", + "src/core/channel/context.h", + "src/core/channel/http_client_filter.h", + "src/core/channel/http_server_filter.h", + "src/core/channel/subchannel_call_holder.h", + "src/core/client_config/client_config.h", + "src/core/client_config/connector.h", + "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/pick_first.h", + "src/core/client_config/lb_policies/round_robin.h", + "src/core/client_config/lb_policy.h", + "src/core/client_config/lb_policy_factory.h", + "src/core/client_config/lb_policy_registry.h", + "src/core/client_config/resolver.h", + "src/core/client_config/resolver_factory.h", + "src/core/client_config/resolver_registry.h", + "src/core/client_config/resolvers/dns_resolver.h", + "src/core/client_config/resolvers/sockaddr_resolver.h", + "src/core/client_config/subchannel.h", + "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.h", + "src/core/client_config/uri_parser.h", + "src/core/compression/algorithm_metadata.h", + "src/core/compression/message_compress.h", + "src/core/debug/trace.h", + "src/core/httpcli/format_request.h", + "src/core/httpcli/httpcli.h", + "src/core/httpcli/parser.h", + "src/core/iomgr/closure.h", + "src/core/iomgr/endpoint.h", + "src/core/iomgr/endpoint_pair.h", + "src/core/iomgr/exec_ctx.h", + "src/core/iomgr/executor.h", + "src/core/iomgr/fd_posix.h", + "src/core/iomgr/iocp_windows.h", + "src/core/iomgr/iomgr.h", + "src/core/iomgr/iomgr_internal.h", + "src/core/iomgr/iomgr_posix.h", + "src/core/iomgr/pollset.h", + "src/core/iomgr/pollset_posix.h", + "src/core/iomgr/pollset_set.h", + "src/core/iomgr/pollset_set_posix.h", + "src/core/iomgr/pollset_set_windows.h", + "src/core/iomgr/pollset_windows.h", + "src/core/iomgr/resolve_address.h", + "src/core/iomgr/sockaddr.h", + "src/core/iomgr/sockaddr_posix.h", + "src/core/iomgr/sockaddr_utils.h", + "src/core/iomgr/sockaddr_win32.h", + "src/core/iomgr/socket_utils_posix.h", + "src/core/iomgr/socket_windows.h", + "src/core/iomgr/tcp_client.h", + "src/core/iomgr/tcp_posix.h", + "src/core/iomgr/tcp_server.h", + "src/core/iomgr/tcp_windows.h", + "src/core/iomgr/time_averaged_stats.h", + "src/core/iomgr/timer.h", + "src/core/iomgr/timer_heap.h", + "src/core/iomgr/timer_internal.h", + "src/core/iomgr/udp_server.h", + "src/core/iomgr/wakeup_fd_pipe.h", + "src/core/iomgr/wakeup_fd_posix.h", + "src/core/iomgr/workqueue.h", + "src/core/iomgr/workqueue_posix.h", + "src/core/iomgr/workqueue_windows.h", + "src/core/json/json.h", + "src/core/json/json_common.h", + "src/core/json/json_reader.h", + "src/core/json/json_writer.h", + "src/core/security/auth_filters.h", + "src/core/security/b64.h", + "src/core/security/credentials.h", + "src/core/security/handshake.h", + "src/core/security/json_token.h", + "src/core/security/jwt_verifier.h", + "src/core/security/secure_endpoint.h", + "src/core/security/security_connector.h", + "src/core/security/security_context.h", + "src/core/statistics/census_interface.h", + "src/core/statistics/census_rpc_stats.h", + "src/core/surface/api_trace.h", + "src/core/surface/call.h", + "src/core/surface/call_test_only.h", + "src/core/surface/channel.h", + "src/core/surface/completion_queue.h", + "src/core/surface/event_string.h", + "src/core/surface/init.h", + "src/core/surface/server.h", + "src/core/surface/surface_trace.h", + "src/core/transport/byte_stream.h", + "src/core/transport/chttp2/alpn.h", + "src/core/transport/chttp2/bin_encoder.h", + "src/core/transport/chttp2/frame.h", + "src/core/transport/chttp2/frame_data.h", + "src/core/transport/chttp2/frame_goaway.h", + "src/core/transport/chttp2/frame_ping.h", + "src/core/transport/chttp2/frame_rst_stream.h", + "src/core/transport/chttp2/frame_settings.h", + "src/core/transport/chttp2/frame_window_update.h", + "src/core/transport/chttp2/hpack_encoder.h", + "src/core/transport/chttp2/hpack_parser.h", + "src/core/transport/chttp2/hpack_table.h", + "src/core/transport/chttp2/http2_errors.h", + "src/core/transport/chttp2/huffsyms.h", + "src/core/transport/chttp2/incoming_metadata.h", + "src/core/transport/chttp2/internal.h", + "src/core/transport/chttp2/status_conversion.h", + "src/core/transport/chttp2/stream_map.h", + "src/core/transport/chttp2/timeout_encoding.h", + "src/core/transport/chttp2/varint.h", + "src/core/transport/chttp2_transport.h", + "src/core/transport/connectivity_state.h", + "src/core/transport/metadata.h", + "src/core/transport/metadata_batch.h", + "src/core/transport/static_metadata.h", + "src/core/transport/transport.h", + "src/core/transport/transport_impl.h", + "src/core/tsi/fake_transport_security.h", + "src/core/tsi/ssl_transport_security.h", + "src/core/tsi/ssl_types.h", + "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_interface.h" + ], + "language": "c", + "name": "grpc", + "src": [ + "include/grpc/byte_buffer.h", + "include/grpc/byte_buffer_reader.h", + "include/grpc/census.h", + "include/grpc/compression.h", + "include/grpc/grpc.h", + "include/grpc/grpc_security.h", + "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/compression_types.h", + "include/grpc/impl/codegen/connectivity_state.h", + "include/grpc/impl/codegen/grpc_types.h", + "include/grpc/impl/codegen/propagation_bits.h", + "include/grpc/impl/codegen/status.h", + "include/grpc/status.h", + "src/core/census/aggregation.h", + "src/core/census/context.c", + "src/core/census/grpc_context.c", + "src/core/census/grpc_filter.c", + "src/core/census/grpc_filter.h", + "src/core/census/initialize.c", + "src/core/census/mlog.c", + "src/core/census/mlog.h", + "src/core/census/operation.c", + "src/core/census/placeholders.c", + "src/core/census/rpc_metric_id.h", + "src/core/census/tracing.c", + "src/core/channel/channel_args.c", + "src/core/channel/channel_args.h", + "src/core/channel/channel_stack.c", + "src/core/channel/channel_stack.h", + "src/core/channel/client_channel.c", + "src/core/channel/client_channel.h", + "src/core/channel/client_uchannel.c", + "src/core/channel/client_uchannel.h", + "src/core/channel/compress_filter.c", + "src/core/channel/compress_filter.h", + "src/core/channel/connected_channel.c", + "src/core/channel/connected_channel.h", + "src/core/channel/context.h", + "src/core/channel/http_client_filter.c", + "src/core/channel/http_client_filter.h", + "src/core/channel/http_server_filter.c", + "src/core/channel/http_server_filter.h", + "src/core/channel/subchannel_call_holder.c", + "src/core/channel/subchannel_call_holder.h", + "src/core/client_config/client_config.c", + "src/core/client_config/client_config.h", + "src/core/client_config/connector.c", + "src/core/client_config/connector.h", + "src/core/client_config/default_initial_connect_string.c", + "src/core/client_config/initial_connect_string.c", + "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/pick_first.c", + "src/core/client_config/lb_policies/pick_first.h", + "src/core/client_config/lb_policies/round_robin.c", + "src/core/client_config/lb_policies/round_robin.h", + "src/core/client_config/lb_policy.c", + "src/core/client_config/lb_policy.h", + "src/core/client_config/lb_policy_factory.c", + "src/core/client_config/lb_policy_factory.h", + "src/core/client_config/lb_policy_registry.c", + "src/core/client_config/lb_policy_registry.h", + "src/core/client_config/resolver.c", + "src/core/client_config/resolver.h", + "src/core/client_config/resolver_factory.c", + "src/core/client_config/resolver_factory.h", + "src/core/client_config/resolver_registry.c", + "src/core/client_config/resolver_registry.h", + "src/core/client_config/resolvers/dns_resolver.c", + "src/core/client_config/resolvers/dns_resolver.h", + "src/core/client_config/resolvers/sockaddr_resolver.c", + "src/core/client_config/resolvers/sockaddr_resolver.h", + "src/core/client_config/subchannel.c", + "src/core/client_config/subchannel.h", + "src/core/client_config/subchannel_factory.c", + "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.c", + "src/core/client_config/subchannel_index.h", + "src/core/client_config/uri_parser.c", + "src/core/client_config/uri_parser.h", + "src/core/compression/algorithm_metadata.h", + "src/core/compression/compression_algorithm.c", + "src/core/compression/message_compress.c", + "src/core/compression/message_compress.h", + "src/core/debug/trace.c", + "src/core/debug/trace.h", + "src/core/httpcli/format_request.c", + "src/core/httpcli/format_request.h", + "src/core/httpcli/httpcli.c", + "src/core/httpcli/httpcli.h", + "src/core/httpcli/httpcli_security_connector.c", + "src/core/httpcli/parser.c", + "src/core/httpcli/parser.h", + "src/core/iomgr/closure.c", + "src/core/iomgr/closure.h", + "src/core/iomgr/endpoint.c", + "src/core/iomgr/endpoint.h", + "src/core/iomgr/endpoint_pair.h", + "src/core/iomgr/endpoint_pair_posix.c", + "src/core/iomgr/endpoint_pair_windows.c", + "src/core/iomgr/exec_ctx.c", + "src/core/iomgr/exec_ctx.h", + "src/core/iomgr/executor.c", + "src/core/iomgr/executor.h", + "src/core/iomgr/fd_posix.c", + "src/core/iomgr/fd_posix.h", + "src/core/iomgr/iocp_windows.c", + "src/core/iomgr/iocp_windows.h", + "src/core/iomgr/iomgr.c", + "src/core/iomgr/iomgr.h", + "src/core/iomgr/iomgr_internal.h", + "src/core/iomgr/iomgr_posix.c", + "src/core/iomgr/iomgr_posix.h", + "src/core/iomgr/iomgr_windows.c", + "src/core/iomgr/pollset.h", + "src/core/iomgr/pollset_multipoller_with_epoll.c", + "src/core/iomgr/pollset_multipoller_with_poll_posix.c", + "src/core/iomgr/pollset_posix.c", + "src/core/iomgr/pollset_posix.h", + "src/core/iomgr/pollset_set.h", + "src/core/iomgr/pollset_set_posix.c", + "src/core/iomgr/pollset_set_posix.h", + "src/core/iomgr/pollset_set_windows.c", + "src/core/iomgr/pollset_set_windows.h", + "src/core/iomgr/pollset_windows.c", + "src/core/iomgr/pollset_windows.h", + "src/core/iomgr/resolve_address.h", + "src/core/iomgr/resolve_address_posix.c", + "src/core/iomgr/resolve_address_windows.c", + "src/core/iomgr/sockaddr.h", + "src/core/iomgr/sockaddr_posix.h", + "src/core/iomgr/sockaddr_utils.c", + "src/core/iomgr/sockaddr_utils.h", + "src/core/iomgr/sockaddr_win32.h", + "src/core/iomgr/socket_utils_common_posix.c", + "src/core/iomgr/socket_utils_linux.c", + "src/core/iomgr/socket_utils_posix.c", + "src/core/iomgr/socket_utils_posix.h", + "src/core/iomgr/socket_windows.c", + "src/core/iomgr/socket_windows.h", + "src/core/iomgr/tcp_client.h", + "src/core/iomgr/tcp_client_posix.c", + "src/core/iomgr/tcp_client_windows.c", + "src/core/iomgr/tcp_posix.c", + "src/core/iomgr/tcp_posix.h", + "src/core/iomgr/tcp_server.h", + "src/core/iomgr/tcp_server_posix.c", + "src/core/iomgr/tcp_server_windows.c", + "src/core/iomgr/tcp_windows.c", + "src/core/iomgr/tcp_windows.h", + "src/core/iomgr/time_averaged_stats.c", + "src/core/iomgr/time_averaged_stats.h", + "src/core/iomgr/timer.c", + "src/core/iomgr/timer.h", + "src/core/iomgr/timer_heap.c", + "src/core/iomgr/timer_heap.h", + "src/core/iomgr/timer_internal.h", + "src/core/iomgr/udp_server.c", + "src/core/iomgr/udp_server.h", + "src/core/iomgr/wakeup_fd_eventfd.c", + "src/core/iomgr/wakeup_fd_nospecial.c", + "src/core/iomgr/wakeup_fd_pipe.c", + "src/core/iomgr/wakeup_fd_pipe.h", + "src/core/iomgr/wakeup_fd_posix.c", + "src/core/iomgr/wakeup_fd_posix.h", + "src/core/iomgr/workqueue.h", + "src/core/iomgr/workqueue_posix.c", + "src/core/iomgr/workqueue_posix.h", + "src/core/iomgr/workqueue_windows.c", + "src/core/iomgr/workqueue_windows.h", + "src/core/json/json.c", + "src/core/json/json.h", + "src/core/json/json_common.h", + "src/core/json/json_reader.c", + "src/core/json/json_reader.h", + "src/core/json/json_string.c", + "src/core/json/json_writer.c", + "src/core/json/json_writer.h", + "src/core/security/auth_filters.h", + "src/core/security/b64.c", + "src/core/security/b64.h", + "src/core/security/client_auth_filter.c", + "src/core/security/credentials.c", + "src/core/security/credentials.h", + "src/core/security/credentials_metadata.c", + "src/core/security/credentials_posix.c", + "src/core/security/credentials_win32.c", + "src/core/security/google_default_credentials.c", + "src/core/security/handshake.c", + "src/core/security/handshake.h", + "src/core/security/json_token.c", + "src/core/security/json_token.h", + "src/core/security/jwt_verifier.c", + "src/core/security/jwt_verifier.h", + "src/core/security/secure_endpoint.c", + "src/core/security/secure_endpoint.h", + "src/core/security/security_connector.c", + "src/core/security/security_connector.h", + "src/core/security/security_context.c", + "src/core/security/security_context.h", + "src/core/security/server_auth_filter.c", + "src/core/security/server_secure_chttp2.c", + "src/core/statistics/census_interface.h", + "src/core/statistics/census_rpc_stats.h", + "src/core/surface/alarm.c", + "src/core/surface/api_trace.c", + "src/core/surface/api_trace.h", + "src/core/surface/byte_buffer.c", + "src/core/surface/byte_buffer_reader.c", + "src/core/surface/call.c", + "src/core/surface/call.h", + "src/core/surface/call_details.c", + "src/core/surface/call_log_batch.c", + "src/core/surface/call_test_only.h", + "src/core/surface/channel.c", + "src/core/surface/channel.h", + "src/core/surface/channel_connectivity.c", + "src/core/surface/channel_create.c", + "src/core/surface/channel_ping.c", + "src/core/surface/completion_queue.c", + "src/core/surface/completion_queue.h", + "src/core/surface/event_string.c", + "src/core/surface/event_string.h", + "src/core/surface/init.c", + "src/core/surface/init.h", + "src/core/surface/init_secure.c", + "src/core/surface/lame_client.c", + "src/core/surface/metadata_array.c", + "src/core/surface/secure_channel_create.c", + "src/core/surface/server.c", + "src/core/surface/server.h", + "src/core/surface/server_chttp2.c", + "src/core/surface/server_create.c", + "src/core/surface/surface_trace.h", + "src/core/surface/validate_metadata.c", + "src/core/surface/version.c", + "src/core/transport/byte_stream.c", + "src/core/transport/byte_stream.h", + "src/core/transport/chttp2/alpn.c", + "src/core/transport/chttp2/alpn.h", + "src/core/transport/chttp2/bin_encoder.c", + "src/core/transport/chttp2/bin_encoder.h", + "src/core/transport/chttp2/frame.h", + "src/core/transport/chttp2/frame_data.c", + "src/core/transport/chttp2/frame_data.h", + "src/core/transport/chttp2/frame_goaway.c", + "src/core/transport/chttp2/frame_goaway.h", + "src/core/transport/chttp2/frame_ping.c", + "src/core/transport/chttp2/frame_ping.h", + "src/core/transport/chttp2/frame_rst_stream.c", + "src/core/transport/chttp2/frame_rst_stream.h", + "src/core/transport/chttp2/frame_settings.c", + "src/core/transport/chttp2/frame_settings.h", + "src/core/transport/chttp2/frame_window_update.c", + "src/core/transport/chttp2/frame_window_update.h", + "src/core/transport/chttp2/hpack_encoder.c", + "src/core/transport/chttp2/hpack_encoder.h", + "src/core/transport/chttp2/hpack_parser.c", + "src/core/transport/chttp2/hpack_parser.h", + "src/core/transport/chttp2/hpack_table.c", + "src/core/transport/chttp2/hpack_table.h", + "src/core/transport/chttp2/http2_errors.h", + "src/core/transport/chttp2/huffsyms.c", + "src/core/transport/chttp2/huffsyms.h", + "src/core/transport/chttp2/incoming_metadata.c", + "src/core/transport/chttp2/incoming_metadata.h", + "src/core/transport/chttp2/internal.h", + "src/core/transport/chttp2/parsing.c", + "src/core/transport/chttp2/status_conversion.c", + "src/core/transport/chttp2/status_conversion.h", + "src/core/transport/chttp2/stream_lists.c", + "src/core/transport/chttp2/stream_map.c", + "src/core/transport/chttp2/stream_map.h", + "src/core/transport/chttp2/timeout_encoding.c", + "src/core/transport/chttp2/timeout_encoding.h", + "src/core/transport/chttp2/varint.c", + "src/core/transport/chttp2/varint.h", + "src/core/transport/chttp2/writing.c", + "src/core/transport/chttp2_transport.c", + "src/core/transport/chttp2_transport.h", + "src/core/transport/connectivity_state.c", + "src/core/transport/connectivity_state.h", + "src/core/transport/metadata.c", + "src/core/transport/metadata.h", + "src/core/transport/metadata_batch.c", + "src/core/transport/metadata_batch.h", + "src/core/transport/static_metadata.c", + "src/core/transport/static_metadata.h", + "src/core/transport/transport.c", + "src/core/transport/transport.h", + "src/core/transport/transport_impl.h", + "src/core/transport/transport_op_string.c", + "src/core/tsi/fake_transport_security.c", + "src/core/tsi/fake_transport_security.h", + "src/core/tsi/ssl_transport_security.c", + "src/core/tsi/ssl_transport_security.h", + "src/core/tsi/ssl_types.h", + "src/core/tsi/transport_security.c", + "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_interface.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr", + "grpc" + ], + "headers": [], + "language": "c", + "name": "grpc_dll", + "src": [], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc" + ], + "headers": [ + "test/core/end2end/cq_verifier.h", + "test/core/end2end/data/ssl_test_data.h", + "test/core/end2end/fixtures/proxy.h", + "test/core/iomgr/endpoint_tests.h", + "test/core/security/oauth2_utils.h", + "test/core/util/grpc_profiler.h", + "test/core/util/parse_hexstring.h", + "test/core/util/port.h", + "test/core/util/slice_splitter.h" + ], + "language": "c", + "name": "grpc_test_util", + "src": [ + "test/core/end2end/cq_verifier.c", + "test/core/end2end/cq_verifier.h", + "test/core/end2end/data/server1_cert.c", + "test/core/end2end/data/server1_key.c", + "test/core/end2end/data/ssl_test_data.h", + "test/core/end2end/data/test_root_cert.c", + "test/core/end2end/fixtures/proxy.c", + "test/core/end2end/fixtures/proxy.h", + "test/core/iomgr/endpoint_tests.c", + "test/core/iomgr/endpoint_tests.h", + "test/core/security/oauth2_utils.c", + "test/core/security/oauth2_utils.h", + "test/core/util/grpc_profiler.c", + "test/core/util/grpc_profiler.h", + "test/core/util/parse_hexstring.c", + "test/core/util/parse_hexstring.h", + "test/core/util/port.h", + "test/core/util/port_posix.c", + "test/core/util/port_windows.c", + "test/core/util/slice_splitter.c", + "test/core/util/slice_splitter.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc_unsecure" + ], + "headers": [ + "test/core/end2end/cq_verifier.h", + "test/core/end2end/fixtures/proxy.h", + "test/core/iomgr/endpoint_tests.h", + "test/core/util/grpc_profiler.h", + "test/core/util/parse_hexstring.h", + "test/core/util/port.h", + "test/core/util/slice_splitter.h" + ], + "language": "c", + "name": "grpc_test_util_unsecure", + "src": [ + "test/core/end2end/cq_verifier.c", + "test/core/end2end/cq_verifier.h", + "test/core/end2end/fixtures/proxy.c", + "test/core/end2end/fixtures/proxy.h", + "test/core/iomgr/endpoint_tests.c", + "test/core/iomgr/endpoint_tests.h", + "test/core/util/grpc_profiler.c", + "test/core/util/grpc_profiler.h", + "test/core/util/parse_hexstring.c", + "test/core/util/parse_hexstring.h", + "test/core/util/port.h", + "test/core/util/port_posix.c", + "test/core/util/port_windows.c", + "test/core/util/slice_splitter.c", + "test/core/util/slice_splitter.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr" + ], + "headers": [ + "include/grpc/byte_buffer.h", + "include/grpc/byte_buffer_reader.h", + "include/grpc/census.h", + "include/grpc/compression.h", + "include/grpc/grpc.h", + "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/compression_types.h", + "include/grpc/impl/codegen/connectivity_state.h", + "include/grpc/impl/codegen/grpc_types.h", + "include/grpc/impl/codegen/propagation_bits.h", + "include/grpc/impl/codegen/status.h", + "include/grpc/status.h", + "src/core/census/aggregation.h", + "src/core/census/grpc_filter.h", + "src/core/census/mlog.h", + "src/core/census/rpc_metric_id.h", + "src/core/channel/channel_args.h", + "src/core/channel/channel_stack.h", + "src/core/channel/client_channel.h", + "src/core/channel/client_uchannel.h", + "src/core/channel/compress_filter.h", + "src/core/channel/connected_channel.h", + "src/core/channel/context.h", + "src/core/channel/http_client_filter.h", + "src/core/channel/http_server_filter.h", + "src/core/channel/subchannel_call_holder.h", + "src/core/client_config/client_config.h", + "src/core/client_config/connector.h", + "src/core/client_config/initial_connect_string.h", + "src/core/client_config/lb_policies/pick_first.h", + "src/core/client_config/lb_policies/round_robin.h", + "src/core/client_config/lb_policy.h", + "src/core/client_config/lb_policy_factory.h", + "src/core/client_config/lb_policy_registry.h", + "src/core/client_config/resolver.h", + "src/core/client_config/resolver_factory.h", + "src/core/client_config/resolver_registry.h", + "src/core/client_config/resolvers/dns_resolver.h", + "src/core/client_config/resolvers/sockaddr_resolver.h", + "src/core/client_config/subchannel.h", + "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.h", + "src/core/client_config/uri_parser.h", + "src/core/compression/algorithm_metadata.h", + "src/core/compression/message_compress.h", + "src/core/debug/trace.h", + "src/core/httpcli/format_request.h", + "src/core/httpcli/httpcli.h", + "src/core/httpcli/parser.h", + "src/core/iomgr/closure.h", + "src/core/iomgr/endpoint.h", + "src/core/iomgr/endpoint_pair.h", + "src/core/iomgr/exec_ctx.h", + "src/core/iomgr/executor.h", + "src/core/iomgr/fd_posix.h", + "src/core/iomgr/iocp_windows.h", + "src/core/iomgr/iomgr.h", + "src/core/iomgr/iomgr_internal.h", + "src/core/iomgr/iomgr_posix.h", + "src/core/iomgr/pollset.h", + "src/core/iomgr/pollset_posix.h", + "src/core/iomgr/pollset_set.h", + "src/core/iomgr/pollset_set_posix.h", + "src/core/iomgr/pollset_set_windows.h", + "src/core/iomgr/pollset_windows.h", + "src/core/iomgr/resolve_address.h", + "src/core/iomgr/sockaddr.h", + "src/core/iomgr/sockaddr_posix.h", + "src/core/iomgr/sockaddr_utils.h", + "src/core/iomgr/sockaddr_win32.h", + "src/core/iomgr/socket_utils_posix.h", + "src/core/iomgr/socket_windows.h", + "src/core/iomgr/tcp_client.h", + "src/core/iomgr/tcp_posix.h", + "src/core/iomgr/tcp_server.h", + "src/core/iomgr/tcp_windows.h", + "src/core/iomgr/time_averaged_stats.h", + "src/core/iomgr/timer.h", + "src/core/iomgr/timer_heap.h", + "src/core/iomgr/timer_internal.h", + "src/core/iomgr/udp_server.h", + "src/core/iomgr/wakeup_fd_pipe.h", + "src/core/iomgr/wakeup_fd_posix.h", + "src/core/iomgr/workqueue.h", + "src/core/iomgr/workqueue_posix.h", + "src/core/iomgr/workqueue_windows.h", + "src/core/json/json.h", + "src/core/json/json_common.h", + "src/core/json/json_reader.h", + "src/core/json/json_writer.h", + "src/core/statistics/census_interface.h", + "src/core/statistics/census_rpc_stats.h", + "src/core/surface/api_trace.h", + "src/core/surface/call.h", + "src/core/surface/call_test_only.h", + "src/core/surface/channel.h", + "src/core/surface/completion_queue.h", + "src/core/surface/event_string.h", + "src/core/surface/init.h", + "src/core/surface/server.h", + "src/core/surface/surface_trace.h", + "src/core/transport/byte_stream.h", + "src/core/transport/chttp2/alpn.h", + "src/core/transport/chttp2/bin_encoder.h", + "src/core/transport/chttp2/frame.h", + "src/core/transport/chttp2/frame_data.h", + "src/core/transport/chttp2/frame_goaway.h", + "src/core/transport/chttp2/frame_ping.h", + "src/core/transport/chttp2/frame_rst_stream.h", + "src/core/transport/chttp2/frame_settings.h", + "src/core/transport/chttp2/frame_window_update.h", + "src/core/transport/chttp2/hpack_encoder.h", + "src/core/transport/chttp2/hpack_parser.h", + "src/core/transport/chttp2/hpack_table.h", + "src/core/transport/chttp2/http2_errors.h", + "src/core/transport/chttp2/huffsyms.h", + "src/core/transport/chttp2/incoming_metadata.h", + "src/core/transport/chttp2/internal.h", + "src/core/transport/chttp2/status_conversion.h", + "src/core/transport/chttp2/stream_map.h", + "src/core/transport/chttp2/timeout_encoding.h", + "src/core/transport/chttp2/varint.h", + "src/core/transport/chttp2_transport.h", + "src/core/transport/connectivity_state.h", + "src/core/transport/metadata.h", + "src/core/transport/metadata_batch.h", + "src/core/transport/static_metadata.h", + "src/core/transport/transport.h", + "src/core/transport/transport_impl.h" + ], + "language": "c", + "name": "grpc_unsecure", + "src": [ + "include/grpc/byte_buffer.h", + "include/grpc/byte_buffer_reader.h", + "include/grpc/census.h", + "include/grpc/compression.h", + "include/grpc/grpc.h", + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", "include/grpc/impl/codegen/grpc_types.h", @@ -3661,8 +4523,8 @@ "src/core/census/grpc_filter.c", "src/core/census/grpc_filter.h", "src/core/census/initialize.c", - "src/core/census/log.c", - "src/core/census/log.h", + "src/core/census/mlog.c", + "src/core/census/mlog.h", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/rpc_metric_id.h", @@ -3721,8 +4583,8 @@ "src/core/client_config/subchannel_index.h", "src/core/client_config/uri_parser.c", "src/core/client_config/uri_parser.h", - "src/core/compression/algorithm.c", "src/core/compression/algorithm_metadata.h", + "src/core/compression/compression_algorithm.c", "src/core/compression/message_compress.c", "src/core/compression/message_compress.h", "src/core/debug/trace.c", @@ -3907,7 +4769,9 @@ "src/core/transport/transport.h", "src/core/transport/transport_impl.h", "src/core/transport/transport_op_string.c" - ] + ], + "third_party": false, + "type": "lib" }, { "deps": [ @@ -3918,52 +4782,322 @@ "include/grpc/grpc_zookeeper.h", "src/core/client_config/resolvers/zookeeper_resolver.h" ], - "language": "c", - "name": "grpc_zookeeper", - "src": [ - "include/grpc/grpc_zookeeper.h", - "src/core/client_config/resolvers/zookeeper_resolver.c", - "src/core/client_config/resolvers/zookeeper_resolver.h" - ] + "language": "c", + "name": "grpc_zookeeper", + "src": [ + "include/grpc/grpc_zookeeper.h", + "src/core/client_config/resolvers/zookeeper_resolver.c", + "src/core/client_config/resolvers/zookeeper_resolver.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util", + "test_tcp_server" + ], + "headers": [ + "test/core/util/reconnect_server.h" + ], + "language": "c", + "name": "reconnect_server", + "src": [ + "test/core/util/reconnect_server.c", + "test/core/util/reconnect_server.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [ + "test/core/util/test_tcp_server.h" + ], + "language": "c", + "name": "test_tcp_server", + "src": [ + "test/core/util/test_tcp_server.c", + "test/core/util/test_tcp_server.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "grpc" + ], + "headers": [ + "include/grpc++/alarm.h", + "include/grpc++/channel.h", + "include/grpc++/client_context.h", + "include/grpc++/completion_queue.h", + "include/grpc++/create_channel.h", + "include/grpc++/generic/async_generic_service.h", + "include/grpc++/generic/generic_stub.h", + "include/grpc++/grpc++.h", + "include/grpc++/impl/call.h", + "include/grpc++/impl/client_unary_call.h", + "include/grpc++/impl/codegen/async_stream.h", + "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/call.h", + "include/grpc++/impl/codegen/call_hook.h", + "include/grpc++/impl/codegen/channel_interface.h", + "include/grpc++/impl/codegen/client_context.h", + "include/grpc++/impl/codegen/client_unary_call.h", + "include/grpc++/impl/codegen/completion_queue.h", + "include/grpc++/impl/codegen/completion_queue_tag.h", + "include/grpc++/impl/codegen/config.h", + "include/grpc++/impl/codegen/config_protobuf.h", + "include/grpc++/impl/codegen/grpc_library.h", + "include/grpc++/impl/codegen/method_handler_impl.h", + "include/grpc++/impl/codegen/proto_utils.h", + "include/grpc++/impl/codegen/rpc_method.h", + "include/grpc++/impl/codegen/rpc_service_method.h", + "include/grpc++/impl/codegen/security/auth_context.h", + "include/grpc++/impl/codegen/serialization_traits.h", + "include/grpc++/impl/codegen/server_context.h", + "include/grpc++/impl/codegen/server_interface.h", + "include/grpc++/impl/codegen/service_type.h", + "include/grpc++/impl/codegen/status.h", + "include/grpc++/impl/codegen/status_code_enum.h", + "include/grpc++/impl/codegen/string_ref.h", + "include/grpc++/impl/codegen/stub_options.h", + "include/grpc++/impl/codegen/sync.h", + "include/grpc++/impl/codegen/sync_cxx11.h", + "include/grpc++/impl/codegen/sync_no_cxx11.h", + "include/grpc++/impl/codegen/sync_stream.h", + "include/grpc++/impl/codegen/time.h", + "include/grpc++/impl/grpc_library.h", + "include/grpc++/impl/method_handler_impl.h", + "include/grpc++/impl/proto_utils.h", + "include/grpc++/impl/rpc_method.h", + "include/grpc++/impl/rpc_service_method.h", + "include/grpc++/impl/serialization_traits.h", + "include/grpc++/impl/server_builder_option.h", + "include/grpc++/impl/service_type.h", + "include/grpc++/impl/sync.h", + "include/grpc++/impl/sync_cxx11.h", + "include/grpc++/impl/sync_no_cxx11.h", + "include/grpc++/impl/thd.h", + "include/grpc++/impl/thd_cxx11.h", + "include/grpc++/impl/thd_no_cxx11.h", + "include/grpc++/security/auth_context.h", + "include/grpc++/security/auth_metadata_processor.h", + "include/grpc++/security/credentials.h", + "include/grpc++/security/server_credentials.h", + "include/grpc++/server.h", + "include/grpc++/server_builder.h", + "include/grpc++/server_context.h", + "include/grpc++/support/async_stream.h", + "include/grpc++/support/async_unary_call.h", + "include/grpc++/support/byte_buffer.h", + "include/grpc++/support/channel_arguments.h", + "include/grpc++/support/config.h", + "include/grpc++/support/config_protobuf.h", + "include/grpc++/support/slice.h", + "include/grpc++/support/status.h", + "include/grpc++/support/status_code_enum.h", + "include/grpc++/support/string_ref.h", + "include/grpc++/support/stub_options.h", + "include/grpc++/support/sync_stream.h", + "include/grpc++/support/time.h", + "src/cpp/client/create_channel_internal.h", + "src/cpp/client/secure_credentials.h", + "src/cpp/common/create_auth_context.h", + "src/cpp/common/secure_auth_context.h", + "src/cpp/server/dynamic_thread_pool.h", + "src/cpp/server/secure_server_credentials.h", + "src/cpp/server/thread_pool_interface.h" + ], + "language": "c++", + "name": "grpc++", + "src": [ + "include/grpc++/alarm.h", + "include/grpc++/channel.h", + "include/grpc++/client_context.h", + "include/grpc++/completion_queue.h", + "include/grpc++/create_channel.h", + "include/grpc++/generic/async_generic_service.h", + "include/grpc++/generic/generic_stub.h", + "include/grpc++/grpc++.h", + "include/grpc++/impl/call.h", + "include/grpc++/impl/client_unary_call.h", + "include/grpc++/impl/codegen/async_stream.h", + "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/call.h", + "include/grpc++/impl/codegen/call_hook.h", + "include/grpc++/impl/codegen/channel_interface.h", + "include/grpc++/impl/codegen/client_context.h", + "include/grpc++/impl/codegen/client_unary_call.h", + "include/grpc++/impl/codegen/completion_queue.h", + "include/grpc++/impl/codegen/completion_queue_tag.h", + "include/grpc++/impl/codegen/config.h", + "include/grpc++/impl/codegen/config_protobuf.h", + "include/grpc++/impl/codegen/grpc_library.h", + "include/grpc++/impl/codegen/method_handler_impl.h", + "include/grpc++/impl/codegen/proto_utils.h", + "include/grpc++/impl/codegen/rpc_method.h", + "include/grpc++/impl/codegen/rpc_service_method.h", + "include/grpc++/impl/codegen/security/auth_context.h", + "include/grpc++/impl/codegen/serialization_traits.h", + "include/grpc++/impl/codegen/server_context.h", + "include/grpc++/impl/codegen/server_interface.h", + "include/grpc++/impl/codegen/service_type.h", + "include/grpc++/impl/codegen/status.h", + "include/grpc++/impl/codegen/status_code_enum.h", + "include/grpc++/impl/codegen/string_ref.h", + "include/grpc++/impl/codegen/stub_options.h", + "include/grpc++/impl/codegen/sync.h", + "include/grpc++/impl/codegen/sync_cxx11.h", + "include/grpc++/impl/codegen/sync_no_cxx11.h", + "include/grpc++/impl/codegen/sync_stream.h", + "include/grpc++/impl/codegen/time.h", + "include/grpc++/impl/grpc_library.h", + "include/grpc++/impl/method_handler_impl.h", + "include/grpc++/impl/proto_utils.h", + "include/grpc++/impl/rpc_method.h", + "include/grpc++/impl/rpc_service_method.h", + "include/grpc++/impl/serialization_traits.h", + "include/grpc++/impl/server_builder_option.h", + "include/grpc++/impl/service_type.h", + "include/grpc++/impl/sync.h", + "include/grpc++/impl/sync_cxx11.h", + "include/grpc++/impl/sync_no_cxx11.h", + "include/grpc++/impl/thd.h", + "include/grpc++/impl/thd_cxx11.h", + "include/grpc++/impl/thd_no_cxx11.h", + "include/grpc++/security/auth_context.h", + "include/grpc++/security/auth_metadata_processor.h", + "include/grpc++/security/credentials.h", + "include/grpc++/security/server_credentials.h", + "include/grpc++/server.h", + "include/grpc++/server_builder.h", + "include/grpc++/server_context.h", + "include/grpc++/support/async_stream.h", + "include/grpc++/support/async_unary_call.h", + "include/grpc++/support/byte_buffer.h", + "include/grpc++/support/channel_arguments.h", + "include/grpc++/support/config.h", + "include/grpc++/support/config_protobuf.h", + "include/grpc++/support/slice.h", + "include/grpc++/support/status.h", + "include/grpc++/support/status_code_enum.h", + "include/grpc++/support/string_ref.h", + "include/grpc++/support/stub_options.h", + "include/grpc++/support/sync_stream.h", + "include/grpc++/support/time.h", + "src/cpp/client/channel.cc", + "src/cpp/client/client_context.cc", + "src/cpp/client/create_channel.cc", + "src/cpp/client/create_channel_internal.cc", + "src/cpp/client/create_channel_internal.h", + "src/cpp/client/credentials.cc", + "src/cpp/client/generic_stub.cc", + "src/cpp/client/insecure_credentials.cc", + "src/cpp/client/secure_credentials.cc", + "src/cpp/client/secure_credentials.h", + "src/cpp/codegen/grpc_library.cc", + "src/cpp/common/alarm.cc", + "src/cpp/common/auth_property_iterator.cc", + "src/cpp/common/call.cc", + "src/cpp/common/channel_arguments.cc", + "src/cpp/common/completion_queue.cc", + "src/cpp/common/create_auth_context.h", + "src/cpp/common/rpc_method.cc", + "src/cpp/common/secure_auth_context.cc", + "src/cpp/common/secure_auth_context.h", + "src/cpp/common/secure_channel_arguments.cc", + "src/cpp/common/secure_create_auth_context.cc", + "src/cpp/proto/proto_utils.cc", + "src/cpp/server/async_generic_service.cc", + "src/cpp/server/create_default_thread_pool.cc", + "src/cpp/server/dynamic_thread_pool.cc", + "src/cpp/server/dynamic_thread_pool.h", + "src/cpp/server/insecure_server_credentials.cc", + "src/cpp/server/secure_server_credentials.cc", + "src/cpp/server/secure_server_credentials.h", + "src/cpp/server/server.cc", + "src/cpp/server/server_builder.cc", + "src/cpp/server/server_context.cc", + "src/cpp/server/server_credentials.cc", + "src/cpp/server/thread_pool_interface.h", + "src/cpp/util/byte_buffer.cc", + "src/cpp/util/slice.cc", + "src/cpp/util/status.cc", + "src/cpp/util/string_ref.cc", + "src/cpp/util/time.cc" + ], + "third_party": false, + "type": "lib" }, { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util", - "test_tcp_server" - ], + "deps": [], "headers": [ - "test/core/util/reconnect_server.h" + "test/cpp/util/test_config.h" ], - "language": "c", - "name": "reconnect_server", + "language": "c++", + "name": "grpc++_test_config", "src": [ - "test/core/util/reconnect_server.c", - "test/core/util/reconnect_server.h" - ] + "test/cpp/util/test_config.cc", + "test/cpp/util/test_config.h" + ], + "third_party": false, + "type": "lib" }, { "deps": [ - "gpr", - "gpr_test_util", - "grpc", + "grpc++", "grpc_test_util" ], "headers": [ - "test/core/util/test_tcp_server.h" + "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h", + "src/proto/grpc/testing/duplicate/echo_duplicate.pb.h", + "src/proto/grpc/testing/echo.grpc.pb.h", + "src/proto/grpc/testing/echo.pb.h", + "src/proto/grpc/testing/echo_messages.grpc.pb.h", + "src/proto/grpc/testing/echo_messages.pb.h", + "test/cpp/end2end/test_service_impl.h", + "test/cpp/util/byte_buffer_proto_helper.h", + "test/cpp/util/cli_call.h", + "test/cpp/util/create_test_channel.h", + "test/cpp/util/string_ref_helper.h", + "test/cpp/util/subprocess.h", + "test/cpp/util/test_credentials_provider.h" ], - "language": "c", - "name": "test_tcp_server", + "language": "c++", + "name": "grpc++_test_util", "src": [ - "test/core/util/test_tcp_server.c", - "test/core/util/test_tcp_server.h" - ] + "test/cpp/end2end/test_service_impl.cc", + "test/cpp/end2end/test_service_impl.h", + "test/cpp/util/byte_buffer_proto_helper.cc", + "test/cpp/util/byte_buffer_proto_helper.h", + "test/cpp/util/cli_call.cc", + "test/cpp/util/cli_call.h", + "test/cpp/util/create_test_channel.cc", + "test/cpp/util/create_test_channel.h", + "test/cpp/util/string_ref_helper.cc", + "test/cpp/util/string_ref_helper.h", + "test/cpp/util/subprocess.cc", + "test/cpp/util/subprocess.h", + "test/cpp/util/test_credentials_provider.cc", + "test/cpp/util/test_credentials_provider.h" + ], + "third_party": false, + "type": "lib" }, { "deps": [ - "grpc" + "gpr", + "grpc_unsecure" ], "headers": [ "include/grpc++/alarm.h", @@ -4041,26 +5175,195 @@ "include/grpc++/support/sync_stream.h", "include/grpc++/support/time.h", "src/cpp/client/create_channel_internal.h", - "src/cpp/client/secure_credentials.h", "src/cpp/common/create_auth_context.h", - "src/cpp/common/secure_auth_context.h", "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/secure_server_credentials.h", "src/cpp/server/thread_pool_interface.h" ], "language": "c++", - "name": "grpc++", + "name": "grpc++_unsecure", + "src": [ + "include/grpc++/alarm.h", + "include/grpc++/channel.h", + "include/grpc++/client_context.h", + "include/grpc++/completion_queue.h", + "include/grpc++/create_channel.h", + "include/grpc++/generic/async_generic_service.h", + "include/grpc++/generic/generic_stub.h", + "include/grpc++/grpc++.h", + "include/grpc++/impl/call.h", + "include/grpc++/impl/client_unary_call.h", + "include/grpc++/impl/codegen/async_stream.h", + "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/call.h", + "include/grpc++/impl/codegen/call_hook.h", + "include/grpc++/impl/codegen/channel_interface.h", + "include/grpc++/impl/codegen/client_context.h", + "include/grpc++/impl/codegen/client_unary_call.h", + "include/grpc++/impl/codegen/completion_queue.h", + "include/grpc++/impl/codegen/completion_queue_tag.h", + "include/grpc++/impl/codegen/config.h", + "include/grpc++/impl/codegen/config_protobuf.h", + "include/grpc++/impl/codegen/grpc_library.h", + "include/grpc++/impl/codegen/method_handler_impl.h", + "include/grpc++/impl/codegen/proto_utils.h", + "include/grpc++/impl/codegen/rpc_method.h", + "include/grpc++/impl/codegen/rpc_service_method.h", + "include/grpc++/impl/codegen/security/auth_context.h", + "include/grpc++/impl/codegen/serialization_traits.h", + "include/grpc++/impl/codegen/server_context.h", + "include/grpc++/impl/codegen/server_interface.h", + "include/grpc++/impl/codegen/service_type.h", + "include/grpc++/impl/codegen/status.h", + "include/grpc++/impl/codegen/status_code_enum.h", + "include/grpc++/impl/codegen/string_ref.h", + "include/grpc++/impl/codegen/stub_options.h", + "include/grpc++/impl/codegen/sync.h", + "include/grpc++/impl/codegen/sync_cxx11.h", + "include/grpc++/impl/codegen/sync_no_cxx11.h", + "include/grpc++/impl/codegen/sync_stream.h", + "include/grpc++/impl/codegen/time.h", + "include/grpc++/impl/grpc_library.h", + "include/grpc++/impl/method_handler_impl.h", + "include/grpc++/impl/proto_utils.h", + "include/grpc++/impl/rpc_method.h", + "include/grpc++/impl/rpc_service_method.h", + "include/grpc++/impl/serialization_traits.h", + "include/grpc++/impl/server_builder_option.h", + "include/grpc++/impl/service_type.h", + "include/grpc++/impl/sync.h", + "include/grpc++/impl/sync_cxx11.h", + "include/grpc++/impl/sync_no_cxx11.h", + "include/grpc++/impl/thd.h", + "include/grpc++/impl/thd_cxx11.h", + "include/grpc++/impl/thd_no_cxx11.h", + "include/grpc++/security/auth_context.h", + "include/grpc++/security/auth_metadata_processor.h", + "include/grpc++/security/credentials.h", + "include/grpc++/security/server_credentials.h", + "include/grpc++/server.h", + "include/grpc++/server_builder.h", + "include/grpc++/server_context.h", + "include/grpc++/support/async_stream.h", + "include/grpc++/support/async_unary_call.h", + "include/grpc++/support/byte_buffer.h", + "include/grpc++/support/channel_arguments.h", + "include/grpc++/support/config.h", + "include/grpc++/support/config_protobuf.h", + "include/grpc++/support/slice.h", + "include/grpc++/support/status.h", + "include/grpc++/support/status_code_enum.h", + "include/grpc++/support/string_ref.h", + "include/grpc++/support/stub_options.h", + "include/grpc++/support/sync_stream.h", + "include/grpc++/support/time.h", + "src/cpp/client/channel.cc", + "src/cpp/client/client_context.cc", + "src/cpp/client/create_channel.cc", + "src/cpp/client/create_channel_internal.cc", + "src/cpp/client/create_channel_internal.h", + "src/cpp/client/credentials.cc", + "src/cpp/client/generic_stub.cc", + "src/cpp/client/insecure_credentials.cc", + "src/cpp/codegen/grpc_library.cc", + "src/cpp/common/alarm.cc", + "src/cpp/common/call.cc", + "src/cpp/common/channel_arguments.cc", + "src/cpp/common/completion_queue.cc", + "src/cpp/common/create_auth_context.h", + "src/cpp/common/insecure_create_auth_context.cc", + "src/cpp/common/rpc_method.cc", + "src/cpp/proto/proto_utils.cc", + "src/cpp/server/async_generic_service.cc", + "src/cpp/server/create_default_thread_pool.cc", + "src/cpp/server/dynamic_thread_pool.cc", + "src/cpp/server/dynamic_thread_pool.h", + "src/cpp/server/insecure_server_credentials.cc", + "src/cpp/server/server.cc", + "src/cpp/server/server_builder.cc", + "src/cpp/server/server_context.cc", + "src/cpp/server/server_credentials.cc", + "src/cpp/server/thread_pool_interface.h", + "src/cpp/util/byte_buffer.cc", + "src/cpp/util/slice.cc", + "src/cpp/util/status.cc", + "src/cpp/util/string_ref.cc", + "src/cpp/util/time.cc" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [], + "headers": [ + "include/grpc++/impl/codegen/async_stream.h", + "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/call.h", + "include/grpc++/impl/codegen/call_hook.h", + "include/grpc++/impl/codegen/channel_interface.h", + "include/grpc++/impl/codegen/client_context.h", + "include/grpc++/impl/codegen/client_unary_call.h", + "include/grpc++/impl/codegen/completion_queue.h", + "include/grpc++/impl/codegen/completion_queue_tag.h", + "include/grpc++/impl/codegen/config.h", + "include/grpc++/impl/codegen/config_protobuf.h", + "include/grpc++/impl/codegen/grpc_library.h", + "include/grpc++/impl/codegen/method_handler_impl.h", + "include/grpc++/impl/codegen/proto_utils.h", + "include/grpc++/impl/codegen/rpc_method.h", + "include/grpc++/impl/codegen/rpc_service_method.h", + "include/grpc++/impl/codegen/security/auth_context.h", + "include/grpc++/impl/codegen/serialization_traits.h", + "include/grpc++/impl/codegen/server_context.h", + "include/grpc++/impl/codegen/server_interface.h", + "include/grpc++/impl/codegen/service_type.h", + "include/grpc++/impl/codegen/status.h", + "include/grpc++/impl/codegen/status_code_enum.h", + "include/grpc++/impl/codegen/string_ref.h", + "include/grpc++/impl/codegen/stub_options.h", + "include/grpc++/impl/codegen/sync.h", + "include/grpc++/impl/codegen/sync_cxx11.h", + "include/grpc++/impl/codegen/sync_no_cxx11.h", + "include/grpc++/impl/codegen/sync_stream.h", + "include/grpc++/impl/codegen/time.h", + "include/grpc++/support/config.h", + "include/grpc++/support/config_protobuf.h", + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/compression_types.h", + "include/grpc/impl/codegen/connectivity_state.h", + "include/grpc/impl/codegen/grpc_types.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/propagation_bits.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/status.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "src/compiler/config.h", + "src/compiler/cpp_generator.h", + "src/compiler/cpp_generator_helpers.h", + "src/compiler/csharp_generator.h", + "src/compiler/csharp_generator_helpers.h", + "src/compiler/generator_helpers.h", + "src/compiler/objective_c_generator.h", + "src/compiler/objective_c_generator_helpers.h", + "src/compiler/python_generator.h", + "src/compiler/ruby_generator.h", + "src/compiler/ruby_generator_helpers-inl.h", + "src/compiler/ruby_generator_map-inl.h", + "src/compiler/ruby_generator_string-inl.h" + ], + "language": "c++", + "name": "grpc_plugin_support", "src": [ - "include/grpc++/alarm.h", - "include/grpc++/channel.h", - "include/grpc++/client_context.h", - "include/grpc++/completion_queue.h", - "include/grpc++/create_channel.h", - "include/grpc++/generic/async_generic_service.h", - "include/grpc++/generic/generic_stub.h", - "include/grpc++/grpc++.h", - "include/grpc++/impl/call.h", - "include/grpc++/impl/client_unary_call.h", "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", "include/grpc++/impl/codegen/call.h", @@ -4091,635 +5394,1308 @@ "include/grpc++/impl/codegen/sync_no_cxx11.h", "include/grpc++/impl/codegen/sync_stream.h", "include/grpc++/impl/codegen/time.h", - "include/grpc++/impl/grpc_library.h", - "include/grpc++/impl/method_handler_impl.h", - "include/grpc++/impl/proto_utils.h", - "include/grpc++/impl/rpc_method.h", - "include/grpc++/impl/rpc_service_method.h", - "include/grpc++/impl/serialization_traits.h", - "include/grpc++/impl/server_builder_option.h", - "include/grpc++/impl/service_type.h", - "include/grpc++/impl/sync.h", - "include/grpc++/impl/sync_cxx11.h", - "include/grpc++/impl/sync_no_cxx11.h", - "include/grpc++/impl/thd.h", - "include/grpc++/impl/thd_cxx11.h", - "include/grpc++/impl/thd_no_cxx11.h", - "include/grpc++/security/auth_context.h", - "include/grpc++/security/auth_metadata_processor.h", - "include/grpc++/security/credentials.h", - "include/grpc++/security/server_credentials.h", - "include/grpc++/server.h", - "include/grpc++/server_builder.h", - "include/grpc++/server_context.h", - "include/grpc++/support/async_stream.h", - "include/grpc++/support/async_unary_call.h", - "include/grpc++/support/byte_buffer.h", - "include/grpc++/support/channel_arguments.h", "include/grpc++/support/config.h", "include/grpc++/support/config_protobuf.h", - "include/grpc++/support/slice.h", - "include/grpc++/support/status.h", - "include/grpc++/support/status_code_enum.h", - "include/grpc++/support/string_ref.h", - "include/grpc++/support/stub_options.h", - "include/grpc++/support/sync_stream.h", - "include/grpc++/support/time.h", - "src/cpp/client/channel.cc", - "src/cpp/client/client_context.cc", - "src/cpp/client/create_channel.cc", - "src/cpp/client/create_channel_internal.cc", - "src/cpp/client/create_channel_internal.h", - "src/cpp/client/credentials.cc", - "src/cpp/client/generic_stub.cc", - "src/cpp/client/insecure_credentials.cc", - "src/cpp/client/secure_credentials.cc", - "src/cpp/client/secure_credentials.h", - "src/cpp/codegen/grpc_library.cc", - "src/cpp/common/alarm.cc", - "src/cpp/common/auth_property_iterator.cc", - "src/cpp/common/call.cc", - "src/cpp/common/channel_arguments.cc", - "src/cpp/common/completion_queue.cc", - "src/cpp/common/create_auth_context.h", - "src/cpp/common/rpc_method.cc", - "src/cpp/common/secure_auth_context.cc", - "src/cpp/common/secure_auth_context.h", - "src/cpp/common/secure_channel_arguments.cc", - "src/cpp/common/secure_create_auth_context.cc", - "src/cpp/proto/proto_utils.cc", - "src/cpp/server/async_generic_service.cc", - "src/cpp/server/create_default_thread_pool.cc", - "src/cpp/server/dynamic_thread_pool.cc", - "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/insecure_server_credentials.cc", - "src/cpp/server/secure_server_credentials.cc", - "src/cpp/server/secure_server_credentials.h", - "src/cpp/server/server.cc", - "src/cpp/server/server_builder.cc", - "src/cpp/server/server_context.cc", - "src/cpp/server/server_credentials.cc", - "src/cpp/server/thread_pool_interface.h", - "src/cpp/util/byte_buffer.cc", - "src/cpp/util/slice.cc", - "src/cpp/util/status.cc", - "src/cpp/util/string_ref.cc", - "src/cpp/util/time.cc" - ] + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/byte_buffer.h", + "include/grpc/impl/codegen/compression_types.h", + "include/grpc/impl/codegen/connectivity_state.h", + "include/grpc/impl/codegen/grpc_types.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/propagation_bits.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/status.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "src/compiler/config.h", + "src/compiler/cpp_generator.cc", + "src/compiler/cpp_generator.h", + "src/compiler/cpp_generator_helpers.h", + "src/compiler/csharp_generator.cc", + "src/compiler/csharp_generator.h", + "src/compiler/csharp_generator_helpers.h", + "src/compiler/generator_helpers.h", + "src/compiler/objective_c_generator.cc", + "src/compiler/objective_c_generator.h", + "src/compiler/objective_c_generator_helpers.h", + "src/compiler/python_generator.cc", + "src/compiler/python_generator.h", + "src/compiler/ruby_generator.cc", + "src/compiler/ruby_generator.h", + "src/compiler/ruby_generator_helpers-inl.h", + "src/compiler/ruby_generator_map-inl.h", + "src/compiler/ruby_generator_string-inl.h", + "src/cpp/codegen/grpc_library.cc" + ], + "third_party": false, + "type": "lib" }, { - "deps": [], + "deps": [ + "gpr", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_test_util" + ], "headers": [ - "test/cpp/util/test_config.h" + "src/proto/grpc/testing/messages.grpc.pb.h", + "src/proto/grpc/testing/messages.pb.h", + "test/cpp/interop/client_helper.h" + ], + "language": "c++", + "name": "interop_client_helper", + "src": [ + "test/cpp/interop/client_helper.cc", + "test/cpp/interop/client_helper.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util", + "interop_client_helper" + ], + "headers": [ + "src/proto/grpc/testing/empty.grpc.pb.h", + "src/proto/grpc/testing/empty.pb.h", + "src/proto/grpc/testing/messages.grpc.pb.h", + "src/proto/grpc/testing/messages.pb.h", + "src/proto/grpc/testing/test.grpc.pb.h", + "src/proto/grpc/testing/test.pb.h", + "test/cpp/interop/interop_client.h" + ], + "language": "c++", + "name": "interop_client_main", + "src": [ + "test/cpp/interop/client.cc", + "test/cpp/interop/interop_client.cc", + "test/cpp/interop/interop_client.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr", + "grpc", + "grpc++", + "grpc_test_util" + ], + "headers": [ + "test/cpp/interop/server_helper.h" + ], + "language": "c++", + "name": "interop_server_helper", + "src": [ + "test/cpp/interop/server_helper.cc", + "test/cpp/interop/server_helper.h" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util", + "interop_server_helper" + ], + "headers": [ + "src/proto/grpc/testing/empty.grpc.pb.h", + "src/proto/grpc/testing/empty.pb.h", + "src/proto/grpc/testing/messages.grpc.pb.h", + "src/proto/grpc/testing/messages.pb.h", + "src/proto/grpc/testing/test.grpc.pb.h", + "src/proto/grpc/testing/test.pb.h" + ], + "language": "c++", + "name": "interop_server_main", + "src": [ + "test/cpp/interop/server_main.cc" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [ + "grpc++", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [ + "src/proto/grpc/testing/control.grpc.pb.h", + "src/proto/grpc/testing/control.pb.h", + "src/proto/grpc/testing/messages.grpc.pb.h", + "src/proto/grpc/testing/messages.pb.h", + "src/proto/grpc/testing/payloads.grpc.pb.h", + "src/proto/grpc/testing/payloads.pb.h", + "src/proto/grpc/testing/perf_db.grpc.pb.h", + "src/proto/grpc/testing/perf_db.pb.h", + "src/proto/grpc/testing/services.grpc.pb.h", + "src/proto/grpc/testing/services.pb.h", + "src/proto/grpc/testing/stats.grpc.pb.h", + "src/proto/grpc/testing/stats.pb.h", + "test/cpp/qps/client.h", + "test/cpp/qps/driver.h", + "test/cpp/qps/histogram.h", + "test/cpp/qps/interarrival.h", + "test/cpp/qps/limit_cores.h", + "test/cpp/qps/perf_db_client.h", + "test/cpp/qps/qps_worker.h", + "test/cpp/qps/report.h", + "test/cpp/qps/server.h", + "test/cpp/qps/stats.h", + "test/cpp/qps/usage_timer.h", + "test/cpp/util/benchmark_config.h" ], "language": "c++", - "name": "grpc++_test_config", + "name": "qps", "src": [ - "test/cpp/util/test_config.cc", - "test/cpp/util/test_config.h" - ] - }, - { - "deps": [ - "grpc++", - "grpc_test_util" - ], - "headers": [ - "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h", - "src/proto/grpc/testing/duplicate/echo_duplicate.pb.h", - "src/proto/grpc/testing/echo.grpc.pb.h", - "src/proto/grpc/testing/echo.pb.h", - "src/proto/grpc/testing/echo_messages.grpc.pb.h", - "src/proto/grpc/testing/echo_messages.pb.h", - "test/cpp/end2end/test_service_impl.h", - "test/cpp/util/byte_buffer_proto_helper.h", - "test/cpp/util/cli_call.h", - "test/cpp/util/create_test_channel.h", - "test/cpp/util/string_ref_helper.h", - "test/cpp/util/subprocess.h", - "test/cpp/util/test_credentials_provider.h" + "test/cpp/qps/client.h", + "test/cpp/qps/client_async.cc", + "test/cpp/qps/client_sync.cc", + "test/cpp/qps/driver.cc", + "test/cpp/qps/driver.h", + "test/cpp/qps/histogram.h", + "test/cpp/qps/interarrival.h", + "test/cpp/qps/limit_cores.cc", + "test/cpp/qps/limit_cores.h", + "test/cpp/qps/perf_db_client.cc", + "test/cpp/qps/perf_db_client.h", + "test/cpp/qps/qps_worker.cc", + "test/cpp/qps/qps_worker.h", + "test/cpp/qps/report.cc", + "test/cpp/qps/report.h", + "test/cpp/qps/server.h", + "test/cpp/qps/server_async.cc", + "test/cpp/qps/server_sync.cc", + "test/cpp/qps/stats.h", + "test/cpp/qps/usage_timer.cc", + "test/cpp/qps/usage_timer.h", + "test/cpp/util/benchmark_config.cc", + "test/cpp/util/benchmark_config.h" ], - "language": "c++", - "name": "grpc++_test_util", - "src": [ - "test/cpp/end2end/test_service_impl.cc", - "test/cpp/end2end/test_service_impl.h", - "test/cpp/util/byte_buffer_proto_helper.cc", - "test/cpp/util/byte_buffer_proto_helper.h", - "test/cpp/util/cli_call.cc", - "test/cpp/util/cli_call.h", - "test/cpp/util/create_test_channel.cc", - "test/cpp/util/create_test_channel.h", - "test/cpp/util/string_ref_helper.cc", - "test/cpp/util/string_ref_helper.h", - "test/cpp/util/subprocess.cc", - "test/cpp/util/subprocess.h", - "test/cpp/util/test_credentials_provider.cc", - "test/cpp/util/test_credentials_provider.h" - ] + "third_party": false, + "type": "lib" }, { "deps": [ "gpr", - "grpc_unsecure" - ], - "headers": [ - "include/grpc++/alarm.h", - "include/grpc++/channel.h", - "include/grpc++/client_context.h", - "include/grpc++/completion_queue.h", - "include/grpc++/create_channel.h", - "include/grpc++/generic/async_generic_service.h", - "include/grpc++/generic/generic_stub.h", - "include/grpc++/grpc++.h", - "include/grpc++/impl/call.h", - "include/grpc++/impl/client_unary_call.h", - "include/grpc++/impl/codegen/async_stream.h", - "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/call.h", - "include/grpc++/impl/codegen/call_hook.h", - "include/grpc++/impl/codegen/channel_interface.h", - "include/grpc++/impl/codegen/client_context.h", - "include/grpc++/impl/codegen/client_unary_call.h", - "include/grpc++/impl/codegen/completion_queue.h", - "include/grpc++/impl/codegen/completion_queue_tag.h", - "include/grpc++/impl/codegen/config.h", - "include/grpc++/impl/codegen/config_protobuf.h", - "include/grpc++/impl/codegen/grpc_library.h", - "include/grpc++/impl/codegen/method_handler_impl.h", - "include/grpc++/impl/codegen/proto_utils.h", - "include/grpc++/impl/codegen/rpc_method.h", - "include/grpc++/impl/codegen/rpc_service_method.h", - "include/grpc++/impl/codegen/security/auth_context.h", - "include/grpc++/impl/codegen/serialization_traits.h", - "include/grpc++/impl/codegen/server_context.h", - "include/grpc++/impl/codegen/server_interface.h", - "include/grpc++/impl/codegen/service_type.h", - "include/grpc++/impl/codegen/status.h", - "include/grpc++/impl/codegen/status_code_enum.h", - "include/grpc++/impl/codegen/string_ref.h", - "include/grpc++/impl/codegen/stub_options.h", - "include/grpc++/impl/codegen/sync.h", - "include/grpc++/impl/codegen/sync_cxx11.h", - "include/grpc++/impl/codegen/sync_no_cxx11.h", - "include/grpc++/impl/codegen/sync_stream.h", - "include/grpc++/impl/codegen/time.h", - "include/grpc++/impl/grpc_library.h", - "include/grpc++/impl/method_handler_impl.h", - "include/grpc++/impl/proto_utils.h", - "include/grpc++/impl/rpc_method.h", - "include/grpc++/impl/rpc_service_method.h", - "include/grpc++/impl/serialization_traits.h", - "include/grpc++/impl/server_builder_option.h", - "include/grpc++/impl/service_type.h", - "include/grpc++/impl/sync.h", - "include/grpc++/impl/sync_cxx11.h", - "include/grpc++/impl/sync_no_cxx11.h", - "include/grpc++/impl/thd.h", - "include/grpc++/impl/thd_cxx11.h", - "include/grpc++/impl/thd_no_cxx11.h", - "include/grpc++/security/auth_context.h", - "include/grpc++/security/auth_metadata_processor.h", - "include/grpc++/security/credentials.h", - "include/grpc++/security/server_credentials.h", - "include/grpc++/server.h", - "include/grpc++/server_builder.h", - "include/grpc++/server_context.h", - "include/grpc++/support/async_stream.h", - "include/grpc++/support/async_unary_call.h", - "include/grpc++/support/byte_buffer.h", - "include/grpc++/support/channel_arguments.h", - "include/grpc++/support/config.h", - "include/grpc++/support/config_protobuf.h", - "include/grpc++/support/slice.h", - "include/grpc++/support/status.h", - "include/grpc++/support/status_code_enum.h", - "include/grpc++/support/string_ref.h", - "include/grpc++/support/stub_options.h", - "include/grpc++/support/sync_stream.h", - "include/grpc++/support/time.h", - "src/cpp/client/create_channel_internal.h", - "src/cpp/common/create_auth_context.h", - "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/thread_pool_interface.h" + "grpc" ], - "language": "c++", - "name": "grpc++_unsecure", - "src": [ - "include/grpc++/alarm.h", - "include/grpc++/channel.h", - "include/grpc++/client_context.h", - "include/grpc++/completion_queue.h", - "include/grpc++/create_channel.h", - "include/grpc++/generic/async_generic_service.h", - "include/grpc++/generic/generic_stub.h", - "include/grpc++/grpc++.h", - "include/grpc++/impl/call.h", - "include/grpc++/impl/client_unary_call.h", - "include/grpc++/impl/codegen/async_stream.h", - "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/call.h", - "include/grpc++/impl/codegen/call_hook.h", - "include/grpc++/impl/codegen/channel_interface.h", - "include/grpc++/impl/codegen/client_context.h", - "include/grpc++/impl/codegen/client_unary_call.h", - "include/grpc++/impl/codegen/completion_queue.h", - "include/grpc++/impl/codegen/completion_queue_tag.h", - "include/grpc++/impl/codegen/config.h", - "include/grpc++/impl/codegen/config_protobuf.h", - "include/grpc++/impl/codegen/grpc_library.h", - "include/grpc++/impl/codegen/method_handler_impl.h", - "include/grpc++/impl/codegen/proto_utils.h", - "include/grpc++/impl/codegen/rpc_method.h", - "include/grpc++/impl/codegen/rpc_service_method.h", - "include/grpc++/impl/codegen/security/auth_context.h", - "include/grpc++/impl/codegen/serialization_traits.h", - "include/grpc++/impl/codegen/server_context.h", - "include/grpc++/impl/codegen/server_interface.h", - "include/grpc++/impl/codegen/service_type.h", - "include/grpc++/impl/codegen/status.h", - "include/grpc++/impl/codegen/status_code_enum.h", - "include/grpc++/impl/codegen/string_ref.h", - "include/grpc++/impl/codegen/stub_options.h", - "include/grpc++/impl/codegen/sync.h", - "include/grpc++/impl/codegen/sync_cxx11.h", - "include/grpc++/impl/codegen/sync_no_cxx11.h", - "include/grpc++/impl/codegen/sync_stream.h", - "include/grpc++/impl/codegen/time.h", - "include/grpc++/impl/grpc_library.h", - "include/grpc++/impl/method_handler_impl.h", - "include/grpc++/impl/proto_utils.h", - "include/grpc++/impl/rpc_method.h", - "include/grpc++/impl/rpc_service_method.h", - "include/grpc++/impl/serialization_traits.h", - "include/grpc++/impl/server_builder_option.h", - "include/grpc++/impl/service_type.h", - "include/grpc++/impl/sync.h", - "include/grpc++/impl/sync_cxx11.h", - "include/grpc++/impl/sync_no_cxx11.h", - "include/grpc++/impl/thd.h", - "include/grpc++/impl/thd_cxx11.h", - "include/grpc++/impl/thd_no_cxx11.h", - "include/grpc++/security/auth_context.h", - "include/grpc++/security/auth_metadata_processor.h", - "include/grpc++/security/credentials.h", - "include/grpc++/security/server_credentials.h", - "include/grpc++/server.h", - "include/grpc++/server_builder.h", - "include/grpc++/server_context.h", - "include/grpc++/support/async_stream.h", - "include/grpc++/support/async_unary_call.h", - "include/grpc++/support/byte_buffer.h", - "include/grpc++/support/channel_arguments.h", - "include/grpc++/support/config.h", - "include/grpc++/support/config_protobuf.h", - "include/grpc++/support/slice.h", - "include/grpc++/support/status.h", - "include/grpc++/support/status_code_enum.h", - "include/grpc++/support/string_ref.h", - "include/grpc++/support/stub_options.h", - "include/grpc++/support/sync_stream.h", - "include/grpc++/support/time.h", - "src/cpp/client/channel.cc", - "src/cpp/client/client_context.cc", - "src/cpp/client/create_channel.cc", - "src/cpp/client/create_channel_internal.cc", - "src/cpp/client/create_channel_internal.h", - "src/cpp/client/credentials.cc", - "src/cpp/client/generic_stub.cc", - "src/cpp/client/insecure_credentials.cc", - "src/cpp/codegen/grpc_library.cc", - "src/cpp/common/alarm.cc", - "src/cpp/common/call.cc", - "src/cpp/common/channel_arguments.cc", - "src/cpp/common/completion_queue.cc", - "src/cpp/common/create_auth_context.h", - "src/cpp/common/insecure_create_auth_context.cc", - "src/cpp/common/rpc_method.cc", - "src/cpp/proto/proto_utils.cc", - "src/cpp/server/async_generic_service.cc", - "src/cpp/server/create_default_thread_pool.cc", - "src/cpp/server/dynamic_thread_pool.cc", - "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/insecure_server_credentials.cc", - "src/cpp/server/server.cc", - "src/cpp/server/server_builder.cc", - "src/cpp/server/server_context.cc", - "src/cpp/server/server_credentials.cc", - "src/cpp/server/thread_pool_interface.h", - "src/cpp/util/byte_buffer.cc", - "src/cpp/util/slice.cc", - "src/cpp/util/status.cc", - "src/cpp/util/string_ref.cc", - "src/cpp/util/time.cc" - ] + "headers": [], + "language": "csharp", + "name": "grpc_csharp_ext", + "src": [ + "src/csharp/ext/grpc_csharp_ext.c" + ], + "third_party": false, + "type": "lib" }, { "deps": [], "headers": [ - "include/grpc++/impl/codegen/async_stream.h", - "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/call.h", - "include/grpc++/impl/codegen/call_hook.h", - "include/grpc++/impl/codegen/channel_interface.h", - "include/grpc++/impl/codegen/client_context.h", - "include/grpc++/impl/codegen/client_unary_call.h", - "include/grpc++/impl/codegen/completion_queue.h", - "include/grpc++/impl/codegen/completion_queue_tag.h", - "include/grpc++/impl/codegen/config.h", - "include/grpc++/impl/codegen/config_protobuf.h", - "include/grpc++/impl/codegen/grpc_library.h", - "include/grpc++/impl/codegen/method_handler_impl.h", - "include/grpc++/impl/codegen/proto_utils.h", - "include/grpc++/impl/codegen/rpc_method.h", - "include/grpc++/impl/codegen/rpc_service_method.h", - "include/grpc++/impl/codegen/security/auth_context.h", - "include/grpc++/impl/codegen/serialization_traits.h", - "include/grpc++/impl/codegen/server_context.h", - "include/grpc++/impl/codegen/server_interface.h", - "include/grpc++/impl/codegen/service_type.h", - "include/grpc++/impl/codegen/status.h", - "include/grpc++/impl/codegen/status_code_enum.h", - "include/grpc++/impl/codegen/string_ref.h", - "include/grpc++/impl/codegen/stub_options.h", - "include/grpc++/impl/codegen/sync.h", - "include/grpc++/impl/codegen/sync_cxx11.h", - "include/grpc++/impl/codegen/sync_no_cxx11.h", - "include/grpc++/impl/codegen/sync_stream.h", - "include/grpc++/impl/codegen/time.h", - "include/grpc++/support/config.h", - "include/grpc++/support/config_protobuf.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "src/compiler/config.h", - "src/compiler/cpp_generator.h", - "src/compiler/cpp_generator_helpers.h", - "src/compiler/csharp_generator.h", - "src/compiler/csharp_generator_helpers.h", - "src/compiler/generator_helpers.h", - "src/compiler/objective_c_generator.h", - "src/compiler/objective_c_generator_helpers.h", - "src/compiler/python_generator.h", - "src/compiler/ruby_generator.h", - "src/compiler/ruby_generator_helpers-inl.h", - "src/compiler/ruby_generator_map-inl.h", - "src/compiler/ruby_generator_string-inl.h" + "third_party/boringssl/crypto/aes/internal.h", + "third_party/boringssl/crypto/asn1/asn1_locl.h", + "third_party/boringssl/crypto/bio/internal.h", + "third_party/boringssl/crypto/bn/internal.h", + "third_party/boringssl/crypto/bn/rsaz_exp.h", + "third_party/boringssl/crypto/bytestring/internal.h", + "third_party/boringssl/crypto/cipher/internal.h", + "third_party/boringssl/crypto/conf/conf_def.h", + "third_party/boringssl/crypto/conf/internal.h", + "third_party/boringssl/crypto/des/internal.h", + "third_party/boringssl/crypto/dh/internal.h", + "third_party/boringssl/crypto/digest/internal.h", + "third_party/boringssl/crypto/digest/md32_common.h", + "third_party/boringssl/crypto/directory.h", + "third_party/boringssl/crypto/dsa/internal.h", + "third_party/boringssl/crypto/ec/internal.h", + "third_party/boringssl/crypto/ec/p256-x86_64-table.h", + "third_party/boringssl/crypto/evp/internal.h", + "third_party/boringssl/crypto/internal.h", + "third_party/boringssl/crypto/modes/internal.h", + "third_party/boringssl/crypto/obj/obj_dat.h", + "third_party/boringssl/crypto/obj/obj_xref.h", + "third_party/boringssl/crypto/pkcs8/internal.h", + "third_party/boringssl/crypto/rand/internal.h", + "third_party/boringssl/crypto/rsa/internal.h", + "third_party/boringssl/crypto/test/scoped_types.h", + "third_party/boringssl/crypto/test/test_util.h", + "third_party/boringssl/crypto/x509/charmap.h", + "third_party/boringssl/crypto/x509/vpm_int.h", + "third_party/boringssl/crypto/x509v3/ext_dat.h", + "third_party/boringssl/crypto/x509v3/pcy_int.h", + "third_party/boringssl/include/openssl/aead.h", + "third_party/boringssl/include/openssl/aes.h", + "third_party/boringssl/include/openssl/arm_arch.h", + "third_party/boringssl/include/openssl/asn1.h", + "third_party/boringssl/include/openssl/asn1_mac.h", + "third_party/boringssl/include/openssl/asn1t.h", + "third_party/boringssl/include/openssl/base.h", + "third_party/boringssl/include/openssl/base64.h", + "third_party/boringssl/include/openssl/bio.h", + "third_party/boringssl/include/openssl/blowfish.h", + "third_party/boringssl/include/openssl/bn.h", + "third_party/boringssl/include/openssl/buf.h", + "third_party/boringssl/include/openssl/buffer.h", + "third_party/boringssl/include/openssl/bytestring.h", + "third_party/boringssl/include/openssl/cast.h", + "third_party/boringssl/include/openssl/chacha.h", + "third_party/boringssl/include/openssl/cipher.h", + "third_party/boringssl/include/openssl/cmac.h", + "third_party/boringssl/include/openssl/conf.h", + "third_party/boringssl/include/openssl/cpu.h", + "third_party/boringssl/include/openssl/crypto.h", + "third_party/boringssl/include/openssl/curve25519.h", + "third_party/boringssl/include/openssl/des.h", + "third_party/boringssl/include/openssl/dh.h", + "third_party/boringssl/include/openssl/digest.h", + "third_party/boringssl/include/openssl/dsa.h", + "third_party/boringssl/include/openssl/dtls1.h", + "third_party/boringssl/include/openssl/ec.h", + "third_party/boringssl/include/openssl/ec_key.h", + "third_party/boringssl/include/openssl/ecdh.h", + "third_party/boringssl/include/openssl/ecdsa.h", + "third_party/boringssl/include/openssl/engine.h", + "third_party/boringssl/include/openssl/err.h", + "third_party/boringssl/include/openssl/evp.h", + "third_party/boringssl/include/openssl/ex_data.h", + "third_party/boringssl/include/openssl/hkdf.h", + "third_party/boringssl/include/openssl/hmac.h", + "third_party/boringssl/include/openssl/lhash.h", + "third_party/boringssl/include/openssl/lhash_macros.h", + "third_party/boringssl/include/openssl/md4.h", + "third_party/boringssl/include/openssl/md5.h", + "third_party/boringssl/include/openssl/mem.h", + "third_party/boringssl/include/openssl/obj.h", + "third_party/boringssl/include/openssl/obj_mac.h", + "third_party/boringssl/include/openssl/objects.h", + "third_party/boringssl/include/openssl/opensslfeatures.h", + "third_party/boringssl/include/openssl/opensslv.h", + "third_party/boringssl/include/openssl/ossl_typ.h", + "third_party/boringssl/include/openssl/pem.h", + "third_party/boringssl/include/openssl/pkcs12.h", + "third_party/boringssl/include/openssl/pkcs7.h", + "third_party/boringssl/include/openssl/pkcs8.h", + "third_party/boringssl/include/openssl/poly1305.h", + "third_party/boringssl/include/openssl/pqueue.h", + "third_party/boringssl/include/openssl/rand.h", + "third_party/boringssl/include/openssl/rc4.h", + "third_party/boringssl/include/openssl/rsa.h", + "third_party/boringssl/include/openssl/safestack.h", + "third_party/boringssl/include/openssl/sha.h", + "third_party/boringssl/include/openssl/srtp.h", + "third_party/boringssl/include/openssl/ssl.h", + "third_party/boringssl/include/openssl/ssl3.h", + "third_party/boringssl/include/openssl/stack.h", + "third_party/boringssl/include/openssl/stack_macros.h", + "third_party/boringssl/include/openssl/thread.h", + "third_party/boringssl/include/openssl/time_support.h", + "third_party/boringssl/include/openssl/tls1.h", + "third_party/boringssl/include/openssl/type_check.h", + "third_party/boringssl/include/openssl/x509.h", + "third_party/boringssl/include/openssl/x509_vfy.h", + "third_party/boringssl/include/openssl/x509v3.h", + "third_party/boringssl/ssl/internal.h", + "third_party/boringssl/ssl/test/async_bio.h", + "third_party/boringssl/ssl/test/packeted_bio.h", + "third_party/boringssl/ssl/test/scoped_types.h", + "third_party/boringssl/ssl/test/test_config.h" + ], + "language": "c", + "name": "boringssl", + "src": [ + "src/boringssl/err_data.c", + "third_party/boringssl/crypto/aes/aes.c", + "third_party/boringssl/crypto/aes/internal.h", + "third_party/boringssl/crypto/aes/mode_wrappers.c", + "third_party/boringssl/crypto/asn1/a_bitstr.c", + "third_party/boringssl/crypto/asn1/a_bool.c", + "third_party/boringssl/crypto/asn1/a_bytes.c", + "third_party/boringssl/crypto/asn1/a_d2i_fp.c", + "third_party/boringssl/crypto/asn1/a_dup.c", + "third_party/boringssl/crypto/asn1/a_enum.c", + "third_party/boringssl/crypto/asn1/a_gentm.c", + "third_party/boringssl/crypto/asn1/a_i2d_fp.c", + "third_party/boringssl/crypto/asn1/a_int.c", + "third_party/boringssl/crypto/asn1/a_mbstr.c", + "third_party/boringssl/crypto/asn1/a_object.c", + "third_party/boringssl/crypto/asn1/a_octet.c", + "third_party/boringssl/crypto/asn1/a_print.c", + "third_party/boringssl/crypto/asn1/a_strnid.c", + "third_party/boringssl/crypto/asn1/a_time.c", + "third_party/boringssl/crypto/asn1/a_type.c", + "third_party/boringssl/crypto/asn1/a_utctm.c", + "third_party/boringssl/crypto/asn1/a_utf8.c", + "third_party/boringssl/crypto/asn1/asn1_lib.c", + "third_party/boringssl/crypto/asn1/asn1_locl.h", + "third_party/boringssl/crypto/asn1/asn1_par.c", + "third_party/boringssl/crypto/asn1/asn_pack.c", + "third_party/boringssl/crypto/asn1/bio_asn1.c", + "third_party/boringssl/crypto/asn1/bio_ndef.c", + "third_party/boringssl/crypto/asn1/f_enum.c", + "third_party/boringssl/crypto/asn1/f_int.c", + "third_party/boringssl/crypto/asn1/f_string.c", + "third_party/boringssl/crypto/asn1/t_bitst.c", + "third_party/boringssl/crypto/asn1/t_pkey.c", + "third_party/boringssl/crypto/asn1/tasn_dec.c", + "third_party/boringssl/crypto/asn1/tasn_enc.c", + "third_party/boringssl/crypto/asn1/tasn_fre.c", + "third_party/boringssl/crypto/asn1/tasn_new.c", + "third_party/boringssl/crypto/asn1/tasn_prn.c", + "third_party/boringssl/crypto/asn1/tasn_typ.c", + "third_party/boringssl/crypto/asn1/tasn_utl.c", + "third_party/boringssl/crypto/asn1/x_bignum.c", + "third_party/boringssl/crypto/asn1/x_long.c", + "third_party/boringssl/crypto/base64/base64.c", + "third_party/boringssl/crypto/bio/bio.c", + "third_party/boringssl/crypto/bio/bio_mem.c", + "third_party/boringssl/crypto/bio/buffer.c", + "third_party/boringssl/crypto/bio/connect.c", + "third_party/boringssl/crypto/bio/fd.c", + "third_party/boringssl/crypto/bio/file.c", + "third_party/boringssl/crypto/bio/hexdump.c", + "third_party/boringssl/crypto/bio/internal.h", + "third_party/boringssl/crypto/bio/pair.c", + "third_party/boringssl/crypto/bio/printf.c", + "third_party/boringssl/crypto/bio/socket.c", + "third_party/boringssl/crypto/bio/socket_helper.c", + "third_party/boringssl/crypto/bn/add.c", + "third_party/boringssl/crypto/bn/asm/x86_64-gcc.c", + "third_party/boringssl/crypto/bn/bn.c", + "third_party/boringssl/crypto/bn/bn_asn1.c", + "third_party/boringssl/crypto/bn/cmp.c", + "third_party/boringssl/crypto/bn/convert.c", + "third_party/boringssl/crypto/bn/ctx.c", + "third_party/boringssl/crypto/bn/div.c", + "third_party/boringssl/crypto/bn/exponentiation.c", + "third_party/boringssl/crypto/bn/gcd.c", + "third_party/boringssl/crypto/bn/generic.c", + "third_party/boringssl/crypto/bn/internal.h", + "third_party/boringssl/crypto/bn/kronecker.c", + "third_party/boringssl/crypto/bn/montgomery.c", + "third_party/boringssl/crypto/bn/mul.c", + "third_party/boringssl/crypto/bn/prime.c", + "third_party/boringssl/crypto/bn/random.c", + "third_party/boringssl/crypto/bn/rsaz_exp.c", + "third_party/boringssl/crypto/bn/rsaz_exp.h", + "third_party/boringssl/crypto/bn/shift.c", + "third_party/boringssl/crypto/bn/sqrt.c", + "third_party/boringssl/crypto/buf/buf.c", + "third_party/boringssl/crypto/bytestring/ber.c", + "third_party/boringssl/crypto/bytestring/cbb.c", + "third_party/boringssl/crypto/bytestring/cbs.c", + "third_party/boringssl/crypto/bytestring/internal.h", + "third_party/boringssl/crypto/chacha/chacha_generic.c", + "third_party/boringssl/crypto/chacha/chacha_vec.c", + "third_party/boringssl/crypto/cipher/aead.c", + "third_party/boringssl/crypto/cipher/cipher.c", + "third_party/boringssl/crypto/cipher/derive_key.c", + "third_party/boringssl/crypto/cipher/e_aes.c", + "third_party/boringssl/crypto/cipher/e_chacha20poly1305.c", + "third_party/boringssl/crypto/cipher/e_des.c", + "third_party/boringssl/crypto/cipher/e_null.c", + "third_party/boringssl/crypto/cipher/e_rc2.c", + "third_party/boringssl/crypto/cipher/e_rc4.c", + "third_party/boringssl/crypto/cipher/e_ssl3.c", + "third_party/boringssl/crypto/cipher/e_tls.c", + "third_party/boringssl/crypto/cipher/internal.h", + "third_party/boringssl/crypto/cipher/tls_cbc.c", + "third_party/boringssl/crypto/cmac/cmac.c", + "third_party/boringssl/crypto/conf/conf.c", + "third_party/boringssl/crypto/conf/conf_def.h", + "third_party/boringssl/crypto/conf/internal.h", + "third_party/boringssl/crypto/cpu-arm.c", + "third_party/boringssl/crypto/cpu-intel.c", + "third_party/boringssl/crypto/crypto.c", + "third_party/boringssl/crypto/curve25519/curve25519.c", + "third_party/boringssl/crypto/des/des.c", + "third_party/boringssl/crypto/des/internal.h", + "third_party/boringssl/crypto/dh/check.c", + "third_party/boringssl/crypto/dh/dh.c", + "third_party/boringssl/crypto/dh/dh_asn1.c", + "third_party/boringssl/crypto/dh/internal.h", + "third_party/boringssl/crypto/dh/params.c", + "third_party/boringssl/crypto/digest/digest.c", + "third_party/boringssl/crypto/digest/digests.c", + "third_party/boringssl/crypto/digest/internal.h", + "third_party/boringssl/crypto/digest/md32_common.h", + "third_party/boringssl/crypto/directory.h", + "third_party/boringssl/crypto/directory_posix.c", + "third_party/boringssl/crypto/directory_win.c", + "third_party/boringssl/crypto/dsa/dsa.c", + "third_party/boringssl/crypto/dsa/dsa_asn1.c", + "third_party/boringssl/crypto/dsa/internal.h", + "third_party/boringssl/crypto/ec/ec.c", + "third_party/boringssl/crypto/ec/ec_asn1.c", + "third_party/boringssl/crypto/ec/ec_key.c", + "third_party/boringssl/crypto/ec/ec_montgomery.c", + "third_party/boringssl/crypto/ec/internal.h", + "third_party/boringssl/crypto/ec/oct.c", + "third_party/boringssl/crypto/ec/p224-64.c", + "third_party/boringssl/crypto/ec/p256-64.c", + "third_party/boringssl/crypto/ec/p256-x86_64-table.h", + "third_party/boringssl/crypto/ec/p256-x86_64.c", + "third_party/boringssl/crypto/ec/simple.c", + "third_party/boringssl/crypto/ec/util-64.c", + "third_party/boringssl/crypto/ec/wnaf.c", + "third_party/boringssl/crypto/ecdh/ecdh.c", + "third_party/boringssl/crypto/ecdsa/ecdsa.c", + "third_party/boringssl/crypto/ecdsa/ecdsa_asn1.c", + "third_party/boringssl/crypto/engine/engine.c", + "third_party/boringssl/crypto/err/err.c", + "third_party/boringssl/crypto/evp/algorithm.c", + "third_party/boringssl/crypto/evp/digestsign.c", + "third_party/boringssl/crypto/evp/evp.c", + "third_party/boringssl/crypto/evp/evp_asn1.c", + "third_party/boringssl/crypto/evp/evp_ctx.c", + "third_party/boringssl/crypto/evp/internal.h", + "third_party/boringssl/crypto/evp/p_dsa_asn1.c", + "third_party/boringssl/crypto/evp/p_ec.c", + "third_party/boringssl/crypto/evp/p_ec_asn1.c", + "third_party/boringssl/crypto/evp/p_rsa.c", + "third_party/boringssl/crypto/evp/p_rsa_asn1.c", + "third_party/boringssl/crypto/evp/pbkdf.c", + "third_party/boringssl/crypto/evp/sign.c", + "third_party/boringssl/crypto/ex_data.c", + "third_party/boringssl/crypto/hkdf/hkdf.c", + "third_party/boringssl/crypto/hmac/hmac.c", + "third_party/boringssl/crypto/internal.h", + "third_party/boringssl/crypto/lhash/lhash.c", + "third_party/boringssl/crypto/md4/md4.c", + "third_party/boringssl/crypto/md5/md5.c", + "third_party/boringssl/crypto/mem.c", + "third_party/boringssl/crypto/modes/cbc.c", + "third_party/boringssl/crypto/modes/cfb.c", + "third_party/boringssl/crypto/modes/ctr.c", + "third_party/boringssl/crypto/modes/gcm.c", + "third_party/boringssl/crypto/modes/internal.h", + "third_party/boringssl/crypto/modes/ofb.c", + "third_party/boringssl/crypto/obj/obj.c", + "third_party/boringssl/crypto/obj/obj_dat.h", + "third_party/boringssl/crypto/obj/obj_xref.c", + "third_party/boringssl/crypto/obj/obj_xref.h", + "third_party/boringssl/crypto/pem/pem_all.c", + "third_party/boringssl/crypto/pem/pem_info.c", + "third_party/boringssl/crypto/pem/pem_lib.c", + "third_party/boringssl/crypto/pem/pem_oth.c", + "third_party/boringssl/crypto/pem/pem_pk8.c", + "third_party/boringssl/crypto/pem/pem_pkey.c", + "third_party/boringssl/crypto/pem/pem_x509.c", + "third_party/boringssl/crypto/pem/pem_xaux.c", + "third_party/boringssl/crypto/pkcs8/internal.h", + "third_party/boringssl/crypto/pkcs8/p5_pbe.c", + "third_party/boringssl/crypto/pkcs8/p5_pbev2.c", + "third_party/boringssl/crypto/pkcs8/p8_pkey.c", + "third_party/boringssl/crypto/pkcs8/pkcs8.c", + "third_party/boringssl/crypto/poly1305/poly1305.c", + "third_party/boringssl/crypto/poly1305/poly1305_arm.c", + "third_party/boringssl/crypto/poly1305/poly1305_vec.c", + "third_party/boringssl/crypto/rand/internal.h", + "third_party/boringssl/crypto/rand/rand.c", + "third_party/boringssl/crypto/rand/urandom.c", + "third_party/boringssl/crypto/rand/windows.c", + "third_party/boringssl/crypto/rc4/rc4.c", + "third_party/boringssl/crypto/refcount_c11.c", + "third_party/boringssl/crypto/refcount_lock.c", + "third_party/boringssl/crypto/rsa/blinding.c", + "third_party/boringssl/crypto/rsa/internal.h", + "third_party/boringssl/crypto/rsa/padding.c", + "third_party/boringssl/crypto/rsa/rsa.c", + "third_party/boringssl/crypto/rsa/rsa_asn1.c", + "third_party/boringssl/crypto/rsa/rsa_impl.c", + "third_party/boringssl/crypto/sha/sha1.c", + "third_party/boringssl/crypto/sha/sha256.c", + "third_party/boringssl/crypto/sha/sha512.c", + "third_party/boringssl/crypto/stack/stack.c", + "third_party/boringssl/crypto/test/scoped_types.h", + "third_party/boringssl/crypto/test/test_util.h", + "third_party/boringssl/crypto/thread.c", + "third_party/boringssl/crypto/thread_none.c", + "third_party/boringssl/crypto/thread_pthread.c", + "third_party/boringssl/crypto/thread_win.c", + "third_party/boringssl/crypto/time_support.c", + "third_party/boringssl/crypto/x509/a_digest.c", + "third_party/boringssl/crypto/x509/a_sign.c", + "third_party/boringssl/crypto/x509/a_strex.c", + "third_party/boringssl/crypto/x509/a_verify.c", + "third_party/boringssl/crypto/x509/asn1_gen.c", + "third_party/boringssl/crypto/x509/by_dir.c", + "third_party/boringssl/crypto/x509/by_file.c", + "third_party/boringssl/crypto/x509/charmap.h", + "third_party/boringssl/crypto/x509/i2d_pr.c", + "third_party/boringssl/crypto/x509/pkcs7.c", + "third_party/boringssl/crypto/x509/t_crl.c", + "third_party/boringssl/crypto/x509/t_req.c", + "third_party/boringssl/crypto/x509/t_x509.c", + "third_party/boringssl/crypto/x509/t_x509a.c", + "third_party/boringssl/crypto/x509/vpm_int.h", + "third_party/boringssl/crypto/x509/x509.c", + "third_party/boringssl/crypto/x509/x509_att.c", + "third_party/boringssl/crypto/x509/x509_cmp.c", + "third_party/boringssl/crypto/x509/x509_d2.c", + "third_party/boringssl/crypto/x509/x509_def.c", + "third_party/boringssl/crypto/x509/x509_ext.c", + "third_party/boringssl/crypto/x509/x509_lu.c", + "third_party/boringssl/crypto/x509/x509_obj.c", + "third_party/boringssl/crypto/x509/x509_r2x.c", + "third_party/boringssl/crypto/x509/x509_req.c", + "third_party/boringssl/crypto/x509/x509_set.c", + "third_party/boringssl/crypto/x509/x509_trs.c", + "third_party/boringssl/crypto/x509/x509_txt.c", + "third_party/boringssl/crypto/x509/x509_v3.c", + "third_party/boringssl/crypto/x509/x509_vfy.c", + "third_party/boringssl/crypto/x509/x509_vpm.c", + "third_party/boringssl/crypto/x509/x509cset.c", + "third_party/boringssl/crypto/x509/x509name.c", + "third_party/boringssl/crypto/x509/x509rset.c", + "third_party/boringssl/crypto/x509/x509spki.c", + "third_party/boringssl/crypto/x509/x509type.c", + "third_party/boringssl/crypto/x509/x_algor.c", + "third_party/boringssl/crypto/x509/x_all.c", + "third_party/boringssl/crypto/x509/x_attrib.c", + "third_party/boringssl/crypto/x509/x_crl.c", + "third_party/boringssl/crypto/x509/x_exten.c", + "third_party/boringssl/crypto/x509/x_info.c", + "third_party/boringssl/crypto/x509/x_name.c", + "third_party/boringssl/crypto/x509/x_pkey.c", + "third_party/boringssl/crypto/x509/x_pubkey.c", + "third_party/boringssl/crypto/x509/x_req.c", + "third_party/boringssl/crypto/x509/x_sig.c", + "third_party/boringssl/crypto/x509/x_spki.c", + "third_party/boringssl/crypto/x509/x_val.c", + "third_party/boringssl/crypto/x509/x_x509.c", + "third_party/boringssl/crypto/x509/x_x509a.c", + "third_party/boringssl/crypto/x509v3/ext_dat.h", + "third_party/boringssl/crypto/x509v3/pcy_cache.c", + "third_party/boringssl/crypto/x509v3/pcy_data.c", + "third_party/boringssl/crypto/x509v3/pcy_int.h", + "third_party/boringssl/crypto/x509v3/pcy_lib.c", + "third_party/boringssl/crypto/x509v3/pcy_map.c", + "third_party/boringssl/crypto/x509v3/pcy_node.c", + "third_party/boringssl/crypto/x509v3/pcy_tree.c", + "third_party/boringssl/crypto/x509v3/v3_akey.c", + "third_party/boringssl/crypto/x509v3/v3_akeya.c", + "third_party/boringssl/crypto/x509v3/v3_alt.c", + "third_party/boringssl/crypto/x509v3/v3_bcons.c", + "third_party/boringssl/crypto/x509v3/v3_bitst.c", + "third_party/boringssl/crypto/x509v3/v3_conf.c", + "third_party/boringssl/crypto/x509v3/v3_cpols.c", + "third_party/boringssl/crypto/x509v3/v3_crld.c", + "third_party/boringssl/crypto/x509v3/v3_enum.c", + "third_party/boringssl/crypto/x509v3/v3_extku.c", + "third_party/boringssl/crypto/x509v3/v3_genn.c", + "third_party/boringssl/crypto/x509v3/v3_ia5.c", + "third_party/boringssl/crypto/x509v3/v3_info.c", + "third_party/boringssl/crypto/x509v3/v3_int.c", + "third_party/boringssl/crypto/x509v3/v3_lib.c", + "third_party/boringssl/crypto/x509v3/v3_ncons.c", + "third_party/boringssl/crypto/x509v3/v3_pci.c", + "third_party/boringssl/crypto/x509v3/v3_pcia.c", + "third_party/boringssl/crypto/x509v3/v3_pcons.c", + "third_party/boringssl/crypto/x509v3/v3_pku.c", + "third_party/boringssl/crypto/x509v3/v3_pmaps.c", + "third_party/boringssl/crypto/x509v3/v3_prn.c", + "third_party/boringssl/crypto/x509v3/v3_purp.c", + "third_party/boringssl/crypto/x509v3/v3_skey.c", + "third_party/boringssl/crypto/x509v3/v3_sxnet.c", + "third_party/boringssl/crypto/x509v3/v3_utl.c", + "third_party/boringssl/include/openssl/aead.h", + "third_party/boringssl/include/openssl/aes.h", + "third_party/boringssl/include/openssl/arm_arch.h", + "third_party/boringssl/include/openssl/asn1.h", + "third_party/boringssl/include/openssl/asn1_mac.h", + "third_party/boringssl/include/openssl/asn1t.h", + "third_party/boringssl/include/openssl/base.h", + "third_party/boringssl/include/openssl/base64.h", + "third_party/boringssl/include/openssl/bio.h", + "third_party/boringssl/include/openssl/blowfish.h", + "third_party/boringssl/include/openssl/bn.h", + "third_party/boringssl/include/openssl/buf.h", + "third_party/boringssl/include/openssl/buffer.h", + "third_party/boringssl/include/openssl/bytestring.h", + "third_party/boringssl/include/openssl/cast.h", + "third_party/boringssl/include/openssl/chacha.h", + "third_party/boringssl/include/openssl/cipher.h", + "third_party/boringssl/include/openssl/cmac.h", + "third_party/boringssl/include/openssl/conf.h", + "third_party/boringssl/include/openssl/cpu.h", + "third_party/boringssl/include/openssl/crypto.h", + "third_party/boringssl/include/openssl/curve25519.h", + "third_party/boringssl/include/openssl/des.h", + "third_party/boringssl/include/openssl/dh.h", + "third_party/boringssl/include/openssl/digest.h", + "third_party/boringssl/include/openssl/dsa.h", + "third_party/boringssl/include/openssl/dtls1.h", + "third_party/boringssl/include/openssl/ec.h", + "third_party/boringssl/include/openssl/ec_key.h", + "third_party/boringssl/include/openssl/ecdh.h", + "third_party/boringssl/include/openssl/ecdsa.h", + "third_party/boringssl/include/openssl/engine.h", + "third_party/boringssl/include/openssl/err.h", + "third_party/boringssl/include/openssl/evp.h", + "third_party/boringssl/include/openssl/ex_data.h", + "third_party/boringssl/include/openssl/hkdf.h", + "third_party/boringssl/include/openssl/hmac.h", + "third_party/boringssl/include/openssl/lhash.h", + "third_party/boringssl/include/openssl/lhash_macros.h", + "third_party/boringssl/include/openssl/md4.h", + "third_party/boringssl/include/openssl/md5.h", + "third_party/boringssl/include/openssl/mem.h", + "third_party/boringssl/include/openssl/obj.h", + "third_party/boringssl/include/openssl/obj_mac.h", + "third_party/boringssl/include/openssl/objects.h", + "third_party/boringssl/include/openssl/opensslfeatures.h", + "third_party/boringssl/include/openssl/opensslv.h", + "third_party/boringssl/include/openssl/ossl_typ.h", + "third_party/boringssl/include/openssl/pem.h", + "third_party/boringssl/include/openssl/pkcs12.h", + "third_party/boringssl/include/openssl/pkcs7.h", + "third_party/boringssl/include/openssl/pkcs8.h", + "third_party/boringssl/include/openssl/poly1305.h", + "third_party/boringssl/include/openssl/pqueue.h", + "third_party/boringssl/include/openssl/rand.h", + "third_party/boringssl/include/openssl/rc4.h", + "third_party/boringssl/include/openssl/rsa.h", + "third_party/boringssl/include/openssl/safestack.h", + "third_party/boringssl/include/openssl/sha.h", + "third_party/boringssl/include/openssl/srtp.h", + "third_party/boringssl/include/openssl/ssl.h", + "third_party/boringssl/include/openssl/ssl3.h", + "third_party/boringssl/include/openssl/stack.h", + "third_party/boringssl/include/openssl/stack_macros.h", + "third_party/boringssl/include/openssl/thread.h", + "third_party/boringssl/include/openssl/time_support.h", + "third_party/boringssl/include/openssl/tls1.h", + "third_party/boringssl/include/openssl/type_check.h", + "third_party/boringssl/include/openssl/x509.h", + "third_party/boringssl/include/openssl/x509_vfy.h", + "third_party/boringssl/include/openssl/x509v3.h", + "third_party/boringssl/ssl/custom_extensions.c", + "third_party/boringssl/ssl/d1_both.c", + "third_party/boringssl/ssl/d1_clnt.c", + "third_party/boringssl/ssl/d1_lib.c", + "third_party/boringssl/ssl/d1_meth.c", + "third_party/boringssl/ssl/d1_pkt.c", + "third_party/boringssl/ssl/d1_srtp.c", + "third_party/boringssl/ssl/d1_srvr.c", + "third_party/boringssl/ssl/dtls_record.c", + "third_party/boringssl/ssl/internal.h", + "third_party/boringssl/ssl/pqueue/pqueue.c", + "third_party/boringssl/ssl/s3_both.c", + "third_party/boringssl/ssl/s3_clnt.c", + "third_party/boringssl/ssl/s3_enc.c", + "third_party/boringssl/ssl/s3_lib.c", + "third_party/boringssl/ssl/s3_meth.c", + "third_party/boringssl/ssl/s3_pkt.c", + "third_party/boringssl/ssl/s3_srvr.c", + "third_party/boringssl/ssl/ssl_aead_ctx.c", + "third_party/boringssl/ssl/ssl_asn1.c", + "third_party/boringssl/ssl/ssl_buffer.c", + "third_party/boringssl/ssl/ssl_cert.c", + "third_party/boringssl/ssl/ssl_cipher.c", + "third_party/boringssl/ssl/ssl_file.c", + "third_party/boringssl/ssl/ssl_lib.c", + "third_party/boringssl/ssl/ssl_rsa.c", + "third_party/boringssl/ssl/ssl_session.c", + "third_party/boringssl/ssl/ssl_stat.c", + "third_party/boringssl/ssl/t1_enc.c", + "third_party/boringssl/ssl/t1_lib.c", + "third_party/boringssl/ssl/test/async_bio.h", + "third_party/boringssl/ssl/test/packeted_bio.h", + "third_party/boringssl/ssl/test/scoped_types.h", + "third_party/boringssl/ssl/test/test_config.h", + "third_party/boringssl/ssl/tls_record.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [], + "headers": [], + "language": "c++", + "name": "boringssl_test_util", + "src": [ + "third_party/boringssl/crypto/test/file_test.cc", + "third_party/boringssl/crypto/test/malloc.cc", + "third_party/boringssl/crypto/test/test_util.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" ], + "headers": [], "language": "c++", - "name": "grpc_plugin_support", + "name": "boringssl_aes_test_lib", "src": [ - "include/grpc++/impl/codegen/async_stream.h", - "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/call.h", - "include/grpc++/impl/codegen/call_hook.h", - "include/grpc++/impl/codegen/channel_interface.h", - "include/grpc++/impl/codegen/client_context.h", - "include/grpc++/impl/codegen/client_unary_call.h", - "include/grpc++/impl/codegen/completion_queue.h", - "include/grpc++/impl/codegen/completion_queue_tag.h", - "include/grpc++/impl/codegen/config.h", - "include/grpc++/impl/codegen/config_protobuf.h", - "include/grpc++/impl/codegen/grpc_library.h", - "include/grpc++/impl/codegen/method_handler_impl.h", - "include/grpc++/impl/codegen/proto_utils.h", - "include/grpc++/impl/codegen/rpc_method.h", - "include/grpc++/impl/codegen/rpc_service_method.h", - "include/grpc++/impl/codegen/security/auth_context.h", - "include/grpc++/impl/codegen/serialization_traits.h", - "include/grpc++/impl/codegen/server_context.h", - "include/grpc++/impl/codegen/server_interface.h", - "include/grpc++/impl/codegen/service_type.h", - "include/grpc++/impl/codegen/status.h", - "include/grpc++/impl/codegen/status_code_enum.h", - "include/grpc++/impl/codegen/string_ref.h", - "include/grpc++/impl/codegen/stub_options.h", - "include/grpc++/impl/codegen/sync.h", - "include/grpc++/impl/codegen/sync_cxx11.h", - "include/grpc++/impl/codegen/sync_no_cxx11.h", - "include/grpc++/impl/codegen/sync_stream.h", - "include/grpc++/impl/codegen/time.h", - "include/grpc++/support/config.h", - "include/grpc++/support/config_protobuf.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/byte_buffer.h", - "include/grpc/impl/codegen/compression_types.h", - "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/grpc_types.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/propagation_bits.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/status.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "src/compiler/config.h", - "src/compiler/cpp_generator.cc", - "src/compiler/cpp_generator.h", - "src/compiler/cpp_generator_helpers.h", - "src/compiler/csharp_generator.cc", - "src/compiler/csharp_generator.h", - "src/compiler/csharp_generator_helpers.h", - "src/compiler/generator_helpers.h", - "src/compiler/objective_c_generator.cc", - "src/compiler/objective_c_generator.h", - "src/compiler/objective_c_generator_helpers.h", - "src/compiler/python_generator.cc", - "src/compiler/python_generator.h", - "src/compiler/ruby_generator.cc", - "src/compiler/ruby_generator.h", - "src/compiler/ruby_generator_helpers-inl.h", - "src/compiler/ruby_generator_map-inl.h", - "src/compiler/ruby_generator_string-inl.h", - "src/cpp/codegen/grpc_library.cc" - ] + "third_party/boringssl/crypto/aes/aes_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_base64_test_lib", + "src": [ + "third_party/boringssl/crypto/base64/base64_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_bio_test_lib", + "src": [ + "third_party/boringssl/crypto/bio/bio_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_bn_test_lib", + "src": [ + "third_party/boringssl/crypto/bn/bn_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_bytestring_test_lib", + "src": [ + "third_party/boringssl/crypto/bytestring/bytestring_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_aead_test_lib", + "src": [ + "third_party/boringssl/crypto/cipher/aead_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_cipher_test_lib", + "src": [ + "third_party/boringssl/crypto/cipher/cipher_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_cmac_test_lib", + "src": [ + "third_party/boringssl/crypto/cmac/cmac_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_constant_time_test_lib", + "src": [ + "third_party/boringssl/crypto/constant_time_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_ed25519_test_lib", + "src": [ + "third_party/boringssl/crypto/curve25519/ed25519_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_x25519_test_lib", + "src": [ + "third_party/boringssl/crypto/curve25519/x25519_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_dh_test_lib", + "src": [ + "third_party/boringssl/crypto/dh/dh_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_digest_test_lib", + "src": [ + "third_party/boringssl/crypto/digest/digest_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_dsa_test_lib", + "src": [ + "third_party/boringssl/crypto/dsa/dsa_test.c" + ], + "third_party": true, + "type": "lib" }, { "deps": [ - "gpr", - "grpc", - "grpc++", - "grpc++_test_util", - "grpc_test_util" + "boringssl", + "boringssl_test_util" ], - "headers": [ - "src/proto/grpc/testing/messages.grpc.pb.h", - "src/proto/grpc/testing/messages.pb.h", - "test/cpp/interop/client_helper.h" + "headers": [], + "language": "c++", + "name": "boringssl_ec_test_lib", + "src": [ + "third_party/boringssl/crypto/ec/ec_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_example_mul_lib", + "src": [ + "third_party/boringssl/crypto/ec/example_mul.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" ], + "headers": [], "language": "c++", - "name": "interop_client_helper", + "name": "boringssl_ecdsa_test_lib", "src": [ - "test/cpp/interop/client_helper.cc", - "test/cpp/interop/client_helper.h" - ] + "third_party/boringssl/crypto/ecdsa/ecdsa_test.cc" + ], + "third_party": true, + "type": "lib" }, { "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc++", - "grpc++_test_config", - "grpc++_test_util", - "grpc_test_util", - "interop_client_helper" + "boringssl", + "boringssl_test_util" ], - "headers": [ - "src/proto/grpc/testing/empty.grpc.pb.h", - "src/proto/grpc/testing/empty.pb.h", - "src/proto/grpc/testing/messages.grpc.pb.h", - "src/proto/grpc/testing/messages.pb.h", - "src/proto/grpc/testing/test.grpc.pb.h", - "src/proto/grpc/testing/test.pb.h", - "test/cpp/interop/interop_client.h" + "headers": [], + "language": "c++", + "name": "boringssl_err_test_lib", + "src": [ + "third_party/boringssl/crypto/err/err_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" ], + "headers": [], "language": "c++", - "name": "interop_client_main", + "name": "boringssl_evp_extra_test_lib", "src": [ - "test/cpp/interop/client.cc", - "test/cpp/interop/interop_client.cc", - "test/cpp/interop/interop_client.h" - ] + "third_party/boringssl/crypto/evp/evp_extra_test.cc" + ], + "third_party": true, + "type": "lib" }, { "deps": [ - "gpr", - "grpc", - "grpc++", - "grpc_test_util" + "boringssl", + "boringssl_test_util" ], - "headers": [ - "test/cpp/interop/server_helper.h" + "headers": [], + "language": "c++", + "name": "boringssl_evp_test_lib", + "src": [ + "third_party/boringssl/crypto/evp/evp_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" ], + "headers": [], "language": "c++", - "name": "interop_server_helper", + "name": "boringssl_pbkdf_test_lib", "src": [ - "test/cpp/interop/server_helper.cc", - "test/cpp/interop/server_helper.h" - ] + "third_party/boringssl/crypto/evp/pbkdf_test.cc" + ], + "third_party": true, + "type": "lib" }, { "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc++", - "grpc++_test_config", - "grpc++_test_util", - "grpc_test_util", - "interop_server_helper" + "boringssl", + "boringssl_test_util" ], - "headers": [ - "src/proto/grpc/testing/empty.grpc.pb.h", - "src/proto/grpc/testing/empty.pb.h", - "src/proto/grpc/testing/messages.grpc.pb.h", - "src/proto/grpc/testing/messages.pb.h", - "src/proto/grpc/testing/test.grpc.pb.h", - "src/proto/grpc/testing/test.pb.h" + "headers": [], + "language": "c", + "name": "boringssl_hkdf_test_lib", + "src": [ + "third_party/boringssl/crypto/hkdf/hkdf_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" ], + "headers": [], "language": "c++", - "name": "interop_server_main", + "name": "boringssl_hmac_test_lib", "src": [ - "test/cpp/interop/server.cc" - ] + "third_party/boringssl/crypto/hmac/hmac_test.cc" + ], + "third_party": true, + "type": "lib" }, { "deps": [ - "grpc++", - "grpc++_test_util", - "grpc_test_util" + "boringssl", + "boringssl_test_util" ], - "headers": [ - "src/proto/grpc/testing/control.grpc.pb.h", - "src/proto/grpc/testing/control.pb.h", - "src/proto/grpc/testing/messages.grpc.pb.h", - "src/proto/grpc/testing/messages.pb.h", - "src/proto/grpc/testing/payloads.grpc.pb.h", - "src/proto/grpc/testing/payloads.pb.h", - "src/proto/grpc/testing/perf_db.grpc.pb.h", - "src/proto/grpc/testing/perf_db.pb.h", - "src/proto/grpc/testing/services.grpc.pb.h", - "src/proto/grpc/testing/services.pb.h", - "src/proto/grpc/testing/stats.grpc.pb.h", - "src/proto/grpc/testing/stats.pb.h", - "test/cpp/qps/client.h", - "test/cpp/qps/driver.h", - "test/cpp/qps/histogram.h", - "test/cpp/qps/interarrival.h", - "test/cpp/qps/limit_cores.h", - "test/cpp/qps/perf_db_client.h", - "test/cpp/qps/qps_worker.h", - "test/cpp/qps/report.h", - "test/cpp/qps/server.h", - "test/cpp/qps/stats.h", - "test/cpp/qps/timer.h", - "test/cpp/util/benchmark_config.h" + "headers": [], + "language": "c", + "name": "boringssl_lhash_test_lib", + "src": [ + "third_party/boringssl/crypto/lhash/lhash_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_gcm_test_lib", + "src": [ + "third_party/boringssl/crypto/modes/gcm_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" ], + "headers": [], "language": "c++", - "name": "qps", + "name": "boringssl_pkcs12_test_lib", "src": [ - "test/cpp/qps/client.h", - "test/cpp/qps/client_async.cc", - "test/cpp/qps/client_sync.cc", - "test/cpp/qps/driver.cc", - "test/cpp/qps/driver.h", - "test/cpp/qps/histogram.h", - "test/cpp/qps/interarrival.h", - "test/cpp/qps/limit_cores.cc", - "test/cpp/qps/limit_cores.h", - "test/cpp/qps/perf_db_client.cc", - "test/cpp/qps/perf_db_client.h", - "test/cpp/qps/qps_worker.cc", - "test/cpp/qps/qps_worker.h", - "test/cpp/qps/report.cc", - "test/cpp/qps/report.h", - "test/cpp/qps/server.h", - "test/cpp/qps/server_async.cc", - "test/cpp/qps/server_sync.cc", - "test/cpp/qps/stats.h", - "test/cpp/qps/timer.cc", - "test/cpp/qps/timer.h", - "test/cpp/util/benchmark_config.cc", - "test/cpp/util/benchmark_config.h" - ] + "third_party/boringssl/crypto/pkcs8/pkcs12_test.cc" + ], + "third_party": true, + "type": "lib" }, { "deps": [ - "gpr", - "grpc" + "boringssl", + "boringssl_test_util" ], "headers": [], - "language": "csharp", - "name": "grpc_csharp_ext", + "language": "c++", + "name": "boringssl_pkcs8_test_lib", "src": [ - "src/csharp/ext/grpc_csharp_ext.c" - ] + "third_party/boringssl/crypto/pkcs8/pkcs8_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_poly1305_test_lib", + "src": [ + "third_party/boringssl/crypto/poly1305/poly1305_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_refcount_test_lib", + "src": [ + "third_party/boringssl/crypto/refcount_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_rsa_test_lib", + "src": [ + "third_party/boringssl/crypto/rsa/rsa_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_thread_test_lib", + "src": [ + "third_party/boringssl/crypto/thread_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_pkcs7_test_lib", + "src": [ + "third_party/boringssl/crypto/x509/pkcs7_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_tab_test_lib", + "src": [ + "third_party/boringssl/crypto/x509v3/tab_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_v3name_test_lib", + "src": [ + "third_party/boringssl/crypto/x509v3/v3name_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c", + "name": "boringssl_pqueue_test_lib", + "src": [ + "third_party/boringssl/ssl/pqueue/pqueue_test.c" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [ + "boringssl", + "boringssl_test_util" + ], + "headers": [], + "language": "c++", + "name": "boringssl_ssl_test_lib", + "src": [ + "third_party/boringssl/ssl/ssl_test.cc" + ], + "third_party": true, + "type": "lib" + }, + { + "deps": [], + "headers": [ + "third_party/zlib/crc32.h", + "third_party/zlib/deflate.h", + "third_party/zlib/gzguts.h", + "third_party/zlib/inffast.h", + "third_party/zlib/inffixed.h", + "third_party/zlib/inflate.h", + "third_party/zlib/inftrees.h", + "third_party/zlib/trees.h", + "third_party/zlib/zconf.h", + "third_party/zlib/zlib.h", + "third_party/zlib/zutil.h" + ], + "language": "c", + "name": "z", + "src": [ + "third_party/zlib/adler32.c", + "third_party/zlib/compress.c", + "third_party/zlib/crc32.c", + "third_party/zlib/crc32.h", + "third_party/zlib/deflate.c", + "third_party/zlib/deflate.h", + "third_party/zlib/gzclose.c", + "third_party/zlib/gzguts.h", + "third_party/zlib/gzlib.c", + "third_party/zlib/gzread.c", + "third_party/zlib/gzwrite.c", + "third_party/zlib/infback.c", + "third_party/zlib/inffast.c", + "third_party/zlib/inffast.h", + "third_party/zlib/inffixed.h", + "third_party/zlib/inflate.c", + "third_party/zlib/inflate.h", + "third_party/zlib/inftrees.c", + "third_party/zlib/inftrees.h", + "third_party/zlib/trees.c", + "third_party/zlib/trees.h", + "third_party/zlib/uncompr.c", + "third_party/zlib/zconf.h", + "third_party/zlib/zlib.h", + "third_party/zlib/zutil.c", + "third_party/zlib/zutil.h" + ], + "third_party": true, + "type": "lib" }, { "deps": [ @@ -4736,7 +6712,9 @@ "src": [ "test/core/bad_client/bad_client.c", "test/core/bad_client/bad_client.h" - ] + ], + "third_party": false, + "type": "lib" }, { "deps": [ @@ -4746,14 +6724,16 @@ "grpc_test_util" ], "headers": [ - "test/core/bad_ssl/server.h" + "test/core/bad_ssl/server_common.h" ], "language": "c", "name": "bad_ssl_test_server", "src": [ - "test/core/bad_ssl/server.c", - "test/core/bad_ssl/server.h" - ] + "test/core/bad_ssl/server_common.c", + "test/core/bad_ssl/server_common.h" + ], + "third_party": false, + "type": "lib" }, { "deps": [ @@ -4782,9 +6762,8 @@ "test/core/end2end/tests/cancel_in_a_vacuum.c", "test/core/end2end/tests/cancel_test_helpers.h", "test/core/end2end/tests/cancel_with_status.c", - "test/core/end2end/tests/channel_connectivity.c", - "test/core/end2end/tests/channel_ping.c", "test/core/end2end/tests/compressed_payload.c", + "test/core/end2end/tests/connectivity.c", "test/core/end2end/tests/default_host.c", "test/core/end2end/tests/disappearing_server.c", "test/core/end2end/tests/empty_batch.c", @@ -4795,10 +6774,10 @@ "test/core/end2end/tests/large_metadata.c", "test/core/end2end/tests/max_concurrent_streams.c", "test/core/end2end/tests/max_message_length.c", - "test/core/end2end/tests/metadata.c", "test/core/end2end/tests/negative_deadline.c", "test/core/end2end/tests/no_op.c", "test/core/end2end/tests/payload.c", + "test/core/end2end/tests/ping.c", "test/core/end2end/tests/ping_pong_streaming.c", "test/core/end2end/tests/registered_call.c", "test/core/end2end/tests/request_with_flags.c", @@ -4807,9 +6786,12 @@ "test/core/end2end/tests/shutdown_finishes_calls.c", "test/core/end2end/tests/shutdown_finishes_tags.c", "test/core/end2end/tests/simple_delayed_request.c", + "test/core/end2end/tests/simple_metadata.c", "test/core/end2end/tests/simple_request.c", "test/core/end2end/tests/trailing_metadata.c" - ] + ], + "third_party": false, + "type": "lib" }, { "deps": [ @@ -4836,9 +6818,8 @@ "test/core/end2end/tests/cancel_in_a_vacuum.c", "test/core/end2end/tests/cancel_test_helpers.h", "test/core/end2end/tests/cancel_with_status.c", - "test/core/end2end/tests/channel_connectivity.c", - "test/core/end2end/tests/channel_ping.c", "test/core/end2end/tests/compressed_payload.c", + "test/core/end2end/tests/connectivity.c", "test/core/end2end/tests/default_host.c", "test/core/end2end/tests/disappearing_server.c", "test/core/end2end/tests/empty_batch.c", @@ -4849,10 +6830,10 @@ "test/core/end2end/tests/large_metadata.c", "test/core/end2end/tests/max_concurrent_streams.c", "test/core/end2end/tests/max_message_length.c", - "test/core/end2end/tests/metadata.c", "test/core/end2end/tests/negative_deadline.c", "test/core/end2end/tests/no_op.c", "test/core/end2end/tests/payload.c", + "test/core/end2end/tests/ping.c", "test/core/end2end/tests/ping_pong_streaming.c", "test/core/end2end/tests/registered_call.c", "test/core/end2end/tests/request_with_flags.c", @@ -4861,9 +6842,12 @@ "test/core/end2end/tests/shutdown_finishes_calls.c", "test/core/end2end/tests/shutdown_finishes_tags.c", "test/core/end2end/tests/simple_delayed_request.c", + "test/core/end2end/tests/simple_metadata.c", "test/core/end2end/tests/simple_request.c", "test/core/end2end/tests/trailing_metadata.c" - ] + ], + "third_party": false, + "type": "lib" }, { "deps": [], @@ -4874,6 +6858,8 @@ "test/core/end2end/data/server1_cert.c", "test/core/end2end/data/server1_key.c", "test/core/end2end/data/test_root_cert.c" - ] + ], + "third_party": false, + "type": "lib" } ] From e32ef7acde174bcf1ffca4f46a3e59b768c9a2ca Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 22 Feb 2016 17:25:51 -0800 Subject: [PATCH 048/236] Make timeout instant as well --- test/cpp/common/alarm_cpp_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cpp/common/alarm_cpp_test.cc b/test/cpp/common/alarm_cpp_test.cc index b81f85a8898..5d7344046ce 100644 --- a/test/cpp/common/alarm_cpp_test.cc +++ b/test/cpp/common/alarm_cpp_test.cc @@ -63,7 +63,7 @@ TEST(AlarmTest, ZeroExpiry) { void* output_tag; bool ok; const CompletionQueue::NextStatus status = cq.AsyncNext( - (void**)&output_tag, &ok, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); + (void**)&output_tag, &ok, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(0)); EXPECT_EQ(status, CompletionQueue::GOT_EVENT); EXPECT_TRUE(ok); @@ -78,7 +78,7 @@ TEST(AlarmTest, NegativeExpiry) { void* output_tag; bool ok; const CompletionQueue::NextStatus status = cq.AsyncNext( - (void**)&output_tag, &ok, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); + (void**)&output_tag, &ok, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(0)); EXPECT_EQ(status, CompletionQueue::GOT_EVENT); EXPECT_TRUE(ok); From 0160873273be3fb502f05d4349e074422f4addcb Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 22 Feb 2016 17:49:45 -0800 Subject: [PATCH 049/236] PR comments addressed --- include/grpc++/alarm.h | 4 ++++ src/core/surface/alarm.c | 5 +++-- src/cpp/common/alarm.cc | 3 --- test/cpp/common/alarm_cpp_test.cc | 17 +++++++++++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/include/grpc++/alarm.h b/include/grpc++/alarm.h index 66904deb48b..59988b8732e 100644 --- a/include/grpc++/alarm.h +++ b/include/grpc++/alarm.h @@ -56,6 +56,10 @@ class Alarm : private GrpcLibrary { /// Once the alarm expires (at \a deadline) or it's cancelled (see \a Cancel), /// an event with tag \a tag will be added to \a cq. If the alarm expired, the /// event's success bit will be true, false otherwise (ie, upon cancellation). + /// \internal We rely on the presence of \a cq for grpc initialization. If \a + /// cq were ever to be removed, a reference to a static + /// internal::GrpcLibraryInitializer instance would need to be introduced + /// here. \endinternal. template Alarm(CompletionQueue* cq, const T& deadline, void* tag) : tag_(tag), diff --git a/src/core/surface/alarm.c b/src/core/surface/alarm.c index fb496f6c474..8169ede0655 100644 --- a/src/core/surface/alarm.c +++ b/src/core/surface/alarm.c @@ -64,8 +64,9 @@ grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline, alarm->tag = tag; grpc_cq_begin_op(cq, tag); - grpc_timer_init(&exec_ctx, &alarm->alarm, deadline, alarm_cb, alarm, - gpr_now(GPR_CLOCK_MONOTONIC)); + grpc_timer_init(&exec_ctx, &alarm->alarm, + gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), + alarm_cb, alarm, gpr_now(GPR_CLOCK_MONOTONIC)); grpc_exec_ctx_finish(&exec_ctx); return alarm; } diff --git a/src/cpp/common/alarm.cc b/src/cpp/common/alarm.cc index 8af17597ef0..0c96be20da7 100644 --- a/src/cpp/common/alarm.cc +++ b/src/cpp/common/alarm.cc @@ -35,10 +35,7 @@ namespace grpc { -static internal::GrpcLibraryInitializer g_gli_initializer; - Alarm::~Alarm() { - g_gli_initializer.summon(); grpc_alarm_destroy(alarm_); } diff --git a/test/cpp/common/alarm_cpp_test.cc b/test/cpp/common/alarm_cpp_test.cc index 4745ef14ec7..874c452fa9a 100644 --- a/test/cpp/common/alarm_cpp_test.cc +++ b/test/cpp/common/alarm_cpp_test.cc @@ -55,6 +55,23 @@ TEST(AlarmTest, RegularExpiry) { EXPECT_EQ(junk, output_tag); } +TEST(AlarmTest, RegularExpiryChrono) { + CompletionQueue cq; + void* junk = reinterpret_cast(1618033); + std::chrono::system_clock::time_point one_sec_deadline = + std::chrono::system_clock::now() + std::chrono::seconds(1); + Alarm alarm(&cq, one_sec_deadline, junk); + + void* output_tag; + bool ok; + const CompletionQueue::NextStatus status = cq.AsyncNext( + (void**)&output_tag, &ok, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(2)); + + EXPECT_EQ(status, CompletionQueue::GOT_EVENT); + EXPECT_TRUE(ok); + EXPECT_EQ(junk, output_tag); +} + TEST(AlarmTest, Cancellation) { CompletionQueue cq; void* junk = reinterpret_cast(1618033); From 5fc09525ed01c8f889247e61efe9a4d9fb8ef2b5 Mon Sep 17 00:00:00 2001 From: Kailash Sethuraman Date: Mon, 22 Feb 2016 17:28:12 -0800 Subject: [PATCH 050/236] Add initial set of SoC ideas --- summerofcode/ideas.md | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/summerofcode/ideas.md b/summerofcode/ideas.md index 073262339b1..b14d3f7b6ad 100644 --- a/summerofcode/ideas.md +++ b/summerofcode/ideas.md @@ -1,4 +1,22 @@ -Google Summer of Code 2016 gRPC Ideas -===================================== +# gRPC Summer of Code Project Ideas -(Skeleton for now.) +C Core: + +1. Port gRPC to one of (Free, Net, Open) BSD platforms and create packages for them. Add kqueue support in the process. +2. Fix gRPC C-core's URI parser. The current parser does not qualify as a standard parser according to [RFC3986]( https://tools.ietf.org/html/rfc3986). Write test suites to verify this and make changes necessary to make the URI parser compliant. +3. HPACK compression efficiency evaluation - Figure out how to benchmark gRPC's compression efficiency (both in terms of bytes on the wire and cpu cycles). Implement benchmarks. Potentially extend this to other standalone implementations -- Java and Go. + + +gRPC Python: + + 1. Evaluate the port of gRPC's Python implementation to PyPy. Investigate the state of [Cython support](http://docs.cython.org/src/userguide/pypy.html) to do this or potentially explore cffi + 2. Develop and test Python 3.5 Support for gRPC. Make necessary changes to port gRPC and package it for supported platforms. + +gRPC Ruby/Java: + +1. jRuby support for gRPC. Develop a jRuby wrapper for gRPC based on grpc-java and ensure that it is API compatible with the existing Ruby implementation and passes all tests. + + +Other: + +1. Develop a Wireshark plugin for the gRPC protocol. Provide documentation and tutorials for this plugin. Bonus: consider set-up and use with the mobile clients. From 19432c3daafd545fbf5d9cc98ad75ec1ffab7169 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 22 Feb 2016 20:58:30 -0800 Subject: [PATCH 051/236] Put a 900-sec timeout on the perf test. --- tools/jenkins/run_performance.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index c80685b23a0..08cc476c6fe 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -31,6 +31,11 @@ # This script is invoked by Jenkins and runs performance smoke test. set -ex +# +# Put a timeout on this test +# +((sleep 900; kill $$)&) + # Enter the gRPC repo root cd $(dirname $0)/../.. From 3c50e70c994ccc67f6cec9c1b7d80fcfb28093ba Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 22 Feb 2016 21:38:36 -0800 Subject: [PATCH 052/236] Start the timeout after the build finishes --- tools/jenkins/run_performance.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 08cc476c6fe..caf0c0bc04c 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -31,11 +31,6 @@ # This script is invoked by Jenkins and runs performance smoke test. set -ex -# -# Put a timeout on this test -# -((sleep 900; kill $$)&) - # Enter the gRPC repo root cd $(dirname $0)/../.. @@ -52,6 +47,11 @@ PID1=$! bins/$config/qps_worker -driver_port 10010 & PID2=$! +# +# Put a timeout on these tests +# +((sleep 900; kill $$)&) + export QPS_WORKERS="localhost:10000,localhost:10010" # big is the size in bytes of large messages (0 is the size otherwise) From 49c5c3308c2a331a59dbcd2f2b57e3ee59d2a87f Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 22 Feb 2016 21:44:33 -0800 Subject: [PATCH 053/236] Some more cleanup on timeout --- tools/jenkins/run_performance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index caf0c0bc04c..fbc078330f7 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -50,7 +50,7 @@ PID2=$! # # Put a timeout on these tests # -((sleep 900; kill $$)&) +((sleep 900; kill $$ && killall qps_worker && rm -f /tmp/qps-test.$$ )&) export QPS_WORKERS="localhost:10000,localhost:10010" From 023759216c4efa34440deb2e7516d1cc9e5b278f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 22 Feb 2016 22:27:28 -0800 Subject: [PATCH 054/236] Add comment --- tools/codegen/core/gen_load_balancing_proto.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/codegen/core/gen_load_balancing_proto.sh b/tools/codegen/core/gen_load_balancing_proto.sh index 6d974ce31b9..fb6a468ee0b 100755 --- a/tools/codegen/core/gen_load_balancing_proto.sh +++ b/tools/codegen/core/gen_load_balancing_proto.sh @@ -110,6 +110,8 @@ virtualenv $VENV_NAME . $VENV_NAME/bin/activate popd +# this should be the same version as the submodule we compile against +# ideally we'd update this as a template to ensure that pip install protobuf==3.0.0b2 pushd "$(dirname $1)" > /dev/null From 6541cabe9db446360f21f0d5fc92318e5486e02a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 22 Feb 2016 22:41:53 -0800 Subject: [PATCH 055/236] clang-fmt --- test/cpp/qps/client_async.cc | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index f8c1fa3b62b..b3b2f926db9 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -84,8 +84,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function< std::unique_ptr>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*)> - start_req, + CompletionQueue*)> start_req, std::function on_done) : context_(), stub_(stub), @@ -142,8 +141,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function next_issue_; std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*)> - start_req_; + CompletionQueue*)> start_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -166,8 +164,7 @@ class AsyncClient : public ClientImpl { AsyncClient(const ClientConfig& config, std::function next_issue, - const RequestType&)> - setup_ctx, + const RequestType&)> setup_ctx, std::function(std::shared_ptr)> create_stub) : ClientImpl(config, create_stub), @@ -280,8 +277,7 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { std::function>( BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, - void*)> - start_req, + void*)> start_req, std::function on_done) : context_(), stub_(stub), @@ -364,10 +360,10 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { State next_state_; std::function callback_; std::function next_issue_; - std::function>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, void*)> - start_req_; + std::function< + std::unique_ptr>( + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, + void*)> start_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -409,8 +405,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function next_issue, std::function( grpc::GenericStub*, grpc::ClientContext*, - const grpc::string& method_name, CompletionQueue*, void*)> - start_req, + const grpc::string& method_name, CompletionQueue*, void*)> start_req, std::function on_done) : context_(), stub_(stub), @@ -498,8 +493,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function next_issue_; std::function( grpc::GenericStub*, grpc::ClientContext*, const grpc::string&, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*, void*)> start_req_; grpc::Status status_; double start_; std::unique_ptr stream_; From 70a57e42f4356677e31eae0ace2a14c45ed03176 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sat, 20 Feb 2016 18:50:27 -0800 Subject: [PATCH 056/236] Build "language" superceded by --build_only flag --- tools/run_tests/run_tests.py | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 0b3efa29e3a..41cf335a812 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -522,39 +522,6 @@ class Sanity(object): return 'sanity' -class Build(object): - - def test_specs(self, config, args): - return [] - - def pre_build_steps(self): - return [] - - def make_targets(self, test_regex): - return ['static'] - - def make_options(self): - return [] - - def build_steps(self): - return [] - - def post_tests_steps(self): - return [] - - def makefile_name(self): - return 'Makefile' - - def supports_multi_config(self): - return True - - def dockerfile_dir(self, config, arch): - return None - - def __str__(self): - return self.make_target - - # different configurations we can run under with open('tools/run_tests/configs.json') as f: _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read())) @@ -570,8 +537,7 @@ _LANGUAGES = { 'ruby': RubyLanguage(), 'csharp': CSharpLanguage(), 'objc' : ObjCLanguage(), - 'sanity': Sanity(), - 'build': Build(), + 'sanity': Sanity() } _WINDOWS_CONFIG = { From 77db432c2d1733b56ed01fcd9c3df1960b62f97e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sat, 20 Feb 2016 20:19:35 -0800 Subject: [PATCH 057/236] simplify run_tests.py --- tools/run_tests/run_tests.py | 257 ++++++++++++++++++----------------- 1 file changed, 135 insertions(+), 122 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 41cf335a812..7f156f7b517 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -118,6 +118,11 @@ def get_c_tests(travis, test_lang) : not (travis and tgt['flaky'])] +def _check_compiler(compiler, supported_compilers): + if compiler not in supported_compilers: + raise Exception('Compiler %s not supported.' % compiler) + + class CLanguage(object): def __init__(self, make_target, test_lang): @@ -125,38 +130,50 @@ class CLanguage(object): self.platform = platform_string() self.test_lang = test_lang - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + if self.platform == 'windows': + _check_compiler(self.args.compiler, ['default', + 'vs2010', + 'vs2013', + 'vs2015']) + else: + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): out = [] - binaries = get_c_tests(args.travis, self.test_lang) + binaries = get_c_tests(self.args.travis, self.test_lang) for target in binaries: - if config.build_config in target['exclude_configs']: + if self.config.build_config in target['exclude_configs']: continue if self.platform == 'windows': binary = 'vsprojects/%s%s/%s.exe' % ( - 'x64/' if args.arch == 'x64' else '', - _WINDOWS_CONFIG[config.build_config], + 'x64/' if self.args.arch == 'x64' else '', + _WINDOWS_CONFIG[self.config.build_config], target['name']) else: - binary = 'bins/%s/%s' % (config.build_config, target['name']) + binary = 'bins/%s/%s' % (self.config.build_config, target['name']) if os.path.isfile(binary): cmdline = [binary] + target['args'] - out.append(config.job_spec(cmdline, [binary], - shortname=' '.join(cmdline), - cpu_cost=target['cpu_cost'], - environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': - os.path.abspath(os.path.dirname( - sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')})) - elif args.regex == '.*' or platform_string() == 'windows': + out.append(self.config.job_spec(cmdline, [binary], + shortname=' '.join(cmdline), + cpu_cost=target['cpu_cost'], + environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': + os.path.abspath(os.path.dirname( + sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')})) + elif self.args.regex == '.*' or self.platform == 'windows': print '\nWARNING: binary not found, skipping', binary return sorted(out) - def make_targets(self, test_regex): - if platform_string() != 'windows' and test_regex != '.*': + def make_targets(self): + test_regex = self.args.regex + if self.platform != 'windows' and self.args.regex != '.*': # use the regex to minimize the number of things to build return [os.path.basename(target['name']) for target in get_c_tests(False, self.test_lang) if re.search(test_regex, '/' + target['name'])] - if platform_string() == 'windows': + if self.platform == 'windows': # don't build tools on windows just yet return ['buildtests_%s' % self.make_target] return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target] @@ -182,11 +199,8 @@ class CLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return True - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return self.make_target @@ -198,13 +212,18 @@ class NodeLanguage(object): self.platform = platform_string() self.node_version = '0.12' - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): if self.platform == 'windows': - return [config.job_spec(['tools\\run_tests\\run_node.bat'], None)] + return [self.config.job_spec(['tools\\run_tests\\run_node.bat'], None)] else: - return [config.job_spec(['tools/run_tests/run_node.sh', self.node_version], - None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + return [self.config.job_spec(['tools/run_tests/run_node.sh', self.node_version], + None, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): if self.platform == 'windows': @@ -212,7 +231,7 @@ class NodeLanguage(object): else: return [['tools/run_tests/pre_build_node.sh', self.node_version]] - def make_targets(self, test_regex): + def make_targets(self): return [] def make_options(self): @@ -230,11 +249,8 @@ class NodeLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'node' @@ -242,14 +258,19 @@ class NodeLanguage(object): class PhpLanguage(object): - def test_specs(self, config, args): - return [config.job_spec(['src/php/bin/run_tests.sh'], None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): + return [self.config.job_spec(['src/php/bin/run_tests.sh'], None, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): return [] - def make_targets(self, test_regex): + def make_targets(self): return ['static_c', 'shared_c'] def make_options(self): @@ -264,11 +285,8 @@ class PhpLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'php' @@ -280,10 +298,15 @@ class PythonLanguage(object): self._build_python_versions = ['2.7'] self._has_python_versions = [] - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) environment['PYVER'] = '2.7' - return [config.job_spec( + return [self.config.job_spec( ['tools/run_tests/run_python.sh'], None, environ=environment, @@ -294,7 +317,7 @@ class PythonLanguage(object): def pre_build_steps(self): return [] - def make_targets(self, test_regex): + def make_targets(self): return ['static_c', 'grpc_python_plugin', 'shared_c'] def make_options(self): @@ -320,11 +343,8 @@ class PythonLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'python' @@ -332,10 +352,15 @@ class PythonLanguage(object): class RubyLanguage(object): - def test_specs(self, config, args): - return [config.job_spec(['tools/run_tests/run_ruby.sh'], None, - timeout_seconds=10*60, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): + return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], None, + timeout_seconds=10*60, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): return [['tools/run_tests/pre_build_ruby.sh']] @@ -355,27 +380,30 @@ class RubyLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'ruby' class CSharpLanguage(object): + def __init__(self): self.platform = platform_string() - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): with open('src/csharp/tests.json') as f: tests_json = json.load(f) assemblies = tests_json['assemblies'] tests = tests_json['tests'] - msbuild_config = _WINDOWS_CONFIG[config.build_config] + msbuild_config = _WINDOWS_CONFIG[self.config.build_config] assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a) for a in assemblies] @@ -387,13 +415,13 @@ class CSharpLanguage(object): else: script_name = 'tools/run_tests/run_csharp.sh' - if config.build_config == 'gcov': + if self.config.build_config == 'gcov': # On Windows, we only collect C# code coverage. # On Linux, we only collect coverage for native extension. # For code coverage all tests need to run as one suite. - return [config.job_spec([script_name] + extra_args, None, - shortname='csharp.coverage', - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + return [self.config.job_spec([script_name] + extra_args, None, + shortname='csharp.coverage', + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] else: specs = [] for test in tests: @@ -402,9 +430,9 @@ class CSharpLanguage(object): # use different output directory for each test to prevent # TestResult.xml clash between parallel test runs. cmdline += ['-work=test-result/%s' % uuid.uuid4()] - specs.append(config.job_spec(cmdline, None, - shortname='csharp.%s' % test, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)) + specs.append(self.config.job_spec(cmdline, None, + shortname='csharp.%s' % test, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)) return specs def pre_build_steps(self): @@ -413,7 +441,7 @@ class CSharpLanguage(object): else: return [['tools/run_tests/pre_build_csharp.sh']] - def make_targets(self, test_regex): + def make_targets(self): # For Windows, this target doesn't really build anything, # everything is build by buildall script later. if self.platform == 'windows': @@ -440,11 +468,8 @@ class CSharpLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): - return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(arch) + def dockerfile_dir(self): + return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(self.args.arch) def __str__(self): return 'csharp' @@ -452,14 +477,19 @@ class CSharpLanguage(object): class ObjCLanguage(object): - def test_specs(self, config, args): - return [config.job_spec(['src/objective-c/tests/run_tests.sh'], None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): + return [self.config.job_spec(['src/objective-c/tests/run_tests.sh'], None, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): return [] - def make_targets(self, test_regex): + def make_targets(self): return ['grpc_objective_c_plugin', 'interop_server'] def make_options(self): @@ -474,10 +504,7 @@ class ObjCLanguage(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): + def dockerfile_dir(self): return None def __str__(self): @@ -486,18 +513,23 @@ class ObjCLanguage(object): class Sanity(object): - def test_specs(self, config, args): + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): import yaml with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f: - return [config.job_spec(cmd['script'].split(), None, - timeout_seconds=None, environ={'TEST': 'true'}, - cpu_cost=cmd.get('cpu_cost', 1)) + return [self.config.job_spec(cmd['script'].split(), None, + timeout_seconds=None, environ={'TEST': 'true'}, + cpu_cost=cmd.get('cpu_cost', 1)) for cmd in yaml.load(f)] def pre_build_steps(self): return [] - def make_targets(self, test_regex): + def make_targets(self): return ['run_dep_checks'] def make_options(self): @@ -512,10 +544,7 @@ class Sanity(object): def makefile_name(self): return 'Makefile' - def supports_multi_config(self): - return False - - def dockerfile_dir(self, config, arch): + def dockerfile_dir(self): return 'tools/dockerfile/test/sanity' def __str__(self): @@ -527,7 +556,6 @@ with open('tools/run_tests/configs.json') as f: _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read())) -_DEFAULT = ['opt'] _LANGUAGES = { 'c++': CLanguage('cxx', 'c++'), 'c': CLanguage('c', 'c'), @@ -540,6 +568,7 @@ _LANGUAGES = { 'sanity': Sanity() } + _WINDOWS_CONFIG = { 'dbg': 'Debug', 'opt': 'Release', @@ -648,9 +677,8 @@ def runs_per_test_type(arg_str): # parse command line argp = argparse.ArgumentParser(description='Run grpc tests.') argp.add_argument('-c', '--config', - choices=['all'] + sorted(_CONFIGS.keys()), - nargs='+', - default=_DEFAULT) + choices=sorted(_CONFIGS.keys()), + default='opt') argp.add_argument('-n', '--runs_per_test', default=1, type=runs_per_test_type, help='A positive integer or "inf". If "inf", all tests will run in an ' 'infinite loop. Especially useful in combination with "-f"') @@ -696,7 +724,7 @@ argp.add_argument('--arch', argp.add_argument('--compiler', choices=['default', 'vs2010', 'vs2013', 'vs2015'], default='default', - help='Selects compiler to use. For some platforms "default" is the only supported choice.') + help='Selects compiler to use. Allowed values depend on the platform and language.') argp.add_argument('--build_only', default=False, action='store_const', @@ -742,11 +770,8 @@ if need_to_regenerate_projects: # grab config -run_configs = set(_CONFIGS[cfg] - for cfg in itertools.chain.from_iterable( - _CONFIGS.iterkeys() if x == 'all' else [x] - for x in args.config)) -build_configs = set(cfg.build_config for cfg in run_configs) +run_config = _CONFIGS[args.config] +build_config = run_config.build_config if args.travis: _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'} @@ -762,12 +787,8 @@ if 'gcov' in args.config: lang_list.remove(bad) languages = set(_LANGUAGES[l] for l in lang_list) - -if len(build_configs) > 1: - for language in languages: - if not language.supports_multi_config(): - print language, 'does not support multiple build configurations' - sys.exit(1) +for l in languages: + l.configure(run_config, args) language_make_options=[] if any(language.make_options() for language in languages): @@ -777,8 +798,8 @@ if any(language.make_options() for language in languages): else: language_make_options = next(iter(languages)).make_options() -if len(languages) != 1 or len(build_configs) != 1: - print 'Multi-language and multi-config testing is not supported.' +if len(languages) != 1: + print 'Multi-language testing is not supported.' sys.exit(1) if args.use_docker: @@ -809,10 +830,6 @@ if args.use_docker: env=env) sys.exit(0) -if platform_string() != 'windows' and args.compiler != 'default': - print 'Compiler %s not supported on current platform.' % args.compiler - sys.exit(1) - _check_arch_option(args.arch) def make_jobspec(cfg, targets, makefile='Makefile'): @@ -852,7 +869,7 @@ make_targets = {} for l in languages: makefile = l.makefile_name() make_targets[makefile] = make_targets.get(makefile, set()).union( - set(l.make_targets(args.regex))) + set(l.make_targets())) def build_step_environ(cfg): environ = {'CONFIG': cfg} @@ -862,22 +879,19 @@ def build_step_environ(cfg): return environ build_steps = list(set( - jobset.JobSpec(cmdline, environ=build_step_environ(cfg), flake_retries=5) - for cfg in build_configs + jobset.JobSpec(cmdline, environ=build_step_environ(build_config), flake_retries=5) for l in languages for cmdline in l.pre_build_steps())) if make_targets: - make_commands = itertools.chain.from_iterable(make_jobspec(cfg, list(targets), makefile) for cfg in build_configs for (makefile, targets) in make_targets.iteritems()) + make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems()) build_steps.extend(set(make_commands)) build_steps.extend(set( - jobset.JobSpec(cmdline, environ=build_step_environ(cfg), timeout_seconds=None) - for cfg in build_configs + jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None) for l in languages for cmdline in l.build_steps())) post_tests_steps = list(set( - jobset.JobSpec(cmdline, environ=build_step_environ(cfg)) - for cfg in build_configs + jobset.JobSpec(cmdline, environ=build_step_environ(build_config)) for l in languages for cmdline in l.post_tests_steps())) runs_per_test = args.runs_per_test @@ -1068,9 +1082,8 @@ def _build_and_run( infinite_runs = runs_per_test == 0 one_run = set( spec - for config in run_configs for language in languages - for spec in language.test_specs(config, args) + for spec in language.test_specs() if re.search(args.regex, spec.shortname)) # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. From c96caf8733ae4aa8e0e7fb3366b67e2fdb8619b1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 22 Feb 2016 17:31:02 -0800 Subject: [PATCH 058/236] compilers for C,C++ --- tools/run_tests/run_tests.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 7f156f7b517..c6b98355cfa 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -134,12 +134,11 @@ class CLanguage(object): self.config = config self.args = args if self.platform == 'windows': - _check_compiler(self.args.compiler, ['default', - 'vs2010', - 'vs2013', - 'vs2015']) + self._make_options = [_windows_toolset_option(self.args.compiler), + _windows_arch_option(self.args.arch)] else: _check_compiler(self.args.compiler, ['default']) + self._make_options = [] def test_specs(self): out = [] @@ -179,7 +178,7 @@ class CLanguage(object): return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target] def make_options(self): - return [] + return self._make_options; def pre_build_steps(self): if self.platform == 'windows': @@ -844,9 +843,7 @@ def make_jobspec(cfg, targets, makefile='Makefile'): return [ jobset.JobSpec([_windows_build_bat(args.compiler), 'vsprojects\\%s.sln' % target, - '/p:Configuration=%s' % _WINDOWS_CONFIG[cfg], - _windows_toolset_option(args.compiler), - _windows_arch_option(args.arch)] + + '/p:Configuration=%s' % _WINDOWS_CONFIG[cfg]] + extra_args + language_make_options, shell=True, timeout_seconds=None) From a2d964c24d91eb79584c0c75794d6e985d05040e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 22 Feb 2016 17:33:09 -0800 Subject: [PATCH 059/236] rename _WINDOWS_CONFIG --- tools/run_tests/run_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index c6b98355cfa..7ce3ed02a39 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -149,7 +149,7 @@ class CLanguage(object): if self.platform == 'windows': binary = 'vsprojects/%s%s/%s.exe' % ( 'x64/' if self.args.arch == 'x64' else '', - _WINDOWS_CONFIG[self.config.build_config], + _MSBUILD_CONFIG[self.config.build_config], target['name']) else: binary = 'bins/%s/%s' % (self.config.build_config, target['name']) @@ -402,7 +402,7 @@ class CSharpLanguage(object): assemblies = tests_json['assemblies'] tests = tests_json['tests'] - msbuild_config = _WINDOWS_CONFIG[self.config.build_config] + msbuild_config = _MSBUILD_CONFIG[self.config.build_config] assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a) for a in assemblies] @@ -568,7 +568,7 @@ _LANGUAGES = { } -_WINDOWS_CONFIG = { +_MSBUILD_CONFIG = { 'dbg': 'Debug', 'opt': 'Release', 'gcov': 'Debug', @@ -843,7 +843,7 @@ def make_jobspec(cfg, targets, makefile='Makefile'): return [ jobset.JobSpec([_windows_build_bat(args.compiler), 'vsprojects\\%s.sln' % target, - '/p:Configuration=%s' % _WINDOWS_CONFIG[cfg]] + + '/p:Configuration=%s' % _MSBUILD_CONFIG[cfg]] + extra_args + language_make_options, shell=True, timeout_seconds=None) @@ -870,7 +870,7 @@ for l in languages: def build_step_environ(cfg): environ = {'CONFIG': cfg} - msbuild_cfg = _WINDOWS_CONFIG.get(cfg) + msbuild_cfg = _MSBUILD_CONFIG.get(cfg) if msbuild_cfg: environ['MSBUILD_CONFIG'] = msbuild_cfg return environ From 3b5121bfe4da57f422015000e675585d5311dc3e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 22 Feb 2016 17:41:05 -0800 Subject: [PATCH 060/236] minor refactoring --- tools/run_tests/run_tests.py | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 7ce3ed02a39..87b91bdc111 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -55,8 +55,8 @@ import report_utils import watch_dirs -ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) -os.chdir(ROOT) +_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(_ROOT) _FORCE_ENVIRON_FOR_WRAPPERS = {} @@ -159,8 +159,7 @@ class CLanguage(object): shortname=' '.join(cmdline), cpu_cost=target['cpu_cost'], environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': - os.path.abspath(os.path.dirname( - sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')})) + _ROOT + '/src/core/tsi/test_creds/ca.pem'})) elif self.args.regex == '.*' or self.platform == 'windows': print '\nWARNING: binary not found, skipping', binary return sorted(out) @@ -645,14 +644,6 @@ def _docker_arch_suffix(arch): sys.exit(1) -def _get_dockerfile_dir(language, cfg, arch): - """Returns dockerfile to use""" - custom = language.dockerfile_dir(cfg, arch) - if custom: - return custom - else: - return 'tools/dockerfile/grpc_tests_multilang_%s' % _docker_arch_suffix(arch) - def runs_per_test_type(arg_str): """Auxilary function to parse the "runs_per_test" flag. @@ -781,7 +772,7 @@ else: lang_list = args.language # We don't support code coverage on some languages if 'gcov' in args.config: - for bad in ['objc', 'sanity', 'build']: + for bad in ['objc', 'sanity']: if bad in lang_list: lang_list.remove(bad) @@ -797,10 +788,6 @@ if any(language.make_options() for language in languages): else: language_make_options = next(iter(languages)).make_options() -if len(languages) != 1: - print 'Multi-language testing is not supported.' - sys.exit(1) - if args.use_docker: if not args.travis: print 'Seen --use_docker flag, will run tests under docker.' @@ -810,14 +797,18 @@ if args.use_docker: print 'copied to the docker environment.' time.sleep(5) + dockerfile_dirs = set([l.dockerfile_dir() for l in languages]) + if len(dockerfile_dirs) > 1: + print 'Languages to be tested require running under different docker images.' + sys.exit(1) + dockerfile_dir = next(iter(dockerfile_dirs)) + child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) env = os.environ.copy() env['RUN_TESTS_COMMAND'] = run_tests_cmd - env['DOCKERFILE_DIR'] = _get_dockerfile_dir(next(iter(languages)), - next(iter(build_configs)), - args.arch) + env['DOCKERFILE_DIR'] = dockerfile_dir env['DOCKER_RUN_SCRIPT'] = 'tools/jenkins/docker_run_tests.sh' if args.xml_report: env['XML_REPORT'] = args.xml_report @@ -1001,7 +992,7 @@ def _start_port_server(port_server_port): print 'last ditch attempt to contact port server succeeded' break except: - traceback.print_exc(); + traceback.print_exc() port_log = open(logfile, 'r').read() print port_log sys.exit(1) @@ -1021,7 +1012,7 @@ def _start_port_server(port_server_port): time.sleep(1) waits += 1 except: - traceback.print_exc(); + traceback.print_exc() port_server.kill() raise From 19fa540100a1b9c28cb9ad6147d1e8d5f9659d82 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 23 Feb 2016 08:19:39 -0800 Subject: [PATCH 061/236] Fix copyrights --- src/core/security/b64.h | 2 +- src/core/support/load_file.c | 2 +- src/core/support/load_file.h | 2 +- src/core/support/tmpfile.h | 2 +- src/core/support/tmpfile_posix.c | 2 +- src/core/support/tmpfile_win32.c | 2 +- test/core/bad_ssl/server_common.c | 2 +- test/core/bad_ssl/server_common.h | 2 +- test/core/bad_ssl/servers/alpn.c | 2 +- test/core/bad_ssl/servers/cert.c | 2 +- test/core/end2end/fixtures/h2_ssl+poll.c | 2 +- test/core/end2end/fixtures/h2_ssl.c | 2 +- test/core/end2end/fixtures/h2_ssl_proxy.c | 2 +- test/core/end2end/tests/connectivity.c | 2 +- test/core/end2end/tests/ping.c | 2 +- test/core/end2end/tests/simple_metadata.c | 2 +- test/core/security/b64_test.c | 2 +- test/core/security/fetch_oauth2.c | 2 +- test/core/support/load_file_test.c | 2 +- test/cpp/qps/usage_timer.cc | 2 +- test/cpp/qps/usage_timer.h | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/core/security/b64.h b/src/core/security/b64.h index 31ae982691a..3e3b5211209 100644 --- a/src/core/security/b64.h +++ b/src/core/security/b64.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/support/load_file.c b/src/core/support/load_file.c index 2ea08a50664..650bd62ccbe 100644 --- a/src/core/support/load_file.c +++ b/src/core/support/load_file.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/support/load_file.h b/src/core/support/load_file.h index fcac94af7e1..746319a50d2 100644 --- a/src/core/support/load_file.h +++ b/src/core/support/load_file.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/support/tmpfile.h b/src/core/support/tmpfile.h index fbe5706d5e2..cecc71e2429 100644 --- a/src/core/support/tmpfile.h +++ b/src/core/support/tmpfile.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/support/tmpfile_posix.c b/src/core/support/tmpfile_posix.c index 70ecfc1b335..b16eeacf9d4 100644 --- a/src/core/support/tmpfile_posix.c +++ b/src/core/support/tmpfile_posix.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/support/tmpfile_win32.c b/src/core/support/tmpfile_win32.c index 834e2684b26..3000f0029fe 100644 --- a/src/core/support/tmpfile_win32.c +++ b/src/core/support/tmpfile_win32.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/bad_ssl/server_common.c b/test/core/bad_ssl/server_common.c index 14b1892c2e9..cde844a5529 100644 --- a/test/core/bad_ssl/server_common.c +++ b/test/core/bad_ssl/server_common.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/bad_ssl/server_common.h b/test/core/bad_ssl/server_common.h index 8ec77555030..2566c259059 100644 --- a/test/core/bad_ssl/server_common.h +++ b/test/core/bad_ssl/server_common.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/bad_ssl/servers/alpn.c b/test/core/bad_ssl/servers/alpn.c index dd0eb793d47..c8cc83b134e 100644 --- a/test/core/bad_ssl/servers/alpn.c +++ b/test/core/bad_ssl/servers/alpn.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/bad_ssl/servers/cert.c b/test/core/bad_ssl/servers/cert.c index d262ce16d81..4edef50b67a 100644 --- a/test/core/bad_ssl/servers/cert.c +++ b/test/core/bad_ssl/servers/cert.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/fixtures/h2_ssl+poll.c b/test/core/end2end/fixtures/h2_ssl+poll.c index 02bf16642d3..0b88876d138 100644 --- a/test/core/end2end/fixtures/h2_ssl+poll.c +++ b/test/core/end2end/fixtures/h2_ssl+poll.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/fixtures/h2_ssl.c b/test/core/end2end/fixtures/h2_ssl.c index 04bfb70303d..e21a3477df2 100644 --- a/test/core/end2end/fixtures/h2_ssl.c +++ b/test/core/end2end/fixtures/h2_ssl.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/fixtures/h2_ssl_proxy.c b/test/core/end2end/fixtures/h2_ssl_proxy.c index 5288318a83b..6340d3f4033 100644 --- a/test/core/end2end/fixtures/h2_ssl_proxy.c +++ b/test/core/end2end/fixtures/h2_ssl_proxy.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/tests/connectivity.c b/test/core/end2end/tests/connectivity.c index edef2ff68b7..975c6207314 100644 --- a/test/core/end2end/tests/connectivity.c +++ b/test/core/end2end/tests/connectivity.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/tests/ping.c b/test/core/end2end/tests/ping.c index f42f1865020..f85df63de79 100644 --- a/test/core/end2end/tests/ping.c +++ b/test/core/end2end/tests/ping.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/end2end/tests/simple_metadata.c b/test/core/end2end/tests/simple_metadata.c index 376ffb670a4..c5084a560ff 100644 --- a/test/core/end2end/tests/simple_metadata.c +++ b/test/core/end2end/tests/simple_metadata.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/security/b64_test.c b/test/core/security/b64_test.c index a45a189457e..772514e1fda 100644 --- a/test/core/security/b64_test.c +++ b/test/core/security/b64_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/security/fetch_oauth2.c b/test/core/security/fetch_oauth2.c index a9e1720a28d..87b54f1a0c3 100644 --- a/test/core/security/fetch_oauth2.c +++ b/test/core/security/fetch_oauth2.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/support/load_file_test.c b/test/core/support/load_file_test.c index 9ba0dc80a17..e6ba6174400 100644 --- a/test/core/support/load_file_test.c +++ b/test/core/support/load_file_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/cpp/qps/usage_timer.cc b/test/cpp/qps/usage_timer.cc index 5a913d498fa..ee64537fedf 100644 --- a/test/cpp/qps/usage_timer.cc +++ b/test/cpp/qps/usage_timer.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/cpp/qps/usage_timer.h b/test/cpp/qps/usage_timer.h index 704d4b02a2e..d5470dba590 100644 --- a/test/cpp/qps/usage_timer.h +++ b/test/cpp/qps/usage_timer.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From c4cbe39889ef482e10750dd4c267acbc66f7ae51 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 22 Feb 2016 19:29:38 -0800 Subject: [PATCH 062/236] add more compilers --- .../tools/dockerfile/apt_get_basic.include | 3 + .../tools/dockerfile/run_tests_addons.include | 3 + .../test/cxx_squeeze_x64/Dockerfile.template | 45 ++++++++++ .../cxx_ubuntu1604_x64/Dockerfile.template | 39 +++++++++ .../test/cxx_squeeze_x64/Dockerfile | 84 ++++++++++++++++++ .../test/cxx_ubuntu1604_x64/Dockerfile | 86 +++++++++++++++++++ tools/run_tests/run_tests.py | 30 ++++++- 7 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template create mode 100644 templates/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile.template create mode 100644 tools/dockerfile/test/cxx_squeeze_x64/Dockerfile create mode 100644 tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile diff --git a/templates/tools/dockerfile/apt_get_basic.include b/templates/tools/dockerfile/apt_get_basic.include index 9237e7dacef..547ce01a300 100644 --- a/templates/tools/dockerfile/apt_get_basic.include +++ b/templates/tools/dockerfile/apt_get_basic.include @@ -1,3 +1,4 @@ +<%page args="skip_golang=False"/>\ # Install Git and basic packages. RUN apt-get update && apt-get install -y ${'\\'} autoconf ${'\\'} @@ -9,7 +10,9 @@ RUN apt-get update && apt-get install -y ${'\\'} gcc ${'\\'} gcc-multilib ${'\\'} git ${'\\'} +% if not skip_golang: golang ${'\\'} +% endif gyp ${'\\'} lcov ${'\\'} libc6 ${'\\'} diff --git a/templates/tools/dockerfile/run_tests_addons.include b/templates/tools/dockerfile/run_tests_addons.include index 27ac67f5d8f..30d22be2980 100644 --- a/templates/tools/dockerfile/run_tests_addons.include +++ b/templates/tools/dockerfile/run_tests_addons.include @@ -1,7 +1,10 @@ +<%page args="skip_zookeeper=False"/>\ <%include file="ccache_setup.include"/> +% if not skip_zookeeper: #====================== # Zookeeper dependencies # TODO(jtattermusch): is zookeeper still needed? RUN apt-get install -y libzookeeper-mt-dev +% endif RUN mkdir /var/local/jenkins diff --git a/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template new file mode 100644 index 00000000000..294ae00b4c7 --- /dev/null +++ b/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template @@ -0,0 +1,45 @@ +%YAML 1.2 +--- | + # Copyright 2015-2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM debian:squeeze + + <%include file="../../apt_get_basic.include" args="skip_golang=True"/> + + # libgflags-dev is not available on squeezy + RUN apt-get update && apt-get -y install libgtest-dev libc++-dev clang && apt-get clean + + RUN apt-get update && apt-get -y install python-pip && apt-get clean + RUN pip install argparse + + <%include file="../../run_tests_addons.include" args="skip_zookeeper=True"/> + # Define the default command. + CMD ["bash"] + \ No newline at end of file diff --git a/templates/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile.template new file mode 100644 index 00000000000..6b302748925 --- /dev/null +++ b/templates/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile.template @@ -0,0 +1,39 @@ +%YAML 1.2 +--- | + # Copyright 2015-2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM ubuntu:16.04 + + <%include file="../../apt_get_basic.include"/> + <%include file="../../cxx_deps.include"/> + <%include file="../../run_tests_addons.include"/> + # Define the default command. + CMD ["bash"] + \ No newline at end of file diff --git a/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile b/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile new file mode 100644 index 00000000000..08488874347 --- /dev/null +++ b/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile @@ -0,0 +1,84 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:squeeze + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + + +# libgflags-dev is not available on squeezy +RUN apt-get update && apt-get -y install libgtest-dev libc++-dev clang && apt-get clean + +RUN apt-get update && apt-get -y install python-pip && apt-get clean +RUN pip install argparse + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile new file mode 100644 index 00000000000..c6fe79b42c0 --- /dev/null +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -0,0 +1,86 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM ubuntu:16.04 + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#====================== +# Zookeeper dependencies +# TODO(jtattermusch): is zookeeper still needed? +RUN apt-get install -y libzookeeper-mt-dev + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 87b91bdc111..09af0552ecc 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -123,6 +123,11 @@ def _check_compiler(compiler, supported_compilers): raise Exception('Compiler %s not supported.' % compiler) +def _is_use_docker_child(): + """Returns True if running running as a --use_docker child.""" + return True if os.getenv('RUN_TESTS_COMMAND') else False + + class CLanguage(object): def __init__(self, make_target, test_lang): @@ -137,8 +142,9 @@ class CLanguage(object): self._make_options = [_windows_toolset_option(self.args.compiler), _windows_arch_option(self.args.arch)] else: - _check_compiler(self.args.compiler, ['default']) self._make_options = [] + self._docker_distro = self._get_docker_distro(self.args.use_docker, + self.args.compiler) def test_specs(self): out = [] @@ -197,8 +203,24 @@ class CLanguage(object): def makefile_name(self): return 'Makefile' + def _get_docker_distro(self, use_docker, compiler): + if _is_use_docker_child(): + return "already_under_docker" + if not use_docker: + _check_compiler(compiler, ['default']) + + if compiler == 'gcc4.9' or compiler == 'default': + return 'jessie' + elif compiler == 'gcc4.4': + return 'squeeze' + elif compiler == 'gcc5.3': + return 'ubuntu1604' + else: + raise Exception('Compiler %s not supported.' % compiler) + def dockerfile_dir(self): - return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(self.args.arch) + return 'tools/dockerfile/test/cxx_%s_%s' % (self._docker_distro, + _docker_arch_suffix(self.args.arch)) def __str__(self): return self.make_target @@ -712,7 +734,9 @@ argp.add_argument('--arch', default='default', help='Selects architecture to target. For some platforms "default" is the only supported choice.') argp.add_argument('--compiler', - choices=['default', 'vs2010', 'vs2013', 'vs2015'], + choices=['default', + 'gcc4.4', 'gcc4.9', 'gcc5.3', + 'vs2010', 'vs2013', 'vs2015'], default='default', help='Selects compiler to use. Allowed values depend on the platform and language.') argp.add_argument('--build_only', From 4651bef8c7629e8cb1c57fee715ada333129e588 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 23 Feb 2016 08:31:25 -0800 Subject: [PATCH 063/236] fix ruby --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 09af0552ecc..7b2bc537162 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -385,7 +385,7 @@ class RubyLanguage(object): def pre_build_steps(self): return [['tools/run_tests/pre_build_ruby.sh']] - def make_targets(self, test_regex): + def make_targets(self): return [] def make_options(self): From 9f23c4a4f6d326463e50868017ad2fd10dda53d9 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Tue, 23 Feb 2016 16:22:32 +0000 Subject: [PATCH 064/236] Further improvements to Summer Of Code Ideas An introductory paragraph, addition of required skills and likely mentors, hyperlinks to other technologies, and copy edits. --- summerofcode/ideas.md | 46 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/summerofcode/ideas.md b/summerofcode/ideas.md index b14d3f7b6ad..4ccbe3b12fc 100644 --- a/summerofcode/ideas.md +++ b/summerofcode/ideas.md @@ -1,22 +1,50 @@ # gRPC Summer of Code Project Ideas -C Core: +Hello students! -1. Port gRPC to one of (Free, Net, Open) BSD platforms and create packages for them. Add kqueue support in the process. -2. Fix gRPC C-core's URI parser. The current parser does not qualify as a standard parser according to [RFC3986]( https://tools.ietf.org/html/rfc3986). Write test suites to verify this and make changes necessary to make the URI parser compliant. -3. HPACK compression efficiency evaluation - Figure out how to benchmark gRPC's compression efficiency (both in terms of bytes on the wire and cpu cycles). Implement benchmarks. Potentially extend this to other standalone implementations -- Java and Go. +We want gRPC to be the universal protocol for all computing platforms and +paradigms, so while these are our ideas of what we think would make good +projects for the summer, we're eager to hear your ideas and proposals as well. +Try us out and get to know the gRPC code and team! + +**Required skills for all projects:** git version control, collaborative +software development on github.com, and software development in at least one +of gRPC's ten languages on at least one of Linux, Mac OS X, and Windows. + +------------------------------------- + +gRPC C Core: + +1. Port gRPC to one of the major BSD platforms ([FreeBSD](https://freebsd.org), [NetBSD](https://netbsd.org), and [OpenBSD](https://openbsd.org)) and create packages for them. Add kqueue support in the process. + * **Required skills:** C programming language, BSD operating system. + * **Likely mentors:** [Craig Tiller](https://github.com/ctiller), [Nicolas Noble](https://github.com/nicolasnoble). +1. Fix gRPC C-core's URI parser. The current parser does not qualify as a standard parser according to [RFC3986]( https://tools.ietf.org/html/rfc3986). Write test suites to verify this and make changes necessary to make the URI parser compliant. + * **Required skills:** C programming language, HTTP standard compliance. + * **Likely mentors:** [Craig Tiller](https://github.com/ctiller). +1. HPACK compression efficiency evaluation - Figure out how to benchmark gRPC's compression efficiency (both in terms of bytes on the wire and cpu cycles). Implement benchmarks. Potentially extend this to other full-stack gRPC implementations (Java and Go). + * **Required skills:** C programming language, software performance benchmarking, potentially Java and Go. + * **Likely mentors:** [Craig Tiller](https://github.com/ctiller). gRPC Python: - 1. Evaluate the port of gRPC's Python implementation to PyPy. Investigate the state of [Cython support](http://docs.cython.org/src/userguide/pypy.html) to do this or potentially explore cffi - 2. Develop and test Python 3.5 Support for gRPC. Make necessary changes to port gRPC and package it for supported platforms. +1. Port gRPC Python to [PyPy](http://pypy.org). Investigate the state of [Cython support](http://docs.cython.org/src/userguide/pypy.html) to do this or potentially explore [cffi](https://cffi.readthedocs.org/en/latest/). + * **Required skills:** Python programming language, PyPy Python interpreter. + * **Likely mentors:** [Nathaniel Manista](https://github.com/nathanielmanistaatgoogle), [Masood Malekghassemi](https://github.com/soltanmm). +1. Develop and test Python 3.5 Support for gRPC. Make necessary changes to port gRPC and package it for supported platforms. + * **Required skills:** Python programming language, PyPy Python interpreter. + * **Likely mentors:** [Nathaniel Manista](https://github.com/nathanielmanistaatgoogle), [Masood Malekghassemi](https://github.com/soltanmm). gRPC Ruby/Java: -1. jRuby support for gRPC. Develop a jRuby wrapper for gRPC based on grpc-java and ensure that it is API compatible with the existing Ruby implementation and passes all tests. +1. [jRuby](http://jruby.org) support for gRPC. Develop a jRuby wrapper for gRPC based on grpc-java and ensure that it is API compatible with the existing Ruby implementation and passes all tests. + * **Required skills:** Java programming language, Ruby programming language. + * **Likely mentors:** [Michael Lumish](https://github.com/murgatroid99), [Eric Anderson](https://github.com/ejona86). -Other: +gRPC Wire Protocol: -1. Develop a Wireshark plugin for the gRPC protocol. Provide documentation and tutorials for this plugin. Bonus: consider set-up and use with the mobile clients. +1. Develop a [Wireshark](https://wireshark.org) plugin for the gRPC protocol. Provide documentation and tutorials for this plugin. + * **Bonus:** consider set-up and use with mobile clients. + * **Required skills:** Wireshark software. + * **Likely mentors:** [Nicolas Noble](https://github.com/nicolasnoble). From 368a66dcbed0472e8a36e14351b6d89d618bbf20 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Tue, 23 Feb 2016 17:17:05 +0000 Subject: [PATCH 065/236] More ideas page tweaks One copy edit, two links. --- summerofcode/ideas.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/summerofcode/ideas.md b/summerofcode/ideas.md index 4ccbe3b12fc..c7f368e7cc2 100644 --- a/summerofcode/ideas.md +++ b/summerofcode/ideas.md @@ -2,10 +2,12 @@ Hello students! -We want gRPC to be the universal protocol for all computing platforms and -paradigms, so while these are our ideas of what we think would make good -projects for the summer, we're eager to hear your ideas and proposals as well. -Try us out and get to know the gRPC code and team! +We want gRPC to be the universal remote procedure call protocol for all +computing platforms and paradigms, so while these are our ideas of what we +think would make good projects for the summer, we're eager to hear your ideas +and proposals as well. +[Try us out](https://github.com/grpc/grpc/blob/master/CONTRIBUTING.md) and get +to know the gRPC code and team! **Required skills for all projects:** git version control, collaborative software development on github.com, and software development in at least one @@ -15,7 +17,7 @@ of gRPC's ten languages on at least one of Linux, Mac OS X, and Windows. gRPC C Core: -1. Port gRPC to one of the major BSD platforms ([FreeBSD](https://freebsd.org), [NetBSD](https://netbsd.org), and [OpenBSD](https://openbsd.org)) and create packages for them. Add kqueue support in the process. +1. Port gRPC to one of the major BSD platforms ([FreeBSD](https://freebsd.org), [NetBSD](https://netbsd.org), and [OpenBSD](https://openbsd.org)) and create packages for them. Add [kqueue](https://www.freebsd.org/cgi/man.cgi?query=kqueue) support in the process. * **Required skills:** C programming language, BSD operating system. * **Likely mentors:** [Craig Tiller](https://github.com/ctiller), [Nicolas Noble](https://github.com/nicolasnoble). 1. Fix gRPC C-core's URI parser. The current parser does not qualify as a standard parser according to [RFC3986]( https://tools.ietf.org/html/rfc3986). Write test suites to verify this and make changes necessary to make the URI parser compliant. From e2f243ba7c8e9492e8a638bf5ac4dc3c6048cf8d Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Feb 2016 09:38:41 -0800 Subject: [PATCH 066/236] clang format --- test/cpp/common/alarm_cpp_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cpp/common/alarm_cpp_test.cc b/test/cpp/common/alarm_cpp_test.cc index 323b7c0fcf5..d4381c05158 100644 --- a/test/cpp/common/alarm_cpp_test.cc +++ b/test/cpp/common/alarm_cpp_test.cc @@ -102,7 +102,6 @@ TEST(AlarmTest, NegativeExpiry) { EXPECT_EQ(junk, output_tag); } - TEST(AlarmTest, Cancellation) { CompletionQueue cq; void* junk = reinterpret_cast(1618033); From 7d037a5ee2ebdba2fb8d0688399151878818c297 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 12 Feb 2016 19:49:23 -0800 Subject: [PATCH 067/236] first version of bigquery python helpers --- tools/bigquery/big_query_utils.py | 181 ++++++++++++++++++ .../grpc_interop_stress_cxx/Dockerfile | 5 + 2 files changed, 186 insertions(+) create mode 100644 tools/bigquery/big_query_utils.py diff --git a/tools/bigquery/big_query_utils.py b/tools/bigquery/big_query_utils.py new file mode 100644 index 00000000000..ebcf9d6ec35 --- /dev/null +++ b/tools/bigquery/big_query_utils.py @@ -0,0 +1,181 @@ +import argparse +import json +import uuid +import httplib2 + +from apiclient import discovery +from apiclient.errors import HttpError +from oauth2client.client import GoogleCredentials + +NUM_RETRIES = 3 + + +def create_bq(): + """Authenticates with cloud platform and gets a BiqQuery service object + """ + creds = GoogleCredentials.get_application_default() + return discovery.build('bigquery', 'v2', credentials=creds) + + +def create_ds(biq_query, project_id, dataset_id): + is_success = True + body = { + 'datasetReference': { + 'projectId': project_id, + 'datasetId': dataset_id + } + } + try: + dataset_req = biq_query.datasets().insert(projectId=project_id, body=body) + dataset_req.execute(num_retries=NUM_RETRIES) + except HttpError as http_error: + if http_error.resp.status == 409: + print 'Warning: The dataset %s already exists' % dataset_id + else: + # Note: For more debugging info, print "http_error.content" + print 'Error in creating dataset: %s. Err: %s' % (dataset_id, http_error) + is_success = False + return is_success + + +def make_field(field_name, field_type, field_description): + return { + 'name': field_name, + 'type': field_type, + 'description': field_description + } + + +def create_table(big_query, project_id, dataset_id, table_id, fields_list, + description): + is_success = True + body = { + 'description': description, + 'schema': { + 'fields': fields_list + }, + 'tableReference': { + 'datasetId': dataset_id, + 'projectId': project_id, + 'tableId': table_id + } + } + try: + table_req = big_query.tables().insert(projectId=project_id, + datasetId=dataset_id, + body=body) + res = table_req.execute(num_retries=NUM_RETRIES) + print 'Successfully created %s "%s"' % (res['kind'], res['id']) + except HttpError as http_error: + if http_error.resp.status == 409: + print 'Warning: Table %s already exists' % table_id + else: + print 'Error in creating table: %s. Err: %s' % (table_id, http_error) + is_success = False + return is_success + + +def insert_rows(big_query, project_id, dataset_id, table_id, rows_list): + is_success = True + body = {'rows': rows_list} + try: + insert_req = big_query.tabledata().insertAll(projectId=project_id, + datasetId=dataset_id, + tableId=table_id, + body=body) + print body + res = insert_req.execute(num_retries=NUM_RETRIES) + print res + except HttpError as http_error: + print 'Error in inserting rows in the table %s' % table_id + is_success = False + return is_success + +##################### + + +def make_emp_row(emp_id, emp_name, emp_email): + return { + 'insertId': str(emp_id), + 'json': { + 'emp_id': emp_id, + 'emp_name': emp_name, + 'emp_email_id': emp_email + } + } + + +def get_emp_table_fields_list(): + return [ + make_field('emp_id', 'INTEGER', 'Employee id'), + make_field('emp_name', 'STRING', 'Employee name'), + make_field('emp_email_id', 'STRING', 'Employee email id') + ] + + +def insert_emp_rows(big_query, project_id, dataset_id, table_id, start_idx, + num_rows): + rows_list = [make_emp_row(i, 'sree_%d' % i, 'sreecha_%d@gmail.com' % i) + for i in range(start_idx, start_idx + num_rows)] + insert_rows(big_query, project_id, dataset_id, table_id, rows_list) + + +def create_emp_table(big_query, project_id, dataset_id, table_id): + fields_list = get_emp_table_fields_list() + description = 'Test table created by sree' + create_table(big_query, project_id, dataset_id, table_id, fields_list, + description) + + +def sync_query(big_query, project_id, query, timeout=5000): + query_data = {'query': query, 'timeoutMs': timeout} + query_job = None + try: + query_job = big_query.jobs().query( + projectId=project_id, + body=query_data).execute(num_retries=NUM_RETRIES) + except HttpError as http_error: + print 'Query execute job failed with error: %s' % http_error + print http_error.content + return query_job + +#[Start query_emp_records] +def query_emp_records(big_query, project_id, dataset_id, table_id): + query = 'SELECT emp_id, emp_name FROM %s.%s ORDER BY emp_id;' % (dataset_id, table_id) + print query + query_job = sync_query(big_query, project_id, query, 5000) + job_id = query_job['jobReference'] + + print query_job + print '**Starting paging **' + #[Start Paging] + page_token = None + while True: + page = big_query.jobs().getQueryResults( + pageToken=page_token, + **query_job['jobReference']).execute(num_retries=NUM_RETRIES) + rows = page['rows'] + for row in rows: + print row['f'][0]['v'], "---", row['f'][1]['v'] + page_token = page.get('pageToken') + if not page_token: + break + #[End Paging] +#[End query_emp_records] + +######################### +DATASET_SEQ_NUM = 1 +TABLE_SEQ_NUM = 11 + +PROJECT_ID = 'sree-gce' +DATASET_ID = 'sree_test_dataset_%d' % DATASET_SEQ_NUM +TABLE_ID = 'sree_test_table_%d' % TABLE_SEQ_NUM + +EMP_ROW_IDX = 10 +EMP_NUM_ROWS = 5 + +bq = create_bq() +create_ds(bq, PROJECT_ID, DATASET_ID) +create_emp_table(bq, PROJECT_ID, DATASET_ID, TABLE_ID) +insert_emp_rows(bq, PROJECT_ID, DATASET_ID, TABLE_ID, EMP_ROW_IDX, EMP_NUM_ROWS) +query_emp_records(bq, PROJECT_ID, DATASET_ID, TABLE_ID) diff --git a/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile b/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile index 58a8c32e344..4123cc1a26a 100644 --- a/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile +++ b/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile @@ -59,6 +59,8 @@ RUN apt-get update && apt-get install -y \ wget \ zip && apt-get clean +RUN easy_install -U pip + # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc RUN ln -s /usr/bin/ccache /usr/local/bin/g++ @@ -71,5 +73,8 @@ RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ # C++ dependencies RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang +# Google Cloud platform API libraries (for BigQuery) +RUN pip install --upgrade google-api-python-client + # Define the default command. CMD ["bash"] From 44ca2c26409a172b80bc9f40f7578f3eaf1d135d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 16 Feb 2016 09:48:36 -0800 Subject: [PATCH 068/236] Examples --- test/cpp/interop/metrics_client.cc | 41 ++++--- .../build_interop_stress.sh | 2 +- tools/{bigquery => gke}/big_query_utils.py | 0 tools/gke/create_client.py | 108 ++++++++++++++++++ tools/gke/create_server.py | 74 ++++++++++++ tools/gke/delete_client.py | 66 +++++++++++ tools/gke/delete_server.py | 58 ++++++++++ tools/gke/kubernetes_api.py | 35 +++--- tools/run_tests/stress_test_wrapper.py | 96 ++++++++++++++++ 9 files changed, 451 insertions(+), 29 deletions(-) rename tools/{bigquery => gke}/big_query_utils.py (100%) create mode 100755 tools/gke/create_client.py create mode 100755 tools/gke/create_server.py create mode 100755 tools/gke/delete_client.py create mode 100755 tools/gke/delete_server.py create mode 100755 tools/run_tests/stress_test_wrapper.py diff --git a/test/cpp/interop/metrics_client.cc b/test/cpp/interop/metrics_client.cc index 0c140ffd85e..cc304f2e895 100644 --- a/test/cpp/interop/metrics_client.cc +++ b/test/cpp/interop/metrics_client.cc @@ -37,39 +37,45 @@ #include #include -#include "test/cpp/util/metrics_server.h" -#include "test/cpp/util/test_config.h" #include "src/proto/grpc/testing/metrics.grpc.pb.h" #include "src/proto/grpc/testing/metrics.pb.h" +#include "test/cpp/util/metrics_server.h" +#include "test/cpp/util/test_config.h" DEFINE_string(metrics_server_address, "", "The metrics server addresses in the fomrat :"); +DEFINE_bool(total_only, false, + "If true, this prints only the total value of all gauges"); + +int kDeadlineSecs = 10; using grpc::testing::EmptyMessage; using grpc::testing::GaugeResponse; using grpc::testing::MetricsService; using grpc::testing::MetricsServiceImpl; -void PrintMetrics(const grpc::string& server_address) { - gpr_log(GPR_INFO, "creating a channel to %s", server_address.c_str()); - std::shared_ptr channel( - grpc::CreateChannel(server_address, grpc::InsecureChannelCredentials())); - - std::unique_ptr stub(MetricsService::NewStub(channel)); - +// Prints the values of all Gauges (unless total_only is set to 'true' in which +// case this only prints the sum of all gauge values). +bool PrintMetrics(std::unique_ptr stub, bool total_only) { grpc::ClientContext context; EmptyMessage message; + std::chrono::system_clock::time_point deadline = + std::chrono::system_clock::now() + std::chrono::seconds(kDeadlineSecs); + + context.set_deadline(deadline); + std::unique_ptr> reader( stub->GetAllGauges(&context, message)); GaugeResponse gauge_response; long overall_qps = 0; - int idx = 0; while (reader->Read(&gauge_response)) { if (gauge_response.value_case() == GaugeResponse::kLongValue) { - gpr_log(GPR_INFO, "Gauge: %d (%s: %ld)", ++idx, - gauge_response.name().c_str(), gauge_response.long_value()); + if (!total_only) { + gpr_log(GPR_INFO, "%s: %ld", gauge_response.name().c_str(), + gauge_response.long_value()); + } overall_qps += gauge_response.long_value(); } else { gpr_log(GPR_INFO, "Gauge %s is not a long value", @@ -77,12 +83,14 @@ void PrintMetrics(const grpc::string& server_address) { } } - gpr_log(GPR_INFO, "OVERALL: %ld", overall_qps); + gpr_log(GPR_INFO, "%ld", overall_qps); const grpc::Status status = reader->Finish(); if (!status.ok()) { gpr_log(GPR_ERROR, "Error in getting metrics from the client"); } + + return status.ok(); } int main(int argc, char** argv) { @@ -97,7 +105,12 @@ int main(int argc, char** argv) { return 1; } - PrintMetrics(FLAGS_metrics_server_address); + std::shared_ptr channel(grpc::CreateChannel( + FLAGS_metrics_server_address, grpc::InsecureChannelCredentials())); + + if (!PrintMetrics(MetricsService::NewStub(channel), FLAGS_total_only)) { + return 1; + } return 0; } diff --git a/tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh index 6ed3ccb3fa0..392bdfccda0 100755 --- a/tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh +++ b/tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh @@ -42,4 +42,4 @@ cd /var/local/git/grpc make install-certs # build C++ interop stress client, interop client and server -make stress_test interop_client interop_server +make stress_test metrics_client interop_client interop_server diff --git a/tools/bigquery/big_query_utils.py b/tools/gke/big_query_utils.py similarity index 100% rename from tools/bigquery/big_query_utils.py rename to tools/gke/big_query_utils.py diff --git a/tools/gke/create_client.py b/tools/gke/create_client.py new file mode 100755 index 00000000000..bc56ef0ef13 --- /dev/null +++ b/tools/gke/create_client.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python2.7 +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse + +import kubernetes_api + +argp = argparse.ArgumentParser(description='Launch Stress tests in GKE') + +argp.add_argument('-n', + '--num_instances', + required=True, + type=int, + help='The number of instances to launch in GKE') +args = argp.parse_args() + +kubernetes_api_server="localhost" +kubernetes_api_port=8001 + + +# Docker image +image_name="gcr.io/sree-gce/grpc_stress_test_2" + +server_address = "stress-server.default.svc.cluster.local:8080" +metrics_server_address = "localhost:8081" + +stress_test_arg_list=[ + "--server_addresses=" + server_address, + "--test_cases=empty_unary:20,large_unary:20", + "--num_stubs_per_channel=10" +] + +metrics_client_arg_list=[ + "--metrics_server_address=" + metrics_server_address, + "--total_only=true"] + +env_dict={ + "GPRC_ROOT": "/var/local/git/grpc", + "STRESS_TEST_IMAGE": "/var/local/git/grpc/bins/opt/stress_test", + "STRESS_TEST_ARGS_STR": ' '.join(stress_test_arg_list), + "METRICS_CLIENT_IMAGE": "/var/local/git/grpc/bins/opt/metrics_client", + "METRICS_CLIENT_ARGS_STR": ' '.join(metrics_client_arg_list)} + +cmd_list=["/var/local/git/grpc/bins/opt/stress_test"] +arg_list=stress_test_arg_list # make this [] in future +port_list=[8081] + +namespace = 'default' +is_headless_service = False # Client is NOT headless service + +print('Creating %d instances of client..' % args.num_instances) + +for i in range(1, args.num_instances + 1): + service_name = 'stress-client-%d' % i + pod_name = service_name # Use the same name for kubernetes Service and Pod + is_success = kubernetes_api.create_pod( + kubernetes_api_server, + kubernetes_api_port, + namespace, + pod_name, + image_name, + port_list, + cmd_list, + arg_list, + env_dict) + if not is_success: + print("Error in creating pod %s" % pod_name) + else: + is_success = kubernetes_api.create_service( + kubernetes_api_server, + kubernetes_api_port, + namespace, + service_name, + pod_name, + port_list, # Service port list + port_list, # Container port list (same as service port list) + is_headless_service) + if not is_success: + print("Error in creating service %s" % service_name) + else: + print("Created client %s" % pod_name) diff --git a/tools/gke/create_server.py b/tools/gke/create_server.py new file mode 100755 index 00000000000..23ab62c205d --- /dev/null +++ b/tools/gke/create_server.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python2.7 +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse + +import kubernetes_api + +service_name = 'stress-server' +pod_name = service_name # Use the same name for kubernetes Service and Pod +namespace = 'default' +is_headless_service = True +cmd_list=['/var/local/git/grpc/bins/opt/interop_server'] +arg_list=['--port=8080'] +port_list=[8080] +image_name='gcr.io/sree-gce/grpc_stress_test_2' +env_dict={} + +# Make sure you run kubectl proxy --port=8001 +kubernetes_api_server='localhost' +kubernetes_api_port=8001 + +is_success = kubernetes_api.create_pod( + kubernetes_api_server, + kubernetes_api_port, + namespace, + pod_name, + image_name, + port_list, + cmd_list, + arg_list, + env_dict) +if not is_success: + print("Error in creating pod") +else: + is_success = kubernetes_api.create_service( + kubernetes_api_server, + kubernetes_api_port, + namespace, + service_name, + pod_name, + port_list, # Service port list + port_list, # Container port list (same as service port list) + is_headless_service) + if not is_success: + print("Error in creating service") + else: + print("Successfully created the Server") diff --git a/tools/gke/delete_client.py b/tools/gke/delete_client.py new file mode 100755 index 00000000000..aa519f26b89 --- /dev/null +++ b/tools/gke/delete_client.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python2.7 +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse + +import kubernetes_api + +argp = argparse.ArgumentParser(description='Delete Stress test clients in GKE') +argp.add_argument('-n', + '--num_instances', + required=True, + type=int, + help='The number of instances currently running') + +args = argp.parse_args() +for i in range(1, args.num_instances + 1): + service_name = 'stress-client-%d' % i + pod_name = service_name + namespace = 'default' + kubernetes_api_server="localhost" + kubernetes_api_port=8001 + + is_success=kubernetes_api.delete_pod( + kubernetes_api_server, + kubernetes_api_port, + namespace, + pod_name) + if not is_success: + print('Error in deleting Pod %s' % pod_name) + else: + is_success= kubernetes_api.delete_service( + kubernetes_api_server, + kubernetes_api_port, + namespace, + service_name) + if not is_success: + print('Error in deleting Service %s' % service_name) + else: + print('Deleted %s' % pod_name) diff --git a/tools/gke/delete_server.py b/tools/gke/delete_server.py new file mode 100755 index 00000000000..6e3fdcc33ba --- /dev/null +++ b/tools/gke/delete_server.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python2.7 +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse + +import kubernetes_api + +service_name = 'stress-server' +pod_name = service_name # Use the same name for kubernetes Service and Pod +namespace = 'default' +is_headless_service = True +kubernetes_api_server="localhost" +kubernetes_api_port=8001 + +is_success = kubernetes_api.delete_pod( + kubernetes_api_server, + kubernetes_api_port, + namespace, + pod_name) +if not is_success: + print("Error in deleting Pod %s" % pod_name) +else: + is_success = kubernetes_api.delete_service( + kubernetes_api_server, + kubernetes_api_port, + namespace, + service_name) + if not is_success: + print("Error in deleting Service %d" % service_name) + else: + print("Deleted server %s" % service_name) diff --git a/tools/gke/kubernetes_api.py b/tools/gke/kubernetes_api.py index 7dd30153650..14d724bd319 100755 --- a/tools/gke/kubernetes_api.py +++ b/tools/gke/kubernetes_api.py @@ -33,8 +33,9 @@ import json _REQUEST_TIMEOUT_SECS = 10 + def _make_pod_config(pod_name, image_name, container_port_list, cmd_list, - arg_list): + arg_list, env_dict): """Creates a string containing the Pod defintion as required by the Kubernetes API""" body = { 'kind': 'Pod', @@ -48,20 +49,21 @@ def _make_pod_config(pod_name, image_name, container_port_list, cmd_list, { 'name': pod_name, 'image': image_name, - 'ports': [] + 'ports': [{'containerPort': port, + 'protocol': 'TCP'} for port in container_port_list] } ] } } - # Populate the 'ports' list - for port in container_port_list: - port_entry = {'containerPort': port, 'protocol': 'TCP'} - body['spec']['containers'][0]['ports'].append(port_entry) + + env_list = [{'name': k, 'value': v} for (k, v) in env_dict.iteritems()] + if len(env_list) > 0: + body['spec']['containers'][0]['env'] = env_list # Add the 'Command' and 'Args' attributes if they are passed. # Note: # - 'Command' overrides the ENTRYPOINT in the Docker Image - # - 'Args' override the COMMAND in Docker image (yes, it is confusing!) + # - 'Args' override the CMD in Docker image (yes, it is confusing!) if len(cmd_list) > 0: body['spec']['containers'][0]['command'] = cmd_list if len(arg_list) > 0: @@ -70,7 +72,7 @@ def _make_pod_config(pod_name, image_name, container_port_list, cmd_list, def _make_service_config(service_name, pod_name, service_port_list, - container_port_list, is_headless): + container_port_list, is_headless): """Creates a string containing the Service definition as required by the Kubernetes API. NOTE: @@ -124,6 +126,7 @@ def _print_connection_error(msg): print('ERROR: Connection failed. Did you remember to run Kubenetes proxy on ' 'localhost (i.e kubectl proxy --port=) ?. Error: %s' % msg) + def _do_post(post_url, api_name, request_body): """Helper to do HTTP POST. @@ -135,7 +138,9 @@ def _do_post(post_url, api_name, request_body): """ is_success = True try: - r = requests.post(post_url, data=request_body, timeout=_REQUEST_TIMEOUT_SECS) + r = requests.post(post_url, + data=request_body, + timeout=_REQUEST_TIMEOUT_SECS) if r.status_code == requests.codes.conflict: print('WARN: Looks like the resource already exists. Api: %s, url: %s' % (api_name, post_url)) @@ -143,7 +148,8 @@ def _do_post(post_url, api_name, request_body): print('ERROR: %s API returned error. HTTP response: (%d) %s' % (api_name, r.status_code, r.text)) is_success = False - except(requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: + except (requests.exceptions.Timeout, + requests.exceptions.ConnectionError) as e: is_success = False _print_connection_error(str(e)) return is_success @@ -165,7 +171,8 @@ def _do_delete(del_url, api_name): print('ERROR: %s API returned error. HTTP response: %s' % (api_name, r.text)) is_success = False - except(requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: + except (requests.exceptions.Timeout, + requests.exceptions.ConnectionError) as e: is_success = False _print_connection_error(str(e)) return is_success @@ -179,12 +186,12 @@ def create_service(kube_host, kube_port, namespace, service_name, pod_name, post_url = 'http://%s:%d/api/v1/namespaces/%s/services' % ( kube_host, kube_port, namespace) request_body = _make_service_config(service_name, pod_name, service_port_list, - container_port_list, is_headless) + container_port_list, is_headless) return _do_post(post_url, 'Create Service', request_body) def create_pod(kube_host, kube_port, namespace, pod_name, image_name, - container_port_list, cmd_list, arg_list): + container_port_list, cmd_list, arg_list, env_dict): """Creates a Kubernetes Pod. Note that it is generally NOT considered a good practice to directly create @@ -200,7 +207,7 @@ def create_pod(kube_host, kube_port, namespace, pod_name, image_name, post_url = 'http://%s:%d/api/v1/namespaces/%s/pods' % (kube_host, kube_port, namespace) request_body = _make_pod_config(pod_name, image_name, container_port_list, - cmd_list, arg_list) + cmd_list, arg_list, env_dict) return _do_post(post_url, 'Create Pod', request_body) diff --git a/tools/run_tests/stress_test_wrapper.py b/tools/run_tests/stress_test_wrapper.py new file mode 100755 index 00000000000..8f1bd2024ec --- /dev/null +++ b/tools/run_tests/stress_test_wrapper.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python2.7 +import os +import re +import select +import subprocess +import sys +import time + +GRPC_ROOT = '/usr/local/google/home/sreek/workspace/grpc/' +STRESS_TEST_IMAGE = GRPC_ROOT + 'bins/opt/stress_test' +STRESS_TEST_ARGS_STR = ' '.join([ + '--server_addresses=localhost:8000', + '--test_cases=empty_unary:1,large_unary:1', '--num_stubs_per_channel=10', + '--test_duration_secs=10']) +METRICS_CLIENT_IMAGE = GRPC_ROOT + 'bins/opt/metrics_client' +METRICS_CLIENT_ARGS_STR = ' '.join([ + '--metrics_server_address=localhost:8081', '--total_only=true']) +LOGFILE_NAME = 'stress_test.log' + + +# TODO (sree): Write a python grpc client to directly query the metrics instead +# of calling metrics_client +def get_qps(metrics_cmd): + qps = 0 + try: + # Note: gpr_log() writes even non-error messages to stderr stream. So it is + # important that we set stderr=subprocess.STDOUT + p = subprocess.Popen(args=metrics_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retcode = p.wait() + (out_str, err_str) = p.communicate() + if retcode != 0: + print 'Error in reading metrics information' + print 'Output: ', out_str + else: + # The overall qps is printed at the end of the line + m = re.search('\d+$', out_str) + qps = int(m.group()) if m else 0 + except Exception as ex: + print 'Exception while reading metrics information: ' + str(ex) + return qps + +def main(argv): + # TODO(sree) Create BigQuery Tables + # (Summary table), (Metrics table) + + # TODO(sree) Update status that the test is starting (in the status table) + # + + metrics_cmd = [METRICS_CLIENT_IMAGE + ] + [x for x in METRICS_CLIENT_ARGS_STR.split()] + + stress_cmd = [STRESS_TEST_IMAGE] + [x for x in STRESS_TEST_ARGS_STR.split()] + # TODO(sree): Add an option to print to stdout if logfilename is absent + logfile = open(LOGFILE_NAME, 'w') + stress_p = subprocess.Popen(args=arg_list, + stdout=logfile, + stderr=subprocess.STDOUT) + + qps_history = [1, 1, 1] # Maintain the last 3 qps + qps_history_idx = 0 # Index into the qps_history list + + is_error = False + while True: + # Check if stress_client is still running. If so, collect metrics and upload + # to BigQuery status table + # + if stress_p is not None: + # TODO(sree) Upload completion status to BiqQuery + is_error = (stress_p.returncode != 0) + break + + # Stress client still running. Get metrics + qps = get_qps(metrics_cmd) + + # If QPS has been zero for the last 3 iterations, flag it as error and exit + qps_history[qps_history_idx] = qps + qps_history_idx = (qps_histor_idx + 1) % len(qps_history) + if sum(a) == 0: + print ('QPS has been zero for the last 3 iterations. Not monitoring ' + 'anymore. The stress test client may be stalled.') + is_error = True + break + + #TODO(sree) Upload qps metrics to BiqQuery + + if is_error: + print 'Waiting indefinitely..' + select.select([],[],[]) + + return 1 + + +if __name__ == '__main__': + main(sys.argv[1:]) From a49335326cf674b1d74fde8777e410aa588ddc1f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 23 Feb 2016 10:45:58 -0800 Subject: [PATCH 069/236] fix C# build --- src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs index ad0c53a3251..cab299a1373 100644 --- a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs +++ b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs @@ -51,7 +51,7 @@ namespace Grpc.Testing public WorkerServiceImpl(Action stopRequestHandler) { - this.stopRequestHandler = Grpc.Core.Utils.Preconditions.CheckNotNull(stopRequestHandler); + this.stopRequestHandler = GrpcPreconditions.CheckNotNull(stopRequestHandler); } public async Task RunServer(IAsyncStreamReader requestStream, IServerStreamWriter responseStream, ServerCallContext context) From 6d5abcb3418d57bbe972d9a1c5e69ede95e6d138 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Feb 2016 13:51:29 -0800 Subject: [PATCH 070/236] cleanup gens/ directory after nanopb check --- tools/distrib/check_nanopb_output.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 9cb7581d87f..99fd6806c49 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -67,3 +67,5 @@ if [ $? != 0 ]; then echo "Outputs differ: $NANOPB_TMP_OUTPUT vs src/core/proto/grpc/lb/v0" exit 1 fi + +rm -Rf $NANOPB_TMP_OUTPUT From 07b4c1cb8d4769c7ea33d5bdff71eb2d0521e095 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Feb 2016 15:13:29 -0800 Subject: [PATCH 071/236] update nanopb version --- third_party/nanopb | 2 +- tools/run_tests/sanity/check_submodules.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/nanopb b/third_party/nanopb index 5497a1dfc91..f8ac4637662 160000 --- a/third_party/nanopb +++ b/third_party/nanopb @@ -1 +1 @@ -Subproject commit 5497a1dfc91a86965383cdd1652e348345400435 +Subproject commit f8ac463766281625ad710900479130c7fcb4d63b diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index c08b382638f..3c6dbb9ea1f 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -44,7 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules 9f897b25800d2f54f5c442ef01a60721aeca6d87 third_party/boringssl (version_for_cocoapods_1.0-67-g9f897b2) 05b155ff59114735ec8cd089f669c4c3d8f59029 third_party/gflags (v2.1.0-45-g05b155f) c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0) - 5497a1dfc91a86965383cdd1652e348345400435 third_party/nanopb (nanopb-0.3.3-10-g5497a1d) + f8ac463766281625ad710900479130c7fcb4d63b third_party/nanopb (nanopb-0.3.4-29-gf8ac463) d5fb408ddc281ffcadeb08699e65bb694656d0bd third_party/protobuf (v3.0.0-beta-2) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) EOF From 4f6683c395a030fe2d9c189817a4265a4ae9a694 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Feb 2016 15:17:58 -0800 Subject: [PATCH 072/236] remove the whole gens/ --- tools/distrib/check_nanopb_output.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 99fd6806c49..241a3a5ce3a 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -68,4 +68,4 @@ if [ $? != 0 ]; then exit 1 fi -rm -Rf $NANOPB_TMP_OUTPUT +rm -Rf "${LOCAL_GIT_ROOT}/gens" From a0414ee12f81db97deaef691563ee98d9e286680 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 23 Feb 2016 16:58:46 -0800 Subject: [PATCH 073/236] Addressed feedback from @jcanizales --- src/objective-c/tests/GRPCClientTests.m | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 1a829939556..8ee6ffa1ec7 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -42,10 +42,12 @@ #import #import #import + static NSString * const kHostAddress = @"localhost:5050"; static NSString * const kPackage = @"grpc.testing"; static NSString * const kService = @"TestService"; static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; + static ProtoMethod *kInexistentMethod; static ProtoMethod *kEmptyCallMethod; static ProtoMethod *kUnaryCallMethod; @@ -127,7 +129,6 @@ static ProtoMethod *kUnaryCallMethod; XCTFail(@"Received unexpected response: %@", value); } completionHandler:^(NSError *errorOrNil) { XCTAssertNotNil(errorOrNil, @"Finished without error!"); - // XCTAssertEqual(errorOrNil.code, 12, @"Finished with unexpected error: %@", errorOrNil); [expectation fulfill]; }]; @@ -257,6 +258,7 @@ static ProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:8 handler:nil]; } +// todo(makaradd): Move to a different file that contains only unit tests - (void)testExceptions { // Try to set userAgentPrefix for host that is nil. This should cause // an exception. From b79c1e112e576cfc9cdbeb95320ba7bb71848887 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 23 Feb 2016 10:00:58 -0800 Subject: [PATCH 074/236] Ensure we can compile boringssl before trying: old compiler compatibility Allow compiling with openssl --- Makefile | 27 ++++++-- build.yaml | 4 ++ templates/Makefile.template | 29 +++++++-- .../test/cxx_squeeze_x64/Dockerfile.template | 16 ++--- .../post-git-setup.sh.template | 37 +++++++++++ .../tools/openssl/use_openssl.sh.template | 63 +++++++++++++++++++ test/build/boringssl.c | 50 +++++++++++++++ .../test/cxx_squeeze_x64/Dockerfile | 5 +- .../test/cxx_squeeze_x64/post-git-setup.sh | 35 +++++++++++ tools/jenkins/docker_run.sh | 2 + tools/jenkins/docker_run_tests.sh | 2 + tools/openssl/use_openssl.sh | 61 ++++++++++++++++++ 12 files changed, 311 insertions(+), 20 deletions(-) create mode 100644 templates/tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh.template create mode 100644 templates/tools/openssl/use_openssl.sh.template create mode 100644 test/build/boringssl.c create mode 100755 tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh create mode 100755 tools/openssl/use_openssl.sh diff --git a/Makefile b/Makefile index 3037af3491c..92d283b3f15 100644 --- a/Makefile +++ b/Makefile @@ -426,6 +426,7 @@ endif OPENSSL_ALPN_CHECK_CMD = $(CC) $(CPPFLAGS) $(CFLAGS) -o $(TMPOUT) test/build/openssl-alpn.c $(addprefix -l, $(OPENSSL_LIBS)) $(LDFLAGS) OPENSSL_NPN_CHECK_CMD = $(CC) $(CPPFLAGS) $(CFLAGS) -o $(TMPOUT) test/build/openssl-npn.c $(addprefix -l, $(OPENSSL_LIBS)) $(LDFLAGS) +BORINGSSL_COMPILE_CHECK_CMD = $(CC) $(CPPFLAGS) -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX $(CFLAGS) -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare -o $(TMPOUT) test/build/boringssl.c $(LDFLAGS) ZLIB_CHECK_CMD = $(CC) $(CPPFLAGS) $(CFLAGS) -o $(TMPOUT) test/build/zlib.c -lz $(LDFLAGS) PROTOBUF_CHECK_CMD = $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $(TMPOUT) test/build/protobuf.cc -lprotobuf $(LDFLAGS) @@ -510,10 +511,13 @@ HAS_ZOOKEEPER = $(shell $(ZOOKEEPER_CHECK_CMD) 2> /dev/null && echo true || echo # Note that for testing purposes, one can do: # make HAS_EMBEDDED_OPENSSL_ALPN=false # to emulate the fact we do not have OpenSSL in the third_party folder. -ifeq ($(wildcard third_party/boringssl/include/openssl/ssl.h),) +ifneq ($(wildcard third_party/openssl-1.0.2f/libssl.a),) +HAS_EMBEDDED_OPENSSL_ALPN = third_party/openssl-1.0.2f +else ifeq ($(wildcard third_party/boringssl/include/openssl/ssl.h),) HAS_EMBEDDED_OPENSSL_ALPN = false else -HAS_EMBEDDED_OPENSSL_ALPN = true +CAN_COMPILE_EMBEDDED_OPENSSL ?= $(shell $(BORINGSSL_COMPILE_CHECK_CMD) 2> /dev/null && echo true || echo false) +HAS_EMBEDDED_OPENSSL_ALPN = $(CAN_COMPILE_EMBEDDED_OPENSSL) endif ifeq ($(wildcard third_party/zlib/zlib.h),) @@ -572,8 +576,8 @@ ifeq ($(HAS_SYSTEM_OPENSSL_ALPN),true) EMBED_OPENSSL ?= false NO_SECURE ?= false else # HAS_SYSTEM_OPENSSL_ALPN=false -ifeq ($(HAS_EMBEDDED_OPENSSL_ALPN),true) -EMBED_OPENSSL ?= true +ifneq ($(HAS_EMBEDDED_OPENSSL_ALPN),false) +EMBED_OPENSSL ?= $(HAS_EMBEDDED_OPENSSL_ALPN) NO_SECURE ?= false else # HAS_EMBEDDED_OPENSSL_ALPN=false ifeq ($(HAS_SYSTEM_OPENSSL_NPN),true) @@ -594,6 +598,12 @@ OPENSSL_MERGE_LIBS += $(LIBDIR)/$(CONFIG)/libboringssl.a OPENSSL_MERGE_OBJS += $(LIBBORINGSSL_OBJS) # need to prefix these to ensure overriding system libraries CPPFLAGS := -Ithird_party/boringssl/include $(CPPFLAGS) +else ifneq ($(EMBED_OPENSSL),false) +OPENSSL_DEP += $(EMBED_OPENSSL)/libssl.a $(EMBED_OPENSSL)/libcrypto.a +OPENSSL_MERGE_LIBS += $(EMBED_OPENSSL)/libssl.a $(EMBED_OPENSSL)/libcrypto.a +OPENSSL_MERGE_OBJS += $(wildcard $(EMBED_OPENSSL)/grpc_obj/*.o) +# need to prefix these to ensure overriding system libraries +CPPFLAGS := -I$(EMBED_OPENSSL)/include $(CPPFLAGS) else # EMBED_OPENSSL=false ifeq ($(HAS_PKG_CONFIG),true) OPENSSL_PKG_CONFIG = true @@ -768,8 +778,9 @@ openssl_dep_message: @echo @echo "DEPENDENCY ERROR" @echo - @echo "The target you are trying to run requires OpenSSL." - @echo "Your system doesn't have it, and neither does the third_party directory." + @echo "The target you are trying to run requires an OpenSSL implementation." + @echo "Your system doesn't have one, and either the third_party directory" + @echo "doesn't have it, or your compiler can't build BoringSSL." @echo @echo "Please consult INSTALL to get more information." @echo @@ -13021,3 +13032,7 @@ test/cpp/util/test_credentials_provider.cc: $(OPENSSL_DEP) endif .PHONY: all strip tools dep_error openssl_dep_error openssl_dep_message git_update stop buildtests buildtests_c buildtests_cxx test test_c test_cxx install install_c install_cxx install-headers install-headers_c install-headers_cxx install-shared install-shared_c install-shared_cxx install-static install-static_c install-static_cxx strip strip-shared strip-static strip_c strip-shared_c strip-static_c strip_cxx strip-shared_cxx strip-static_cxx dep_c dep_cxx bins_dep_c bins_dep_cxx clean + +.PHONY: printvars +printvars: + @$(foreach V,$(sort $(.VARIABLES)), $(if $(filter-out environment% default automatic, $(origin $V)),$(warning $V=$($V) ($(value $V))))) diff --git a/build.yaml b/build.yaml index 4d76bab1e11..8e966781b16 100644 --- a/build.yaml +++ b/build.yaml @@ -2787,6 +2787,10 @@ node_modules: - src/node/ext/server.cc - src/node/ext/server_credentials.cc - src/node/ext/timeval.cc +openssl_fallback: + base_uri: http://openssl.org/source/ + extraction_dir: openssl-1.0.2f + tarball: openssl-1.0.2f.tar.gz python_dependencies: deps: - grpc diff --git a/templates/Makefile.template b/templates/Makefile.template index 7aa6ad71e26..453cb45ffd8 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -352,6 +352,7 @@ OPENSSL_ALPN_CHECK_CMD = $(CC) $(CPPFLAGS) $(CFLAGS) -o $(TMPOUT) test/build/openssl-alpn.c $(addprefix -l, $(OPENSSL_LIBS)) $(LDFLAGS) OPENSSL_NPN_CHECK_CMD = $(CC) $(CPPFLAGS) $(CFLAGS) -o $(TMPOUT) test/build/openssl-npn.c $(addprefix -l, $(OPENSSL_LIBS)) $(LDFLAGS) + BORINGSSL_COMPILE_CHECK_CMD = $(CC) $(CPPFLAGS) ${defaults.boringssl.CPPFLAGS} $(CFLAGS) ${defaults.boringssl.CFLAGS} -o $(TMPOUT) test/build/boringssl.c $(LDFLAGS) ZLIB_CHECK_CMD = $(CC) $(CPPFLAGS) $(CFLAGS) -o $(TMPOUT) test/build/zlib.c -lz $(LDFLAGS) PROTOBUF_CHECK_CMD = $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $(TMPOUT) test/build/protobuf.cc -lprotobuf $(LDFLAGS) @@ -436,10 +437,13 @@ # Note that for testing purposes, one can do: # make HAS_EMBEDDED_OPENSSL_ALPN=false # to emulate the fact we do not have OpenSSL in the third_party folder. - ifeq ($(wildcard third_party/boringssl/include/openssl/ssl.h),) + ifneq ($(wildcard third_party/${openssl_fallback.extraction_dir}/libssl.a),) + HAS_EMBEDDED_OPENSSL_ALPN = third_party/${openssl_fallback.extraction_dir} + else ifeq ($(wildcard third_party/boringssl/include/openssl/ssl.h),) HAS_EMBEDDED_OPENSSL_ALPN = false else - HAS_EMBEDDED_OPENSSL_ALPN = true + CAN_COMPILE_EMBEDDED_OPENSSL ?= $(shell $(BORINGSSL_COMPILE_CHECK_CMD) 2> /dev/null && echo true || echo false) + HAS_EMBEDDED_OPENSSL_ALPN = $(CAN_COMPILE_EMBEDDED_OPENSSL) endif ifeq ($(wildcard third_party/zlib/zlib.h),) @@ -498,8 +502,8 @@ EMBED_OPENSSL ?= false NO_SECURE ?= false else # HAS_SYSTEM_OPENSSL_ALPN=false - ifeq ($(HAS_EMBEDDED_OPENSSL_ALPN),true) - EMBED_OPENSSL ?= true + ifneq ($(HAS_EMBEDDED_OPENSSL_ALPN),false) + EMBED_OPENSSL ?= $(HAS_EMBEDDED_OPENSSL_ALPN) NO_SECURE ?= false else # HAS_EMBEDDED_OPENSSL_ALPN=false ifeq ($(HAS_SYSTEM_OPENSSL_NPN),true) @@ -520,6 +524,12 @@ OPENSSL_MERGE_OBJS += $(LIBBORINGSSL_OBJS) # need to prefix these to ensure overriding system libraries CPPFLAGS := -Ithird_party/boringssl/include $(CPPFLAGS) + else ifneq ($(EMBED_OPENSSL),false) + OPENSSL_DEP += $(EMBED_OPENSSL)/libssl.a $(EMBED_OPENSSL)/libcrypto.a + OPENSSL_MERGE_LIBS += $(EMBED_OPENSSL)/libssl.a $(EMBED_OPENSSL)/libcrypto.a + OPENSSL_MERGE_OBJS += $(wildcard $(EMBED_OPENSSL)/grpc_obj/*.o) + # need to prefix these to ensure overriding system libraries + CPPFLAGS := -I$(EMBED_OPENSSL)/include $(CPPFLAGS) else # EMBED_OPENSSL=false ifeq ($(HAS_PKG_CONFIG),true) OPENSSL_PKG_CONFIG = true @@ -706,8 +716,9 @@ @echo @echo "DEPENDENCY ERROR" @echo - @echo "The target you are trying to run requires OpenSSL." - @echo "Your system doesn't have it, and neither does the third_party directory." + @echo "The target you are trying to run requires an OpenSSL implementation." + @echo "Your system doesn't have one, and either the third_party directory" + @echo "doesn't have it, or your compiler can't build BoringSSL." @echo @echo "Please consult INSTALL to get more information." @echo @@ -1846,3 +1857,9 @@ strip_cxx strip-shared_cxx strip-static_cxx \ dep_c dep_cxx bins_dep_c bins_dep_cxx \ clean + + .PHONY: printvars + printvars: + @$(foreach V,$(sort $(.VARIABLES)), \ + $(if $(filter-out environment% default automatic, \ + $(origin $V)),$(warning $V=$($V) ($(value $V))))) diff --git a/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template index 294ae00b4c7..ba8f03862c8 100644 --- a/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template @@ -1,6 +1,6 @@ %YAML 1.2 --- | - # Copyright 2015-2016, Google Inc. + # Copyright 2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,18 +28,20 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - + FROM debian:squeeze - + <%include file="../../apt_get_basic.include" args="skip_golang=True"/> - + # libgflags-dev is not available on squeezy RUN apt-get update && apt-get -y install libgtest-dev libc++-dev clang && apt-get clean - + RUN apt-get update && apt-get -y install python-pip && apt-get clean RUN pip install argparse - + + RUN wget ${openssl_fallback.base_uri + openssl_fallback.tarball} + ADD post-git-setup.sh / + <%include file="../../run_tests_addons.include" args="skip_zookeeper=True"/> # Define the default command. CMD ["bash"] - \ No newline at end of file diff --git a/templates/tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh.template b/templates/tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh.template new file mode 100644 index 00000000000..b8851017484 --- /dev/null +++ b/templates/tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh.template @@ -0,0 +1,37 @@ +%YAML 1.2 +--- | + #!/bin/bash + # Copyright 2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + set -ex + + cd /var/local/git/grpc + cp /${openssl_fallback.tarball} third_party + ./tools/openssl/use_openssl.sh diff --git a/templates/tools/openssl/use_openssl.sh.template b/templates/tools/openssl/use_openssl.sh.template new file mode 100644 index 00000000000..5fb377154a1 --- /dev/null +++ b/templates/tools/openssl/use_openssl.sh.template @@ -0,0 +1,63 @@ +%YAML 1.2 +--- | + #!/bin/bash + + # Copyright 2015-2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + set -ex + + cd $(dirname $0)/../.. + set root=`pwd` + CC=${"${CC:-cc}"} + + # allow openssl to be pre-downloaded + if [ ! -e third_party/${openssl_fallback.tarball} ] + then + echo "Downloading ${openssl_fallback.base_uri + openssl_fallback.tarball} to third_party/${openssl_fallback.tarball}" + wget ${openssl_fallback.base_uri + openssl_fallback.tarball} -O third_party/${openssl_fallback.tarball} + fi + + # clean openssl directory + rm -rf third_party/${openssl_fallback.extraction_dir} + + # extract archive + cd third_party + tar xfz ${openssl_fallback.tarball} + + # build openssl + cd ${openssl_fallback.extraction_dir} + CC="$CC -fPIC -fvisibility=hidden" ./config no-asm + make + + # generate the 'grpc_obj' directory needed by the makefile + mkdir grpc_obj + cd grpc_obj + ar x ../libcrypto.a + ar x ../libssl.a diff --git a/test/build/boringssl.c b/test/build/boringssl.c new file mode 100644 index 00000000000..0c906751ba7 --- /dev/null +++ b/test/build/boringssl.c @@ -0,0 +1,50 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// Check that boringssl is going to compile + +#include + +// boringssl uses anonymous unions +struct foo { + union { + int a; + int b; + }; +}; + +int main(void) { + const char *close = "this should not shadow"; + printf("%s\n", close); + return 0; +} diff --git a/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile b/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile index 08488874347..35a3131d6c8 100644 --- a/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015-2016, Google Inc. +# Copyright 2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -69,6 +69,9 @@ RUN apt-get update && apt-get -y install libgtest-dev libc++-dev clang && apt-ge RUN apt-get update && apt-get -y install python-pip && apt-get clean RUN pip install argparse +RUN wget http://openssl.org/source/openssl-1.0.2f.tar.gz +ADD post-git-setup.sh / + # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc RUN ln -s /usr/bin/ccache /usr/local/bin/g++ diff --git a/tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh b/tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh new file mode 100755 index 00000000000..dfde93b1bd2 --- /dev/null +++ b/tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd /var/local/git/grpc +cp /openssl-1.0.2f.tar.gz third_party +./tools/openssl/use_openssl.sh diff --git a/tools/jenkins/docker_run.sh b/tools/jenkins/docker_run.sh index df7b6571d7a..850249a9a2b 100755 --- a/tools/jenkins/docker_run.sh +++ b/tools/jenkins/docker_run.sh @@ -42,6 +42,8 @@ else cp -r "$EXTERNAL_GIT_ROOT/$RELATIVE_COPY_PATH"/* "/var/local/git/grpc/$RELATIVE_COPY_PATH" fi +[ -e /post-git-setup.sh ] && /post-git-setup.sh + if [ -x "$(command -v rvm)" ] then rvm use ruby-2.1 diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 282b8573511..c3a606edcb9 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -43,6 +43,8 @@ chown $(whoami) $XDG_CACHE_HOME mkdir -p /var/local/git git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc +[ -e /post-git-setup.sh ] && /post-git-setup.sh + mkdir -p reports exit_code=0 diff --git a/tools/openssl/use_openssl.sh b/tools/openssl/use_openssl.sh new file mode 100755 index 00000000000..3098217ec1b --- /dev/null +++ b/tools/openssl/use_openssl.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../.. +set root=`pwd` +CC=${CC:-cc} + +# allow openssl to be pre-downloaded +if [ ! -e third_party/openssl-1.0.2f.tar.gz ] +then + echo "Downloading http://openssl.org/source/openssl-1.0.2f.tar.gz to third_party/openssl-1.0.2f.tar.gz" + wget http://openssl.org/source/openssl-1.0.2f.tar.gz -O third_party/openssl-1.0.2f.tar.gz +fi + +# clean openssl directory +rm -rf third_party/openssl-1.0.2f + +# extract archive +cd third_party +tar xfz openssl-1.0.2f.tar.gz + +# build openssl +cd openssl-1.0.2f +CC="$CC -fPIC -fvisibility=hidden" ./config no-asm +make + +# generate the 'grpc_obj' directory needed by the makefile +mkdir grpc_obj +cd grpc_obj +ar x ../libcrypto.a +ar x ../libssl.a From 804b85534be9ecf9f64da56b9ca46ca5f2eb6211 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 23 Feb 2016 16:50:24 -0800 Subject: [PATCH 075/236] Test for a working -Wshadow --- Makefile | 8 ++++++- templates/Makefile.template | 8 ++++++- test/build/boringssl.c | 1 + test/build/shadow.c | 43 +++++++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 test/build/shadow.c diff --git a/Makefile b/Makefile index 92d283b3f15..8a005609b26 100644 --- a/Makefile +++ b/Makefile @@ -275,6 +275,12 @@ endif CXX11_CHECK_CMD = $(CXX) -std=c++11 -o $(TMPOUT) -c test/build/c++11.cc HAS_CXX11 = $(shell $(CXX11_CHECK_CMD) 2> /dev/null && echo true || echo false) +CHECK_SHADOW_WORKS_CMD = $(CC) -std=c99 -Werror -Wshadow -o $(TMPOUT) -c test/build/shadow.c +HAS_WORKING_SHADOW = $(shell $(CHECK_SHADOW_WORKS_CMD) 2> /dev/null && echo true || echo false) +ifeq ($(HAS_WORKING_SHADOW),true) +W_SHADOW=-Wshadow +endif + CHECK_NO_SHIFT_NEGATIVE_VALUE_CMD = $(CC) -std=c99 -Werror -Wno-shift-negative-value -o $(TMPOUT) -c test/build/empty.c HAS_NO_SHIFT_NEGATIVE_VALUE = $(shell $(CHECK_NO_SHIFT_NEGATIVE_VALUE_CMD) 2> /dev/null && echo true || echo false) ifeq ($(HAS_NO_SHIFT_NEGATIVE_VALUE),true) @@ -295,7 +301,7 @@ ifdef EXTRA_DEFINES DEFINES += $(EXTRA_DEFINES) endif -CFLAGS += -std=c99 -Wsign-conversion -Wconversion -Wshadow +CFLAGS += -std=c99 -Wsign-conversion -Wconversion $(W_SHADOW) ifeq ($(HAS_CXX11),true) CXXFLAGS += -std=c++11 else diff --git a/templates/Makefile.template b/templates/Makefile.template index 453cb45ffd8..c54c146620f 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -187,6 +187,12 @@ CXX11_CHECK_CMD = $(CXX) -std=c++11 -o $(TMPOUT) -c test/build/c++11.cc HAS_CXX11 = $(shell $(CXX11_CHECK_CMD) 2> /dev/null && echo true || echo false) + CHECK_SHADOW_WORKS_CMD = $(CC) -std=c99 -Werror -Wshadow -o $(TMPOUT) -c test/build/shadow.c + HAS_WORKING_SHADOW = $(shell $(CHECK_SHADOW_WORKS_CMD) 2> /dev/null && echo true || echo false) + ifeq ($(HAS_WORKING_SHADOW),true) + W_SHADOW=-Wshadow + endif + CHECK_NO_SHIFT_NEGATIVE_VALUE_CMD = $(CC) -std=c99 -Werror -Wno-shift-negative-value -o $(TMPOUT) -c test/build/empty.c HAS_NO_SHIFT_NEGATIVE_VALUE = $(shell $(CHECK_NO_SHIFT_NEGATIVE_VALUE_CMD) 2> /dev/null && echo true || echo false) ifeq ($(HAS_NO_SHIFT_NEGATIVE_VALUE),true) @@ -207,7 +213,7 @@ DEFINES += $(EXTRA_DEFINES) endif - CFLAGS += -std=c99 -Wsign-conversion -Wconversion -Wshadow + CFLAGS += -std=c99 -Wsign-conversion -Wconversion $(W_SHADOW) ifeq ($(HAS_CXX11),true) CXXFLAGS += -std=c++11 else diff --git a/test/build/boringssl.c b/test/build/boringssl.c index 0c906751ba7..a31d4bf3964 100644 --- a/test/build/boringssl.c +++ b/test/build/boringssl.c @@ -33,6 +33,7 @@ // Check that boringssl is going to compile +#include #include // boringssl uses anonymous unions diff --git a/test/build/shadow.c b/test/build/shadow.c new file mode 100644 index 00000000000..51d4f9e3852 --- /dev/null +++ b/test/build/shadow.c @@ -0,0 +1,43 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// Check that boringssl is going to compile + +#include +#include + +int main(void) { + const char *close = "this should not shadow"; + printf("%s\n", close); + return 0; +} From 5b18682adda061060f3e68d91a56e0c5b484c1ad Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 23 Feb 2016 17:00:09 -0800 Subject: [PATCH 076/236] Add fallback for secure_getenv (again) --- src/core/support/env_linux.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core/support/env_linux.c b/src/core/support/env_linux.c index 1ca6fa1affe..365f9673be2 100644 --- a/src/core/support/env_linux.c +++ b/src/core/support/env_linux.c @@ -43,6 +43,7 @@ #include "src/core/support/env.h" #include +#include #include #include @@ -60,12 +61,19 @@ char *gpr_getenv(const char *name) { const char *names[] = {"secure_getenv", "__secure_getenv", "getenv"}; for (size_t i = 0; getenv_func == NULL && i < GPR_ARRAY_SIZE(names); i++) { getenv_func = (getenv_type)dlsym(RTLD_DEFAULT, names[i]); + if (getenv_func != NULL && strstr(names[i], "secure") == NULL) { + gpr_log(GPR_DEBUG, "Warning: insecure environment read function '%s' used", names[i]); + } } char *result = getenv_func(name); return result == NULL ? result : gpr_strdup(result); -#else +#elif __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) char *result = secure_getenv(name); return result == NULL ? result : gpr_strdup(result); +#else + gpr_log(GPR_DEBUG, "Warning: insecure environment read function '%s' used", "getenv"); + char *result = getenv(name); + return result == NULL ? result : gpr_strdup(result); #endif } From 0d2df65c4120aebbe63f84cd316fc7b8068cf067 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Feb 2016 17:37:47 -0800 Subject: [PATCH 077/236] updated generated lb proto code to latest nanopb version --- src/core/proto/grpc/lb/v0/load_balancer.pb.c | 2 +- src/core/proto/grpc/lb/v0/load_balancer.pb.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.c b/src/core/proto/grpc/lb/v0/load_balancer.pb.c index abd4b1cefc6..59aae30cff9 100644 --- a/src/core/proto/grpc/lb/v0/load_balancer.pb.c +++ b/src/core/proto/grpc/lb/v0/load_balancer.pb.c @@ -31,7 +31,7 @@ * */ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.3.4-dev */ +/* Generated by nanopb-0.3.5-dev */ #include "src/core/proto/grpc/lb/v0/load_balancer.pb.h" diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.h b/src/core/proto/grpc/lb/v0/load_balancer.pb.h index 87037213998..3599f881bb1 100644 --- a/src/core/proto/grpc/lb/v0/load_balancer.pb.h +++ b/src/core/proto/grpc/lb/v0/load_balancer.pb.h @@ -31,7 +31,7 @@ * */ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.3.4-dev */ +/* Generated by nanopb-0.3.5-dev */ #ifndef PB_LOAD_BALANCER_PB_H_INCLUDED #define PB_LOAD_BALANCER_PB_H_INCLUDED @@ -44,7 +44,6 @@ extern "C" { #endif -/* Enum definitions */ /* Struct definitions */ typedef struct _grpc_lb_v0_ClientStats { bool has_total_requests; @@ -164,6 +163,7 @@ extern const pb_field_t grpc_lb_v0_Server_fields[5]; #define grpc_lb_v0_LoadBalanceRequest_size 169 #define grpc_lb_v0_InitialLoadBalanceRequest_size 131 #define grpc_lb_v0_ClientStats_size 33 +#define grpc_lb_v0_LoadBalanceResponse_size (165 + grpc_lb_v0_ServerList_size) #define grpc_lb_v0_InitialLoadBalanceResponse_size 156 #define grpc_lb_v0_Server_size 127 From e86b4573fefc73f90c4136ebde9395bd15346276 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Feb 2016 17:38:01 -0800 Subject: [PATCH 078/236] fixed check nanopb sanity script --- tools/distrib/check_nanopb_output.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 241a3a5ce3a..5f49ebb93e6 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -32,7 +32,7 @@ set -ex apt-get install -y autoconf automake libtool curl python-virtualenv -readonly NANOPB_TMP_OUTPUT="${LOCAL_GIT_ROOT}/gens/src/proto/grpc/lb/v0" +readonly NANOPB_TMP_OUTPUT="$(mktemp -d)" # install protoc version 3 pushd third_party/protobuf @@ -62,10 +62,7 @@ PATH="$PROTOC_PATH:$PATH" ./tools/codegen/core/gen_load_balancing_proto.sh \ $NANOPB_TMP_OUTPUT # compare outputs to checked compiled code -diff -rq $NANOPB_TMP_OUTPUT src/core/proto/grpc/lb/v0 -if [ $? != 0 ]; then +if ! diff -r $NANOPB_TMP_OUTPUT src/core/proto/grpc/lb/v0; then echo "Outputs differ: $NANOPB_TMP_OUTPUT vs src/core/proto/grpc/lb/v0" - exit 1 + exit 2 fi - -rm -Rf "${LOCAL_GIT_ROOT}/gens" From 5adb71fb9add555ac161ebf745e5ac104fe3f847 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Sat, 13 Feb 2016 00:03:02 -0800 Subject: [PATCH 079/236] php: simplify installation --- build.yaml | 28 + config.m4 | 533 ++++++++++ package.xml | 984 ++++++++++++++++++ templates/config.m4.template | 30 + templates/package.xml.template | 159 +++ test/distrib/php/distribtest.php | 13 + test/distrib/php/run_distrib_test.sh | 39 + .../distribtest/php_jessie_x64/Dockerfile | 32 + .../grpc_artifact_linux_x64/Dockerfile | 7 + tools/run_tests/artifact_targets.py | 21 +- tools/run_tests/build_artifact_php.sh | 40 + tools/run_tests/build_package_php.sh | 36 + tools/run_tests/distribtest_targets.py | 31 +- tools/run_tests/package_targets.py | 20 +- 14 files changed, 1970 insertions(+), 3 deletions(-) create mode 100644 config.m4 create mode 100644 package.xml create mode 100644 templates/config.m4.template create mode 100644 templates/package.xml.template create mode 100644 test/distrib/php/distribtest.php create mode 100755 test/distrib/php/run_distrib_test.sh create mode 100644 tools/dockerfile/distribtest/php_jessie_x64/Dockerfile create mode 100755 tools/run_tests/build_artifact_php.sh create mode 100755 tools/run_tests/build_package_php.sh diff --git a/build.yaml b/build.yaml index 625c7ab46f1..b16cb162d4d 100644 --- a/build.yaml +++ b/build.yaml @@ -2815,6 +2815,34 @@ node_modules: - src/node/ext/server.cc - src/node/ext/server_credentials.cc - src/node/ext/timeval.cc +php_config_m4: + deps: + - grpc + - gpr + - boringssl + - z + headers: + - src/php/ext/grpc/byte_buffer.h + - src/php/ext/grpc/call.h + - src/php/ext/grpc/call_credentials.h + - src/php/ext/grpc/channel.h + - src/php/ext/grpc/channel_credentials.h + - src/php/ext/grpc/completion_queue.h + - src/php/ext/grpc/php_grpc.h + - src/php/ext/grpc/server.h + - src/php/ext/grpc/server_credentials.h + - src/php/ext/grpc/timeval.h + src: + - src/php/ext/grpc/byte_buffer.c + - src/php/ext/grpc/call.c + - src/php/ext/grpc/call_credentials.c + - src/php/ext/grpc/channel.c + - src/php/ext/grpc/channel_credentials.c + - src/php/ext/grpc/completion_queue.c + - src/php/ext/grpc/php_grpc.c + - src/php/ext/grpc/server.c + - src/php/ext/grpc/server_credentials.c + - src/php/ext/grpc/timeval.c python_dependencies: deps: - grpc diff --git a/config.m4 b/config.m4 new file mode 100644 index 00000000000..6d04add8e94 --- /dev/null +++ b/config.m4 @@ -0,0 +1,533 @@ +PHP_ARG_ENABLE(grpc, whether to enable grpc support, +[ --enable-grpc Enable grpc support]) + +if test "$PHP_GRPC" != "no"; then + dnl Write more examples of tests here... + + dnl # --with-grpc -> add include path + PHP_ADD_INCLUDE(../../grpc/include) + PHP_ADD_INCLUDE(../../grpc/src/php/ext/grpc) + PHP_ADD_INCLUDE(../../grpc/third_party/boringssl/include) + + PHP_ADD_LIBRARY(pthread) + + PHP_NEW_EXTENSION(grpc, + src/php/ext/grpc/byte_buffer.c \ + src/php/ext/grpc/call.c \ + src/php/ext/grpc/call_credentials.c \ + src/php/ext/grpc/channel.c \ + src/php/ext/grpc/channel_credentials.c \ + src/php/ext/grpc/completion_queue.c \ + src/php/ext/grpc/php_grpc.c \ + src/php/ext/grpc/server.c \ + src/php/ext/grpc/server_credentials.c \ + src/php/ext/grpc/timeval.c \ + src/core/profiling/basic_timers.c \ + src/core/profiling/stap_timers.c \ + src/core/support/alloc.c \ + src/core/support/avl.c \ + src/core/support/cmdline.c \ + src/core/support/cpu_iphone.c \ + src/core/support/cpu_linux.c \ + src/core/support/cpu_posix.c \ + src/core/support/cpu_windows.c \ + src/core/support/env_linux.c \ + src/core/support/env_posix.c \ + src/core/support/env_win32.c \ + src/core/support/file.c \ + src/core/support/file_posix.c \ + src/core/support/file_win32.c \ + src/core/support/histogram.c \ + src/core/support/host_port.c \ + src/core/support/log.c \ + src/core/support/log_android.c \ + src/core/support/log_linux.c \ + src/core/support/log_posix.c \ + src/core/support/log_win32.c \ + src/core/support/murmur_hash.c \ + src/core/support/slice.c \ + src/core/support/slice_buffer.c \ + src/core/support/stack_lockfree.c \ + src/core/support/string.c \ + src/core/support/string_posix.c \ + src/core/support/string_win32.c \ + src/core/support/subprocess_posix.c \ + src/core/support/subprocess_windows.c \ + src/core/support/sync.c \ + src/core/support/sync_posix.c \ + src/core/support/sync_win32.c \ + src/core/support/thd.c \ + src/core/support/thd_posix.c \ + src/core/support/thd_win32.c \ + src/core/support/time.c \ + src/core/support/time_posix.c \ + src/core/support/time_precise.c \ + src/core/support/time_win32.c \ + src/core/support/tls_pthread.c \ + src/core/support/wrap_memcpy.c \ + src/core/httpcli/httpcli_security_connector.c \ + src/core/security/base64.c \ + src/core/security/client_auth_filter.c \ + src/core/security/credentials.c \ + src/core/security/credentials_metadata.c \ + src/core/security/credentials_posix.c \ + src/core/security/credentials_win32.c \ + src/core/security/google_default_credentials.c \ + src/core/security/handshake.c \ + src/core/security/json_token.c \ + src/core/security/jwt_verifier.c \ + src/core/security/secure_endpoint.c \ + src/core/security/security_connector.c \ + src/core/security/security_context.c \ + src/core/security/server_auth_filter.c \ + src/core/security/server_secure_chttp2.c \ + src/core/surface/init_secure.c \ + src/core/surface/secure_channel_create.c \ + src/core/tsi/fake_transport_security.c \ + src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security.c \ + src/core/census/grpc_context.c \ + src/core/census/grpc_filter.c \ + src/core/channel/channel_args.c \ + src/core/channel/channel_stack.c \ + src/core/channel/client_channel.c \ + src/core/channel/client_uchannel.c \ + src/core/channel/compress_filter.c \ + src/core/channel/connected_channel.c \ + src/core/channel/http_client_filter.c \ + src/core/channel/http_server_filter.c \ + src/core/channel/subchannel_call_holder.c \ + src/core/client_config/client_config.c \ + src/core/client_config/connector.c \ + src/core/client_config/default_initial_connect_string.c \ + src/core/client_config/initial_connect_string.c \ + src/core/client_config/lb_policies/pick_first.c \ + src/core/client_config/lb_policies/round_robin.c \ + src/core/client_config/lb_policy.c \ + src/core/client_config/lb_policy_factory.c \ + src/core/client_config/lb_policy_registry.c \ + src/core/client_config/resolver.c \ + src/core/client_config/resolver_factory.c \ + src/core/client_config/resolver_registry.c \ + src/core/client_config/resolvers/dns_resolver.c \ + src/core/client_config/resolvers/sockaddr_resolver.c \ + src/core/client_config/subchannel.c \ + src/core/client_config/subchannel_factory.c \ + src/core/client_config/uri_parser.c \ + src/core/compression/algorithm.c \ + src/core/compression/message_compress.c \ + src/core/debug/trace.c \ + src/core/httpcli/format_request.c \ + src/core/httpcli/httpcli.c \ + src/core/httpcli/parser.c \ + src/core/iomgr/closure.c \ + src/core/iomgr/endpoint.c \ + src/core/iomgr/endpoint_pair_posix.c \ + src/core/iomgr/endpoint_pair_windows.c \ + src/core/iomgr/exec_ctx.c \ + src/core/iomgr/executor.c \ + src/core/iomgr/fd_posix.c \ + src/core/iomgr/iocp_windows.c \ + src/core/iomgr/iomgr.c \ + src/core/iomgr/iomgr_posix.c \ + src/core/iomgr/iomgr_windows.c \ + src/core/iomgr/pollset_multipoller_with_epoll.c \ + src/core/iomgr/pollset_multipoller_with_poll_posix.c \ + src/core/iomgr/pollset_posix.c \ + src/core/iomgr/pollset_set_posix.c \ + src/core/iomgr/pollset_set_windows.c \ + src/core/iomgr/pollset_windows.c \ + src/core/iomgr/resolve_address_posix.c \ + src/core/iomgr/resolve_address_windows.c \ + src/core/iomgr/sockaddr_utils.c \ + src/core/iomgr/socket_utils_common_posix.c \ + src/core/iomgr/socket_utils_linux.c \ + src/core/iomgr/socket_utils_posix.c \ + src/core/iomgr/socket_windows.c \ + src/core/iomgr/tcp_client_posix.c \ + src/core/iomgr/tcp_client_windows.c \ + src/core/iomgr/tcp_posix.c \ + src/core/iomgr/tcp_server_posix.c \ + src/core/iomgr/tcp_server_windows.c \ + src/core/iomgr/tcp_windows.c \ + src/core/iomgr/time_averaged_stats.c \ + src/core/iomgr/timer.c \ + src/core/iomgr/timer_heap.c \ + src/core/iomgr/udp_server.c \ + src/core/iomgr/wakeup_fd_eventfd.c \ + src/core/iomgr/wakeup_fd_nospecial.c \ + src/core/iomgr/wakeup_fd_pipe.c \ + src/core/iomgr/wakeup_fd_posix.c \ + src/core/iomgr/workqueue_posix.c \ + src/core/iomgr/workqueue_windows.c \ + src/core/json/json.c \ + src/core/json/json_reader.c \ + src/core/json/json_string.c \ + src/core/json/json_writer.c \ + src/core/surface/alarm.c \ + src/core/surface/api_trace.c \ + src/core/surface/byte_buffer.c \ + src/core/surface/byte_buffer_reader.c \ + src/core/surface/call.c \ + src/core/surface/call_details.c \ + src/core/surface/call_log_batch.c \ + src/core/surface/channel.c \ + src/core/surface/channel_connectivity.c \ + src/core/surface/channel_create.c \ + src/core/surface/channel_ping.c \ + src/core/surface/completion_queue.c \ + src/core/surface/event_string.c \ + src/core/surface/init.c \ + src/core/surface/lame_client.c \ + src/core/surface/metadata_array.c \ + src/core/surface/server.c \ + src/core/surface/server_chttp2.c \ + src/core/surface/server_create.c \ + src/core/surface/validate_metadata.c \ + src/core/surface/version.c \ + src/core/transport/byte_stream.c \ + src/core/transport/chttp2/alpn.c \ + src/core/transport/chttp2/bin_encoder.c \ + src/core/transport/chttp2/frame_data.c \ + src/core/transport/chttp2/frame_goaway.c \ + src/core/transport/chttp2/frame_ping.c \ + src/core/transport/chttp2/frame_rst_stream.c \ + src/core/transport/chttp2/frame_settings.c \ + src/core/transport/chttp2/frame_window_update.c \ + src/core/transport/chttp2/hpack_encoder.c \ + src/core/transport/chttp2/hpack_parser.c \ + src/core/transport/chttp2/hpack_table.c \ + src/core/transport/chttp2/huffsyms.c \ + src/core/transport/chttp2/incoming_metadata.c \ + src/core/transport/chttp2/parsing.c \ + src/core/transport/chttp2/status_conversion.c \ + src/core/transport/chttp2/stream_lists.c \ + src/core/transport/chttp2/stream_map.c \ + src/core/transport/chttp2/timeout_encoding.c \ + src/core/transport/chttp2/varint.c \ + src/core/transport/chttp2/writing.c \ + src/core/transport/chttp2_transport.c \ + src/core/transport/connectivity_state.c \ + src/core/transport/metadata.c \ + src/core/transport/metadata_batch.c \ + src/core/transport/static_metadata.c \ + src/core/transport/transport.c \ + src/core/transport/transport_op_string.c \ + src/core/census/context.c \ + src/core/census/initialize.c \ + src/core/census/operation.c \ + src/core/census/placeholders.c \ + src/core/census/tracing.c \ + src/boringssl/err_data.c \ + third_party/boringssl/crypto/aes/aes.c \ + third_party/boringssl/crypto/aes/mode_wrappers.c \ + third_party/boringssl/crypto/asn1/a_bitstr.c \ + third_party/boringssl/crypto/asn1/a_bool.c \ + third_party/boringssl/crypto/asn1/a_bytes.c \ + third_party/boringssl/crypto/asn1/a_d2i_fp.c \ + third_party/boringssl/crypto/asn1/a_dup.c \ + third_party/boringssl/crypto/asn1/a_enum.c \ + third_party/boringssl/crypto/asn1/a_gentm.c \ + third_party/boringssl/crypto/asn1/a_i2d_fp.c \ + third_party/boringssl/crypto/asn1/a_int.c \ + third_party/boringssl/crypto/asn1/a_mbstr.c \ + third_party/boringssl/crypto/asn1/a_object.c \ + third_party/boringssl/crypto/asn1/a_octet.c \ + third_party/boringssl/crypto/asn1/a_print.c \ + third_party/boringssl/crypto/asn1/a_strnid.c \ + third_party/boringssl/crypto/asn1/a_time.c \ + third_party/boringssl/crypto/asn1/a_type.c \ + third_party/boringssl/crypto/asn1/a_utctm.c \ + third_party/boringssl/crypto/asn1/a_utf8.c \ + third_party/boringssl/crypto/asn1/asn1_lib.c \ + third_party/boringssl/crypto/asn1/asn1_par.c \ + third_party/boringssl/crypto/asn1/asn_pack.c \ + third_party/boringssl/crypto/asn1/bio_asn1.c \ + third_party/boringssl/crypto/asn1/bio_ndef.c \ + third_party/boringssl/crypto/asn1/f_enum.c \ + third_party/boringssl/crypto/asn1/f_int.c \ + third_party/boringssl/crypto/asn1/f_string.c \ + third_party/boringssl/crypto/asn1/t_bitst.c \ + third_party/boringssl/crypto/asn1/t_pkey.c \ + third_party/boringssl/crypto/asn1/tasn_dec.c \ + third_party/boringssl/crypto/asn1/tasn_enc.c \ + third_party/boringssl/crypto/asn1/tasn_fre.c \ + third_party/boringssl/crypto/asn1/tasn_new.c \ + third_party/boringssl/crypto/asn1/tasn_prn.c \ + third_party/boringssl/crypto/asn1/tasn_typ.c \ + third_party/boringssl/crypto/asn1/tasn_utl.c \ + third_party/boringssl/crypto/asn1/x_bignum.c \ + third_party/boringssl/crypto/asn1/x_long.c \ + third_party/boringssl/crypto/base64/base64.c \ + third_party/boringssl/crypto/bio/bio.c \ + third_party/boringssl/crypto/bio/bio_mem.c \ + third_party/boringssl/crypto/bio/buffer.c \ + third_party/boringssl/crypto/bio/connect.c \ + third_party/boringssl/crypto/bio/fd.c \ + third_party/boringssl/crypto/bio/file.c \ + third_party/boringssl/crypto/bio/hexdump.c \ + third_party/boringssl/crypto/bio/pair.c \ + third_party/boringssl/crypto/bio/printf.c \ + third_party/boringssl/crypto/bio/socket.c \ + third_party/boringssl/crypto/bio/socket_helper.c \ + third_party/boringssl/crypto/bn/add.c \ + third_party/boringssl/crypto/bn/asm/x86_64-gcc.c \ + third_party/boringssl/crypto/bn/bn.c \ + third_party/boringssl/crypto/bn/bn_asn1.c \ + third_party/boringssl/crypto/bn/cmp.c \ + third_party/boringssl/crypto/bn/convert.c \ + third_party/boringssl/crypto/bn/ctx.c \ + third_party/boringssl/crypto/bn/div.c \ + third_party/boringssl/crypto/bn/exponentiation.c \ + third_party/boringssl/crypto/bn/gcd.c \ + third_party/boringssl/crypto/bn/generic.c \ + third_party/boringssl/crypto/bn/kronecker.c \ + third_party/boringssl/crypto/bn/montgomery.c \ + third_party/boringssl/crypto/bn/mul.c \ + third_party/boringssl/crypto/bn/prime.c \ + third_party/boringssl/crypto/bn/random.c \ + third_party/boringssl/crypto/bn/rsaz_exp.c \ + third_party/boringssl/crypto/bn/shift.c \ + third_party/boringssl/crypto/bn/sqrt.c \ + third_party/boringssl/crypto/buf/buf.c \ + third_party/boringssl/crypto/bytestring/ber.c \ + third_party/boringssl/crypto/bytestring/cbb.c \ + third_party/boringssl/crypto/bytestring/cbs.c \ + third_party/boringssl/crypto/chacha/chacha_generic.c \ + third_party/boringssl/crypto/chacha/chacha_vec.c \ + third_party/boringssl/crypto/cipher/aead.c \ + third_party/boringssl/crypto/cipher/cipher.c \ + third_party/boringssl/crypto/cipher/derive_key.c \ + third_party/boringssl/crypto/cipher/e_aes.c \ + third_party/boringssl/crypto/cipher/e_chacha20poly1305.c \ + third_party/boringssl/crypto/cipher/e_des.c \ + third_party/boringssl/crypto/cipher/e_null.c \ + third_party/boringssl/crypto/cipher/e_rc2.c \ + third_party/boringssl/crypto/cipher/e_rc4.c \ + third_party/boringssl/crypto/cipher/e_ssl3.c \ + third_party/boringssl/crypto/cipher/e_tls.c \ + third_party/boringssl/crypto/cipher/tls_cbc.c \ + third_party/boringssl/crypto/cmac/cmac.c \ + third_party/boringssl/crypto/conf/conf.c \ + third_party/boringssl/crypto/cpu-arm.c \ + third_party/boringssl/crypto/cpu-intel.c \ + third_party/boringssl/crypto/crypto.c \ + third_party/boringssl/crypto/curve25519/curve25519.c \ + third_party/boringssl/crypto/des/des.c \ + third_party/boringssl/crypto/dh/check.c \ + third_party/boringssl/crypto/dh/dh.c \ + third_party/boringssl/crypto/dh/dh_asn1.c \ + third_party/boringssl/crypto/dh/params.c \ + third_party/boringssl/crypto/digest/digest.c \ + third_party/boringssl/crypto/digest/digests.c \ + third_party/boringssl/crypto/directory_posix.c \ + third_party/boringssl/crypto/directory_win.c \ + third_party/boringssl/crypto/dsa/dsa.c \ + third_party/boringssl/crypto/dsa/dsa_asn1.c \ + third_party/boringssl/crypto/ec/ec.c \ + third_party/boringssl/crypto/ec/ec_asn1.c \ + third_party/boringssl/crypto/ec/ec_key.c \ + third_party/boringssl/crypto/ec/ec_montgomery.c \ + third_party/boringssl/crypto/ec/oct.c \ + third_party/boringssl/crypto/ec/p224-64.c \ + third_party/boringssl/crypto/ec/p256-64.c \ + third_party/boringssl/crypto/ec/p256-x86_64.c \ + third_party/boringssl/crypto/ec/simple.c \ + third_party/boringssl/crypto/ec/util-64.c \ + third_party/boringssl/crypto/ec/wnaf.c \ + third_party/boringssl/crypto/ecdh/ecdh.c \ + third_party/boringssl/crypto/ecdsa/ecdsa.c \ + third_party/boringssl/crypto/ecdsa/ecdsa_asn1.c \ + third_party/boringssl/crypto/engine/engine.c \ + third_party/boringssl/crypto/err/err.c \ + third_party/boringssl/crypto/evp/algorithm.c \ + third_party/boringssl/crypto/evp/digestsign.c \ + third_party/boringssl/crypto/evp/evp.c \ + third_party/boringssl/crypto/evp/evp_asn1.c \ + third_party/boringssl/crypto/evp/evp_ctx.c \ + third_party/boringssl/crypto/evp/p_dsa_asn1.c \ + third_party/boringssl/crypto/evp/p_ec.c \ + third_party/boringssl/crypto/evp/p_ec_asn1.c \ + third_party/boringssl/crypto/evp/p_rsa.c \ + third_party/boringssl/crypto/evp/p_rsa_asn1.c \ + third_party/boringssl/crypto/evp/pbkdf.c \ + third_party/boringssl/crypto/evp/sign.c \ + third_party/boringssl/crypto/ex_data.c \ + third_party/boringssl/crypto/hkdf/hkdf.c \ + third_party/boringssl/crypto/hmac/hmac.c \ + third_party/boringssl/crypto/lhash/lhash.c \ + third_party/boringssl/crypto/md4/md4.c \ + third_party/boringssl/crypto/md5/md5.c \ + third_party/boringssl/crypto/mem.c \ + third_party/boringssl/crypto/modes/cbc.c \ + third_party/boringssl/crypto/modes/cfb.c \ + third_party/boringssl/crypto/modes/ctr.c \ + third_party/boringssl/crypto/modes/gcm.c \ + third_party/boringssl/crypto/modes/ofb.c \ + third_party/boringssl/crypto/obj/obj.c \ + third_party/boringssl/crypto/obj/obj_xref.c \ + third_party/boringssl/crypto/pem/pem_all.c \ + third_party/boringssl/crypto/pem/pem_info.c \ + third_party/boringssl/crypto/pem/pem_lib.c \ + third_party/boringssl/crypto/pem/pem_oth.c \ + third_party/boringssl/crypto/pem/pem_pk8.c \ + third_party/boringssl/crypto/pem/pem_pkey.c \ + third_party/boringssl/crypto/pem/pem_x509.c \ + third_party/boringssl/crypto/pem/pem_xaux.c \ + third_party/boringssl/crypto/pkcs8/p5_pbe.c \ + third_party/boringssl/crypto/pkcs8/p5_pbev2.c \ + third_party/boringssl/crypto/pkcs8/p8_pkey.c \ + third_party/boringssl/crypto/pkcs8/pkcs8.c \ + third_party/boringssl/crypto/poly1305/poly1305.c \ + third_party/boringssl/crypto/poly1305/poly1305_arm.c \ + third_party/boringssl/crypto/poly1305/poly1305_vec.c \ + third_party/boringssl/crypto/rand/rand.c \ + third_party/boringssl/crypto/rand/urandom.c \ + third_party/boringssl/crypto/rand/windows.c \ + third_party/boringssl/crypto/rc4/rc4.c \ + third_party/boringssl/crypto/refcount_c11.c \ + third_party/boringssl/crypto/refcount_lock.c \ + third_party/boringssl/crypto/rsa/blinding.c \ + third_party/boringssl/crypto/rsa/padding.c \ + third_party/boringssl/crypto/rsa/rsa.c \ + third_party/boringssl/crypto/rsa/rsa_asn1.c \ + third_party/boringssl/crypto/rsa/rsa_impl.c \ + third_party/boringssl/crypto/sha/sha1.c \ + third_party/boringssl/crypto/sha/sha256.c \ + third_party/boringssl/crypto/sha/sha512.c \ + third_party/boringssl/crypto/stack/stack.c \ + third_party/boringssl/crypto/thread.c \ + third_party/boringssl/crypto/thread_none.c \ + third_party/boringssl/crypto/thread_pthread.c \ + third_party/boringssl/crypto/thread_win.c \ + third_party/boringssl/crypto/time_support.c \ + third_party/boringssl/crypto/x509/a_digest.c \ + third_party/boringssl/crypto/x509/a_sign.c \ + third_party/boringssl/crypto/x509/a_strex.c \ + third_party/boringssl/crypto/x509/a_verify.c \ + third_party/boringssl/crypto/x509/asn1_gen.c \ + third_party/boringssl/crypto/x509/by_dir.c \ + third_party/boringssl/crypto/x509/by_file.c \ + third_party/boringssl/crypto/x509/i2d_pr.c \ + third_party/boringssl/crypto/x509/pkcs7.c \ + third_party/boringssl/crypto/x509/t_crl.c \ + third_party/boringssl/crypto/x509/t_req.c \ + third_party/boringssl/crypto/x509/t_x509.c \ + third_party/boringssl/crypto/x509/t_x509a.c \ + third_party/boringssl/crypto/x509/x509.c \ + third_party/boringssl/crypto/x509/x509_att.c \ + third_party/boringssl/crypto/x509/x509_cmp.c \ + third_party/boringssl/crypto/x509/x509_d2.c \ + third_party/boringssl/crypto/x509/x509_def.c \ + third_party/boringssl/crypto/x509/x509_ext.c \ + third_party/boringssl/crypto/x509/x509_lu.c \ + third_party/boringssl/crypto/x509/x509_obj.c \ + third_party/boringssl/crypto/x509/x509_r2x.c \ + third_party/boringssl/crypto/x509/x509_req.c \ + third_party/boringssl/crypto/x509/x509_set.c \ + third_party/boringssl/crypto/x509/x509_trs.c \ + third_party/boringssl/crypto/x509/x509_txt.c \ + third_party/boringssl/crypto/x509/x509_v3.c \ + third_party/boringssl/crypto/x509/x509_vfy.c \ + third_party/boringssl/crypto/x509/x509_vpm.c \ + third_party/boringssl/crypto/x509/x509cset.c \ + third_party/boringssl/crypto/x509/x509name.c \ + third_party/boringssl/crypto/x509/x509rset.c \ + third_party/boringssl/crypto/x509/x509spki.c \ + third_party/boringssl/crypto/x509/x509type.c \ + third_party/boringssl/crypto/x509/x_algor.c \ + third_party/boringssl/crypto/x509/x_all.c \ + third_party/boringssl/crypto/x509/x_attrib.c \ + third_party/boringssl/crypto/x509/x_crl.c \ + third_party/boringssl/crypto/x509/x_exten.c \ + third_party/boringssl/crypto/x509/x_info.c \ + third_party/boringssl/crypto/x509/x_name.c \ + third_party/boringssl/crypto/x509/x_pkey.c \ + third_party/boringssl/crypto/x509/x_pubkey.c \ + third_party/boringssl/crypto/x509/x_req.c \ + third_party/boringssl/crypto/x509/x_sig.c \ + third_party/boringssl/crypto/x509/x_spki.c \ + third_party/boringssl/crypto/x509/x_val.c \ + third_party/boringssl/crypto/x509/x_x509.c \ + third_party/boringssl/crypto/x509/x_x509a.c \ + third_party/boringssl/crypto/x509v3/pcy_cache.c \ + third_party/boringssl/crypto/x509v3/pcy_data.c \ + third_party/boringssl/crypto/x509v3/pcy_lib.c \ + third_party/boringssl/crypto/x509v3/pcy_map.c \ + third_party/boringssl/crypto/x509v3/pcy_node.c \ + third_party/boringssl/crypto/x509v3/pcy_tree.c \ + third_party/boringssl/crypto/x509v3/v3_akey.c \ + third_party/boringssl/crypto/x509v3/v3_akeya.c \ + third_party/boringssl/crypto/x509v3/v3_alt.c \ + third_party/boringssl/crypto/x509v3/v3_bcons.c \ + third_party/boringssl/crypto/x509v3/v3_bitst.c \ + third_party/boringssl/crypto/x509v3/v3_conf.c \ + third_party/boringssl/crypto/x509v3/v3_cpols.c \ + third_party/boringssl/crypto/x509v3/v3_crld.c \ + third_party/boringssl/crypto/x509v3/v3_enum.c \ + third_party/boringssl/crypto/x509v3/v3_extku.c \ + third_party/boringssl/crypto/x509v3/v3_genn.c \ + third_party/boringssl/crypto/x509v3/v3_ia5.c \ + third_party/boringssl/crypto/x509v3/v3_info.c \ + third_party/boringssl/crypto/x509v3/v3_int.c \ + third_party/boringssl/crypto/x509v3/v3_lib.c \ + third_party/boringssl/crypto/x509v3/v3_ncons.c \ + third_party/boringssl/crypto/x509v3/v3_pci.c \ + third_party/boringssl/crypto/x509v3/v3_pcia.c \ + third_party/boringssl/crypto/x509v3/v3_pcons.c \ + third_party/boringssl/crypto/x509v3/v3_pku.c \ + third_party/boringssl/crypto/x509v3/v3_pmaps.c \ + third_party/boringssl/crypto/x509v3/v3_prn.c \ + third_party/boringssl/crypto/x509v3/v3_purp.c \ + third_party/boringssl/crypto/x509v3/v3_skey.c \ + third_party/boringssl/crypto/x509v3/v3_sxnet.c \ + third_party/boringssl/crypto/x509v3/v3_utl.c \ + third_party/boringssl/ssl/custom_extensions.c \ + third_party/boringssl/ssl/d1_both.c \ + third_party/boringssl/ssl/d1_clnt.c \ + third_party/boringssl/ssl/d1_lib.c \ + third_party/boringssl/ssl/d1_meth.c \ + third_party/boringssl/ssl/d1_pkt.c \ + third_party/boringssl/ssl/d1_srtp.c \ + third_party/boringssl/ssl/d1_srvr.c \ + third_party/boringssl/ssl/dtls_record.c \ + third_party/boringssl/ssl/pqueue/pqueue.c \ + third_party/boringssl/ssl/s3_both.c \ + third_party/boringssl/ssl/s3_clnt.c \ + third_party/boringssl/ssl/s3_enc.c \ + third_party/boringssl/ssl/s3_lib.c \ + third_party/boringssl/ssl/s3_meth.c \ + third_party/boringssl/ssl/s3_pkt.c \ + third_party/boringssl/ssl/s3_srvr.c \ + third_party/boringssl/ssl/ssl_aead_ctx.c \ + third_party/boringssl/ssl/ssl_asn1.c \ + third_party/boringssl/ssl/ssl_buffer.c \ + third_party/boringssl/ssl/ssl_cert.c \ + third_party/boringssl/ssl/ssl_cipher.c \ + third_party/boringssl/ssl/ssl_file.c \ + third_party/boringssl/ssl/ssl_lib.c \ + third_party/boringssl/ssl/ssl_rsa.c \ + third_party/boringssl/ssl/ssl_session.c \ + third_party/boringssl/ssl/ssl_stat.c \ + third_party/boringssl/ssl/t1_enc.c \ + third_party/boringssl/ssl/t1_lib.c \ + third_party/boringssl/ssl/tls_record.c \ + third_party/zlib/adler32.c \ + third_party/zlib/compress.c \ + third_party/zlib/crc32.c \ + third_party/zlib/deflate.c \ + third_party/zlib/gzclose.c \ + third_party/zlib/gzlib.c \ + third_party/zlib/gzread.c \ + third_party/zlib/gzwrite.c \ + third_party/zlib/infback.c \ + third_party/zlib/inffast.c \ + third_party/zlib/inflate.c \ + third_party/zlib/inftrees.c \ + third_party/zlib/trees.c \ + third_party/zlib/uncompr.c \ + third_party/zlib/zutil.c \ + , $ext_shared, , -Wall -Werror -std=c11 \ + -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN \ + -D_HAS_EXCEPTIONS=0 -DNOMINMAX) +fi diff --git a/package.xml b/package.xml new file mode 100644 index 00000000000..0d672512e5c --- /dev/null +++ b/package.xml @@ -0,0 +1,984 @@ + + + grpc + pecl.php.net + A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. + Remote Procedure Calls (RPCs) provide a useful abstraction for building distributed applications and services. The libraries in this repository provide a concrete implementation of the gRPC protocol, layered over HTTP/2. These libraries enable communication between clients and servers using any combination of the supported languages. + + Stanley Cheung + stanleycheung + grpc-packages@google.com + yes + + 2016-02-12 + + + 0.8.0 + 0.8.0 + + + beta + beta + + BSD + +- Simplify gRPC PHP installation #4517 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5.5.0 + + + 1.4.0 + + + + grpc + + + + + 0.5.0 + 0.5.0 + + + alpha + alpha + + 2015-06-16 + BSD + +First alpha release + + + + + 0.5.1 + 0.5.1 + + + alpha + alpha + + 2015-07-09 + BSD + +Update to wrap gRPC C Core version 0.10.0 + + + + + 0.6.0 + 0.6.0 + + + beta + beta + + 2015-09-24 + BSD + +- support per message compression disable +- expose per-call host override option +- expose connectivity API +- expose channel target and call peer +- add user-agent +- update to wrap gRPC C core library beta version 0.11.0 + + + + + 0.6.1 + 0.6.0 + + + beta + beta + + 2015-10-21 + BSD + +- fixed undefined constant fatal error when run with apache/nginx #2275 + + + + + 0.7.0 + 0.7.0 + + + beta + beta + + 2016-01-13 + BSD + +- Breaking change to Credentials class (removed) #3765 +- Replaced by ChannelCredentials and CallCredentials class #3765 +- New plugin based metadata auth API #4394 +- Explicit ChannelCredentials::createInsecure() call + + + + + 0.8.0 + 0.8.0 + + + beta + beta + + 2016-02-12 + BSD + +- Simplify gRPC PHP installation #4517 + + + + diff --git a/templates/config.m4.template b/templates/config.m4.template new file mode 100644 index 00000000000..1f8c1d9ca49 --- /dev/null +++ b/templates/config.m4.template @@ -0,0 +1,30 @@ +%YAML 1.2 +--- | + PHP_ARG_ENABLE(grpc, whether to enable grpc support, + [ --enable-grpc Enable grpc support]) + + if test "$PHP_GRPC" != "no"; then + dnl Write more examples of tests here... + + dnl # --with-grpc -> add include path + PHP_ADD_INCLUDE(../../grpc/include) + PHP_ADD_INCLUDE(../../grpc/src/php/ext/grpc) + PHP_ADD_INCLUDE(../../grpc/third_party/boringssl/include) + + PHP_ADD_LIBRARY(pthread) + + PHP_NEW_EXTENSION(grpc, + % for source in php_config_m4.src: + ${source} ${"\\"} + % endfor + % for lib in libs: + % if lib.name in php_config_m4.get('deps', []): + % for source in lib.src: + ${source} ${"\\"} + % endfor + % endif + % endfor + , $ext_shared, , -Wall -Werror -std=c11 ${"\\"} + -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN ${"\\"} + -D_HAS_EXCEPTIONS=0 -DNOMINMAX) + fi diff --git a/templates/package.xml.template b/templates/package.xml.template new file mode 100644 index 00000000000..455b002e64a --- /dev/null +++ b/templates/package.xml.template @@ -0,0 +1,159 @@ +%YAML 1.2 +--- | + + + grpc + pecl.php.net + A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. + Remote Procedure Calls (RPCs) provide a useful abstraction for building distributed applications and services. The libraries in this repository provide a concrete implementation of the gRPC protocol, layered over HTTP/2. These libraries enable communication between clients and servers using any combination of the supported languages. + + Stanley Cheung + stanleycheung + grpc-packages@google.com + yes + + <%! from time import strftime %>${"%Y-%m-%d" | strftime} + + + 0.8.0 + 0.8.0 + + + beta + beta + + BSD + + - Simplify gRPC PHP installation #4517 + + + + + + + + % for source in php_config_m4.src + php_config_m4.headers: + + % endfor + % for lib in libs: + % if lib.name in php_config_m4.get('deps', []): + % for source in lib.get('public_headers', []) + lib.headers + lib.src: + + % endfor + % endif + % endfor + + + + + + 5.5.0 + + + 1.4.0 + + + + grpc + + + + + 0.5.0 + 0.5.0 + + + alpha + alpha + + 2015-06-16 + BSD + + First alpha release + + + + + 0.5.1 + 0.5.1 + + + alpha + alpha + + 2015-07-09 + BSD + + Update to wrap gRPC C Core version 0.10.0 + + + + + 0.6.0 + 0.6.0 + + + beta + beta + + 2015-09-24 + BSD + + - support per message compression disable + - expose per-call host override option + - expose connectivity API + - expose channel target and call peer + - add user-agent + - update to wrap gRPC C core library beta version 0.11.0 + + + + + 0.6.1 + 0.6.0 + + + beta + beta + + 2015-10-21 + BSD + + - fixed undefined constant fatal error when run with apache/nginx #2275 + + + + + 0.7.0 + 0.7.0 + + + beta + beta + + 2016-01-13 + BSD + + - Breaking change to Credentials class (removed) #3765 + - Replaced by ChannelCredentials and CallCredentials class #3765 + - New plugin based metadata auth API #4394 + - Explicit ChannelCredentials::createInsecure() call + + + + + 0.8.0 + 0.8.0 + + + beta + beta + + ${"%Y-%m-%d" | strftime} + BSD + + - Simplify gRPC PHP installation #4517 + + + + diff --git a/test/distrib/php/distribtest.php b/test/distrib/php/distribtest.php new file mode 100644 index 00000000000..318c6f9cf08 --- /dev/null +++ b/test/distrib/php/distribtest.php @@ -0,0 +1,13 @@ + Grpc\ChannelCredentials::createInsecure() +]); + +$deadline = Grpc\Timeval::infFuture(); +$call = new Grpc\Call($channel, + 'dummy_method', + $deadline); + +$call->cancel(); +$channel->close(); diff --git a/test/distrib/php/run_distrib_test.sh b/test/distrib/php/run_distrib_test.sh new file mode 100755 index 00000000000..43b28d84290 --- /dev/null +++ b/test/distrib/php/run_distrib_test.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0) + +cp -r $EXTERNAL_GIT_ROOT/input_artifacts/grpc-php.tgz . + +pecl install grpc-php.tgz + +php -d extension=grpc.so -d max_execution_time=300 distribtest.php diff --git a/tools/dockerfile/distribtest/php_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/php_jessie_x64/Dockerfile new file mode 100644 index 00000000000..d5d3fd626ff --- /dev/null +++ b/tools/dockerfile/distribtest/php_jessie_x64/Dockerfile @@ -0,0 +1,32 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +RUN apt-get update && apt-get install -y php5 php5-dev php-pear phpunit diff --git a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile index 41dc4274541..d048b725c85 100644 --- a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile @@ -91,6 +91,13 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +################## +# PHP dependencies + +RUN apt-get update && apt-get install -y \ + php5 php5-dev php-pear phpunit + RUN mkdir /var/local/jenkins # Define the default command. diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index f84d35580bb..15a1e2a681f 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -241,6 +241,23 @@ class NodeExtArtifact: ['tools/run_tests/build_artifact_node.sh', self.gyp_arch]) +class PHPExtArtifact: + """Builds PHP native extension""" + + def __init__(self, platform, arch): + self.name = 'php_ext_{0}_{1}'.format(platform, arch) + self.platform = platform + self.arch = arch + self.labels = ['artifact', 'php', platform, arch] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), + 'tools/run_tests/build_artifact_php.sh') class ProtocArtifact: """Builds protoc and protoc-plugin artifacts""" @@ -299,4 +316,6 @@ def targets(): PythonArtifact('windows', 'x64'), RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), - RubyArtifact('macos', 'x64')]) + RubyArtifact('macos', 'x64'), + PHPExtArtifact('linux', 'x64'), + PHPExtArtifact('macos', 'x64')]) diff --git a/tools/run_tests/build_artifact_php.sh b/tools/run_tests/build_artifact_php.sh new file mode 100755 index 00000000000..50bf0ea8215 --- /dev/null +++ b/tools/run_tests/build_artifact_php.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +PHP_TARGET_ARCH=$1 +set -ex + +cd $(dirname $0)/../.. + +mkdir -p artifacts + +pear package + +cp -r grpc-*.tgz artifacts/grpc-php.tgz diff --git a/tools/run_tests/build_package_php.sh b/tools/run_tests/build_package_php.sh new file mode 100755 index 00000000000..56e3319ed9b --- /dev/null +++ b/tools/run_tests/build_package_php.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../.. + +mkdir -p artifacts/ +cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=php,platform={windows,linux,macos}/artifacts/* artifacts/ || true diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index 261f44bc6d9..0c02344d90d 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -198,6 +198,33 @@ class RubyDistribTest(object): return self.name +class PHPDistribTest(object): + """Tests PHP package""" + + def __init__(self, platform, arch, docker_suffix): + self.name = 'php_%s_%s_%s' % (platform, arch, docker_suffix) + self.platform = platform + self.arch = arch + self.docker_suffix = docker_suffix + self.labels = ['distribtest', 'php', platform, arch, docker_suffix] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + if not self.platform == 'linux': + raise Exception("Not supported yet.") + + return create_docker_jobspec(self.name, + 'tools/dockerfile/distribtest/php_%s_%s' % ( + self.docker_suffix, + self.arch), + 'test/distrib/php/run_distrib_test.sh') + + def __str__(self): + return self.name + + def targets(): """Gets list of supported targets""" return [CSharpDistribTest('linux', 'x64', 'wheezy'), @@ -241,7 +268,9 @@ def targets(): RubyDistribTest('linux', 'x64', 'ubuntu1510'), RubyDistribTest('linux', 'x64', 'ubuntu1604'), NodeDistribTest('macos', 'x64', None, '4'), - NodeDistribTest('linux', 'x86', 'jessie', '4') + NodeDistribTest('macos', 'x64', None, '5'), + NodeDistribTest('linux', 'x86', 'jessie', '4'), + PHPDistribTest('linux', 'x64', 'jessie'), ] + [ NodeDistribTest('linux', 'x64', os, version) for os in ('wheezy', 'jessie', 'ubuntu1204', 'ubuntu1404', diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py index 4ca8279f1bf..8aee1f86dec 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/package_targets.py @@ -139,9 +139,27 @@ class PythonPackage: 'tools/run_tests/build_package_python.sh') +class PHPPackage: + """Builds PHP PECL package and collects precompiled package""" + + def __init__(self): + self.name = 'php_package' + self.labels = ['package', 'php', 'linux'] + + def pre_build_jobspecs(self): + return [] + + def build_jobspec(self): + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_x64', + 'tools/run_tests/build_package_php.sh') + + def targets(): """Gets list of supported targets""" return [CSharpPackage(), NodePackage(), RubyPackage(), - PythonPackage()] + PythonPackage(), + PHPPackage()] From bf74d69ed601796fca2d6e737519cc1c2de7c60c Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 23 Feb 2016 22:39:25 -0800 Subject: [PATCH 080/236] fix php artifact name and update config.m4 template --- config.m4 | 63 +++++++++++++------- package.xml | 90 +++++++++++++++++------------ templates/config.m4.template | 14 +++++ tools/run_tests/artifact_targets.py | 10 ++-- tools/run_tests/package_targets.py | 2 +- 5 files changed, 114 insertions(+), 65 deletions(-) diff --git a/config.m4 b/config.m4 index 6d04add8e94..0323d5d2ad3 100644 --- a/config.m4 +++ b/config.m4 @@ -9,8 +9,22 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_INCLUDE(../../grpc/src/php/ext/grpc) PHP_ADD_INCLUDE(../../grpc/third_party/boringssl/include) + LIBS="-lpthread $LIBS" + + GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" PHP_ADD_LIBRARY(pthread) + PHP_ADD_LIBRARY(dl,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(dl) + + case $host in + *darwin*) ;; + *) + PHP_ADD_LIBRARY(rt,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(rt) + ;; + esac + PHP_NEW_EXTENSION(grpc, src/php/ext/grpc/byte_buffer.c \ src/php/ext/grpc/call.c \ @@ -65,27 +79,6 @@ if test "$PHP_GRPC" != "no"; then src/core/support/time_win32.c \ src/core/support/tls_pthread.c \ src/core/support/wrap_memcpy.c \ - src/core/httpcli/httpcli_security_connector.c \ - src/core/security/base64.c \ - src/core/security/client_auth_filter.c \ - src/core/security/credentials.c \ - src/core/security/credentials_metadata.c \ - src/core/security/credentials_posix.c \ - src/core/security/credentials_win32.c \ - src/core/security/google_default_credentials.c \ - src/core/security/handshake.c \ - src/core/security/json_token.c \ - src/core/security/jwt_verifier.c \ - src/core/security/secure_endpoint.c \ - src/core/security/security_connector.c \ - src/core/security/security_context.c \ - src/core/security/server_auth_filter.c \ - src/core/security/server_secure_chttp2.c \ - src/core/surface/init_secure.c \ - src/core/surface/secure_channel_create.c \ - src/core/tsi/fake_transport_security.c \ - src/core/tsi/ssl_transport_security.c \ - src/core/tsi/transport_security.c \ src/core/census/grpc_context.c \ src/core/census/grpc_filter.c \ src/core/channel/channel_args.c \ @@ -101,6 +94,7 @@ if test "$PHP_GRPC" != "no"; then src/core/client_config/connector.c \ src/core/client_config/default_initial_connect_string.c \ src/core/client_config/initial_connect_string.c \ + src/core/client_config/lb_policies/load_balancer_api.c \ src/core/client_config/lb_policies/pick_first.c \ src/core/client_config/lb_policies/round_robin.c \ src/core/client_config/lb_policy.c \ @@ -113,6 +107,7 @@ if test "$PHP_GRPC" != "no"; then src/core/client_config/resolvers/sockaddr_resolver.c \ src/core/client_config/subchannel.c \ src/core/client_config/subchannel_factory.c \ + src/core/client_config/subchannel_index.c \ src/core/client_config/uri_parser.c \ src/core/compression/algorithm.c \ src/core/compression/message_compress.c \ @@ -164,6 +159,7 @@ if test "$PHP_GRPC" != "no"; then src/core/json/json_reader.c \ src/core/json/json_string.c \ src/core/json/json_writer.c \ + src/core/proto/grpc/lb/v0/load_balancer.pb.c \ src/core/surface/alarm.c \ src/core/surface/api_trace.c \ src/core/surface/byte_buffer.c \ @@ -213,11 +209,36 @@ if test "$PHP_GRPC" != "no"; then src/core/transport/static_metadata.c \ src/core/transport/transport.c \ src/core/transport/transport_op_string.c \ + src/core/httpcli/httpcli_security_connector.c \ + src/core/security/base64.c \ + src/core/security/client_auth_filter.c \ + src/core/security/credentials.c \ + src/core/security/credentials_metadata.c \ + src/core/security/credentials_posix.c \ + src/core/security/credentials_win32.c \ + src/core/security/google_default_credentials.c \ + src/core/security/handshake.c \ + src/core/security/json_token.c \ + src/core/security/jwt_verifier.c \ + src/core/security/secure_endpoint.c \ + src/core/security/security_connector.c \ + src/core/security/security_context.c \ + src/core/security/server_auth_filter.c \ + src/core/security/server_secure_chttp2.c \ + src/core/surface/init_secure.c \ + src/core/surface/secure_channel_create.c \ + src/core/tsi/fake_transport_security.c \ + src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security.c \ src/core/census/context.c \ src/core/census/initialize.c \ + src/core/census/mlog.c \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tracing.c \ + third_party/nanopb/pb_common.c \ + third_party/nanopb/pb_decode.c \ + third_party/nanopb/pb_encode.c \ src/boringssl/err_data.c \ third_party/boringssl/crypto/aes/aes.c \ third_party/boringssl/crypto/aes/mode_wrappers.c \ diff --git a/package.xml b/package.xml index 0d672512e5c..99109ada8fd 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ grpc-packages@google.com yes - 2016-02-12 + 2016-02-23 0.8.0 @@ -158,20 +158,6 @@ - - - - - - - - - - - - - - @@ -186,6 +172,7 @@ + @@ -198,6 +185,7 @@ + @@ -235,7 +223,6 @@ - @@ -246,6 +233,7 @@ + @@ -285,29 +273,27 @@ + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + @@ -323,6 +309,7 @@ + @@ -335,6 +322,7 @@ + @@ -386,6 +374,7 @@ + @@ -435,11 +424,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -974,7 +988,7 @@ Update to wrap gRPC C Core version 0.10.0 beta beta - 2016-02-12 + 2016-02-23 BSD - Simplify gRPC PHP installation #4517 diff --git a/templates/config.m4.template b/templates/config.m4.template index 1f8c1d9ca49..dbc12188dc0 100644 --- a/templates/config.m4.template +++ b/templates/config.m4.template @@ -11,8 +11,22 @@ PHP_ADD_INCLUDE(../../grpc/src/php/ext/grpc) PHP_ADD_INCLUDE(../../grpc/third_party/boringssl/include) + LIBS="-lpthread $LIBS" + + GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" PHP_ADD_LIBRARY(pthread) + PHP_ADD_LIBRARY(dl,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(dl) + + case $host in + *darwin*) ;; + *) + PHP_ADD_LIBRARY(rt,,GRPC_SHARED_LIBADD) + PHP_ADD_LIBRARY(rt) + ;; + esac + PHP_NEW_EXTENSION(grpc, % for source in php_config_m4.src: ${source} ${"\\"} diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 15a1e2a681f..803d3d106b2 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -241,11 +241,11 @@ class NodeExtArtifact: ['tools/run_tests/build_artifact_node.sh', self.gyp_arch]) -class PHPExtArtifact: - """Builds PHP native extension""" +class PHPArtifact: + """Builds PHP PECL package""" def __init__(self, platform, arch): - self.name = 'php_ext_{0}_{1}'.format(platform, arch) + self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) self.platform = platform self.arch = arch self.labels = ['artifact', 'php', platform, arch] @@ -317,5 +317,5 @@ def targets(): RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), - PHPExtArtifact('linux', 'x64'), - PHPExtArtifact('macos', 'x64')]) + PHPArtifact('linux', 'x64'), + PHPArtifact('macos', 'x64')]) diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py index 8aee1f86dec..87bc4865ce3 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/package_targets.py @@ -140,7 +140,7 @@ class PythonPackage: class PHPPackage: - """Builds PHP PECL package and collects precompiled package""" + """Copy PHP PECL package artifact""" def __init__(self): self.name = 'php_package' From 9114a142c9211e0f3401166c0b753349573caf14 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 08:40:30 -0800 Subject: [PATCH 081/236] Port pollset worker changes to windows --- src/core/iomgr/pollset_windows.c | 2 +- test/core/util/port_windows.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index 8f35a465091..bbce23b46a7 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -192,7 +192,7 @@ done: remove_worker(&worker, GRPC_POLLSET_WORKER_LINK_GLOBAL); remove_worker(&worker, GRPC_POLLSET_WORKER_LINK_POLLSET); } - gpr_cv_destroy(&worker->cv); + gpr_cv_destroy(&worker.cv); *worker_hdl = NULL; } diff --git a/test/core/util/port_windows.c b/test/core/util/port_windows.c index 85d7c0ce077..b5bd0168ad2 100644 --- a/test/core/util/port_windows.c +++ b/test/core/util/port_windows.c @@ -180,7 +180,7 @@ static int pick_port_using_server(char *server) { &pr); gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); while (pr.port == -1) { - grpc_pollset_worker worker; + grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); From c46beaaa29af84f676bde9d013a85dffed1d58c9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 09:17:19 -0800 Subject: [PATCH 082/236] Add an implementation firewall against pollset_set So multiple implementations can exist in one binary --- src/core/channel/client_channel.c | 45 +++++++++++-------- .../client_config/lb_policies/pick_first.c | 31 +++++++------ .../client_config/lb_policies/round_robin.c | 25 ++++++----- src/core/client_config/lb_policy.c | 4 +- src/core/client_config/lb_policy.h | 3 +- src/core/client_config/subchannel.c | 30 ++++++------- src/core/httpcli/httpcli.c | 22 ++++----- src/core/httpcli/httpcli.h | 3 +- src/core/iomgr/pollset_set.h | 10 +---- src/core/iomgr/pollset_set_posix.c | 23 +++++++++- src/core/iomgr/pollset_set_posix.h | 17 +------ src/core/iomgr/pollset_set_windows.c | 4 +- src/core/iomgr/pollset_set_windows.h | 2 +- src/core/iomgr/tcp_client_posix.c | 12 ++--- src/core/iomgr/tcp_posix.c | 5 ++- 15 files changed, 128 insertions(+), 108 deletions(-) diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index 7176c01b05f..a96b49ac12f 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -78,8 +78,8 @@ typedef struct client_channel_channel_data { int exit_idle_when_lb_policy_arrives; /** owning stack */ grpc_channel_stack *owning_stack; - /** interested parties */ - grpc_pollset_set interested_parties; + /** interested parties (owned) */ + grpc_pollset_set *interested_parties; } channel_data; /** We create one watcher for each new lb_policy that is returned from a @@ -183,8 +183,8 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, chand->incoming_configuration = NULL; if (lb_policy != NULL) { - grpc_pollset_set_add_pollset_set(exec_ctx, &lb_policy->interested_parties, - &chand->interested_parties); + grpc_pollset_set_add_pollset_set(exec_ctx, lb_policy->interested_parties, + chand->interested_parties); } gpr_mu_lock(&chand->mu_config); @@ -231,9 +231,8 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, } if (old_lb_policy != NULL) { - grpc_pollset_set_del_pollset_set(exec_ctx, - &old_lb_policy->interested_parties, - &chand->interested_parties); + grpc_pollset_set_del_pollset_set( + exec_ctx, old_lb_policy->interested_parties, chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, old_lb_policy, "channel"); } @@ -254,7 +253,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, GPR_ASSERT(op->set_accept_stream == NULL); if (op->bind_pollset != NULL) { - grpc_pollset_set_add_pollset(exec_ctx, &chand->interested_parties, + grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, op->bind_pollset); } @@ -284,8 +283,8 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, chand->resolver = NULL; if (chand->lb_policy != NULL) { grpc_pollset_set_del_pollset_set(exec_ctx, - &chand->lb_policy->interested_parties, - &chand->interested_parties); + chand->lb_policy->interested_parties, + chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel"); chand->lb_policy = NULL; } @@ -411,7 +410,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE, "client_channel"); - grpc_pollset_set_init(&chand->interested_parties); + chand->interested_parties = grpc_pollset_set_create(); } /* Destructor for channel_data */ @@ -425,12 +424,12 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, } if (chand->lb_policy != NULL) { grpc_pollset_set_del_pollset_set(exec_ctx, - &chand->lb_policy->interested_parties, - &chand->interested_parties); + chand->lb_policy->interested_parties, + chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel"); } grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker); - grpc_pollset_set_destroy(&chand->interested_parties); + grpc_pollset_set_destroy(chand->interested_parties); gpr_mu_destroy(&chand->mu_config); } @@ -441,9 +440,17 @@ static void cc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, } const grpc_channel_filter grpc_client_channel_filter = { - cc_start_transport_stream_op, cc_start_transport_op, sizeof(call_data), - init_call_elem, cc_set_pollset, destroy_call_elem, sizeof(channel_data), - init_channel_elem, destroy_channel_elem, cc_get_peer, "client-channel", + cc_start_transport_stream_op, + cc_start_transport_op, + sizeof(call_data), + init_call_elem, + cc_set_pollset, + destroy_call_elem, + sizeof(channel_data), + init_channel_elem, + destroy_channel_elem, + cc_get_peer, + "client-channel", }; void grpc_client_channel_set_resolver(grpc_exec_ctx *exec_ctx, @@ -501,7 +508,7 @@ static void on_external_watch_complete(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) { external_connectivity_watcher *w = arg; grpc_closure *follow_up = w->on_complete; - grpc_pollset_set_del_pollset(exec_ctx, &w->chand->interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, w->chand->interested_parties, w->pollset); GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "external_connectivity_watcher"); @@ -517,7 +524,7 @@ void grpc_client_channel_watch_connectivity_state( w->chand = chand; w->pollset = pollset; w->on_complete = on_complete; - grpc_pollset_set_add_pollset(exec_ctx, &chand->interested_parties, pollset); + grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, pollset); grpc_closure_init(&w->my_closure, on_external_watch_complete, w); GRPC_CHANNEL_STACK_REF(w->chand->owning_stack, "external_connectivity_watcher"); diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c index 459bbebb68a..9f38f398d8a 100644 --- a/src/core/client_config/lb_policies/pick_first.c +++ b/src/core/client_config/lb_policies/pick_first.c @@ -31,8 +31,8 @@ * */ -#include "src/core/client_config/lb_policy_factory.h" #include "src/core/client_config/lb_policies/pick_first.h" +#include "src/core/client_config/lb_policy_factory.h" #include @@ -119,7 +119,7 @@ void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { while (pp != NULL) { pending_pick *next = pp->next; *pp->target = NULL; - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL); gpr_free(pp); @@ -137,7 +137,7 @@ static void pf_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, while (pp != NULL) { pending_pick *next = pp->next; if (pp->target == target) { - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); *target = NULL; grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL); @@ -158,7 +158,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, pick_first_lb_policy *p) { GRPC_LB_POLICY_WEAK_REF(&p->base, "pick_first_connectivity"); grpc_subchannel_notify_on_state_change( exec_ctx, p->subchannels[p->checking_subchannel], - &p->base.interested_parties, &p->checking_connectivity, + p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); } @@ -195,8 +195,7 @@ int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset, if (!p->started_picking) { start_picking(exec_ctx, p); } - grpc_pollset_set_add_pollset(exec_ctx, &p->base.interested_parties, - pollset); + grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset); pp = gpr_malloc(sizeof(*pp)); pp->next = p->pending_picks; pp->pollset = pollset; @@ -253,7 +252,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, p->checking_connectivity, "selected_changed"); if (p->checking_connectivity != GRPC_CHANNEL_FATAL_FAILURE) { grpc_connected_subchannel_notify_on_state_change( - exec_ctx, selected, &p->base.interested_parties, + exec_ctx, selected, p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); } else { GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "pick_first_connectivity"); @@ -278,13 +277,13 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, while ((pp = p->pending_picks)) { p->pending_picks = pp->next; *pp->target = selected; - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL); gpr_free(pp); } grpc_connected_subchannel_notify_on_state_change( - exec_ctx, selected, &p->base.interested_parties, + exec_ctx, selected, p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); break; case GRPC_CHANNEL_TRANSIENT_FAILURE: @@ -298,7 +297,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, if (p->checking_connectivity == GRPC_CHANNEL_TRANSIENT_FAILURE) { grpc_subchannel_notify_on_state_change( exec_ctx, p->subchannels[p->checking_subchannel], - &p->base.interested_parties, &p->checking_connectivity, + p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); } else { goto loop; @@ -311,7 +310,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, "connecting_changed"); grpc_subchannel_notify_on_state_change( exec_ctx, p->subchannels[p->checking_subchannel], - &p->base.interested_parties, &p->checking_connectivity, + p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); break; case GRPC_CHANNEL_FATAL_FAILURE: @@ -379,8 +378,14 @@ void pf_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } static const grpc_lb_policy_vtable pick_first_lb_policy_vtable = { - pf_destroy, pf_shutdown, pf_pick, pf_cancel_pick, pf_ping_one, pf_exit_idle, - pf_check_connectivity, pf_notify_on_state_change}; + pf_destroy, + pf_shutdown, + pf_pick, + pf_cancel_pick, + pf_ping_one, + pf_exit_idle, + pf_check_connectivity, + pf_notify_on_state_change}; static void pick_first_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/lb_policies/round_robin.c b/src/core/client_config/lb_policies/round_robin.c index b1171c45b00..114ece6e4d9 100644 --- a/src/core/client_config/lb_policies/round_robin.c +++ b/src/core/client_config/lb_policies/round_robin.c @@ -260,7 +260,7 @@ static void rr_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, while (pp != NULL) { pending_pick *next = pp->next; if (pp->target == target) { - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); *target = NULL; grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL); @@ -285,7 +285,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) { subchannel_data *sd = p->subchannels[i]; sd->connectivity_state = GRPC_CHANNEL_IDLE; grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, &p->base.interested_parties, + exec_ctx, sd->subchannel, p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); GRPC_LB_POLICY_WEAK_REF(&p->base, "round_robin_connectivity"); } @@ -322,8 +322,7 @@ int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset, if (!p->started_picking) { start_picking(exec_ctx, p); } - grpc_pollset_set_add_pollset(exec_ctx, &p->base.interested_parties, - pollset); + grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset); pp = gpr_malloc(sizeof(*pp)); pp->next = p->pending_picks; pp->pollset = pollset; @@ -374,13 +373,13 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)", selected->subchannel, selected); } - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL); gpr_free(pp); } grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, &p->base.interested_parties, + exec_ctx, sd->subchannel, p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); break; case GRPC_CHANNEL_CONNECTING: @@ -389,13 +388,13 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, sd->connectivity_state, "connecting_changed"); grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, &p->base.interested_parties, + exec_ctx, sd->subchannel, p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); break; case GRPC_CHANNEL_TRANSIENT_FAILURE: /* renew state notification */ grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, &p->base.interested_parties, + exec_ctx, sd->subchannel, p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); /* remove from ready list if still present */ @@ -484,8 +483,14 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } static const grpc_lb_policy_vtable round_robin_lb_policy_vtable = { - rr_destroy, rr_shutdown, rr_pick, rr_cancel_pick, rr_ping_one, rr_exit_idle, - rr_check_connectivity, rr_notify_on_state_change}; + rr_destroy, + rr_shutdown, + rr_pick, + rr_cancel_pick, + rr_ping_one, + rr_exit_idle, + rr_check_connectivity, + rr_notify_on_state_change}; static void round_robin_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/lb_policy.c b/src/core/client_config/lb_policy.c index d4672f6b255..5ff623e0069 100644 --- a/src/core/client_config/lb_policy.c +++ b/src/core/client_config/lb_policy.c @@ -39,7 +39,7 @@ void grpc_lb_policy_init(grpc_lb_policy *policy, const grpc_lb_policy_vtable *vtable) { policy->vtable = vtable; gpr_atm_no_barrier_store(&policy->ref_pair, 1 << WEAK_REF_BITS); - grpc_pollset_set_init(&policy->interested_parties); + policy->interested_parties = grpc_pollset_set_create(); } #ifdef GRPC_LB_POLICY_REFCOUNT_DEBUG @@ -93,7 +93,7 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, gpr_atm old_val = ref_mutate(policy, -(gpr_atm)1, 1 REF_MUTATE_PASS_ARGS("WEAK_UNREF")); if (old_val == 1) { - grpc_pollset_set_destroy(&policy->interested_parties); + grpc_pollset_set_destroy(policy->interested_parties); policy->vtable->destroy(exec_ctx, policy); } } diff --git a/src/core/client_config/lb_policy.h b/src/core/client_config/lb_policy.h index db5238c8ca0..4fbb12da394 100644 --- a/src/core/client_config/lb_policy.h +++ b/src/core/client_config/lb_policy.h @@ -48,7 +48,8 @@ typedef void (*grpc_lb_completion)(void *cb_arg, grpc_subchannel *subchannel, struct grpc_lb_policy { const grpc_lb_policy_vtable *vtable; gpr_atm ref_pair; - grpc_pollset_set interested_parties; + /* owned pointer to interested parties in load balancing decisions */ + grpc_pollset_set *interested_parties; }; struct grpc_lb_policy_vtable { diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index 6599c75dba7..291ad3472cc 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -108,7 +108,7 @@ struct grpc_subchannel { /** pollset_set tracking who's interested in a connection being setup */ - grpc_pollset_set pollset_set; + grpc_pollset_set *pollset_set; /** active connection, or null; of type grpc_connected_subchannel */ gpr_atm connected_subchannel; @@ -184,8 +184,8 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(c); } -void grpc_connected_subchannel_ref(grpc_connected_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_connected_subchannel_ref( + grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON); } @@ -209,7 +209,7 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_slice_unref(c->initial_connect_string); grpc_connectivity_state_destroy(exec_ctx, &c->state_tracker); grpc_connector_unref(exec_ctx, c->connector); - grpc_pollset_set_destroy(&c->pollset_set); + grpc_pollset_set_destroy(c->pollset_set); grpc_subchannel_key_destroy(exec_ctx, c->key); gpr_free(c); } @@ -226,8 +226,8 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta, return old_val; } -grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_ref( + grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS), 0 REF_MUTATE_PURPOSE("STRONG_REF")); @@ -235,8 +235,8 @@ grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c return c; } -grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_weak_ref( + grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF")); GPR_ASSERT(old_refs != 0); @@ -326,7 +326,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, } c->addr = gpr_malloc(args->addr_len); memcpy(c->addr, args->addr, args->addr_len); - grpc_pollset_set_init(&c->pollset_set); + c->pollset_set = grpc_pollset_set_create(); c->addr_len = args->addr_len; grpc_set_initial_connect_string(&c->addr, &c->addr_len, &c->initial_connect_string); @@ -345,7 +345,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { grpc_connect_in_args args; - args.interested_parties = &c->pollset_set; + args.interested_parties = c->pollset_set; args.addr = c->addr; args.addr_len = c->addr_len; args.deadline = compute_connect_deadline(c); @@ -379,7 +379,7 @@ static void on_external_state_watcher_done(grpc_exec_ctx *exec_ctx, void *arg, external_state_watcher *w = arg; grpc_closure *follow_up = w->notify; if (w->pollset_set != NULL) { - grpc_pollset_set_del_pollset_set(exec_ctx, &w->subchannel->pollset_set, + grpc_pollset_set_del_pollset_set(exec_ctx, w->subchannel->pollset_set, w->pollset_set); } gpr_mu_lock(&w->subchannel->mu); @@ -415,7 +415,7 @@ void grpc_subchannel_notify_on_state_change( w->notify = notify; grpc_closure_init(&w->closure, on_external_state_watcher_done, w); if (interested_parties != NULL) { - grpc_pollset_set_add_pollset_set(exec_ctx, &c->pollset_set, + grpc_pollset_set_add_pollset_set(exec_ctx, c->pollset_set, interested_parties); } GRPC_SUBCHANNEL_WEAK_REF(c, "external_state_watcher"); @@ -573,7 +573,7 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher"); GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); grpc_connected_subchannel_notify_on_state_change( - exec_ctx, con, &c->pollset_set, &sw_subchannel->connectivity_state, + exec_ctx, con, c->pollset_set, &sw_subchannel->connectivity_state, &sw_subchannel->closure); /* signal completion */ @@ -690,8 +690,8 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); } -void grpc_subchannel_call_ref(grpc_subchannel_call *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_subchannel_call_ref( + grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } diff --git a/src/core/httpcli/httpcli.c b/src/core/httpcli/httpcli.c index 71237bb6140..68fa8c8cf5a 100644 --- a/src/core/httpcli/httpcli.c +++ b/src/core/httpcli/httpcli.c @@ -31,20 +31,20 @@ * */ -#include "src/core/iomgr/sockaddr.h" #include "src/core/httpcli/httpcli.h" +#include "src/core/iomgr/sockaddr.h" #include +#include +#include +#include +#include "src/core/httpcli/format_request.h" +#include "src/core/httpcli/parser.h" #include "src/core/iomgr/endpoint.h" #include "src/core/iomgr/resolve_address.h" #include "src/core/iomgr/tcp_client.h" -#include "src/core/httpcli/format_request.h" -#include "src/core/httpcli/parser.h" #include "src/core/support/string.h" -#include -#include -#include typedef struct { gpr_slice request_text; @@ -84,18 +84,18 @@ const grpc_httpcli_handshaker grpc_httpcli_plaintext = {"http", plaintext_handshake}; void grpc_httpcli_context_init(grpc_httpcli_context *context) { - grpc_pollset_set_init(&context->pollset_set); + context->pollset_set = grpc_pollset_set_create(); } void grpc_httpcli_context_destroy(grpc_httpcli_context *context) { - grpc_pollset_set_destroy(&context->pollset_set); + grpc_pollset_set_destroy(context->pollset_set); } static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req); static void finish(grpc_exec_ctx *exec_ctx, internal_request *req, int success) { - grpc_pollset_set_del_pollset(exec_ctx, &req->context->pollset_set, + grpc_pollset_set_del_pollset(exec_ctx, req->context->pollset_set, req->pollset); req->on_response(exec_ctx, req->user_data, success ? &req->parser.r : NULL); grpc_httpcli_parser_destroy(&req->parser); @@ -197,7 +197,7 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req) { addr = &req->addresses->addrs[req->next_address++]; grpc_closure_init(&req->connected, on_connected, req); grpc_tcp_client_connect( - exec_ctx, &req->connected, &req->ep, &req->context->pollset_set, + exec_ctx, &req->connected, &req->ep, req->context->pollset_set, (struct sockaddr *)&addr->addr, addr->len, req->deadline); } @@ -237,7 +237,7 @@ static void internal_request_begin( req->host = gpr_strdup(request->host); req->ssl_host_override = gpr_strdup(request->ssl_host_override); - grpc_pollset_set_add_pollset(exec_ctx, &req->context->pollset_set, + grpc_pollset_set_add_pollset(exec_ctx, req->context->pollset_set, req->pollset); grpc_resolve_address(request->host, req->handshaker->default_port, on_resolved, req); diff --git a/src/core/httpcli/httpcli.h b/src/core/httpcli/httpcli.h index 30875d71f13..86e17c1d697 100644 --- a/src/core/httpcli/httpcli.h +++ b/src/core/httpcli/httpcli.h @@ -39,6 +39,7 @@ #include #include "src/core/iomgr/endpoint.h" +#include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/pollset_set.h" /* User agent this library reports */ @@ -56,7 +57,7 @@ typedef struct grpc_httpcli_header { TODO(ctiller): allow caching and capturing multiple requests for the same content and combining them */ typedef struct grpc_httpcli_context { - grpc_pollset_set pollset_set; + grpc_pollset_set *pollset_set; } grpc_httpcli_context; typedef struct { diff --git a/src/core/iomgr/pollset_set.h b/src/core/iomgr/pollset_set.h index 09c04438f71..9591bf0d32c 100644 --- a/src/core/iomgr/pollset_set.h +++ b/src/core/iomgr/pollset_set.h @@ -41,15 +41,9 @@ fd's (etc) that have been registered with the set_set to that pollset. Registering fd's automatically adds them to all current pollsets. */ -#ifdef GPR_POSIX_SOCKET -#include "src/core/iomgr/pollset_set_posix.h" -#endif +typedef struct grpc_pollset_set grpc_pollset_set; -#ifdef GPR_WIN32 -#include "src/core/iomgr/pollset_set_windows.h" -#endif - -void grpc_pollset_set_init(grpc_pollset_set *pollset_set); +grpc_pollset_set *grpc_pollset_set_create(void); void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set); void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/iomgr/pollset_set_posix.c index 3610e36b586..9dc9aff4a8d 100644 --- a/src/core/iomgr/pollset_set_posix.c +++ b/src/core/iomgr/pollset_set_posix.c @@ -42,11 +42,29 @@ #include #include "src/core/iomgr/pollset_posix.h" -#include "src/core/iomgr/pollset_set.h" +#include "src/core/iomgr/pollset_set_posix.h" -void grpc_pollset_set_init(grpc_pollset_set *pollset_set) { +struct grpc_pollset_set { + gpr_mu mu; + + size_t pollset_count; + size_t pollset_capacity; + grpc_pollset **pollsets; + + size_t pollset_set_count; + size_t pollset_set_capacity; + struct grpc_pollset_set **pollset_sets; + + size_t fd_count; + size_t fd_capacity; + grpc_fd **fds; +}; + +grpc_pollset_set *grpc_pollset_set_create(void) { + grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set)); memset(pollset_set, 0, sizeof(*pollset_set)); gpr_mu_init(&pollset_set->mu); + return pollset_set; } void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) { @@ -58,6 +76,7 @@ void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) { gpr_free(pollset_set->pollsets); gpr_free(pollset_set->pollset_sets); gpr_free(pollset_set->fds); + gpr_free(pollset_set); } void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, diff --git a/src/core/iomgr/pollset_set_posix.h b/src/core/iomgr/pollset_set_posix.h index 5ee83b5dd6a..7d1aaf41817 100644 --- a/src/core/iomgr/pollset_set_posix.h +++ b/src/core/iomgr/pollset_set_posix.h @@ -35,22 +35,7 @@ #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_POSIX_H #include "src/core/iomgr/fd_posix.h" - -typedef struct grpc_pollset_set { - gpr_mu mu; - - size_t pollset_count; - size_t pollset_capacity; - grpc_pollset **pollsets; - - size_t pollset_set_count; - size_t pollset_set_capacity; - struct grpc_pollset_set **pollset_sets; - - size_t fd_count; - size_t fd_capacity; - grpc_fd **fds; -} grpc_pollset_set; +#include "src/core/iomgr/pollset_set.h" void grpc_pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, grpc_fd *fd); diff --git a/src/core/iomgr/pollset_set_windows.c b/src/core/iomgr/pollset_set_windows.c index 157b46ec32a..9cf8fd4472f 100644 --- a/src/core/iomgr/pollset_set_windows.c +++ b/src/core/iomgr/pollset_set_windows.c @@ -35,9 +35,9 @@ #ifdef GPR_WINSOCK_SOCKET -#include "src/core/iomgr/pollset_set.h" +#include "src/core/iomgr/pollset_set_windows.h" -void grpc_pollset_set_init(grpc_pollset_set* pollset_set) {} +grpc_pollset_set* grpc_pollset_set_create(pollset_set) { return NULL; } void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {} diff --git a/src/core/iomgr/pollset_set_windows.h b/src/core/iomgr/pollset_set_windows.h index cada0d2b61f..aa5abe91338 100644 --- a/src/core/iomgr/pollset_set_windows.h +++ b/src/core/iomgr/pollset_set_windows.h @@ -34,6 +34,6 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H -typedef struct grpc_pollset_set { void *unused; } grpc_pollset_set; +#include "src/core/iomgr/pollset_set.h" #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/tcp_client_posix.c b/src/core/iomgr/tcp_client_posix.c index c76c2e3b0f2..15727856abf 100644 --- a/src/core/iomgr/tcp_client_posix.c +++ b/src/core/iomgr/tcp_client_posix.c @@ -42,17 +42,19 @@ #include #include -#include "src/core/iomgr/timer.h" +#include +#include +#include +#include + #include "src/core/iomgr/iomgr_posix.h" #include "src/core/iomgr/pollset_posix.h" +#include "src/core/iomgr/pollset_set_posix.h" #include "src/core/iomgr/sockaddr_utils.h" #include "src/core/iomgr/socket_utils_posix.h" #include "src/core/iomgr/tcp_posix.h" +#include "src/core/iomgr/timer.h" #include "src/core/support/string.h" -#include -#include -#include -#include extern int grpc_tcp_trace; diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index fba35634276..e8f73811cec 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -53,6 +53,7 @@ #include "src/core/debug/trace.h" #include "src/core/iomgr/pollset_posix.h" +#include "src/core/iomgr/pollset_set_posix.h" #include "src/core/profiling/timers.h" #include "src/core/support/string.h" @@ -296,7 +297,7 @@ static flush_result tcp_flush(grpc_tcp *tcp) { unwind_slice_idx = tcp->outgoing_slice_idx; unwind_byte_idx = tcp->outgoing_byte_idx; for (iov_size = 0; tcp->outgoing_slice_idx != tcp->outgoing_buffer->count && - iov_size != MAX_WRITE_IOVEC; + iov_size != MAX_WRITE_IOVEC; iov_size++) { iov[iov_size].iov_base = GPR_SLICE_START_PTR( @@ -445,7 +446,7 @@ static char *tcp_get_peer(grpc_endpoint *ep) { } static const grpc_endpoint_vtable vtable = { - tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set, + tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set, tcp_shutdown, tcp_destroy, tcp_get_peer}; grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size, From 56042ce22d318bd4292d29b35ce4a98e837109aa Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 24 Feb 2016 09:24:29 -0800 Subject: [PATCH 083/236] Remove outdated ref to openssl from Mac instructions --- INSTALL | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/INSTALL b/INSTALL index a0df57dcd35..e33f8970a95 100644 --- a/INSTALL +++ b/INSTALL @@ -141,15 +141,7 @@ Then execute the following for all the needed build dependencies $ make gtest.a gtest_main.a $ sudo cp libgtest.a libgtest_main.a /opt/local/lib $ sudo mkdir /opt/local/include/gtest - $ sudo cp -pr ../gtest-svn/include/gtest /opt/local/include/gtest - -We will also need to make openssl and install it appropriately - - $ cd - $ cd third_party/openssl - $ ./config - $ sudo make install - $ cd ../../ + $ sudo cp -pr ../gtest-svn/include/gtest /opt/local/include/gtest If you are going to make changes and need to regenerate the projects file, you will need to install certain modules for python. From 6483c32f7cfd10b4ae68d7d1e7f73d40af391f2f Mon Sep 17 00:00:00 2001 From: vjpai Date: Wed, 24 Feb 2016 09:29:12 -0800 Subject: [PATCH 084/236] BSD platforms (such as Mac) are likely to have /bin/sh refer to traditional Bourne shell syntax, not Bash syntax. Change "|&" to "2>&1 |" since that's the traditional way to redirect stdout and stderr to a pipe --- test/cpp/qps/qps-sweep.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/qps-sweep.sh b/test/cpp/qps/qps-sweep.sh index 7a357888497..9d3f053a7be 100755 --- a/test/cpp/qps/qps-sweep.sh +++ b/test/cpp/qps/qps-sweep.sh @@ -72,7 +72,7 @@ for secure in true false; do --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ - --num_servers=1 --num_clients=0 |& tee /tmp/qps-test.$$ + --num_servers=1 --num_clients=0 2>&1 | tee /tmp/qps-test.$$ # Scenario 2b: QPS with a single server core "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ From 7c0715af6c5ee47fe786e9b63400961f0c3ae220 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 09:57:21 -0800 Subject: [PATCH 085/236] Properly initialize TLS var --- src/core/client_config/subchannel_index.c | 2 ++ third_party/nanopb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/client_config/subchannel_index.c b/src/core/client_config/subchannel_index.c index f78a7fd588c..3f948998f9c 100644 --- a/src/core/client_config/subchannel_index.c +++ b/src/core/client_config/subchannel_index.c @@ -149,11 +149,13 @@ static const gpr_avl_vtable subchannel_avl_vtable = { void grpc_subchannel_index_init(void) { g_subchannel_index = gpr_avl_create(&subchannel_avl_vtable); gpr_mu_init(&g_mu); + gpr_tls_init(&subchannel_index_exec_ctx); } void grpc_subchannel_index_shutdown(void) { gpr_mu_destroy(&g_mu); gpr_avl_unref(g_subchannel_index); + gpr_tls_destroy(&subchannel_index_exec_ctx); } grpc_subchannel *grpc_subchannel_index_find(grpc_exec_ctx *exec_ctx, diff --git a/third_party/nanopb b/third_party/nanopb index f8ac4637662..5497a1dfc91 160000 --- a/third_party/nanopb +++ b/third_party/nanopb @@ -1 +1 @@ -Subproject commit f8ac463766281625ad710900479130c7fcb4d63b +Subproject commit 5497a1dfc91a86965383cdd1652e348345400435 From 559e45becd0a50bd6af850900abbb2b5759f8719 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 19 Feb 2016 03:02:16 -0800 Subject: [PATCH 086/236] Scripts to launch stress tests in GKE --- test/cpp/util/metrics_server.cc | 2 +- tools/{gke => big_query}/big_query_utils.py | 135 +++--- tools/gke/create_client.py | 108 ----- tools/gke/create_server.py | 74 ---- tools/gke/delete_client.py | 66 --- tools/gke/delete_server.py | 58 --- tools/gke/kubernetes_api.py | 5 +- tools/gke/run_stress_tests_on_gke.py | 389 ++++++++++++++++++ tools/run_tests/stress_test/run_client.py | 188 +++++++++ tools/run_tests/stress_test/run_server.py | 115 ++++++ .../stress_test/stress_test_utils.py | 192 +++++++++ tools/run_tests/stress_test_wrapper.py | 96 ----- 12 files changed, 935 insertions(+), 493 deletions(-) rename tools/{gke => big_query}/big_query_utils.py (51%) mode change 100644 => 100755 delete mode 100755 tools/gke/create_client.py delete mode 100755 tools/gke/create_server.py delete mode 100755 tools/gke/delete_client.py delete mode 100755 tools/gke/delete_server.py create mode 100755 tools/gke/run_stress_tests_on_gke.py create mode 100755 tools/run_tests/stress_test/run_client.py create mode 100755 tools/run_tests/stress_test/run_server.py create mode 100755 tools/run_tests/stress_test/stress_test_utils.py delete mode 100755 tools/run_tests/stress_test_wrapper.py diff --git a/test/cpp/util/metrics_server.cc b/test/cpp/util/metrics_server.cc index 07978d0bdbf..34d51eb3169 100644 --- a/test/cpp/util/metrics_server.cc +++ b/test/cpp/util/metrics_server.cc @@ -57,7 +57,7 @@ long Gauge::Get() { grpc::Status MetricsServiceImpl::GetAllGauges( ServerContext* context, const EmptyMessage* request, ServerWriter* writer) { - gpr_log(GPR_INFO, "GetAllGauges called"); + gpr_log(GPR_DEBUG, "GetAllGauges called"); std::lock_guard lock(mu_); for (auto it = gauges_.begin(); it != gauges_.end(); it++) { diff --git a/tools/gke/big_query_utils.py b/tools/big_query/big_query_utils.py old mode 100644 new mode 100755 similarity index 51% rename from tools/gke/big_query_utils.py rename to tools/big_query/big_query_utils.py index ebcf9d6ec35..267d0198509 --- a/tools/gke/big_query_utils.py +++ b/tools/big_query/big_query_utils.py @@ -1,3 +1,33 @@ +#!/usr/bin/env python2.7 +# Copyright 2015-2016 Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import argparse import json import uuid @@ -10,14 +40,14 @@ from oauth2client.client import GoogleCredentials NUM_RETRIES = 3 -def create_bq(): +def create_big_query(): """Authenticates with cloud platform and gets a BiqQuery service object """ creds = GoogleCredentials.get_application_default() return discovery.build('bigquery', 'v2', credentials=creds) -def create_ds(biq_query, project_id, dataset_id): +def create_dataset(biq_query, project_id, dataset_id): is_success = True body = { 'datasetReference': { @@ -25,6 +55,7 @@ def create_ds(biq_query, project_id, dataset_id): 'datasetId': dataset_id } } + try: dataset_req = biq_query.datasets().insert(projectId=project_id, body=body) dataset_req.execute(num_retries=NUM_RETRIES) @@ -38,21 +69,18 @@ def create_ds(biq_query, project_id, dataset_id): return is_success -def make_field(field_name, field_type, field_description): - return { - 'name': field_name, - 'type': field_type, - 'description': field_description - } - - -def create_table(big_query, project_id, dataset_id, table_id, fields_list, +def create_table(big_query, project_id, dataset_id, table_id, table_schema, description): is_success = True + body = { 'description': description, 'schema': { - 'fields': fields_list + 'fields': [{ + 'name': field_name, + 'type': field_type, + 'description': field_description + } for (field_name, field_type, field_description) in table_schema] }, 'tableReference': { 'datasetId': dataset_id, @@ -60,6 +88,7 @@ def create_table(big_query, project_id, dataset_id, table_id, fields_list, 'tableId': table_id } } + try: table_req = big_query.tables().insert(projectId=project_id, datasetId=dataset_id, @@ -91,43 +120,8 @@ def insert_rows(big_query, project_id, dataset_id, table_id, rows_list): is_success = False return is_success -##################### - -def make_emp_row(emp_id, emp_name, emp_email): - return { - 'insertId': str(emp_id), - 'json': { - 'emp_id': emp_id, - 'emp_name': emp_name, - 'emp_email_id': emp_email - } - } - - -def get_emp_table_fields_list(): - return [ - make_field('emp_id', 'INTEGER', 'Employee id'), - make_field('emp_name', 'STRING', 'Employee name'), - make_field('emp_email_id', 'STRING', 'Employee email id') - ] - - -def insert_emp_rows(big_query, project_id, dataset_id, table_id, start_idx, - num_rows): - rows_list = [make_emp_row(i, 'sree_%d' % i, 'sreecha_%d@gmail.com' % i) - for i in range(start_idx, start_idx + num_rows)] - insert_rows(big_query, project_id, dataset_id, table_id, rows_list) - - -def create_emp_table(big_query, project_id, dataset_id, table_id): - fields_list = get_emp_table_fields_list() - description = 'Test table created by sree' - create_table(big_query, project_id, dataset_id, table_id, fields_list, - description) - - -def sync_query(big_query, project_id, query, timeout=5000): +def sync_query_job(big_query, project_id, query, timeout=5000): query_data = {'query': query, 'timeoutMs': timeout} query_job = None try: @@ -139,43 +133,8 @@ def sync_query(big_query, project_id, query, timeout=5000): print http_error.content return query_job -#[Start query_emp_records] -def query_emp_records(big_query, project_id, dataset_id, table_id): - query = 'SELECT emp_id, emp_name FROM %s.%s ORDER BY emp_id;' % (dataset_id, table_id) - print query - query_job = sync_query(big_query, project_id, query, 5000) - job_id = query_job['jobReference'] - - print query_job - print '**Starting paging **' - #[Start Paging] - page_token = None - while True: - page = big_query.jobs().getQueryResults( - pageToken=page_token, - **query_job['jobReference']).execute(num_retries=NUM_RETRIES) - rows = page['rows'] - for row in rows: - print row['f'][0]['v'], "---", row['f'][1]['v'] - page_token = page.get('pageToken') - if not page_token: - break - #[End Paging] -#[End query_emp_records] - -######################### -DATASET_SEQ_NUM = 1 -TABLE_SEQ_NUM = 11 - -PROJECT_ID = 'sree-gce' -DATASET_ID = 'sree_test_dataset_%d' % DATASET_SEQ_NUM -TABLE_ID = 'sree_test_table_%d' % TABLE_SEQ_NUM - -EMP_ROW_IDX = 10 -EMP_NUM_ROWS = 5 - -bq = create_bq() -create_ds(bq, PROJECT_ID, DATASET_ID) -create_emp_table(bq, PROJECT_ID, DATASET_ID, TABLE_ID) -insert_emp_rows(bq, PROJECT_ID, DATASET_ID, TABLE_ID, EMP_ROW_IDX, EMP_NUM_ROWS) -query_emp_records(bq, PROJECT_ID, DATASET_ID, TABLE_ID) + # List of (column name, column type, description) tuples +def make_row(unique_row_id, row_values_dict): + """row_values_dict is a dictionar of column name and column value. + """ + return {'insertId': unique_row_id, 'json': row_values_dict} diff --git a/tools/gke/create_client.py b/tools/gke/create_client.py deleted file mode 100755 index bc56ef0ef13..00000000000 --- a/tools/gke/create_client.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import argparse - -import kubernetes_api - -argp = argparse.ArgumentParser(description='Launch Stress tests in GKE') - -argp.add_argument('-n', - '--num_instances', - required=True, - type=int, - help='The number of instances to launch in GKE') -args = argp.parse_args() - -kubernetes_api_server="localhost" -kubernetes_api_port=8001 - - -# Docker image -image_name="gcr.io/sree-gce/grpc_stress_test_2" - -server_address = "stress-server.default.svc.cluster.local:8080" -metrics_server_address = "localhost:8081" - -stress_test_arg_list=[ - "--server_addresses=" + server_address, - "--test_cases=empty_unary:20,large_unary:20", - "--num_stubs_per_channel=10" -] - -metrics_client_arg_list=[ - "--metrics_server_address=" + metrics_server_address, - "--total_only=true"] - -env_dict={ - "GPRC_ROOT": "/var/local/git/grpc", - "STRESS_TEST_IMAGE": "/var/local/git/grpc/bins/opt/stress_test", - "STRESS_TEST_ARGS_STR": ' '.join(stress_test_arg_list), - "METRICS_CLIENT_IMAGE": "/var/local/git/grpc/bins/opt/metrics_client", - "METRICS_CLIENT_ARGS_STR": ' '.join(metrics_client_arg_list)} - -cmd_list=["/var/local/git/grpc/bins/opt/stress_test"] -arg_list=stress_test_arg_list # make this [] in future -port_list=[8081] - -namespace = 'default' -is_headless_service = False # Client is NOT headless service - -print('Creating %d instances of client..' % args.num_instances) - -for i in range(1, args.num_instances + 1): - service_name = 'stress-client-%d' % i - pod_name = service_name # Use the same name for kubernetes Service and Pod - is_success = kubernetes_api.create_pod( - kubernetes_api_server, - kubernetes_api_port, - namespace, - pod_name, - image_name, - port_list, - cmd_list, - arg_list, - env_dict) - if not is_success: - print("Error in creating pod %s" % pod_name) - else: - is_success = kubernetes_api.create_service( - kubernetes_api_server, - kubernetes_api_port, - namespace, - service_name, - pod_name, - port_list, # Service port list - port_list, # Container port list (same as service port list) - is_headless_service) - if not is_success: - print("Error in creating service %s" % service_name) - else: - print("Created client %s" % pod_name) diff --git a/tools/gke/create_server.py b/tools/gke/create_server.py deleted file mode 100755 index 23ab62c205d..00000000000 --- a/tools/gke/create_server.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import argparse - -import kubernetes_api - -service_name = 'stress-server' -pod_name = service_name # Use the same name for kubernetes Service and Pod -namespace = 'default' -is_headless_service = True -cmd_list=['/var/local/git/grpc/bins/opt/interop_server'] -arg_list=['--port=8080'] -port_list=[8080] -image_name='gcr.io/sree-gce/grpc_stress_test_2' -env_dict={} - -# Make sure you run kubectl proxy --port=8001 -kubernetes_api_server='localhost' -kubernetes_api_port=8001 - -is_success = kubernetes_api.create_pod( - kubernetes_api_server, - kubernetes_api_port, - namespace, - pod_name, - image_name, - port_list, - cmd_list, - arg_list, - env_dict) -if not is_success: - print("Error in creating pod") -else: - is_success = kubernetes_api.create_service( - kubernetes_api_server, - kubernetes_api_port, - namespace, - service_name, - pod_name, - port_list, # Service port list - port_list, # Container port list (same as service port list) - is_headless_service) - if not is_success: - print("Error in creating service") - else: - print("Successfully created the Server") diff --git a/tools/gke/delete_client.py b/tools/gke/delete_client.py deleted file mode 100755 index aa519f26b89..00000000000 --- a/tools/gke/delete_client.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import argparse - -import kubernetes_api - -argp = argparse.ArgumentParser(description='Delete Stress test clients in GKE') -argp.add_argument('-n', - '--num_instances', - required=True, - type=int, - help='The number of instances currently running') - -args = argp.parse_args() -for i in range(1, args.num_instances + 1): - service_name = 'stress-client-%d' % i - pod_name = service_name - namespace = 'default' - kubernetes_api_server="localhost" - kubernetes_api_port=8001 - - is_success=kubernetes_api.delete_pod( - kubernetes_api_server, - kubernetes_api_port, - namespace, - pod_name) - if not is_success: - print('Error in deleting Pod %s' % pod_name) - else: - is_success= kubernetes_api.delete_service( - kubernetes_api_server, - kubernetes_api_port, - namespace, - service_name) - if not is_success: - print('Error in deleting Service %s' % service_name) - else: - print('Deleted %s' % pod_name) diff --git a/tools/gke/delete_server.py b/tools/gke/delete_server.py deleted file mode 100755 index 6e3fdcc33ba..00000000000 --- a/tools/gke/delete_server.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import argparse - -import kubernetes_api - -service_name = 'stress-server' -pod_name = service_name # Use the same name for kubernetes Service and Pod -namespace = 'default' -is_headless_service = True -kubernetes_api_server="localhost" -kubernetes_api_port=8001 - -is_success = kubernetes_api.delete_pod( - kubernetes_api_server, - kubernetes_api_port, - namespace, - pod_name) -if not is_success: - print("Error in deleting Pod %s" % pod_name) -else: - is_success = kubernetes_api.delete_service( - kubernetes_api_server, - kubernetes_api_port, - namespace, - service_name) - if not is_success: - print("Error in deleting Service %d" % service_name) - else: - print("Deleted server %s" % service_name) diff --git a/tools/gke/kubernetes_api.py b/tools/gke/kubernetes_api.py index 14d724bd319..d14c26ad6ac 100755 --- a/tools/gke/kubernetes_api.py +++ b/tools/gke/kubernetes_api.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. +# Copyright 2015-2016 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -50,7 +50,8 @@ def _make_pod_config(pod_name, image_name, container_port_list, cmd_list, 'name': pod_name, 'image': image_name, 'ports': [{'containerPort': port, - 'protocol': 'TCP'} for port in container_port_list] + 'protocol': 'TCP'} for port in container_port_list], + 'imagePullPolicy': 'Always' } ] } diff --git a/tools/gke/run_stress_tests_on_gke.py b/tools/gke/run_stress_tests_on_gke.py new file mode 100755 index 00000000000..d0c3887a422 --- /dev/null +++ b/tools/gke/run_stress_tests_on_gke.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python2.7 +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import datetime +import os +import subprocess +import sys +import time + +import kubernetes_api + +GRPC_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(GRPC_ROOT) + +class BigQuerySettings: + + def __init__(self, run_id, dataset_id, summary_table_id, qps_table_id): + self.run_id = run_id + self.dataset_id = dataset_id + self.summary_table_id = summary_table_id + self.qps_table_id = qps_table_id + + +class KubernetesProxy: + """ Class to start a proxy on localhost to the Kubernetes API server """ + + def __init__(self, api_port): + self.port = api_port + self.p = None + self.started = False + + def start(self): + cmd = ['kubectl', 'proxy', '--port=%d' % self.port] + self.p = subprocess.Popen(args=cmd) + self.started = True + time.sleep(2) + print '..Started' + + def get_port(self): + return self.port + + def is_started(self): + return self.started + + def __del__(self): + if self.p is not None: + self.p.kill() + + +def _build_docker_image(image_name, tag_name): + """ Build the docker image and add a tag """ + os.environ['INTEROP_IMAGE'] = image_name + # Note that 'BASE_NAME' HAS to be 'grpc_interop_stress_cxx' since the script + # build_interop_stress_image.sh invokes the following script: + # tools/dockerfile/$BASE_NAME/build_interop_stress.sh + os.environ['BASE_NAME'] = 'grpc_interop_stress_cxx' + cmd = ['tools/jenkins/build_interop_stress_image.sh'] + p = subprocess.Popen(args=cmd) + retcode = p.wait() + if retcode != 0: + print 'Error in building docker image' + return False + + cmd = ['docker', 'tag', '-f', image_name, tag_name] + p = subprocess.Popen(args=cmd) + retcode = p.wait() + if retcode != 0: + print 'Error in creating the tag %s for %s' % (tag_name, image_name) + return False + + return True + + +def _push_docker_image_to_gke_registry(docker_tag_name): + """Executes 'gcloud docker push ' to push the image to GKE registry""" + cmd = ['gcloud', 'docker', 'push', docker_tag_name] + print 'Pushing %s to GKE registry..' % docker_tag_name + p = subprocess.Popen(args=cmd) + retcode = p.wait() + if retcode != 0: + print 'Error in pushing docker image %s to the GKE registry' % docker_tag_name + return False + return True + + +def _launch_image_on_gke(kubernetes_api_server, kubernetes_api_port, namespace, + pod_name, image_name, port_list, cmd_list, arg_list, + env_dict, is_headless_service): + """Creates a GKE Pod and a Service object for a given image by calling Kubernetes API""" + is_success = kubernetes_api.create_pod( + kubernetes_api_server, + kubernetes_api_port, + namespace, + pod_name, + image_name, + port_list, # The ports to be exposed on this container/pod + cmd_list, # The command that launches the stress server + arg_list, + env_dict # Environment variables to be passed to the pod + ) + if not is_success: + print 'Error in creating Pod' + return False + + is_success = kubernetes_api.create_service( + kubernetes_api_server, + kubernetes_api_port, + namespace, + pod_name, # Use the pod name for service name as well + pod_name, + port_list, # Service port list + port_list, # Container port list (same as service port list) + is_headless_service) + if not is_success: + print 'Error in creating Service' + return False + + print 'Successfully created the pod/service %s' % pod_name + return True + + +def _delete_image_on_gke(kubernetes_proxy, pod_name_list): + """Deletes a GKE Pod and Service object for given list of Pods by calling Kubernetes API""" + if not kubernetes_proxy.is_started: + print 'Kubernetes proxy must be started before calling this function' + return False + + is_success = True + for pod_name in pod_name_list: + is_success = kubernetes_api.delete_pod( + 'localhost', kubernetes_proxy.get_port(), 'default', pod_name) + if not is_success: + print 'Error in deleting pod %s' % pod_name + break + + is_success = kubernetes_api.delete_service( + 'localhost', kubernetes_proxy.get_port(), 'default', + pod_name) # service name same as pod name + if not is_success: + print 'Error in deleting service %s' % pod_name + break + + if is_success: + print 'Successfully deleted the Pods/Services: %s' % ','.join(pod_name_list) + + return is_success + + +def _launch_server(gcp_project_id, docker_image_name, bq_settings, + kubernetes_proxy, server_pod_name, server_port): + """ Launches a stress test server instance in GKE cluster """ + if not kubernetes_proxy.is_started: + print 'Kubernetes proxy must be started before calling this function' + return False + + server_cmd_list = [ + '/var/local/git/grpc/tools/run_tests/stress_test/run_server.py' + ] # Process that is launched + server_arg_list = [] # run_server.py does not take any args (for now) + + # == Parameters to the server process launched in GKE == + server_env = { + 'STRESS_TEST_IMAGE_TYPE': 'SERVER', + 'STRESS_TEST_IMAGE': '/var/local/git/grpc/bins/opt/interop_server', + 'STRESS_TEST_ARGS_STR': '--port=%s' % server_port, + 'RUN_ID': bq_settings.run_id, + 'POD_NAME': server_pod_name, + 'GCP_PROJECT_ID': gcp_project_id, + 'DATASET_ID': bq_settings.dataset_id, + 'SUMMARY_TABLE_ID': bq_settings.summary_table_id, + 'QPS_TABLE_ID': bq_settings.qps_table_id + } + + # Launch Server + is_success = _launch_image_on_gke( + 'localhost', + kubernetes_proxy.get_port(), + 'default', + server_pod_name, + docker_image_name, + [server_port], # Port that should be exposed on the container + server_cmd_list, + server_arg_list, + server_env, + True # Headless = True for server. Since we want DNS records to be greated by GKE + ) + + return is_success + + +def _launch_client(gcp_project_id, docker_image_name, bq_settings, + kubernetes_proxy, num_instances, client_pod_name_prefix, + server_pod_name, server_port): + """ Launches a configurable number of stress test clients on GKE cluster """ + if not kubernetes_proxy.is_started: + print 'Kubernetes proxy must be started before calling this function' + return False + + server_address = '%s.default.svc.cluster.local:%d' % (server_pod_name, + server_port) + #TODO(sree) Make the whole client args configurable + test_cases_str = 'empty_unary:1,large_unary:1' + stress_client_arg_list = [ + '--server_addresses=%s' % server_address, + '--test_cases=%s' % test_cases_str, '--num_stubs_per_channel=10' + ] + + client_cmd_list = [ + '/var/local/git/grpc/tools/run_tests/stress_test/run_client.py' + ] + # run_client.py takes no args. All args are passed as env variables + client_arg_list = [] + + # TODO(sree) Make this configurable (and also less frequent) + poll_interval_secs = 5 + + metrics_port = 8081 + metrics_server_address = 'localhost:%d' % metrics_port + metrics_client_arg_list = [ + '--metrics_server_address=%s' % metrics_server_address, + '--total_only=true' + ] + + client_env = { + 'STRESS_TEST_IMAGE_TYPE': 'CLIENT', + 'STRESS_TEST_IMAGE': '/var/local/git/grpc/bins/opt/stress_test', + 'STRESS_TEST_ARGS_STR': ' '.join(stress_client_arg_list), + 'METRICS_CLIENT_IMAGE': '/var/local/git/grpc/bins/opt/metrics_client', + 'METRICS_CLIENT_ARGS_STR': ' '.join(metrics_client_arg_list), + 'RUN_ID': bq_settings.run_id, + 'POLL_INTERVAL_SECS': str(poll_interval_secs), + 'GCP_PROJECT_ID': gcp_project_id, + 'DATASET_ID': bq_settings.dataset_id, + 'SUMMARY_TABLE_ID': bq_settings.summary_table_id, + 'QPS_TABLE_ID': bq_settings.qps_table_id + } + + for i in range(1, num_instances + 1): + pod_name = '%s-%d' % (client_pod_name_prefix, i) + client_env['POD_NAME'] = pod_name + is_success = _launch_image_on_gke( + 'localhost', + kubernetes_proxy.get_port(), + 'default', + pod_name, + docker_image_name, + [metrics_port], # Client pods expose metrics port + client_cmd_list, + client_arg_list, + client_env, + False # Client is not a headless service. + ) + if not is_success: + print 'Error in launching client %s' % pod_name + return False + + return True + + +def _launch_server_and_client(gcp_project_id, docker_image_name, + num_client_instances): + # == Big Query tables related settings (Common for both server and client) == + + # Create a unique id for this run (Note: Using timestamp instead of UUID to + # make it easier to deduce the date/time of the run just by looking at the run + # run id. This is useful in debugging when looking at records in Biq query) + run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + + dataset_id = 'stress_test_%s' % run_id + summary_table_id = 'summary' + qps_table_id = 'qps' + + bq_settings = BigQuerySettings(run_id, dataset_id, summary_table_id, + qps_table_id) + + # Start kubernetes proxy + kubernetes_api_port = 9001 + kubernetes_proxy = KubernetesProxy(kubernetes_api_port) + kubernetes_proxy.start() + + server_pod_name = 'stress-server' + server_port = 8080 + is_success = _launch_server(gcp_project_id, docker_image_name, bq_settings, + kubernetes_proxy, server_pod_name, server_port) + if not is_success: + print 'Error in launching server' + return False + + # Server takes a while to start. + # TODO(sree) Use Kubernetes API to query the status of the server instead of + # sleeping + time.sleep(60) + + # Launch client + server_address = '%s.default.svc.cluster.local:%d' % (server_pod_name, + server_port) + client_pod_name_prefix = 'stress-client' + is_success = _launch_client(gcp_project_id, docker_image_name, bq_settings, + kubernetes_proxy, num_client_instances, + client_pod_name_prefix, server_pod_name, + server_port) + if not is_success: + print 'Error in launching client(s)' + return False + + return True + + +def _delete_server_and_client(num_client_instances): + kubernetes_api_port = 9001 + kubernetes_proxy = KubernetesProxy(kubernetes_api_port) + kubernetes_proxy.start() + + # Delete clients first + client_pod_names = ['stress-client-%d' % i + for i in range(1, num_client_instances + 1)] + + is_success = _delete_image_on_gke(kubernetes_proxy, client_pod_names) + if not is_success: + return False + + # Delete server + server_pod_name = 'stress-server' + return _delete_image_on_gke(kubernetes_proxy, [server_pod_name]) + + +def _build_and_push_docker_image(gcp_project_id, docker_image_name, tag_name): + is_success = _build_docker_image(docker_image_name, tag_name) + if not is_success: + return False + return _push_docker_image_to_gke_registry(tag_name) + + +# TODO(sree): This is just to test the above APIs. Rewrite this to make +# everything configurable (like image names / number of instances etc) +def test_run(): + image_name = 'grpc_stress_test' + gcp_project_id = 'sree-gce' + tag_name = 'gcr.io/%s/%s' % (gcp_project_id, image_name) + num_client_instances = 3 + + is_success = _build_docker_image(image_name, tag_name) + if not is_success: + return + + is_success = _push_docker_image_to_gke_registry(tag_name) + if not is_success: + return + + is_success = _launch_server_and_client(gcp_project_id, tag_name, + num_client_instances) + + # Run the test for 2 mins + time.sleep(120) + + is_success = _delete_server_and_client(num_client_instances) + + if not is_success: + return + + +if __name__ == '__main__': + test_run() diff --git a/tools/run_tests/stress_test/run_client.py b/tools/run_tests/stress_test/run_client.py new file mode 100755 index 00000000000..33958bce496 --- /dev/null +++ b/tools/run_tests/stress_test/run_client.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python2.7 +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import datetime +import os +import re +import select +import subprocess +import sys +import time + +from stress_test_utils import EventType +from stress_test_utils import BigQueryHelper + + +# TODO (sree): Write a python grpc client to directly query the metrics instead +# of calling metrics_client +def _get_qps(metrics_cmd): + qps = 0 + try: + # Note: gpr_log() writes even non-error messages to stderr stream. So it is + # important that we set stderr=subprocess.STDOUT + p = subprocess.Popen(args=metrics_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retcode = p.wait() + (out_str, err_str) = p.communicate() + if retcode != 0: + print 'Error in reading metrics information' + print 'Output: ', out_str + else: + # The overall qps is printed at the end of the line + m = re.search('\d+$', out_str) + qps = int(m.group()) if m else 0 + except Exception as ex: + print 'Exception while reading metrics information: ' + str(ex) + return qps + + +def run_client(): + """This is a wrapper around the stress test client and performs the following: + 1) Create the following two tables in Big Query: + (i) Summary table: To record events like the test started, completed + successfully or failed + (ii) Qps table: To periodically record the QPS sent by this client + 2) Start the stress test client and add a row in the Big Query summary + table + 3) Once every few seconds (as specificed by the poll_interval_secs) poll + the status of the stress test client process and perform the + following: + 3.1) If the process is still running, get the current qps by invoking + the metrics client program and add a row in the Big Query + Qps table. Sleep for a duration specified by poll_interval_secs + 3.2) If the process exited successfully, add a row in the Big Query + Summary table and exit + 3.3) If the process failed, add a row in Big Query summary table and + wait forever. + NOTE: This script typically runs inside a GKE pod which means + that the pod gets destroyed when the script exits. However, in + case the stress test client fails, we would not want the pod to + be destroyed (since we might want to connect to the pod for + examining logs). This is the reason why the script waits forever + in case of failures + """ + env = dict(os.environ) + image_type = env['STRESS_TEST_IMAGE_TYPE'] + image_name = env['STRESS_TEST_IMAGE'] + args_str = env['STRESS_TEST_ARGS_STR'] + metrics_client_image = env['METRICS_CLIENT_IMAGE'] + metrics_client_args_str = env['METRICS_CLIENT_ARGS_STR'] + run_id = env['RUN_ID'] + pod_name = env['POD_NAME'] + logfile_name = env.get('LOGFILE_NAME') + poll_interval_secs = float(env['POLL_INTERVAL_SECS']) + project_id = env['GCP_PROJECT_ID'] + dataset_id = env['DATASET_ID'] + summary_table_id = env['SUMMARY_TABLE_ID'] + qps_table_id = env['QPS_TABLE_ID'] + + bq_helper = BigQueryHelper(run_id, image_type, pod_name, project_id, + dataset_id, summary_table_id, qps_table_id) + bq_helper.initialize() + + # Create BigQuery Dataset and Tables: Summary Table and Metrics Table + if not bq_helper.setup_tables(): + print 'Error in creating BigQuery tables' + return + + start_time = datetime.datetime.now() + + logfile = None + details = 'Logging to stdout' + if logfile_name is not None: + print 'Opening logfile: %s ...' % logfile_name + details = 'Logfile: %s' % logfile_name + logfile = open(logfile_name, 'w') + + # Update status that the test is starting (in the status table) + bq_helper.insert_summary_row(EventType.STARTING, details) + + metrics_cmd = [metrics_client_image + ] + [x for x in metrics_client_args_str.split()] + stress_cmd = [image_name] + [x for x in args_str.split()] + + print 'Launching process %s ...' % stress_cmd + stress_p = subprocess.Popen(args=stress_cmd, + stdout=logfile, + stderr=subprocess.STDOUT) + + qps_history = [1, 1, 1] # Maintain the last 3 qps readings + qps_history_idx = 0 # Index into the qps_history list + + is_error = False + while True: + # Check if stress_client is still running. If so, collect metrics and upload + # to BigQuery status table + if stress_p.poll() is not None: + # TODO(sree) Upload completion status to BigQuery + end_time = datetime.datetime.now().isoformat() + event_type = EventType.SUCCESS + details = 'End time: %s' % end_time + if stress_p.returncode != 0: + event_type = EventType.FAILURE + details = 'Return code = %d. End time: %s' % (stress_p.returncode, + end_time) + is_error = True + bq_helper.insert_summary_row(event_type, details) + print details + break + + # Stress client still running. Get metrics + qps = _get_qps(metrics_cmd) + qps_recorded_at = datetime.datetime.now().isoformat() + print 'qps: %d at %s' % (qps, qps_recorded_at) + + # If QPS has been zero for the last 3 iterations, flag it as error and exit + qps_history[qps_history_idx] = qps + qps_history_idx = (qps_history_idx + 1) % len(qps_history) + if sum(qps_history) == 0: + details = 'QPS has been zero for the last %d seconds - as of : %s' % ( + poll_interval_secs * 3, qps_recorded_at) + is_error = True + bq_helper.insert_summary_row(EventType.FAILURE, details) + print details + break + + # Upload qps metrics to BiqQuery + bq_helper.insert_qps_row(qps, qps_recorded_at) + + time.sleep(poll_interval_secs) + + if is_error: + print 'Waiting indefinitely..' + select.select([], [], []) + + print 'Completed' + return + + +if __name__ == '__main__': + run_client() diff --git a/tools/run_tests/stress_test/run_server.py b/tools/run_tests/stress_test/run_server.py new file mode 100755 index 00000000000..9ad8d636385 --- /dev/null +++ b/tools/run_tests/stress_test/run_server.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python2.7 +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import datetime +import os +import select +import subprocess +import sys +import time + +from stress_test_utils import BigQueryHelper +from stress_test_utils import EventType + + +def run_server(): + """This is a wrapper around the interop server and performs the following: + 1) Create a 'Summary table' in Big Query to record events like the server + started, completed successfully or failed. NOTE: This also creates + another table called the QPS table which is currently NOT needed on the + server (it is needed on the stress test clients) + 2) Start the server process and add a row in Big Query summary table + 3) Wait for the server process to terminate. The server process does not + terminate unless there is an error. + If the server process terminated with a failure, add a row in Big Query + and wait forever. + NOTE: This script typically runs inside a GKE pod which means that the + pod gets destroyed when the script exits. However, in case the server + process fails, we would not want the pod to be destroyed (since we + might want to connect to the pod for examining logs). This is the + reason why the script waits forever in case of failures. + """ + + # Read the parameters from environment variables + env = dict(os.environ) + + run_id = env['RUN_ID'] # The unique run id for this test + image_type = env['STRESS_TEST_IMAGE_TYPE'] + image_name = env['STRESS_TEST_IMAGE'] + args_str = env['STRESS_TEST_ARGS_STR'] + pod_name = env['POD_NAME'] + project_id = env['GCP_PROJECT_ID'] + dataset_id = env['DATASET_ID'] + summary_table_id = env['SUMMARY_TABLE_ID'] + qps_table_id = env['QPS_TABLE_ID'] + + logfile_name = env.get('LOGFILE_NAME') + + bq_helper = BigQueryHelper(run_id, image_type, pod_name, project_id, + dataset_id, summary_table_id, qps_table_id) + bq_helper.initialize() + + # Create BigQuery Dataset and Tables: Summary Table and Metrics Table + if not bq_helper.setup_tables(): + print 'Error in creating BigQuery tables' + return + + start_time = datetime.datetime.now() + + logfile = None + details = 'Logging to stdout' + if logfile_name is not None: + print 'Opening log file: ', logfile_name + logfile = open(logfile_name, 'w') + details = 'Logfile: %s' % logfile_name + + # Update status that the test is starting (in the status table) + bq_helper.insert_summary_row(EventType.STARTING, details) + + stress_cmd = [image_name] + [x for x in args_str.split()] + + print 'Launching process %s ...' % stress_cmd + stress_p = subprocess.Popen(args=stress_cmd, + stdout=logfile, + stderr=subprocess.STDOUT) + + returncode = stress_p.wait() + if returncode != 0: + end_time = datetime.datetime.now().isoformat() + event_type = EventType.FAILURE + details = 'Returncode: %d; End time: %s' % (returncode, end_time) + bq_helper.insert_summary_row(event_type, details) + print 'Waiting indefinitely..' + select.select([], [], []) + return returncode + + +if __name__ == '__main__': + run_server() diff --git a/tools/run_tests/stress_test/stress_test_utils.py b/tools/run_tests/stress_test/stress_test_utils.py new file mode 100755 index 00000000000..a0626ce3aca --- /dev/null +++ b/tools/run_tests/stress_test/stress_test_utils.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python2.7 +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import datetime +import json +import os +import re +import select +import subprocess +import sys +import time + +# Import big_query_utils module +bq_utils_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../../big_query')) +sys.path.append(bq_utils_dir) +import big_query_utils as bq_utils + +class EventType: + STARTING = 'STARTING' + SUCCESS = 'SUCCESS' + FAILURE = 'FAILURE' + +class BigQueryHelper: + """Helper class for the stress test wrappers to interact with BigQuery. + """ + + def __init__(self, run_id, image_type, pod_name, project_id, dataset_id, + summary_table_id, qps_table_id): + self.run_id = run_id + self.image_type = image_type + self.pod_name = pod_name + self.project_id = project_id + self.dataset_id = dataset_id + self.summary_table_id = summary_table_id + self.qps_table_id = qps_table_id + + def initialize(self): + self.bq = bq_utils.create_big_query() + + def setup_tables(self): + return bq_utils.create_dataset(self.bq, self.project_id, self.dataset_id) \ + and self.__create_summary_table() \ + and self.__create_qps_table() + + def insert_summary_row(self, event_type, details): + row_values_dict = { + 'run_id': self.run_id, + 'image_type': self.image_type, + 'pod_name': self.pod_name, + 'event_date': datetime.datetime.now().isoformat(), + 'event_type': event_type, + 'details': details + } + # Something that uniquely identifies the row (Biquery needs it for duplicate + # detection). + row_unique_id = '%s_%s_%s' % (self.run_id, self.pod_name, event_type) + + row = bq_utils.make_row(row_unique_id, row_values_dict) + return bq_utils.insert_rows(self.bq, self.project_id, self.dataset_id, + self.summary_table_id, [row]) + + def insert_qps_row(self, qps, recorded_at): + row_values_dict = { + 'run_id': self.run_id, + 'pod_name': self.pod_name, + 'recorded_at': recorded_at, + 'qps': qps + } + + row_unique_id = '%s_%s_%s' % (self.run_id, self.pod_name, recorded_at) + row = bq_utils.make_row(row_unique_id, row_values_dict) + return bq_utils.insert_rows(self.bq, self.project_id, self.dataset_id, + self.qps_table_id, [row]) + + def check_if_any_tests_failed(self, num_query_retries=3): + query = ('SELECT event_type FROM %s.%s WHERE run_id = %s AND ' + 'event_type="%s"') % (self.dataset_id, self.summary_table_id, + self.run_id, EventType.FAILURE) + query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) + page = self.bq.jobs().getQueryResults(**query_job['jobReference']).execute( + num_retries=num_query_retries) + print page + num_failures = int(page['totalRows']) + print 'num rows: ', num_failures + return num_failures > 0 + + def print_summary_records(self, num_query_retries=3): + line = '-' * 120 + print line + print 'Summary records' + print 'Run Id', self.run_id + print line + query = ('SELECT pod_name, image_type, event_type, event_date, details' + ' FROM %s.%s WHERE run_id = %s ORDER by event_date;') % ( + self.dataset_id, self.summary_table_id, self.run_id) + query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) + + print '{:<25} {:<12} {:<12} {:<30} {}'.format( + 'Pod name', 'Image type', 'Event type', 'Date', 'Details') + print line + page_token = None + while True: + page = self.bq.jobs().getQueryResults( + pageToken=page_token, + **query_job['jobReference']).execute(num_retries=num_query_retries) + rows = page.get('rows', []) + for row in rows: + print '{:<25} {:<12} {:<12} {:<30} {}'.format( + row['f'][0]['v'], row['f'][1]['v'], row['f'][2]['v'], + row['f'][3]['v'], row['f'][4]['v']) + page_token = page.get('pageToken') + if not page_token: + break + + def print_qps_records(self, num_query_retries=3): + line = '-' * 80 + print line + print 'QPS Summary' + print 'Run Id: ', self.run_id + print line + query = ( + 'SELECT pod_name, recorded_at, qps FROM %s.%s WHERE run_id = %s ORDER ' + 'by recorded_at;') % (self.dataset_id, self.qps_table_id, self.run_id) + query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) + print '{:<25} {:30} {}'.format('Pod name', 'Recorded at', 'Qps') + print line + page_token = None + while True: + page = self.bq.jobs().getQueryResults( + pageToken=page_token, + **query_job['jobReference']).execute(num_retries=num_query_retries) + rows = page.get('rows', []) + for row in rows: + print '{:<25} {:30} {}'.format(row['f'][0]['v'], row['f'][1]['v'], + row['f'][2]['v']) + page_token = page.get('pageToken') + if not page_token: + break + + def __create_summary_table(self): + summary_table_schema = [ + ('run_id', 'INTEGER', 'Test run id'), + ('image_type', 'STRING', 'Client or Server?'), + ('pod_name', 'STRING', 'GKE pod hosting this image'), + ('event_date', 'STRING', 'The date of this event'), + ('event_type', 'STRING', 'STARTED/SUCCESS/FAILURE'), + ('details', 'STRING', 'Any other relevant details') + ] + desc = ('The table that contains START/SUCCESS/FAILURE events for ' + ' the stress test clients and servers') + return bq_utils.create_table(self.bq, self.project_id, self.dataset_id, + self.summary_table_id, summary_table_schema, + desc) + + def __create_qps_table(self): + qps_table_schema = [ + ('run_id', 'INTEGER', 'Test run id'), + ('pod_name', 'STRING', 'GKE pod hosting this image'), + ('recorded_at', 'STRING', 'Metrics recorded at time'), + ('qps', 'INTEGER', 'Queries per second') + ] + desc = 'The table that cointains the qps recorded at various intervals' + return bq_utils.create_table(self.bq, self.project_id, self.dataset_id, + self.qps_table_id, qps_table_schema, desc) diff --git a/tools/run_tests/stress_test_wrapper.py b/tools/run_tests/stress_test_wrapper.py deleted file mode 100755 index 8f1bd2024ec..00000000000 --- a/tools/run_tests/stress_test_wrapper.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python2.7 -import os -import re -import select -import subprocess -import sys -import time - -GRPC_ROOT = '/usr/local/google/home/sreek/workspace/grpc/' -STRESS_TEST_IMAGE = GRPC_ROOT + 'bins/opt/stress_test' -STRESS_TEST_ARGS_STR = ' '.join([ - '--server_addresses=localhost:8000', - '--test_cases=empty_unary:1,large_unary:1', '--num_stubs_per_channel=10', - '--test_duration_secs=10']) -METRICS_CLIENT_IMAGE = GRPC_ROOT + 'bins/opt/metrics_client' -METRICS_CLIENT_ARGS_STR = ' '.join([ - '--metrics_server_address=localhost:8081', '--total_only=true']) -LOGFILE_NAME = 'stress_test.log' - - -# TODO (sree): Write a python grpc client to directly query the metrics instead -# of calling metrics_client -def get_qps(metrics_cmd): - qps = 0 - try: - # Note: gpr_log() writes even non-error messages to stderr stream. So it is - # important that we set stderr=subprocess.STDOUT - p = subprocess.Popen(args=metrics_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - retcode = p.wait() - (out_str, err_str) = p.communicate() - if retcode != 0: - print 'Error in reading metrics information' - print 'Output: ', out_str - else: - # The overall qps is printed at the end of the line - m = re.search('\d+$', out_str) - qps = int(m.group()) if m else 0 - except Exception as ex: - print 'Exception while reading metrics information: ' + str(ex) - return qps - -def main(argv): - # TODO(sree) Create BigQuery Tables - # (Summary table), (Metrics table) - - # TODO(sree) Update status that the test is starting (in the status table) - # - - metrics_cmd = [METRICS_CLIENT_IMAGE - ] + [x for x in METRICS_CLIENT_ARGS_STR.split()] - - stress_cmd = [STRESS_TEST_IMAGE] + [x for x in STRESS_TEST_ARGS_STR.split()] - # TODO(sree): Add an option to print to stdout if logfilename is absent - logfile = open(LOGFILE_NAME, 'w') - stress_p = subprocess.Popen(args=arg_list, - stdout=logfile, - stderr=subprocess.STDOUT) - - qps_history = [1, 1, 1] # Maintain the last 3 qps - qps_history_idx = 0 # Index into the qps_history list - - is_error = False - while True: - # Check if stress_client is still running. If so, collect metrics and upload - # to BigQuery status table - # - if stress_p is not None: - # TODO(sree) Upload completion status to BiqQuery - is_error = (stress_p.returncode != 0) - break - - # Stress client still running. Get metrics - qps = get_qps(metrics_cmd) - - # If QPS has been zero for the last 3 iterations, flag it as error and exit - qps_history[qps_history_idx] = qps - qps_history_idx = (qps_histor_idx + 1) % len(qps_history) - if sum(a) == 0: - print ('QPS has been zero for the last 3 iterations. Not monitoring ' - 'anymore. The stress test client may be stalled.') - is_error = True - break - - #TODO(sree) Upload qps metrics to BiqQuery - - if is_error: - print 'Waiting indefinitely..' - select.select([],[],[]) - - return 1 - - -if __name__ == '__main__': - main(sys.argv[1:]) From 2173edfe1912e350d30138f5c38fcec80625680a Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 24 Feb 2016 12:06:01 -0800 Subject: [PATCH 087/236] redo the nanopb upgrade --- third_party/nanopb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/nanopb b/third_party/nanopb index 5497a1dfc91..f8ac4637662 160000 --- a/third_party/nanopb +++ b/third_party/nanopb @@ -1 +1 @@ -Subproject commit 5497a1dfc91a86965383cdd1652e348345400435 +Subproject commit f8ac463766281625ad710900479130c7fcb4d63b From e2d39e08f9d65b89b0e0f842bc8b8431022f1435 Mon Sep 17 00:00:00 2001 From: vjpai Date: Wed, 24 Feb 2016 12:36:15 -0800 Subject: [PATCH 088/236] Timer->UsageTimer consistently --- test/cpp/qps/client.h | 8 ++++---- test/cpp/qps/client_async.cc | 18 ++++++++---------- test/cpp/qps/client_sync.cc | 8 ++++---- test/cpp/qps/server.h | 8 ++++---- test/cpp/qps/usage_timer.cc | 8 ++++---- test/cpp/qps/usage_timer.h | 2 +- 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index aecbae95f9e..2dc83f0f292 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -112,12 +112,12 @@ class ClientRequestCreator { class Client { public: - Client() : timer_(new Timer), interarrival_timer_() {} + Client() : timer_(new UsageTimer), interarrival_timer_() {} virtual ~Client() {} ClientStats Mark(bool reset) { Histogram latencies; - Timer::Result timer_result; + UsageTimer::Result timer_result; // avoid std::vector for old compilers that expect a copy constructor if (reset) { @@ -125,7 +125,7 @@ class Client { for (size_t i = 0; i < threads_.size(); i++) { threads_[i]->BeginSwap(&to_merge[i]); } - std::unique_ptr timer(new Timer); + std::unique_ptr timer(new UsageTimer); timer_.swap(timer); for (size_t i = 0; i < threads_.size(); i++) { threads_[i]->EndSwap(); @@ -294,7 +294,7 @@ class Client { }; std::vector> threads_; - std::unique_ptr timer_; + std::unique_ptr timer_; InterarrivalTimer interarrival_timer_; std::vector next_time_; diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index b3b2f926db9..9e9da9909af 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -107,14 +107,14 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { bool RunNextState(bool ok, Histogram* hist) GRPC_OVERRIDE { switch (next_state_) { case State::READY: - start_ = Timer::Now(); + start_ = UsageTimer::Now(); response_reader_ = start_req_(stub_, &context_, req_, cq_); response_reader_->Finish(&response_, &status_, ClientRpcContext::tag(this)); next_state_ = State::RESP_DONE; return true; case State::RESP_DONE: - hist->Add((Timer::Now() - start_) * 1e9); + hist->Add((UsageTimer::Now() - start_) * 1e9); callback_(status_, &response_); next_state_ = State::INVALID; return false; @@ -287,8 +287,7 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req), - start_(Timer::Now()) {} + start_req_(start_req) {} ~ClientRpcContextStreamingImpl() GRPC_OVERRIDE {} void Start(CompletionQueue* cq) GRPC_OVERRIDE { cq_ = cq; @@ -314,7 +313,7 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { if (!ok) { return false; } - start_ = Timer::Now(); + start_ = UsageTimer::Now(); next_state_ = State::WRITE_DONE; stream_->Write(req_, ClientRpcContext::tag(this)); return true; @@ -327,7 +326,7 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { return true; break; case State::READ_DONE: - hist->Add((Timer::Now() - start_) * 1e9); + hist->Add((UsageTimer::Now() - start_) * 1e9); callback_(status_, &response_); next_state_ = State::STREAM_IDLE; break; // loop around @@ -415,8 +414,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req), - start_(Timer::Now()) {} + start_req_(start_req) {} ~ClientRpcContextGenericStreamingImpl() GRPC_OVERRIDE {} void Start(CompletionQueue* cq) GRPC_OVERRIDE { cq_ = cq; @@ -445,7 +443,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { if (!ok) { return false; } - start_ = Timer::Now(); + start_ = UsageTimer::Now(); next_state_ = State::WRITE_DONE; stream_->Write(req_, ClientRpcContext::tag(this)); return true; @@ -458,7 +456,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { return true; break; case State::READ_DONE: - hist->Add((Timer::Now() - start_) * 1e9); + hist->Add((UsageTimer::Now() - start_) * 1e9); callback_(status_, &response_); next_state_ = State::STREAM_IDLE; break; // loop around diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc index e39768b4981..4284e07bd4c 100644 --- a/test/cpp/qps/client_sync.cc +++ b/test/cpp/qps/client_sync.cc @@ -104,12 +104,12 @@ class SynchronousUnaryClient GRPC_FINAL : public SynchronousClient { bool ThreadFunc(Histogram* histogram, size_t thread_idx) GRPC_OVERRIDE { WaitToIssue(thread_idx); auto* stub = channels_[thread_idx % channels_.size()].get_stub(); - double start = Timer::Now(); + double start = UsageTimer::Now(); GPR_TIMER_SCOPE("SynchronousUnaryClient::ThreadFunc", 0); grpc::ClientContext context; grpc::Status s = stub->UnaryCall(&context, request_, &responses_[thread_idx]); - histogram->Add((Timer::Now() - start) * 1e9); + histogram->Add((UsageTimer::Now() - start) * 1e9); return s.ok(); } }; @@ -143,10 +143,10 @@ class SynchronousStreamingClient GRPC_FINAL : public SynchronousClient { bool ThreadFunc(Histogram* histogram, size_t thread_idx) GRPC_OVERRIDE { WaitToIssue(thread_idx); GPR_TIMER_SCOPE("SynchronousStreamingClient::ThreadFunc", 0); - double start = Timer::Now(); + double start = UsageTimer::Now(); if (stream_[thread_idx]->Write(request_) && stream_[thread_idx]->Read(&responses_[thread_idx])) { - histogram->Add((Timer::Now() - start) * 1e9); + histogram->Add((UsageTimer::Now() - start) * 1e9); return true; } return false; diff --git a/test/cpp/qps/server.h b/test/cpp/qps/server.h index 3227347e3ff..de46452c3d8 100644 --- a/test/cpp/qps/server.h +++ b/test/cpp/qps/server.h @@ -50,7 +50,7 @@ namespace testing { class Server { public: - explicit Server(const ServerConfig& config) : timer_(new Timer) { + explicit Server(const ServerConfig& config) : timer_(new UsageTimer) { cores_ = LimitCores(config.core_list().data(), config.core_list_size()); if (config.port()) { port_ = config.port(); @@ -62,9 +62,9 @@ class Server { virtual ~Server() {} ServerStats Mark(bool reset) { - Timer::Result timer_result; + UsageTimer::Result timer_result; if (reset) { - std::unique_ptr timer(new Timer); + std::unique_ptr timer(new UsageTimer); timer.swap(timer_); timer_result = timer->Mark(); } else { @@ -108,7 +108,7 @@ class Server { private: int port_; int cores_; - std::unique_ptr timer_; + std::unique_ptr timer_; }; std::unique_ptr CreateSynchronousServer(const ServerConfig& config); diff --git a/test/cpp/qps/usage_timer.cc b/test/cpp/qps/usage_timer.cc index ee64537fedf..6663a9ac103 100644 --- a/test/cpp/qps/usage_timer.cc +++ b/test/cpp/qps/usage_timer.cc @@ -37,9 +37,9 @@ #include #include -Timer::Timer() : start_(Sample()) {} +UsageTimer::UsageTimer() : start_(Sample()) {} -double Timer::Now() { +double UsageTimer::Now() { auto ts = gpr_now(GPR_CLOCK_REALTIME); return ts.tv_sec + 1e-9 * ts.tv_nsec; } @@ -48,7 +48,7 @@ static double time_double(struct timeval* tv) { return tv->tv_sec + 1e-6 * tv->tv_usec; } -Timer::Result Timer::Sample() { +UsageTimer::Result UsageTimer::Sample() { struct rusage usage; struct timeval tv; gettimeofday(&tv, NULL); @@ -61,7 +61,7 @@ Timer::Result Timer::Sample() { return r; } -Timer::Result Timer::Mark() const { +UsageTimer::Result UsageTimer::Mark() const { Result s = Sample(); Result r; r.wall = s.wall - start_.wall; diff --git a/test/cpp/qps/usage_timer.h b/test/cpp/qps/usage_timer.h index d5470dba590..d19f8205649 100644 --- a/test/cpp/qps/usage_timer.h +++ b/test/cpp/qps/usage_timer.h @@ -36,7 +36,7 @@ class UsageTimer { public: - Timer(); + UsageTimer(); struct Result { double wall; From 85b9cd9ebde69ee1a168806d1a8c4477a1a27f6f Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 24 Feb 2016 13:18:47 -0800 Subject: [PATCH 089/236] Fix name of report.xml in zip execution --- tools/jenkins/docker_run_tests.sh | 2 +- tools/run_tests/run_node.bat | 2 +- tools/run_tests/run_node.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 2e40f8fd65a..3e4689265d9 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -60,6 +60,6 @@ echo '' >> index.html cd .. zip -r reports.zip reports -find . -name reports.xml | xargs zip reports.zip +find . -name report.xml | xargs zip reports.zip exit $exit_code diff --git a/tools/run_tests/run_node.bat b/tools/run_tests/run_node.bat index ad9ca14b8b9..41777363568 100644 --- a/tools/run_tests/run_node.bat +++ b/tools/run_tests/run_node.bat @@ -27,6 +27,6 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set JUNIT_REPORT_PATH=src\node\reports.xml +set JUNIT_REPORT_PATH=src\node\report.xml set JUNIT_REPORT_STACK=1 .\node_modules\.bin\mocha.cmd --reporter mocha-jenkins-reporter --timeout 8000 src\node\test \ No newline at end of file diff --git a/tools/run_tests/run_node.sh b/tools/run_tests/run_node.sh index 178584ae8ed..d33890068dd 100755 --- a/tools/run_tests/run_node.sh +++ b/tools/run_tests/run_node.sh @@ -58,7 +58,7 @@ then echo '' > \ ../reports/node_coverage/index.html else - JUNIT_REPORT_PATH=src/node/reports.xml JUNIT_REPORT_STACK=1 \ + JUNIT_REPORT_PATH=src/node/report.xml JUNIT_REPORT_STACK=1 \ ./node_modules/.bin/mocha --timeout $timeout \ --reporter mocha-jenkins-reporter $test_directory fi From 63f2b5b2d04404687b20c0728a226d8296542ba2 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 24 Feb 2016 13:54:48 -0800 Subject: [PATCH 090/236] Fixed nits. --- src/objective-c/tests/GRPCClientTests.m | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 8ee6ffa1ec7..6554f06e90f 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -192,7 +192,6 @@ static ProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:8 handler:nil]; } -// TODO(jcanizales): Activate this test against the remote server. - (void)testMetadata { __weak XCTestExpectation *expectation = [self expectationWithDescription:@"RPC unauthorized."]; @@ -258,7 +257,7 @@ static ProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:8 handler:nil]; } -// todo(makaradd): Move to a different file that contains only unit tests +// TODO(makarandd): Move to a different file that contains only unit tests - (void)testExceptions { // Try to set userAgentPrefix for host that is nil. This should cause // an exception. From 54a3cc64b8367a5b5447e200338491e05d7607fe Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 24 Feb 2016 14:10:28 -0800 Subject: [PATCH 091/236] Reverted a change, removed empty test. --- src/objective-c/tests/InteropTestsLocalCleartext.m | 2 +- src/objective-c/tests/RxLibraryUnitTests.m | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/objective-c/tests/InteropTestsLocalCleartext.m b/src/objective-c/tests/InteropTestsLocalCleartext.m index f4542851104..56927a8af6d 100644 --- a/src/objective-c/tests/InteropTestsLocalCleartext.m +++ b/src/objective-c/tests/InteropTestsLocalCleartext.m @@ -49,7 +49,7 @@ static NSString * const kLocalCleartextHost = @"localhost:5050"; - (void)setUp { // Register test server as non-SSL. - [GRPCCall useInsecureConnectionsForHost:nil]; + [GRPCCall useInsecureConnectionsForHost:kLocalCleartextHost]; [super setUp]; } diff --git a/src/objective-c/tests/RxLibraryUnitTests.m b/src/objective-c/tests/RxLibraryUnitTests.m index ba79191dc48..c94dcf533a4 100644 --- a/src/objective-c/tests/RxLibraryUnitTests.m +++ b/src/objective-c/tests/RxLibraryUnitTests.m @@ -157,7 +157,4 @@ XCTAssertEqualObjects(handler.errorOrNil, nil); } -- (void)testBufferedPipeSetState { -} - @end From 1c2890d6fb559282692daffa5b3c2e65220cd5e3 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 24 Feb 2016 14:35:46 -0800 Subject: [PATCH 092/236] Order change of includes. --- src/objective-c/tests/GRPCClientTests.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 6554f06e90f..619f2cf56d5 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -35,13 +35,13 @@ #import #import +#import #import #import #import #import #import #import -#import static NSString * const kHostAddress = @"localhost:5050"; static NSString * const kPackage = @"grpc.testing"; From f30941cd95be22aa7108e2b0b14d657ab8c46856 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Wed, 24 Feb 2016 14:17:26 -0800 Subject: [PATCH 093/236] Injectable test credentials provider interface. --- test/cpp/util/test_credentials_provider.cc | 117 ++++++++++++++++----- test/cpp/util/test_credentials_provider.h | 16 +++ 2 files changed, 106 insertions(+), 27 deletions(-) diff --git a/test/cpp/util/test_credentials_provider.cc b/test/cpp/util/test_credentials_provider.cc index 1086e14258b..3efe3b48ba7 100644 --- a/test/cpp/util/test_credentials_provider.cc +++ b/test/cpp/util/test_credentials_provider.cc @@ -34,48 +34,111 @@ #include "test/cpp/util/test_credentials_provider.h" +#include "include/grpc/impl/codegen/sync_posix.h" +#include "include/grpc++/impl/codegen/sync_no_cxx11.h" #include "test/core/end2end/data/ssl_test_data.h" +namespace { + +using grpc::ChannelArguments; +using grpc::ChannelCredentials; +using grpc::InsecureChannelCredentials; +using grpc::InsecureServerCredentials; +using grpc::ServerCredentials; +using grpc::SslCredentialsOptions; +using grpc::SslServerCredentialsOptions; +using grpc::testing::CredentialsProvider; + +class DefaultCredentialsProvider : public CredentialsProvider { + public: + ~DefaultCredentialsProvider() override {} + + std::shared_ptr GetChannelCredentials( + const grpc::string& type, ChannelArguments* args) override { + if (type == grpc::testing::kInsecureCredentialsType) { + return InsecureChannelCredentials(); + } else if (type == grpc::testing::kTlsCredentialsType) { + SslCredentialsOptions ssl_opts = {test_root_cert, "", ""}; + args->SetSslTargetNameOverride("foo.test.google.fr"); + return SslCredentials(ssl_opts); + } else { + gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); + } + return nullptr; + } + + std::shared_ptr GetServerCredentials( + const grpc::string& type) override { + if (type == grpc::testing::kInsecureCredentialsType) { + return InsecureServerCredentials(); + } else if (type == grpc::testing::kTlsCredentialsType) { + SslServerCredentialsOptions::PemKeyCertPair pkcp = {test_server1_key, + test_server1_cert}; + SslServerCredentialsOptions ssl_opts; + ssl_opts.pem_root_certs = ""; + ssl_opts.pem_key_cert_pairs.push_back(pkcp); + return SslServerCredentials(ssl_opts); + } else { + gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); + } + return nullptr; + } + std::vector GetSecureCredentialsTypeList() override { + std::vector types; + types.push_back(grpc::testing::kTlsCredentialsType); + return types; + } +}; + +gpr_once g_once_init_provider_mu = GPR_ONCE_INIT; +grpc::mutex* g_provider_mu; +CredentialsProvider* g_provider = nullptr; + +void InitProviderMu() { + g_provider_mu = new grpc::mutex; +} + +grpc::mutex& GetMu() { + gpr_once_init(&g_once_init_provider_mu, &InitProviderMu); + return *g_provider_mu; +} + +CredentialsProvider* GetProvider() { + grpc::unique_lock lock(GetMu()); + if (g_provider == nullptr) { + g_provider = new DefaultCredentialsProvider; + } + return g_provider; +} + +} // namespace + namespace grpc { namespace testing { -const char kTlsCredentialsType[] = "TLS_CREDENTIALS"; +// Note that it is not thread-safe to set a provider while concurrently using +// the previously set provider, as this deletes and replaces it. nullptr may be +// given to reset to the default. +void SetTestCredentialsProvider(std::unique_ptr provider) { + grpc::unique_lock lock(GetMu()); + if (g_provider != nullptr) { + delete g_provider; + } + g_provider = provider.release(); +} std::shared_ptr GetChannelCredentials( const grpc::string& type, ChannelArguments* args) { - if (type == kInsecureCredentialsType) { - return InsecureChannelCredentials(); - } else if (type == kTlsCredentialsType) { - SslCredentialsOptions ssl_opts = {test_root_cert, "", ""}; - args->SetSslTargetNameOverride("foo.test.google.fr"); - return SslCredentials(ssl_opts); - } else { - gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); - } - return nullptr; + return GetProvider()->GetChannelCredentials(type, args); } std::shared_ptr GetServerCredentials( const grpc::string& type) { - if (type == kInsecureCredentialsType) { - return InsecureServerCredentials(); - } else if (type == kTlsCredentialsType) { - SslServerCredentialsOptions::PemKeyCertPair pkcp = {test_server1_key, - test_server1_cert}; - SslServerCredentialsOptions ssl_opts; - ssl_opts.pem_root_certs = ""; - ssl_opts.pem_key_cert_pairs.push_back(pkcp); - return SslServerCredentials(ssl_opts); - } else { - gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); - } - return nullptr; + return GetProvider()->GetServerCredentials(type); } std::vector GetSecureCredentialsTypeList() { - std::vector types; - types.push_back(kTlsCredentialsType); - return types; + return GetProvider()->GetSecureCredentialsTypeList(); } } // namespace testing diff --git a/test/cpp/util/test_credentials_provider.h b/test/cpp/util/test_credentials_provider.h index f7253051a97..a6b547cb070 100644 --- a/test/cpp/util/test_credentials_provider.h +++ b/test/cpp/util/test_credentials_provider.h @@ -44,6 +44,22 @@ namespace grpc { namespace testing { const char kInsecureCredentialsType[] = "INSECURE_CREDENTIALS"; +const char kTlsCredentialsType[] = "TLS_CREDENTIALS"; + +class CredentialsProvider { + public: + virtual ~CredentialsProvider() {} + + virtual std::shared_ptr GetChannelCredentials( + const grpc::string& type, ChannelArguments* args) = 0; + virtual std::shared_ptr GetServerCredentials( + const grpc::string& type) = 0; + virtual std::vector GetSecureCredentialsTypeList() = 0; +}; + +// Set the CredentialsProvider used by the other functions in this file. If this +// is not set, a default provider will be used. +void SetTestCredentialsProvider(std::unique_ptr provider); // Provide channel credentials according to the given type. Alter the channel // arguments if needed. From 7275f364119db1d6af0f9487aabc05d84eb8e834 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Wed, 24 Feb 2016 15:09:16 -0800 Subject: [PATCH 094/236] Initialize mutex to nullptr. --- test/cpp/util/test_credentials_provider.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/util/test_credentials_provider.cc b/test/cpp/util/test_credentials_provider.cc index 3efe3b48ba7..5c9816f8537 100644 --- a/test/cpp/util/test_credentials_provider.cc +++ b/test/cpp/util/test_credentials_provider.cc @@ -91,7 +91,7 @@ class DefaultCredentialsProvider : public CredentialsProvider { }; gpr_once g_once_init_provider_mu = GPR_ONCE_INIT; -grpc::mutex* g_provider_mu; +grpc::mutex* g_provider_mu = nullptr; CredentialsProvider* g_provider = nullptr; void InitProviderMu() { From 2715a39a2e7b5a47ff6726daadeac63c4664550e Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 24 Feb 2016 12:01:52 -0800 Subject: [PATCH 095/236] Change RUN_ID type to string to allow for a non-numeric run_id --- tools/big_query/big_query_utils.py | 2 +- tools/gke/run_stress_tests_on_gke.py | 103 ++++++++++++------ tools/run_tests/stress_test/run_server.py | 5 + .../stress_test/stress_test_utils.py | 17 +-- 4 files changed, 85 insertions(+), 42 deletions(-) diff --git a/tools/big_query/big_query_utils.py b/tools/big_query/big_query_utils.py index 267d0198509..e2379fd1aa7 100755 --- a/tools/big_query/big_query_utils.py +++ b/tools/big_query/big_query_utils.py @@ -135,6 +135,6 @@ def sync_query_job(big_query, project_id, query, timeout=5000): # List of (column name, column type, description) tuples def make_row(unique_row_id, row_values_dict): - """row_values_dict is a dictionar of column name and column value. + """row_values_dict is a dictionary of column name and column value. """ return {'insertId': unique_row_id, 'json': row_values_dict} diff --git a/tools/gke/run_stress_tests_on_gke.py b/tools/gke/run_stress_tests_on_gke.py index d0c3887a422..0ea7b7fcc10 100755 --- a/tools/gke/run_stress_tests_on_gke.py +++ b/tools/gke/run_stress_tests_on_gke.py @@ -33,11 +33,17 @@ import subprocess import sys import time +stress_test_utils_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../run_tests/stress_test')) +sys.path.append(stress_test_utils_dir) +from stress_test_utils import BigQueryHelper + import kubernetes_api GRPC_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(GRPC_ROOT) + class BigQuerySettings: def __init__(self, run_id, dataset_id, summary_table_id, qps_table_id): @@ -283,27 +289,16 @@ def _launch_client(gcp_project_id, docker_image_name, bq_settings, return True -def _launch_server_and_client(gcp_project_id, docker_image_name, +def _launch_server_and_client(bq_settings, gcp_project_id, docker_image_name, num_client_instances): - # == Big Query tables related settings (Common for both server and client) == - - # Create a unique id for this run (Note: Using timestamp instead of UUID to - # make it easier to deduce the date/time of the run just by looking at the run - # run id. This is useful in debugging when looking at records in Biq query) - run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') - - dataset_id = 'stress_test_%s' % run_id - summary_table_id = 'summary' - qps_table_id = 'qps' - - bq_settings = BigQuerySettings(run_id, dataset_id, summary_table_id, - qps_table_id) - # Start kubernetes proxy kubernetes_api_port = 9001 kubernetes_proxy = KubernetesProxy(kubernetes_api_port) kubernetes_proxy.start() + # num of seconds to wait for the GKE image to start and warmup + image_warmp_secs = 60 + server_pod_name = 'stress-server' server_port = 8080 is_success = _launch_server(gcp_project_id, docker_image_name, bq_settings, @@ -315,7 +310,8 @@ def _launch_server_and_client(gcp_project_id, docker_image_name, # Server takes a while to start. # TODO(sree) Use Kubernetes API to query the status of the server instead of # sleeping - time.sleep(60) + print 'Waiting for %s seconds for the server to start...' % image_warmp_secs + time.sleep(image_warmp_secs) # Launch client server_address = '%s.default.svc.cluster.local:%d' % (server_pod_name, @@ -329,6 +325,8 @@ def _launch_server_and_client(gcp_project_id, docker_image_name, print 'Error in launching client(s)' return False + print 'Waiting for %s seconds for the client images to start...' % image_warmp_secs + time.sleep(image_warmp_secs) return True @@ -359,31 +357,68 @@ def _build_and_push_docker_image(gcp_project_id, docker_image_name, tag_name): # TODO(sree): This is just to test the above APIs. Rewrite this to make # everything configurable (like image names / number of instances etc) -def test_run(): - image_name = 'grpc_stress_test' - gcp_project_id = 'sree-gce' - tag_name = 'gcr.io/%s/%s' % (gcp_project_id, image_name) - num_client_instances = 3 +def run_test(skip_building_image, gcp_project_id, image_name, tag_name, + num_client_instances, poll_interval_secs, total_duration_secs): + if not skip_building_image: + is_success = _build_docker_image(image_name, tag_name) + if not is_success: + return False - is_success = _build_docker_image(image_name, tag_name) - if not is_success: - return + is_success = _push_docker_image_to_gke_registry(tag_name) + if not is_success: + return False - is_success = _push_docker_image_to_gke_registry(tag_name) - if not is_success: - return + # == Big Query tables related settings (Common for both server and client) == + + # Create a unique id for this run (Note: Using timestamp instead of UUID to + # make it easier to deduce the date/time of the run just by looking at the run + # run id. This is useful in debugging when looking at records in Biq query) + run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + dataset_id = 'stress_test_%s' % run_id + summary_table_id = 'summary' + qps_table_id = 'qps' + bq_settings = BigQuerySettings(run_id, dataset_id, summary_table_id, + qps_table_id) - is_success = _launch_server_and_client(gcp_project_id, tag_name, + bq_helper = BigQueryHelper(run_id, '', '', gcp_project_id, dataset_id, + summary_table_id, qps_table_id) + bq_helper.initialize() + is_success = _launch_server_and_client(bq_settings, gcp_project_id, tag_name, num_client_instances) + if not is_success: + return False + + start_time = datetime.datetime.now() + end_time = start_time + datetime.timedelta(seconds=total_duration_secs) - # Run the test for 2 mins - time.sleep(120) + while True: + if datetime.datetime.now() > end_time: + print 'Test was run for %d seconds' % total_duration_secs + break - is_success = _delete_server_and_client(num_client_instances) + # Check if either stress server or clients have failed + if not bq_helper.check_if_any_tests_failed(): + is_success = False + print 'Some tests failed.' + break + # Things seem to be running fine. Wait until next poll time to check the + # status + time.sleep(poll_interval_secs) - if not is_success: - return + # Print BiqQuery tables + bq_helper.print_summary_records() + bq_helper.print_qps_records() + + _delete_server_and_client(num_client_instances) + return is_success if __name__ == '__main__': - test_run() + image_name = 'grpc_stress_test' + gcp_project_id = 'sree-gce' + tag_name = 'gcr.io/%s/%s' % (gcp_project_id, image_name) + num_client_instances = 3 + poll_interval_secs = 5, + test_duration_secs = 150 + run_test(True, gcp_project_id, image_name, tag_name, num_client_instances, + poll_interval_secs, test_duration_secs) diff --git a/tools/run_tests/stress_test/run_server.py b/tools/run_tests/stress_test/run_server.py index 9ad8d636385..64322f61004 100755 --- a/tools/run_tests/stress_test/run_server.py +++ b/tools/run_tests/stress_test/run_server.py @@ -72,6 +72,11 @@ def run_server(): logfile_name = env.get('LOGFILE_NAME') + print('pod_name: %s, project_id: %s, run_id: %s, dataset_id: %s, ' + 'summary_table_id: %s, qps_table_id: %s') % ( + pod_name, project_id, run_id, dataset_id, summary_table_id, + qps_table_id) + bq_helper = BigQueryHelper(run_id, image_type, pod_name, project_id, dataset_id, summary_table_id, qps_table_id) bq_helper.initialize() diff --git a/tools/run_tests/stress_test/stress_test_utils.py b/tools/run_tests/stress_test/stress_test_utils.py index a0626ce3aca..71f0dcd9211 100755 --- a/tools/run_tests/stress_test/stress_test_utils.py +++ b/tools/run_tests/stress_test/stress_test_utils.py @@ -43,11 +43,13 @@ bq_utils_dir = os.path.abspath(os.path.join( sys.path.append(bq_utils_dir) import big_query_utils as bq_utils + class EventType: STARTING = 'STARTING' SUCCESS = 'SUCCESS' FAILURE = 'FAILURE' + class BigQueryHelper: """Helper class for the stress test wrappers to interact with BigQuery. """ @@ -101,9 +103,9 @@ class BigQueryHelper: self.qps_table_id, [row]) def check_if_any_tests_failed(self, num_query_retries=3): - query = ('SELECT event_type FROM %s.%s WHERE run_id = %s AND ' + query = ('SELECT event_type FROM %s.%s WHERE run_id = \'%s\' AND ' 'event_type="%s"') % (self.dataset_id, self.summary_table_id, - self.run_id, EventType.FAILURE) + self.run_id, EventType.FAILURE) query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) page = self.bq.jobs().getQueryResults(**query_job['jobReference']).execute( num_retries=num_query_retries) @@ -119,7 +121,7 @@ class BigQueryHelper: print 'Run Id', self.run_id print line query = ('SELECT pod_name, image_type, event_type, event_date, details' - ' FROM %s.%s WHERE run_id = %s ORDER by event_date;') % ( + ' FROM %s.%s WHERE run_id = \'%s\' ORDER by event_date;') % ( self.dataset_id, self.summary_table_id, self.run_id) query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) @@ -147,8 +149,9 @@ class BigQueryHelper: print 'Run Id: ', self.run_id print line query = ( - 'SELECT pod_name, recorded_at, qps FROM %s.%s WHERE run_id = %s ORDER ' - 'by recorded_at;') % (self.dataset_id, self.qps_table_id, self.run_id) + 'SELECT pod_name, recorded_at, qps FROM %s.%s WHERE run_id = \'%s\' ' + 'ORDER by recorded_at;') % (self.dataset_id, self.qps_table_id, + self.run_id) query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) print '{:<25} {:30} {}'.format('Pod name', 'Recorded at', 'Qps') print line @@ -167,7 +170,7 @@ class BigQueryHelper: def __create_summary_table(self): summary_table_schema = [ - ('run_id', 'INTEGER', 'Test run id'), + ('run_id', 'STRING', 'Test run id'), ('image_type', 'STRING', 'Client or Server?'), ('pod_name', 'STRING', 'GKE pod hosting this image'), ('event_date', 'STRING', 'The date of this event'), @@ -182,7 +185,7 @@ class BigQueryHelper: def __create_qps_table(self): qps_table_schema = [ - ('run_id', 'INTEGER', 'Test run id'), + ('run_id', 'STRING', 'Test run id'), ('pod_name', 'STRING', 'GKE pod hosting this image'), ('recorded_at', 'STRING', 'Metrics recorded at time'), ('qps', 'INTEGER', 'Queries per second') From d14d1033879bbb3e8e757dfb39d78fba6d93e144 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 15:39:55 -0800 Subject: [PATCH 096/236] Use env-var for post git step --- .../tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template | 3 ++- tools/jenkins/docker_run.sh | 2 -- tools/jenkins/docker_run_tests.sh | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template index ba8f03862c8..49371aaa3bb 100644 --- a/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile.template @@ -40,7 +40,8 @@ RUN pip install argparse RUN wget ${openssl_fallback.base_uri + openssl_fallback.tarball} - ADD post-git-setup.sh / + + ENV POST_GIT_STEP tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh <%include file="../../run_tests_addons.include" args="skip_zookeeper=True"/> # Define the default command. diff --git a/tools/jenkins/docker_run.sh b/tools/jenkins/docker_run.sh index 850249a9a2b..df7b6571d7a 100755 --- a/tools/jenkins/docker_run.sh +++ b/tools/jenkins/docker_run.sh @@ -42,8 +42,6 @@ else cp -r "$EXTERNAL_GIT_ROOT/$RELATIVE_COPY_PATH"/* "/var/local/git/grpc/$RELATIVE_COPY_PATH" fi -[ -e /post-git-setup.sh ] && /post-git-setup.sh - if [ -x "$(command -v rvm)" ] then rvm use ruby-2.1 diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 178f2736418..1b93b1d4925 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -43,10 +43,10 @@ chown $(whoami) $XDG_CACHE_HOME mkdir -p /var/local/git git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc -[ -e /post-git-setup.sh ] && /post-git-setup.sh - mkdir -p reports +$POST_GIT_STEP + exit_code=0 $RUN_TESTS_COMMAND || exit_code=$? From 8b7876075d5840326fca99bc2239a0e54a1167c7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 15:41:37 -0800 Subject: [PATCH 097/236] huh --- tools/jenkins/docker_run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/jenkins/docker_run.sh b/tools/jenkins/docker_run.sh index df7b6571d7a..f04b1cfb55b 100755 --- a/tools/jenkins/docker_run.sh +++ b/tools/jenkins/docker_run.sh @@ -42,6 +42,8 @@ else cp -r "$EXTERNAL_GIT_ROOT/$RELATIVE_COPY_PATH"/* "/var/local/git/grpc/$RELATIVE_COPY_PATH" fi +$POST_GIT_STEP + if [ -x "$(command -v rvm)" ] then rvm use ruby-2.1 From 53a94ef4e7f7f71158da3286cf447ad36dcc9633 Mon Sep 17 00:00:00 2001 From: Dan Born Date: Wed, 24 Feb 2016 15:44:43 -0800 Subject: [PATCH 098/236] Use standard include paths. --- test/cpp/util/test_credentials_provider.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/cpp/util/test_credentials_provider.cc b/test/cpp/util/test_credentials_provider.cc index 5c9816f8537..cfd3ebbb111 100644 --- a/test/cpp/util/test_credentials_provider.cc +++ b/test/cpp/util/test_credentials_provider.cc @@ -34,8 +34,9 @@ #include "test/cpp/util/test_credentials_provider.h" -#include "include/grpc/impl/codegen/sync_posix.h" -#include "include/grpc++/impl/codegen/sync_no_cxx11.h" +#include +#include + #include "test/core/end2end/data/ssl_test_data.h" namespace { From ec8bcb8c7cbb377784efede0315855d90d45ed18 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 15:44:53 -0800 Subject: [PATCH 099/236] Actually update dockerfile --- tools/dockerfile/test/cxx_squeeze_x64/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile b/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile index 35a3131d6c8..b7f95aaa8d9 100644 --- a/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_squeeze_x64/Dockerfile @@ -70,7 +70,8 @@ RUN apt-get update && apt-get -y install python-pip && apt-get clean RUN pip install argparse RUN wget http://openssl.org/source/openssl-1.0.2f.tar.gz -ADD post-git-setup.sh / + +ENV POST_GIT_STEP tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc From d734167648a197f12ac2ece7d6e8083709e3e50f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 15:49:17 -0800 Subject: [PATCH 100/236] Fix bug --- src/core/support/env_linux.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/support/env_linux.c b/src/core/support/env_linux.c index 365f9673be2..5863ad5682f 100644 --- a/src/core/support/env_linux.c +++ b/src/core/support/env_linux.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include From 8bcbee815c5d7aa6cda13371241536db6dc03fbb Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 24 Feb 2016 15:52:54 -0800 Subject: [PATCH 101/236] Fix a bug in failure check --- tools/gke/run_stress_tests_on_gke.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/gke/run_stress_tests_on_gke.py b/tools/gke/run_stress_tests_on_gke.py index 0ea7b7fcc10..9a8a33cac5e 100755 --- a/tools/gke/run_stress_tests_on_gke.py +++ b/tools/gke/run_stress_tests_on_gke.py @@ -244,7 +244,7 @@ def _launch_client(gcp_project_id, docker_image_name, bq_settings, client_arg_list = [] # TODO(sree) Make this configurable (and also less frequent) - poll_interval_secs = 5 + poll_interval_secs = 30 metrics_port = 8081 metrics_server_address = 'localhost:%d' % metrics_port @@ -397,7 +397,7 @@ def run_test(skip_building_image, gcp_project_id, image_name, tag_name, break # Check if either stress server or clients have failed - if not bq_helper.check_if_any_tests_failed(): + if bq_helper.check_if_any_tests_failed(): is_success = False print 'Some tests failed.' break From 560c9017c4ce73322cc541eef421e607bd1856a0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 16:34:38 -0800 Subject: [PATCH 102/236] Faster code generation --- .gitignore | 1 + config.m4 | 10 +- package.json | 192 ++++++++++++++-------------- package.xml | 19 +-- tools/buildgen/generate_projects.py | 31 +++-- tools/buildgen/mako_renderer.py | 40 ++++-- 6 files changed, 160 insertions(+), 133 deletions(-) diff --git a/.gitignore b/.gitignore index 9a1dc008505..502483f4569 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ coverage # cache for run_tests.py .run_tests_cache +.preprocessed_build # emacs temp files *~ diff --git a/config.m4 b/config.m4 index 0323d5d2ad3..3a690c99f76 100644 --- a/config.m4 +++ b/config.m4 @@ -48,11 +48,9 @@ if test "$PHP_GRPC" != "no"; then src/core/support/env_linux.c \ src/core/support/env_posix.c \ src/core/support/env_win32.c \ - src/core/support/file.c \ - src/core/support/file_posix.c \ - src/core/support/file_win32.c \ src/core/support/histogram.c \ src/core/support/host_port.c \ + src/core/support/load_file.c \ src/core/support/log.c \ src/core/support/log_android.c \ src/core/support/log_linux.c \ @@ -78,6 +76,8 @@ if test "$PHP_GRPC" != "no"; then src/core/support/time_precise.c \ src/core/support/time_win32.c \ src/core/support/tls_pthread.c \ + src/core/support/tmpfile_posix.c \ + src/core/support/tmpfile_win32.c \ src/core/support/wrap_memcpy.c \ src/core/census/grpc_context.c \ src/core/census/grpc_filter.c \ @@ -109,7 +109,7 @@ if test "$PHP_GRPC" != "no"; then src/core/client_config/subchannel_factory.c \ src/core/client_config/subchannel_index.c \ src/core/client_config/uri_parser.c \ - src/core/compression/algorithm.c \ + src/core/compression/compression_algorithm.c \ src/core/compression/message_compress.c \ src/core/debug/trace.c \ src/core/httpcli/format_request.c \ @@ -210,7 +210,7 @@ if test "$PHP_GRPC" != "no"; then src/core/transport/transport.c \ src/core/transport/transport_op_string.c \ src/core/httpcli/httpcli_security_connector.c \ - src/core/security/base64.c \ + src/core/security/b64.c \ src/core/security/client_auth_filter.c \ src/core/security/credentials.c \ src/core/security/credentials_metadata.c \ diff --git a/package.json b/package.json index 4e7e83b02bf..db7f299e136 100644 --- a/package.json +++ b/package.json @@ -421,102 +421,6 @@ "third_party/zlib/trees.c", "third_party/zlib/uncompr.c", "third_party/zlib/zutil.c", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/atm_gcc_atomic.h", - "include/grpc/support/atm_gcc_sync.h", - "include/grpc/support/atm_win32.h", - "include/grpc/support/avl.h", - "include/grpc/support/cmdline.h", - "include/grpc/support/cpu.h", - "include/grpc/support/histogram.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/log_win32.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/subprocess.h", - "include/grpc/support/sync.h", - "include/grpc/support/sync_generic.h", - "include/grpc/support/sync_posix.h", - "include/grpc/support/sync_win32.h", - "include/grpc/support/thd.h", - "include/grpc/support/time.h", - "include/grpc/support/tls.h", - "include/grpc/support/tls_gcc.h", - "include/grpc/support/tls_msvc.h", - "include/grpc/support/tls_pthread.h", - "include/grpc/support/useful.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "src/core/profiling/timers.h", - "src/core/support/block_annotate.h", - "src/core/support/env.h", - "src/core/support/load_file.h", - "src/core/support/murmur_hash.h", - "src/core/support/stack_lockfree.h", - "src/core/support/string.h", - "src/core/support/string_win32.h", - "src/core/support/thd_internal.h", - "src/core/support/time_precise.h", - "src/core/support/tmpfile.h", - "src/core/profiling/basic_timers.c", - "src/core/profiling/stap_timers.c", - "src/core/support/alloc.c", - "src/core/support/avl.c", - "src/core/support/cmdline.c", - "src/core/support/cpu_iphone.c", - "src/core/support/cpu_linux.c", - "src/core/support/cpu_posix.c", - "src/core/support/cpu_windows.c", - "src/core/support/env_linux.c", - "src/core/support/env_posix.c", - "src/core/support/env_win32.c", - "src/core/support/histogram.c", - "src/core/support/host_port.c", - "src/core/support/load_file.c", - "src/core/support/log.c", - "src/core/support/log_android.c", - "src/core/support/log_linux.c", - "src/core/support/log_posix.c", - "src/core/support/log_win32.c", - "src/core/support/murmur_hash.c", - "src/core/support/slice.c", - "src/core/support/slice_buffer.c", - "src/core/support/stack_lockfree.c", - "src/core/support/string.c", - "src/core/support/string_posix.c", - "src/core/support/string_win32.c", - "src/core/support/subprocess_posix.c", - "src/core/support/subprocess_windows.c", - "src/core/support/sync.c", - "src/core/support/sync_posix.c", - "src/core/support/sync_win32.c", - "src/core/support/thd.c", - "src/core/support/thd_posix.c", - "src/core/support/thd_win32.c", - "src/core/support/time.c", - "src/core/support/time_posix.c", - "src/core/support/time_precise.c", - "src/core/support/time_win32.c", - "src/core/support/tls_pthread.c", - "src/core/support/tmpfile_posix.c", - "src/core/support/tmpfile_win32.c", - "src/core/support/wrap_memcpy.c", "third_party/boringssl/crypto/aes/internal.h", "third_party/boringssl/crypto/asn1/asn1_locl.h", "third_party/boringssl/crypto/bio/internal.h", @@ -918,6 +822,102 @@ "third_party/boringssl/ssl/t1_enc.c", "third_party/boringssl/ssl/t1_lib.c", "third_party/boringssl/ssl/tls_record.c", + "include/grpc/support/alloc.h", + "include/grpc/support/atm.h", + "include/grpc/support/atm_gcc_atomic.h", + "include/grpc/support/atm_gcc_sync.h", + "include/grpc/support/atm_win32.h", + "include/grpc/support/avl.h", + "include/grpc/support/cmdline.h", + "include/grpc/support/cpu.h", + "include/grpc/support/histogram.h", + "include/grpc/support/host_port.h", + "include/grpc/support/log.h", + "include/grpc/support/log_win32.h", + "include/grpc/support/port_platform.h", + "include/grpc/support/slice.h", + "include/grpc/support/slice_buffer.h", + "include/grpc/support/string_util.h", + "include/grpc/support/subprocess.h", + "include/grpc/support/sync.h", + "include/grpc/support/sync_generic.h", + "include/grpc/support/sync_posix.h", + "include/grpc/support/sync_win32.h", + "include/grpc/support/thd.h", + "include/grpc/support/time.h", + "include/grpc/support/tls.h", + "include/grpc/support/tls_gcc.h", + "include/grpc/support/tls_msvc.h", + "include/grpc/support/tls_pthread.h", + "include/grpc/support/useful.h", + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "src/core/profiling/timers.h", + "src/core/support/block_annotate.h", + "src/core/support/env.h", + "src/core/support/load_file.h", + "src/core/support/murmur_hash.h", + "src/core/support/stack_lockfree.h", + "src/core/support/string.h", + "src/core/support/string_win32.h", + "src/core/support/thd_internal.h", + "src/core/support/time_precise.h", + "src/core/support/tmpfile.h", + "src/core/profiling/basic_timers.c", + "src/core/profiling/stap_timers.c", + "src/core/support/alloc.c", + "src/core/support/avl.c", + "src/core/support/cmdline.c", + "src/core/support/cpu_iphone.c", + "src/core/support/cpu_linux.c", + "src/core/support/cpu_posix.c", + "src/core/support/cpu_windows.c", + "src/core/support/env_linux.c", + "src/core/support/env_posix.c", + "src/core/support/env_win32.c", + "src/core/support/histogram.c", + "src/core/support/host_port.c", + "src/core/support/load_file.c", + "src/core/support/log.c", + "src/core/support/log_android.c", + "src/core/support/log_linux.c", + "src/core/support/log_posix.c", + "src/core/support/log_win32.c", + "src/core/support/murmur_hash.c", + "src/core/support/slice.c", + "src/core/support/slice_buffer.c", + "src/core/support/stack_lockfree.c", + "src/core/support/string.c", + "src/core/support/string_posix.c", + "src/core/support/string_win32.c", + "src/core/support/subprocess_posix.c", + "src/core/support/subprocess_windows.c", + "src/core/support/sync.c", + "src/core/support/sync_posix.c", + "src/core/support/sync_win32.c", + "src/core/support/thd.c", + "src/core/support/thd_posix.c", + "src/core/support/thd_win32.c", + "src/core/support/time.c", + "src/core/support/time_posix.c", + "src/core/support/time_precise.c", + "src/core/support/time_win32.c", + "src/core/support/tls_pthread.c", + "src/core/support/tmpfile_posix.c", + "src/core/support/tmpfile_win32.c", + "src/core/support/wrap_memcpy.c", "binding.gyp" ], "main": "src/node/index.js", diff --git a/package.xml b/package.xml index 99109ada8fd..7e987d91c49 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ grpc-packages@google.com yes - 2016-02-23 + 2016-02-24 0.8.0 @@ -95,13 +95,14 @@ - + + @@ -114,11 +115,9 @@ - - - + @@ -144,6 +143,8 @@ + + @@ -274,7 +275,7 @@ - + @@ -324,7 +325,7 @@ - + @@ -425,7 +426,7 @@ - + @@ -988,7 +989,7 @@ Update to wrap gRPC C Core version 0.10.0 beta beta - 2016-02-23 + 2016-02-24 BSD - Simplify gRPC PHP installation #4517 diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 965dd292afb..0602d93e563 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -45,12 +45,12 @@ import jobset os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) argp = argparse.ArgumentParser() -argp.add_argument('json', nargs='+') +argp.add_argument('build_files', nargs='+', default=[]) argp.add_argument('--templates', nargs='+', default=[]) argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) args = argp.parse_args() -json = args.json +json = args.build_files test = {} if 'TEST' in os.environ else None @@ -62,21 +62,31 @@ if not templates: for f in files: templates.append(os.path.join(root, f)) +pre_jobs = [] +base_cmd = ['python2.7', 'tools/buildgen/mako_renderer.py'] +cmd = base_cmd[:] +for plugin in plugins: + cmd.append('-p') + cmd.append(plugin) +for js in json: + cmd.append('-d') + cmd.append(js) +cmd.append('-w') +preprocessed_build = '.preprocessed_build' +cmd.append(preprocessed_build) +pre_jobs.append(jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None)) + jobs = [] -for template in templates: +for template in reversed(sorted(templates)): root, f = os.path.split(template) if os.path.splitext(f)[1] == '.template': out_dir = '.' + root[len('templates'):] out = out_dir + '/' + os.path.splitext(f)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) - cmd = ['python2.7', 'tools/buildgen/mako_renderer.py'] - for plugin in plugins: - cmd.append('-p') - cmd.append(plugin) - for js in json: - cmd.append('-d') - cmd.append(js) + cmd = base_cmd[:] + cmd.append('-P') + cmd.append(preprocessed_build) cmd.append('-o') if test is None: cmd.append(out) @@ -88,6 +98,7 @@ for template in templates: cmd.append(root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) +jobset.run(pre_jobs, maxjobs=args.jobs) jobset.run(jobs, maxjobs=args.jobs) if test is not None: diff --git a/tools/buildgen/mako_renderer.py b/tools/buildgen/mako_renderer.py index f1b28d352e7..f629e68eb92 100755 --- a/tools/buildgen/mako_renderer.py +++ b/tools/buildgen/mako_renderer.py @@ -38,6 +38,7 @@ Just a wrapper around the mako rendering library. import getopt import imp import os +import cPickle as pickle import shutil import sys @@ -66,21 +67,23 @@ def out(msg): def showhelp(): - out('mako-renderer.py [-o out] [-m cache] [-d dict] [-d dict...] template') + out('mako-renderer.py [-o out] [-m cache] [-P preprocessed_input] [-d dict] [-d dict...]' + ' [-t template] [-w preprocessed_output]') def main(argv): got_input = False module_directory = None + preprocessed_output = None dictionary = {} json_dict = {} got_output = False - output_file = sys.stdout plugins = [] output_name = None + got_preprocessed_input = False try: - opts, args = getopt.getopt(argv, 'hm:d:o:p:') + opts, args = getopt.getopt(argv, 'hm:d:o:p:t:P:w:') except getopt.GetoptError: out('Unknown option') showhelp() @@ -104,18 +107,31 @@ def main(argv): showhelp() sys.exit(4) module_directory = arg + elif opt == '-P': + assert not got_preprocessed_input + assert json_dict == {} + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'plugins'))) + with open(arg, 'r') as dict_file: + dictionary = pickle.load(dict_file) + got_preprocessed_input = True elif opt == '-d': - dict_file = open(arg, 'r') - bunch.merge_json(json_dict, yaml.load(dict_file.read())) - dict_file.close() + assert not got_preprocessed_input + with open(arg, 'r') as dict_file: + bunch.merge_json(json_dict, yaml.load(dict_file.read())) elif opt == '-p': plugins.append(import_plugin(arg)) + elif opt == '-w': + preprocessed_output = arg - for plugin in plugins: - plugin.mako_plugin(json_dict) + if not got_preprocessed_input: + for plugin in plugins: + plugin.mako_plugin(json_dict) + for k, v in json_dict.items(): + dictionary[k] = bunch.to_bunch(v) - for k, v in json_dict.items(): - dictionary[k] = bunch.to_bunch(v) + if preprocessed_output: + with open(preprocessed_output, 'w') as dict_file: + pickle.dump(dictionary, dict_file) cleared_dir = False for arg in args: @@ -168,11 +184,9 @@ def main(argv): with open(item_output_name, 'w') as output_file: template.render_context(Context(output_file, **args)) - if not got_input: + if not got_input and not preprocessed_output: out('Got nothing to do') showhelp() - output_file.close() - if __name__ == '__main__': main(sys.argv[1:]) From 1b5a264eb80817a6fc362637a7a0497cbc0611ec Mon Sep 17 00:00:00 2001 From: Dan Born Date: Wed, 24 Feb 2016 18:52:39 -0800 Subject: [PATCH 103/236] Allow new credential types to be added to tests. --- test/cpp/util/test_credentials_provider.cc | 84 +++++++++++++++------- test/cpp/util/test_credentials_provider.h | 19 ++--- 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/test/cpp/util/test_credentials_provider.cc b/test/cpp/util/test_credentials_provider.cc index cfd3ebbb111..65d32057673 100644 --- a/test/cpp/util/test_credentials_provider.cc +++ b/test/cpp/util/test_credentials_provider.cc @@ -34,6 +34,8 @@ #include "test/cpp/util/test_credentials_provider.h" +#include + #include #include @@ -48,12 +50,36 @@ using grpc::InsecureServerCredentials; using grpc::ServerCredentials; using grpc::SslCredentialsOptions; using grpc::SslServerCredentialsOptions; -using grpc::testing::CredentialsProvider; +using grpc::testing::CredentialTypeProvider; + +// Provide test credentials. Thread-safe. +class CredentialsProvider { + public: + virtual ~CredentialsProvider() {} + + virtual void AddSecureType( + const grpc::string& type, + std::unique_ptr type_provider) = 0; + virtual std::shared_ptr GetChannelCredentials( + const grpc::string& type, ChannelArguments* args) = 0; + virtual std::shared_ptr GetServerCredentials( + const grpc::string& type) = 0; + virtual std::vector GetSecureCredentialsTypeList() = 0; +}; class DefaultCredentialsProvider : public CredentialsProvider { public: ~DefaultCredentialsProvider() override {} + void AddSecureType( + const grpc::string& type, + std::unique_ptr type_provider) override { + // This clobbers any existing entry for type, except the defaults, which + // can't be clobbered. + grpc::unique_lock lock(mu_); + added_secure_types_[type] = std::move(type_provider); + } + std::shared_ptr GetChannelCredentials( const grpc::string& type, ChannelArguments* args) override { if (type == grpc::testing::kInsecureCredentialsType) { @@ -63,9 +89,14 @@ class DefaultCredentialsProvider : public CredentialsProvider { args->SetSslTargetNameOverride("foo.test.google.fr"); return SslCredentials(ssl_opts); } else { - gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); + grpc::unique_lock lock(mu_); + auto it(added_secure_types_.find(type)); + if (it == added_secure_types_.end()) { + gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); + return nullptr; + } + return it->second->GetChannelCredentials(args); } - return nullptr; } std::shared_ptr GetServerCredentials( @@ -80,35 +111,40 @@ class DefaultCredentialsProvider : public CredentialsProvider { ssl_opts.pem_key_cert_pairs.push_back(pkcp); return SslServerCredentials(ssl_opts); } else { - gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); + grpc::unique_lock lock(mu_); + auto it(added_secure_types_.find(type)); + if (it == added_secure_types_.end()) { + gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); + return nullptr; + } + return it->second->GetServerCredentials(); } - return nullptr; } std::vector GetSecureCredentialsTypeList() override { std::vector types; types.push_back(grpc::testing::kTlsCredentialsType); + grpc::unique_lock lock(mu_); + for (const auto& type_pair : added_secure_types_) { + types.push_back(type_pair.first); + } return types; } + + private: + grpc::mutex mu_; + std::unordered_map > + added_secure_types_; }; -gpr_once g_once_init_provider_mu = GPR_ONCE_INIT; -grpc::mutex* g_provider_mu = nullptr; +gpr_once g_once_init_provider = GPR_ONCE_INIT; CredentialsProvider* g_provider = nullptr; -void InitProviderMu() { - g_provider_mu = new grpc::mutex; -} - -grpc::mutex& GetMu() { - gpr_once_init(&g_once_init_provider_mu, &InitProviderMu); - return *g_provider_mu; +void CreateDefaultProvider() { + g_provider = new DefaultCredentialsProvider; } CredentialsProvider* GetProvider() { - grpc::unique_lock lock(GetMu()); - if (g_provider == nullptr) { - g_provider = new DefaultCredentialsProvider; - } + gpr_once_init(&g_once_init_provider, &CreateDefaultProvider); return g_provider; } @@ -117,15 +153,9 @@ CredentialsProvider* GetProvider() { namespace grpc { namespace testing { -// Note that it is not thread-safe to set a provider while concurrently using -// the previously set provider, as this deletes and replaces it. nullptr may be -// given to reset to the default. -void SetTestCredentialsProvider(std::unique_ptr provider) { - grpc::unique_lock lock(GetMu()); - if (g_provider != nullptr) { - delete g_provider; - } - g_provider = provider.release(); +void AddSecureType(const grpc::string& type, + std::unique_ptr type_provider) { + GetProvider()->AddSecureType(type, std::move(type_provider)); } std::shared_ptr GetChannelCredentials( diff --git a/test/cpp/util/test_credentials_provider.h b/test/cpp/util/test_credentials_provider.h index a6b547cb070..50fadb53a24 100644 --- a/test/cpp/util/test_credentials_provider.h +++ b/test/cpp/util/test_credentials_provider.h @@ -46,20 +46,21 @@ namespace testing { const char kInsecureCredentialsType[] = "INSECURE_CREDENTIALS"; const char kTlsCredentialsType[] = "TLS_CREDENTIALS"; -class CredentialsProvider { +// Provide test credentials of a particular type. +class CredentialTypeProvider { public: - virtual ~CredentialsProvider() {} + virtual ~CredentialTypeProvider() {} virtual std::shared_ptr GetChannelCredentials( - const grpc::string& type, ChannelArguments* args) = 0; - virtual std::shared_ptr GetServerCredentials( - const grpc::string& type) = 0; - virtual std::vector GetSecureCredentialsTypeList() = 0; + ChannelArguments* args) = 0; + virtual std::shared_ptr GetServerCredentials() = 0; }; -// Set the CredentialsProvider used by the other functions in this file. If this -// is not set, a default provider will be used. -void SetTestCredentialsProvider(std::unique_ptr provider); +// Add a secure type in addition to the defaults above +// (kInsecureCredentialsType, kTlsCredentialsType) that can be returned from the +// functions below. +void AddSecureType(const grpc::string& type, + std::unique_ptr type_provider); // Provide channel credentials according to the given type. Alter the channel // arguments if needed. From 80db5be7b515bc2f86c8cd2d6ce86f6aebf19f00 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Wed, 24 Feb 2016 21:35:56 -0800 Subject: [PATCH 104/236] fix bug with pecl install on mac --- build.yaml | 1 - config.m4 | 81 +++++++++++++++++++++----- package.xml | 26 --------- templates/config.m4.template | 14 +++++ tools/run_tests/artifact_targets.py | 12 ++-- tools/run_tests/distribtest_targets.py | 21 ++++--- 6 files changed, 101 insertions(+), 54 deletions(-) diff --git a/build.yaml b/build.yaml index 8fcd31ef752..651fe95f18d 100644 --- a/build.yaml +++ b/build.yaml @@ -2825,7 +2825,6 @@ php_config_m4: - grpc - gpr - boringssl - - z headers: - src/php/ext/grpc/byte_buffer.h - src/php/ext/grpc/call.h diff --git a/config.m4 b/config.m4 index 3a690c99f76..f09ddcb40ec 100644 --- a/config.m4 +++ b/config.m4 @@ -533,22 +533,73 @@ if test "$PHP_GRPC" != "no"; then third_party/boringssl/ssl/t1_enc.c \ third_party/boringssl/ssl/t1_lib.c \ third_party/boringssl/ssl/tls_record.c \ - third_party/zlib/adler32.c \ - third_party/zlib/compress.c \ - third_party/zlib/crc32.c \ - third_party/zlib/deflate.c \ - third_party/zlib/gzclose.c \ - third_party/zlib/gzlib.c \ - third_party/zlib/gzread.c \ - third_party/zlib/gzwrite.c \ - third_party/zlib/infback.c \ - third_party/zlib/inffast.c \ - third_party/zlib/inflate.c \ - third_party/zlib/inftrees.c \ - third_party/zlib/trees.c \ - third_party/zlib/uncompr.c \ - third_party/zlib/zutil.c \ , $ext_shared, , -Wall -Werror -std=c11 \ -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN \ -D_HAS_EXCEPTIONS=0 -DNOMINMAX) + + PHP_ADD_BUILD_DIR($ext_builddir/src/php/ext/grpc) + + PHP_ADD_BUILD_DIR($ext_builddir/src/boringssl) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/census) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/channel) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_config) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_config/lb_policies) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_config/resolvers) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/compression) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/debug) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/httpcli) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/iomgr) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/json) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/profiling) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/proto/grpc/lb/v0) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/security) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/support) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/surface) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/transport) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/transport/chttp2) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/aes) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/asn1) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/base64) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bio) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bn) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bn/asm) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/buf) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/bytestring) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/chacha) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/cipher) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/cmac) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/conf) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/curve25519) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/des) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/dh) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/digest) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/dsa) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/ec) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/ecdh) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/ecdsa) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/engine) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/err) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/evp) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/hkdf) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/hmac) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/lhash) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/md4) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/md5) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/modes) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/obj) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/pem) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/pkcs8) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/poly1305) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/rand) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/rc4) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/rsa) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/sha) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/stack) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/x509) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/x509v3) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/ssl) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/ssl/pqueue) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/nanopb) fi diff --git a/package.xml b/package.xml index 7e987d91c49..42a0361df0a 100644 --- a/package.xml +++ b/package.xml @@ -856,32 +856,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/templates/config.m4.template b/templates/config.m4.template index dbc12188dc0..5e73901efa9 100644 --- a/templates/config.m4.template +++ b/templates/config.m4.template @@ -41,4 +41,18 @@ , $ext_shared, , -Wall -Werror -std=c11 ${"\\"} -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN ${"\\"} -D_HAS_EXCEPTIONS=0 -DNOMINMAX) + + PHP_ADD_BUILD_DIR($ext_builddir/src/php/ext/grpc) + <% + dirs = {} + for lib in libs: + if lib.name in php_config_m4.get('deps', []): + for source in lib.src: + dirs[source[:source.rfind('/')]] = 1 + dirs = dirs.keys() + dirs.sort() + %> + % for dir in dirs: + PHP_ADD_BUILD_DIR($ext_builddir/${dir}) + % endfor fi diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 803d3d106b2..288a3f01542 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -254,10 +254,14 @@ class PHPArtifact: return [] def build_jobspec(self): - return create_docker_jobspec( - self.name, - 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), - 'tools/run_tests/build_artifact_php.sh') + if self.platform == 'linux': + return create_docker_jobspec( + self.name, + 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), + 'tools/run_tests/build_artifact_php.sh') + else: + return create_jobspec(self.name, + ['tools/run_tests/build_artifact_php.sh']) class ProtocArtifact: """Builds protoc and protoc-plugin artifacts""" diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index 0c02344d90d..933103f0a05 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -201,7 +201,7 @@ class RubyDistribTest(object): class PHPDistribTest(object): """Tests PHP package""" - def __init__(self, platform, arch, docker_suffix): + def __init__(self, platform, arch, docker_suffix=None): self.name = 'php_%s_%s_%s' % (platform, arch, docker_suffix) self.platform = platform self.arch = arch @@ -212,15 +212,19 @@ class PHPDistribTest(object): return [] def build_jobspec(self): - if not self.platform == 'linux': + if self.platform == 'linux': + return create_docker_jobspec(self.name, + 'tools/dockerfile/distribtest/php_%s_%s' % ( + self.docker_suffix, + self.arch), + 'test/distrib/php/run_distrib_test.sh') + elif self.platform == 'macos': + return create_jobspec(self.name, + ['test/distrib/php/run_distrib_test.sh'], + environ={'EXTERNAL_GIT_ROOT': '../../..'}) + else: raise Exception("Not supported yet.") - return create_docker_jobspec(self.name, - 'tools/dockerfile/distribtest/php_%s_%s' % ( - self.docker_suffix, - self.arch), - 'test/distrib/php/run_distrib_test.sh') - def __str__(self): return self.name @@ -271,6 +275,7 @@ def targets(): NodeDistribTest('macos', 'x64', None, '5'), NodeDistribTest('linux', 'x86', 'jessie', '4'), PHPDistribTest('linux', 'x64', 'jessie'), + PHPDistribTest('macos', 'x64'), ] + [ NodeDistribTest('linux', 'x64', os, version) for os in ('wheezy', 'jessie', 'ubuntu1204', 'ubuntu1404', From 6b57328ead62e8444eab96ca5ca00ebfeb72d8f1 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Wed, 24 Feb 2016 21:44:47 -0800 Subject: [PATCH 105/236] remove current date from package.xml; --- templates/package.xml.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/package.xml.template b/templates/package.xml.template index 455b002e64a..067c8839d5a 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -12,7 +12,7 @@ grpc-packages@google.com yes - <%! from time import strftime %>${"%Y-%m-%d" | strftime} + 2016-02-24 0.8.0 @@ -149,7 +149,7 @@ beta beta - ${"%Y-%m-%d" | strftime} + 2016-02-24 BSD - Simplify gRPC PHP installation #4517 From 4f4d37cbde484f7058a03339d85428087490259b Mon Sep 17 00:00:00 2001 From: Julien Boeuf Date: Wed, 24 Feb 2016 22:07:36 -0800 Subject: [PATCH 106/236] Have a dedicated server security connector. That was overdue and the handshake is now slightly different for clients (channels) and servers. --- src/core/httpcli/httpcli_security_connector.c | 14 +- src/core/security/client_auth_filter.c | 1 - src/core/security/credentials.c | 6 +- src/core/security/credentials.h | 6 +- src/core/security/handshake.c | 22 ++- src/core/security/handshake.h | 3 +- src/core/security/security_connector.c | 125 ++++++++++-------- src/core/security/security_connector.h | 64 ++++++--- src/core/security/server_secure_chttp2.c | 14 +- src/core/surface/secure_channel_create.c | 11 +- 10 files changed, 154 insertions(+), 112 deletions(-) diff --git a/src/core/httpcli/httpcli_security_connector.c b/src/core/httpcli/httpcli_security_connector.c index 41ad1de6c00..156961a377a 100644 --- a/src/core/httpcli/httpcli_security_connector.c +++ b/src/core/httpcli/httpcli_security_connector.c @@ -59,7 +59,7 @@ static void httpcli_ssl_destroy(grpc_security_connector *sc) { } static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx, - grpc_security_connector *sc, + grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, void *user_data) { @@ -78,8 +78,8 @@ static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx, tsi_result_to_string(result)); cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); } else { - grpc_do_security_handshake(exec_ctx, handshaker, sc, nonsecure_endpoint, cb, - user_data); + grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true, + nonsecure_endpoint, cb, user_data); } } @@ -103,7 +103,7 @@ static void httpcli_ssl_check_peer(grpc_exec_ctx *exec_ctx, } static grpc_security_connector_vtable httpcli_ssl_vtable = { - httpcli_ssl_destroy, httpcli_ssl_do_handshake, httpcli_ssl_check_peer}; + httpcli_ssl_destroy, httpcli_ssl_check_peer}; static grpc_security_status httpcli_ssl_channel_security_connector_create( const unsigned char *pem_root_certs, size_t pem_root_certs_size, @@ -121,7 +121,6 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create( memset(c, 0, sizeof(grpc_httpcli_ssl_channel_security_connector)); gpr_ref_init(&c->base.base.refcount, 1); - c->base.base.is_client_side = 1; c->base.base.vtable = &httpcli_ssl_vtable; if (secure_peer_name != NULL) { c->secure_peer_name = gpr_strdup(secure_peer_name); @@ -136,6 +135,7 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create( *sc = NULL; return GRPC_SECURITY_ERROR; } + c->base.do_handshake = httpcli_ssl_do_handshake; *sc = &c->base; return GRPC_SECURITY_OK; } @@ -180,8 +180,8 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg, GPR_ASSERT(httpcli_ssl_channel_security_connector_create( pem_root_certs, pem_root_certs_size, host, &sc) == GRPC_SECURITY_OK); - grpc_security_connector_do_handshake(exec_ctx, &sc->base, tcp, - on_secure_transport_setup_done, c); + grpc_channel_security_connector_do_handshake( + exec_ctx, sc, tcp, on_secure_transport_setup_done, c); GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli"); } diff --git a/src/core/security/client_auth_filter.c b/src/core/security/client_auth_filter.c index 57b367d00ff..332d4259d28 100644 --- a/src/core/security/client_auth_filter.c +++ b/src/core/security/client_auth_filter.c @@ -310,7 +310,6 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(auth_context != NULL); /* initialize members */ - GPR_ASSERT(sc->is_client_side); chand->security_connector = (grpc_channel_security_connector *)GRPC_SECURITY_CONNECTOR_REF( sc, "client_auth_filter"); diff --git a/src/core/security/credentials.c b/src/core/security/credentials.c index c58574bd6d8..b4fa616fa7e 100644 --- a/src/core/security/credentials.c +++ b/src/core/security/credentials.c @@ -166,7 +166,7 @@ void grpc_server_credentials_release(grpc_server_credentials *creds) { } grpc_security_status grpc_server_credentials_create_security_connector( - grpc_server_credentials *creds, grpc_security_connector **sc) { + grpc_server_credentials *creds, grpc_server_security_connector **sc) { if (creds == NULL || creds->vtable->create_security_connector == NULL) { gpr_log(GPR_ERROR, "Server credentials cannot create security context."); return GRPC_SECURITY_ERROR; @@ -298,7 +298,7 @@ static grpc_security_status ssl_create_security_connector( } static grpc_security_status ssl_server_create_security_connector( - grpc_server_credentials *creds, grpc_security_connector **sc) { + grpc_server_credentials *creds, grpc_server_security_connector **sc) { grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds; return grpc_ssl_server_security_connector_create(&c->config, sc); } @@ -894,7 +894,7 @@ static grpc_security_status fake_transport_security_create_security_connector( static grpc_security_status fake_transport_security_server_create_security_connector( - grpc_server_credentials *c, grpc_security_connector **sc) { + grpc_server_credentials *c, grpc_server_security_connector **sc) { *sc = grpc_fake_server_security_connector_create(); return GRPC_SECURITY_OK; } diff --git a/src/core/security/credentials.h b/src/core/security/credentials.h index 3cd652cd574..0de4cd94689 100644 --- a/src/core/security/credentials.h +++ b/src/core/security/credentials.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -234,7 +234,7 @@ grpc_refresh_token_credentials_create_from_auth_refresh_token( typedef struct { void (*destruct)(grpc_server_credentials *c); grpc_security_status (*create_security_connector)( - grpc_server_credentials *c, grpc_security_connector **sc); + grpc_server_credentials *c, grpc_server_security_connector **sc); } grpc_server_credentials_vtable; struct grpc_server_credentials { @@ -245,7 +245,7 @@ struct grpc_server_credentials { }; grpc_security_status grpc_server_credentials_create_security_connector( - grpc_server_credentials *creds, grpc_security_connector **sc); + grpc_server_credentials *creds, grpc_server_security_connector **sc); grpc_server_credentials *grpc_server_credentials_ref( grpc_server_credentials *creds); diff --git a/src/core/security/handshake.c b/src/core/security/handshake.c index a8b2fef6297..b5bb6667a71 100644 --- a/src/core/security/handshake.c +++ b/src/core/security/handshake.c @@ -33,6 +33,7 @@ #include "src/core/security/handshake.h" +#include #include #include "src/core/security/security_context.h" @@ -46,6 +47,7 @@ typedef struct { grpc_security_connector *connector; tsi_handshaker *handshaker; + bool is_client_side; unsigned char *handshake_buffer; size_t handshake_buffer_size; grpc_endpoint *wrapped_endpoint; @@ -67,9 +69,11 @@ static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx, void *setup, bool success); static void security_connector_remove_handshake(grpc_security_handshake *h) { + GPR_ASSERT(!h->is_client_side); grpc_security_connector_handshake_list *node; grpc_security_connector_handshake_list *tmp; - grpc_security_connector *sc = h->connector; + grpc_server_security_connector *sc = + (grpc_server_security_connector *)h->connector; gpr_mu_lock(&sc->mu); node = sc->handshaking_handshakes; if (node && node->handshake == h) { @@ -94,7 +98,7 @@ static void security_connector_remove_handshake(grpc_security_handshake *h) { static void security_handshake_done(grpc_exec_ctx *exec_ctx, grpc_security_handshake *h, int is_success) { - if (!h->connector->is_client_side) { + if (!h->is_client_side) { security_connector_remove_handshake(h); } if (is_success) { @@ -290,6 +294,7 @@ static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx, void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker, grpc_security_connector *connector, + bool is_client_side, grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, void *user_data) { @@ -298,6 +303,7 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx, memset(h, 0, sizeof(grpc_security_handshake)); h->handshaker = handshaker; h->connector = GRPC_SECURITY_CONNECTOR_REF(connector, "handshake"); + h->is_client_side = is_client_side; h->handshake_buffer_size = GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE; h->handshake_buffer = gpr_malloc(h->handshake_buffer_size); h->wrapped_endpoint = nonsecure_endpoint; @@ -310,13 +316,15 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx, gpr_slice_buffer_init(&h->left_overs); gpr_slice_buffer_init(&h->outgoing); gpr_slice_buffer_init(&h->incoming); - if (!connector->is_client_side) { + if (!is_client_side) { + grpc_server_security_connector *server_connector = + (grpc_server_security_connector *)connector; handshake_node = gpr_malloc(sizeof(grpc_security_connector_handshake_list)); handshake_node->handshake = h; - gpr_mu_lock(&connector->mu); - handshake_node->next = connector->handshaking_handshakes; - connector->handshaking_handshakes = handshake_node; - gpr_mu_unlock(&connector->mu); + gpr_mu_lock(&server_connector->mu); + handshake_node->next = server_connector->handshaking_handshakes; + server_connector->handshaking_handshakes = handshake_node; + gpr_mu_unlock(&server_connector->mu); } send_handshake_bytes_to_peer(exec_ctx, h); } diff --git a/src/core/security/handshake.h b/src/core/security/handshake.h index 44215d16ef0..db8b3749215 100644 --- a/src/core/security/handshake.h +++ b/src/core/security/handshake.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,6 +41,7 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker, grpc_security_connector *connector, + bool is_client_side, grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, void *user_data); diff --git a/src/core/security/security_connector.c b/src/core/security/security_connector.c index 51a99b4492d..33c62a20c20 100644 --- a/src/core/security/security_connector.c +++ b/src/core/security/security_connector.c @@ -33,6 +33,7 @@ #include "src/core/security/security_connector.h" +#include #include #include @@ -110,31 +111,39 @@ const tsi_peer_property *tsi_peer_get_property_by_name(const tsi_peer *peer, return NULL; } -void grpc_security_connector_shutdown(grpc_exec_ctx *exec_ctx, - grpc_security_connector *connector) { +void grpc_server_security_connector_shutdown( + grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector) { grpc_security_connector_handshake_list *tmp; - if (!connector->is_client_side) { - gpr_mu_lock(&connector->mu); - while (connector->handshaking_handshakes) { - tmp = connector->handshaking_handshakes; - grpc_security_handshake_shutdown( - exec_ctx, connector->handshaking_handshakes->handshake); - connector->handshaking_handshakes = tmp->next; - gpr_free(tmp); - } - gpr_mu_unlock(&connector->mu); + gpr_mu_lock(&connector->mu); + while (connector->handshaking_handshakes) { + tmp = connector->handshaking_handshakes; + grpc_security_handshake_shutdown( + exec_ctx, connector->handshaking_handshakes->handshake); + connector->handshaking_handshakes = tmp->next; + gpr_free(tmp); + } + gpr_mu_unlock(&connector->mu); +} + +void grpc_channel_security_connector_do_handshake( + grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, + grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, + void *user_data) { + if (sc == NULL || nonsecure_endpoint == NULL) { + cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); + } else { + sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, cb, user_data); } } -void grpc_security_connector_do_handshake(grpc_exec_ctx *exec_ctx, - grpc_security_connector *sc, - grpc_endpoint *nonsecure_endpoint, - grpc_security_handshake_done_cb cb, - void *user_data) { +void grpc_server_security_connector_do_handshake( + grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, + grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, + grpc_security_handshake_done_cb cb, void *user_data) { if (sc == NULL || nonsecure_endpoint == NULL) { cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); } else { - sc->vtable->do_handshake(exec_ctx, sc, nonsecure_endpoint, cb, user_data); + sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, cb, user_data); } } @@ -248,7 +257,8 @@ static void fake_channel_destroy(grpc_security_connector *sc) { } static void fake_server_destroy(grpc_security_connector *sc) { - gpr_mu_destroy(&sc->mu); + grpc_server_security_connector *c = (grpc_server_security_connector *)sc; + gpr_mu_destroy(&c->mu); gpr_free(sc); } @@ -298,49 +308,52 @@ static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx, } static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx, - grpc_security_connector *sc, + grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, void *user_data) { - grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), sc, - nonsecure_endpoint, cb, user_data); + grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), &sc->base, + true, nonsecure_endpoint, cb, user_data); } static void fake_server_do_handshake(grpc_exec_ctx *exec_ctx, - grpc_security_connector *sc, + grpc_server_security_connector *sc, + grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, void *user_data) { - grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), sc, - nonsecure_endpoint, cb, user_data); + grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base, + false, nonsecure_endpoint, cb, user_data); } static grpc_security_connector_vtable fake_channel_vtable = { - fake_channel_destroy, fake_channel_do_handshake, fake_check_peer}; + fake_channel_destroy, fake_check_peer}; -static grpc_security_connector_vtable fake_server_vtable = { - fake_server_destroy, fake_server_do_handshake, fake_check_peer}; +static grpc_security_connector_vtable fake_server_vtable = {fake_server_destroy, + fake_check_peer}; grpc_channel_security_connector *grpc_fake_channel_security_connector_create( grpc_call_credentials *request_metadata_creds) { grpc_channel_security_connector *c = gpr_malloc(sizeof(*c)); memset(c, 0, sizeof(*c)); gpr_ref_init(&c->base.refcount, 1); - c->base.is_client_side = 1; c->base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME; c->base.vtable = &fake_channel_vtable; c->request_metadata_creds = grpc_call_credentials_ref(request_metadata_creds); c->check_call_host = fake_channel_check_call_host; + c->do_handshake = fake_channel_do_handshake; return c; } -grpc_security_connector *grpc_fake_server_security_connector_create(void) { - grpc_security_connector *c = gpr_malloc(sizeof(grpc_security_connector)); - memset(c, 0, sizeof(grpc_security_connector)); - gpr_ref_init(&c->refcount, 1); - c->is_client_side = 0; - c->vtable = &fake_server_vtable; - c->url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME; +grpc_server_security_connector *grpc_fake_server_security_connector_create( + void) { + grpc_server_security_connector *c = + gpr_malloc(sizeof(grpc_server_security_connector)); + memset(c, 0, sizeof(*c)); + gpr_ref_init(&c->base.refcount, 1); + c->base.vtable = &fake_server_vtable; + c->base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME; + c->do_handshake = fake_server_do_handshake; gpr_mu_init(&c->mu); return c; } @@ -355,7 +368,7 @@ typedef struct { } grpc_ssl_channel_security_connector; typedef struct { - grpc_security_connector base; + grpc_server_security_connector base; tsi_ssl_handshaker_factory *handshaker_factory; } grpc_ssl_server_security_connector; @@ -378,12 +391,12 @@ static void ssl_server_destroy(grpc_security_connector *sc) { if (c->handshaker_factory != NULL) { tsi_ssl_handshaker_factory_destroy(c->handshaker_factory); } - gpr_mu_destroy(&sc->mu); + gpr_mu_destroy(&c->base.mu); gpr_free(sc); } static grpc_security_status ssl_create_handshaker( - tsi_ssl_handshaker_factory *handshaker_factory, int is_client, + tsi_ssl_handshaker_factory *handshaker_factory, bool is_client, const char *peer_name, tsi_handshaker **handshaker) { tsi_result result = TSI_OK; if (handshaker_factory == NULL) return GRPC_SECURITY_ERROR; @@ -398,7 +411,7 @@ static grpc_security_status ssl_create_handshaker( } static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx, - grpc_security_connector *sc, + grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, void *user_data) { @@ -406,20 +419,21 @@ static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx, (grpc_ssl_channel_security_connector *)sc; tsi_handshaker *handshaker; grpc_security_status status = ssl_create_handshaker( - c->handshaker_factory, 1, + c->handshaker_factory, true, c->overridden_target_name != NULL ? c->overridden_target_name : c->target_name, &handshaker); if (status != GRPC_SECURITY_OK) { cb(exec_ctx, user_data, status, NULL, NULL); } else { - grpc_do_security_handshake(exec_ctx, handshaker, sc, nonsecure_endpoint, cb, - user_data); + grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true, + nonsecure_endpoint, cb, user_data); } } static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx, - grpc_security_connector *sc, + grpc_server_security_connector *sc, + grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, void *user_data) { @@ -427,12 +441,12 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx, (grpc_ssl_server_security_connector *)sc; tsi_handshaker *handshaker; grpc_security_status status = - ssl_create_handshaker(c->handshaker_factory, 0, NULL, &handshaker); + ssl_create_handshaker(c->handshaker_factory, false, NULL, &handshaker); if (status != GRPC_SECURITY_OK) { cb(exec_ctx, user_data, status, NULL, NULL); } else { - grpc_do_security_handshake(exec_ctx, handshaker, sc, nonsecure_endpoint, cb, - user_data); + grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, false, + nonsecure_endpoint, cb, user_data); } } @@ -603,10 +617,10 @@ static void ssl_channel_check_call_host(grpc_exec_ctx *exec_ctx, } static grpc_security_connector_vtable ssl_channel_vtable = { - ssl_channel_destroy, ssl_channel_do_handshake, ssl_channel_check_peer}; + ssl_channel_destroy, ssl_channel_check_peer}; static grpc_security_connector_vtable ssl_server_vtable = { - ssl_server_destroy, ssl_server_do_handshake, ssl_server_check_peer}; + ssl_server_destroy, ssl_server_check_peer}; static gpr_slice compute_default_pem_root_certs_once(void) { gpr_slice result = gpr_empty_slice(); @@ -700,11 +714,11 @@ grpc_security_status grpc_ssl_channel_security_connector_create( gpr_ref_init(&c->base.base.refcount, 1); c->base.base.vtable = &ssl_channel_vtable; - c->base.base.is_client_side = 1; c->base.base.url_scheme = GRPC_SSL_URL_SCHEME; c->base.request_metadata_creds = grpc_call_credentials_ref(request_metadata_creds); c->base.check_call_host = ssl_channel_check_call_host; + c->base.do_handshake = ssl_channel_do_handshake; gpr_split_host_port(target_name, &c->target_name, &port); gpr_free(port); if (overridden_target_name != NULL) { @@ -735,7 +749,7 @@ error: } grpc_security_status grpc_ssl_server_security_connector_create( - const grpc_ssl_server_config *config, grpc_security_connector **sc) { + const grpc_ssl_server_config *config, grpc_server_security_connector **sc) { size_t num_alpn_protocols = grpc_chttp2_num_alpn_versions(); const unsigned char **alpn_protocol_strings = gpr_malloc(sizeof(const char *) * num_alpn_protocols); @@ -759,9 +773,9 @@ grpc_security_status grpc_ssl_server_security_connector_create( c = gpr_malloc(sizeof(grpc_ssl_server_security_connector)); memset(c, 0, sizeof(grpc_ssl_server_security_connector)); - gpr_ref_init(&c->base.refcount, 1); - c->base.url_scheme = GRPC_SSL_URL_SCHEME; - c->base.vtable = &ssl_server_vtable; + gpr_ref_init(&c->base.base.refcount, 1); + c->base.base.url_scheme = GRPC_SSL_URL_SCHEME; + c->base.base.vtable = &ssl_server_vtable; result = tsi_create_ssl_server_handshaker_factory( (const unsigned char **)config->pem_private_keys, config->pem_private_keys_sizes, @@ -774,11 +788,12 @@ grpc_security_status grpc_ssl_server_security_connector_create( if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); - ssl_server_destroy(&c->base); + ssl_server_destroy(&c->base.base); *sc = NULL; goto error; } gpr_mu_init(&c->base.mu); + c->base.do_handshake = ssl_server_do_handshake; *sc = &c->base; gpr_free((void *)alpn_protocol_strings); gpr_free(alpn_protocol_string_lengths); diff --git a/src/core/security/security_connector.h b/src/core/security/security_connector.h index 39df7821f0c..1e35d3f9b7c 100644 --- a/src/core/security/security_connector.h +++ b/src/core/security/security_connector.h @@ -36,6 +36,7 @@ #include #include "src/core/iomgr/endpoint.h" +#include "src/core/iomgr/tcp_server.h" #include "src/core/tsi/transport_security_interface.h" /* --- status enum. --- */ @@ -68,9 +69,6 @@ typedef void (*grpc_security_handshake_done_cb)( typedef struct { void (*destroy)(grpc_security_connector *sc); - void (*do_handshake)(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc, - grpc_endpoint *nonsecure_endpoint, - grpc_security_handshake_done_cb cb, void *user_data); void (*check_peer)(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc, tsi_peer peer, grpc_security_peer_check_cb cb, void *user_data); @@ -84,13 +82,7 @@ typedef struct grpc_security_connector_handshake_list { struct grpc_security_connector { const grpc_security_connector_vtable *vtable; gpr_refcount refcount; - int is_client_side; const char *url_scheme; - /* Used on server side only. */ - /* TODO(yangg): Create a grpc_server_security_connector with these. */ - gpr_mu mu; - grpc_security_connector_handshake_list *handshaking_handshakes; - const grpc_channel_args *channel_args; }; /* Refcounting. */ @@ -113,13 +105,6 @@ grpc_security_connector *grpc_security_connector_ref( void grpc_security_connector_unref(grpc_security_connector *policy); #endif -/* Handshake. */ -void grpc_security_connector_do_handshake(grpc_exec_ctx *exec_ctx, - grpc_security_connector *connector, - grpc_endpoint *nonsecure_endpoint, - grpc_security_handshake_done_cb cb, - void *user_data); - /* Check the peer. Callee takes ownership of the peer object. The callback will include the resulting auth_context. */ void grpc_security_connector_check_peer(grpc_exec_ctx *exec_ctx, @@ -128,9 +113,6 @@ void grpc_security_connector_check_peer(grpc_exec_ctx *exec_ctx, grpc_security_peer_check_cb cb, void *user_data); -void grpc_security_connector_shutdown(grpc_exec_ctx *exec_ctx, - grpc_security_connector *connector); - /* Util to encapsulate the connector in a channel arg. */ grpc_arg grpc_security_connector_to_arg(grpc_security_connector *sc); @@ -153,12 +135,16 @@ typedef void (*grpc_security_call_host_check_cb)(grpc_exec_ctx *exec_ctx, grpc_security_status status); struct grpc_channel_security_connector { - grpc_security_connector base; /* requires is_client_side to be non 0. */ + grpc_security_connector base; grpc_call_credentials *request_metadata_creds; void (*check_call_host)(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, const char *host, grpc_auth_context *auth_context, grpc_security_call_host_check_cb cb, void *user_data); + void (*do_handshake)(grpc_exec_ctx *exec_ctx, + grpc_channel_security_connector *sc, + grpc_endpoint *nonsecure_endpoint, + grpc_security_handshake_done_cb cb, void *user_data); }; /* Checks that the host that will be set for a call is acceptable. */ @@ -167,6 +153,39 @@ void grpc_channel_security_connector_check_call_host( const char *host, grpc_auth_context *auth_context, grpc_security_call_host_check_cb cb, void *user_data); +/* Handshake. */ +void grpc_channel_security_connector_do_handshake( + grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector, + grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, + void *user_data); + +/* --- server_security_connector object. --- + + A server security connector object represents away to configure the + underlying transport security mechanism on the server side. */ + +typedef struct grpc_server_security_connector grpc_server_security_connector; + +struct grpc_server_security_connector { + grpc_security_connector base; + gpr_mu mu; + grpc_security_connector_handshake_list *handshaking_handshakes; + const grpc_channel_args *channel_args; + void (*do_handshake)(grpc_exec_ctx *exec_ctx, + grpc_server_security_connector *sc, + grpc_tcp_server_acceptor *acceptor, + grpc_endpoint *nonsecure_endpoint, + grpc_security_handshake_done_cb cb, void *user_data); +}; + +void grpc_server_security_connector_do_handshake( + grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, + grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, + grpc_security_handshake_done_cb cb, void *user_data); + +void grpc_server_security_connector_shutdown( + grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector); + /* --- Creation security connectors. --- */ /* For TESTING ONLY! @@ -176,7 +195,8 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create( /* For TESTING ONLY! Creates a fake connector that emulates real server security. */ -grpc_security_connector *grpc_fake_server_security_connector_create(void); +grpc_server_security_connector *grpc_fake_server_security_connector_create( + void); /* Config for ssl clients. */ typedef struct { @@ -231,7 +251,7 @@ typedef struct { specific error code otherwise. */ grpc_security_status grpc_ssl_server_security_connector_create( - const grpc_ssl_server_config *config, grpc_security_connector **sc); + const grpc_ssl_server_config *config, grpc_server_security_connector **sc); /* Util. */ const tsi_peer_property *tsi_peer_get_property_by_name(const tsi_peer *peer, diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c index 84a883390c2..91547eb26e9 100644 --- a/src/core/security/server_secure_chttp2.c +++ b/src/core/security/server_secure_chttp2.c @@ -55,7 +55,7 @@ typedef struct grpc_server_secure_state { grpc_server *server; grpc_tcp_server *tcp; - grpc_security_connector *sc; + grpc_server_security_connector *sc; grpc_server_credentials *creds; int is_shutdown; gpr_mu mu; @@ -74,7 +74,7 @@ static void state_unref(grpc_server_secure_state *state) { gpr_mu_lock(&state->mu); gpr_mu_unlock(&state->mu); /* clean up */ - GRPC_SECURITY_CONNECTOR_UNREF(state->sc, "server"); + GRPC_SECURITY_CONNECTOR_UNREF(&state->sc->base, "server"); grpc_server_credentials_unref(state->creds); gpr_free(state); } @@ -130,8 +130,8 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp, grpc_tcp_server_acceptor *acceptor) { grpc_server_secure_state *state = statep; state_ref(state); - grpc_security_connector_do_handshake(exec_ctx, state->sc, tcp, - on_secure_handshake_done, state); + grpc_server_security_connector_do_handshake( + exec_ctx, state->sc, acceptor, tcp, on_secure_handshake_done, state); } /* Server callback: start listening on our ports */ @@ -148,7 +148,7 @@ static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, bool success) { state->destroy_callback->cb(exec_ctx, state->destroy_callback->cb_arg, success); } - grpc_security_connector_shutdown(exec_ctx, state->sc); + grpc_server_security_connector_shutdown(exec_ctx, state->sc); state_unref(state); } @@ -176,7 +176,7 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, int port_num = -1; int port_temp; grpc_security_status status = GRPC_SECURITY_ERROR; - grpc_security_connector *sc = NULL; + grpc_server_security_connector *sc = NULL; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GRPC_API_TRACE( @@ -256,7 +256,7 @@ error: grpc_tcp_server_unref(&exec_ctx, tcp); } else { if (sc) { - GRPC_SECURITY_CONNECTOR_UNREF(sc, "server"); + GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "server"); } if (state) { gpr_free(state); diff --git a/src/core/surface/secure_channel_create.c b/src/core/surface/secure_channel_create.c index 9c04426d872..aadfac4c91f 100644 --- a/src/core/surface/secure_channel_create.c +++ b/src/core/surface/secure_channel_create.c @@ -130,9 +130,9 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg, bool success) { connector *c = arg; - grpc_security_connector_do_handshake(exec_ctx, &c->security_connector->base, - c->connecting_endpoint, - on_secure_handshake_done, c); + grpc_channel_security_connector_do_handshake(exec_ctx, c->security_connector, + c->connecting_endpoint, + on_secure_handshake_done, c); } static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) { @@ -153,9 +153,8 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) { grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer, &c->initial_string_sent); } else { - grpc_security_connector_do_handshake(exec_ctx, - &c->security_connector->base, tcp, - on_secure_handshake_done, c); + grpc_channel_security_connector_do_handshake( + exec_ctx, c->security_connector, tcp, on_secure_handshake_done, c); } } else { memset(c->result, 0, sizeof(*c->result)); From ca62ff014b4a33f9ce6bf5f75786c1b282f48c60 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 22:22:57 -0800 Subject: [PATCH 107/236] Expand gtest suites into individual run_tests tests --- build.yaml | 22 +++ templates/tools/run_tests/tests.json.template | 3 +- tools/buildgen/build-cleaner.py | 2 +- tools/buildgen/plugins/expand_bin_attrs.py | 1 + tools/run_tests/run_tests.py | 34 ++++- tools/run_tests/tests.json | 130 ++++++++++++++++++ 6 files changed, 184 insertions(+), 8 deletions(-) diff --git a/build.yaml b/build.yaml index 8fcd31ef752..2ef7a03919f 100644 --- a/build.yaml +++ b/build.yaml @@ -1954,6 +1954,7 @@ targets: - linux - posix - name: alarm_cpp_test + gtest: true build: test language: c++ src: @@ -1966,6 +1967,7 @@ targets: - gpr_test_util - gpr - name: async_end2end_test + gtest: true build: test language: c++ src: @@ -2012,6 +2014,7 @@ targets: - linux - posix - name: auth_property_iterator_test + gtest: true build: test language: c++ src: @@ -2024,6 +2027,7 @@ targets: - gpr_test_util - gpr - name: channel_arguments_test + gtest: true build: test language: c++ src: @@ -2033,6 +2037,7 @@ targets: - grpc - gpr - name: cli_call_test + gtest: true build: test language: c++ src: @@ -2045,6 +2050,7 @@ targets: - gpr_test_util - gpr - name: client_crash_test + gtest: true cpu_cost: 0.1 build: test language: c++ @@ -2075,6 +2081,7 @@ targets: - gpr_test_util - gpr - name: credentials_test + gtest: true build: test language: c++ src: @@ -2084,6 +2091,7 @@ targets: - grpc - gpr - name: cxx_byte_buffer_test + gtest: true build: test language: c++ src: @@ -2095,6 +2103,7 @@ targets: - gpr_test_util - gpr - name: cxx_slice_test + gtest: true build: test language: c++ src: @@ -2106,6 +2115,7 @@ targets: - gpr_test_util - gpr - name: cxx_string_ref_test + gtest: true build: test language: c++ src: @@ -2113,6 +2123,7 @@ targets: deps: - grpc++ - name: cxx_time_test + gtest: true build: test language: c++ src: @@ -2124,6 +2135,7 @@ targets: - gpr_test_util - gpr - name: end2end_test + gtest: true cpu_cost: 0.5 build: test language: c++ @@ -2154,6 +2166,7 @@ targets: - linux - posix - name: generic_end2end_test + gtest: true build: test language: c++ src: @@ -2230,6 +2243,7 @@ targets: vs_config_type: Application vs_project_guid: '{069E9D05-B78B-4751-9252-D21EBAE7DE8E}' - name: grpclb_api_test + gtest: true build: test language: c++ src: @@ -2241,6 +2255,7 @@ targets: - grpc++ - grpc - name: hybrid_end2end_test + gtest: true build: test language: c++ src: @@ -2320,6 +2335,7 @@ targets: - gpr - grpc++_test_config - name: mock_test + gtest: true build: test language: c++ src: @@ -2455,6 +2471,7 @@ targets: - gpr - grpc++_test_config - name: secure_auth_context_test + gtest: true build: test language: c++ src: @@ -2484,6 +2501,7 @@ targets: - linux - posix - name: server_crash_test + gtest: true cpu_cost: 0.1 build: test language: c++ @@ -2514,6 +2532,7 @@ targets: - gpr_test_util - gpr - name: shutdown_test + gtest: true build: test language: c++ src: @@ -2537,6 +2556,7 @@ targets: - gpr_test_util - gpr - name: streaming_throughput_test + gtest: true build: test language: c++ src: @@ -2613,6 +2633,7 @@ targets: - linux - posix - name: thread_stress_test + gtest: true cpu_cost: 100 build: test language: c++ @@ -2626,6 +2647,7 @@ targets: - gpr_test_util - gpr - name: zookeeper_test + gtest: true build: test run: false language: c++ diff --git a/templates/tools/run_tests/tests.json.template b/templates/tools/run_tests/tests.json.template index 9a84783467a..5690874415a 100644 --- a/templates/tools/run_tests/tests.json.template +++ b/templates/tools/run_tests/tests.json.template @@ -3,11 +3,12 @@ <%! import json %> - + ${json.dumps([{"name": tgt.name, "language": tgt.language, "platforms": tgt.platforms, "ci_platforms": tgt.ci_platforms, + "gtest": tgt.gtest, "exclude_configs": tgt.get("exclude_configs", []), "args": [], "flaky": tgt.flaky, diff --git a/tools/buildgen/build-cleaner.py b/tools/buildgen/build-cleaner.py index 49a36441235..12054da238e 100755 --- a/tools/buildgen/build-cleaner.py +++ b/tools/buildgen/build-cleaner.py @@ -40,6 +40,7 @@ TEST = (os.environ.get('TEST', 'false') == 'true') _TOP_LEVEL_KEYS = ['settings', 'proto_deps', 'filegroups', 'libs', 'targets', 'vspackages'] _ELEM_KEYS = [ 'name', + 'gtest', 'cpu_cost', 'flaky', 'build', @@ -98,4 +99,3 @@ for filename in sys.argv[1:]: else: with open(filename, 'w') as f: f.write(output) - diff --git a/tools/buildgen/plugins/expand_bin_attrs.py b/tools/buildgen/plugins/expand_bin_attrs.py index 735c60ea995..c30df2ad892 100755 --- a/tools/buildgen/plugins/expand_bin_attrs.py +++ b/tools/buildgen/plugins/expand_bin_attrs.py @@ -52,6 +52,7 @@ def mako_plugin(dictionary): tgt['ci_platforms'] = sorted(tgt.get('ci_platforms', tgt['platforms'])) tgt['boringssl'] = tgt.get('boringssl', False) tgt['zlib'] = tgt.get('zlib', False) + tgt['gtest'] = tgt.get('gtest', False) libs = dictionary.get('libs') for lib in libs: diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 7b2bc537162..bdda3686743 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -160,12 +160,34 @@ class CLanguage(object): else: binary = 'bins/%s/%s' % (self.config.build_config, target['name']) if os.path.isfile(binary): - cmdline = [binary] + target['args'] - out.append(self.config.job_spec(cmdline, [binary], - shortname=' '.join(cmdline), - cpu_cost=target['cpu_cost'], - environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': - _ROOT + '/src/core/tsi/test_creds/ca.pem'})) + if 'gtest' in target and target['gtest']: + with open(os.devnull, 'w') as fnull: + tests = subprocess.check_output([binary, '--gtest_list_tests'], + stderr=fnull) + base = None + for line in tests.split('\n'): + i = line.find('#') + if i >= 0: line = line[:i] + if not line: continue + if line[0] != ' ': + base = line + else: + assert base is not None + assert line[1] == ' ' + test = base + line[2:] + cmdline = [binary] + ['--gtest_filter=%s' % test] + out.append(self.config.job_spec(cmdline, [binary], + shortname='%s:%s' % (binary, test), + cpu_cost=target['cpu_cost'], + environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': + _ROOT + '/src/core/tsi/test_creds/ca.pem'})) + else: + cmdline = [binary] + target['args'] + out.append(self.config.job_spec(cmdline, [binary], + shortname=' '.join(cmdline), + cpu_cost=target['cpu_cost'], + environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': + _ROOT + '/src/core/tsi/test_creds/ca.pem'})) elif self.args.regex == '.*' or self.platform == 'windows': print '\nWARNING: binary not found, skipping', binary return sorted(out) diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 742005c43ef..d91245cd06a 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -12,6 +12,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "alarm_test", "platforms": [ @@ -32,6 +33,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "algorithm_test", "platforms": [ @@ -52,6 +54,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "alloc_test", "platforms": [ @@ -72,6 +75,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "alpn_test", "platforms": [ @@ -92,6 +96,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "bin_encoder_test", "platforms": [ @@ -112,6 +117,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "census_context_test", "platforms": [ @@ -132,6 +138,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "channel_create_test", "platforms": [ @@ -152,6 +159,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "chttp2_hpack_encoder_test", "platforms": [ @@ -172,6 +180,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "chttp2_status_conversion_test", "platforms": [ @@ -192,6 +201,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "chttp2_stream_map_test", "platforms": [ @@ -212,6 +222,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "chttp2_varint_test", "platforms": [ @@ -232,6 +243,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "compression_test", "platforms": [ @@ -252,6 +264,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "dns_resolver_test", "platforms": [ @@ -271,6 +284,7 @@ "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "dualstack_socket_test", "platforms": [ @@ -290,6 +304,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "endpoint_pair_test", "platforms": [ @@ -309,6 +324,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "fd_conservation_posix_test", "platforms": [ @@ -327,6 +343,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "fd_posix_test", "platforms": [ @@ -345,6 +362,7 @@ "cpu_cost": 2, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "fling_stream_test", "platforms": [ @@ -363,6 +381,7 @@ "cpu_cost": 2, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "fling_test", "platforms": [ @@ -382,6 +401,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_avl_test", "platforms": [ @@ -402,6 +422,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_cmdline_test", "platforms": [ @@ -422,6 +443,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_cpu_test", "platforms": [ @@ -442,6 +464,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_env_test", "platforms": [ @@ -462,6 +485,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_histogram_test", "platforms": [ @@ -482,6 +506,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_host_port_test", "platforms": [ @@ -502,6 +527,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_load_file_test", "platforms": [ @@ -522,6 +548,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_log_test", "platforms": [ @@ -542,6 +569,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_slice_buffer_test", "platforms": [ @@ -562,6 +590,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_slice_test", "platforms": [ @@ -582,6 +611,7 @@ "cpu_cost": 10, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_stack_lockfree_test", "platforms": [ @@ -602,6 +632,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_string_test", "platforms": [ @@ -622,6 +653,7 @@ "cpu_cost": 10, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_sync_test", "platforms": [ @@ -642,6 +674,7 @@ "cpu_cost": 10, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_thd_test", "platforms": [ @@ -662,6 +695,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_time_test", "platforms": [ @@ -682,6 +716,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_tls_test", "platforms": [ @@ -702,6 +737,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "gpr_useful_test", "platforms": [ @@ -722,6 +758,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_auth_context_test", "platforms": [ @@ -742,6 +779,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_b64_test", "platforms": [ @@ -762,6 +800,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_byte_buffer_reader_test", "platforms": [ @@ -782,6 +821,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_channel_args_test", "platforms": [ @@ -802,6 +842,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_channel_stack_test", "platforms": [ @@ -822,6 +863,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_completion_queue_test", "platforms": [ @@ -842,6 +884,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_credentials_test", "platforms": [ @@ -862,6 +905,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_invalid_channel_args_test", "platforms": [ @@ -881,6 +925,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_json_token_test", "platforms": [ @@ -900,6 +945,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_jwt_verifier_test", "platforms": [ @@ -920,6 +966,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "grpc_security_connector_test", "platforms": [ @@ -940,6 +987,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "hpack_parser_test", "platforms": [ @@ -960,6 +1008,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "hpack_table_test", "platforms": [ @@ -980,6 +1029,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "httpcli_format_request_test", "platforms": [ @@ -1000,6 +1050,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "httpcli_parser_test", "platforms": [ @@ -1019,6 +1070,7 @@ "cpu_cost": 0.5, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "httpcli_test", "platforms": [ @@ -1035,6 +1087,7 @@ "cpu_cost": 0.5, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "httpscli_test", "platforms": [ @@ -1052,6 +1105,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "init_test", "platforms": [ @@ -1072,6 +1126,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "invalid_call_argument_test", "platforms": [ @@ -1092,6 +1147,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "json_rewrite_test", "platforms": [ @@ -1112,6 +1168,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "json_stream_error_test", "platforms": [ @@ -1132,6 +1189,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "json_test", "platforms": [ @@ -1152,6 +1210,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "lame_client_test", "platforms": [ @@ -1172,6 +1231,7 @@ "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "lb_policies_test", "platforms": [ @@ -1192,6 +1252,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "message_compress_test", "platforms": [ @@ -1212,6 +1273,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "mlog_test", "platforms": [ @@ -1232,6 +1294,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "multiple_server_queues_test", "platforms": [ @@ -1252,6 +1315,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "murmur_hash_test", "platforms": [ @@ -1272,6 +1336,7 @@ "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "no_server_test", "platforms": [ @@ -1292,6 +1357,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "resolve_address_test", "platforms": [ @@ -1312,6 +1378,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "secure_channel_create_test", "platforms": [ @@ -1332,6 +1399,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "secure_endpoint_test", "platforms": [ @@ -1352,6 +1420,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "server_chttp2_test", "platforms": [ @@ -1372,6 +1441,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "server_test", "platforms": [ @@ -1392,6 +1462,7 @@ "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "set_initial_connect_string_test", "platforms": [ @@ -1412,6 +1483,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "sockaddr_resolver_test", "platforms": [ @@ -1432,6 +1504,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "sockaddr_utils_test", "platforms": [ @@ -1451,6 +1524,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "socket_utils_test", "platforms": [ @@ -1469,6 +1543,7 @@ "cpu_cost": 0.5, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "tcp_client_posix_test", "platforms": [ @@ -1487,6 +1562,7 @@ "cpu_cost": 0.5, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "tcp_posix_test", "platforms": [ @@ -1505,6 +1581,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "tcp_server_posix_test", "platforms": [ @@ -1524,6 +1601,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "time_averaged_stats_test", "platforms": [ @@ -1544,6 +1622,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "timeout_encoding_test", "platforms": [ @@ -1564,6 +1643,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "timer_heap_test", "platforms": [ @@ -1584,6 +1664,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "timer_list_test", "platforms": [ @@ -1604,6 +1685,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "timers_test", "platforms": [ @@ -1624,6 +1706,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "transport_connectivity_state_test", "platforms": [ @@ -1644,6 +1727,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "transport_metadata_test", "platforms": [ @@ -1663,6 +1747,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "transport_security_test", "platforms": [ @@ -1681,6 +1766,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "udp_server_test", "platforms": [ @@ -1700,6 +1786,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "uri_parser_test", "platforms": [ @@ -1719,6 +1806,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "workqueue_test", "platforms": [ @@ -1738,6 +1826,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "alarm_cpp_test", "platforms": [ @@ -1758,6 +1847,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "async_end2end_test", "platforms": [ @@ -1777,6 +1867,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "async_streaming_ping_pong_test", "platforms": [ @@ -1795,6 +1886,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "async_unary_ping_pong_test", "platforms": [ @@ -1814,6 +1906,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "auth_property_iterator_test", "platforms": [ @@ -1834,6 +1927,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "channel_arguments_test", "platforms": [ @@ -1854,6 +1948,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "cli_call_test", "platforms": [ @@ -1873,6 +1968,7 @@ "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "client_crash_test", "platforms": [ @@ -1892,6 +1988,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "credentials_test", "platforms": [ @@ -1912,6 +2009,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "cxx_byte_buffer_test", "platforms": [ @@ -1932,6 +2030,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "cxx_slice_test", "platforms": [ @@ -1952,6 +2051,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "cxx_string_ref_test", "platforms": [ @@ -1972,6 +2072,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "cxx_time_test", "platforms": [ @@ -1992,6 +2093,7 @@ "cpu_cost": 0.5, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "end2end_test", "platforms": [ @@ -2011,6 +2113,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "generic_async_streaming_ping_pong_test", "platforms": [ @@ -2030,6 +2133,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "generic_end2end_test", "platforms": [ @@ -2050,6 +2154,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "grpclb_api_test", "platforms": [ @@ -2070,6 +2175,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "hybrid_end2end_test", "platforms": [ @@ -2089,6 +2195,7 @@ "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "interop_test", "platforms": [ @@ -2108,6 +2215,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "mock_test", "platforms": [ @@ -2127,6 +2235,7 @@ "cpu_cost": 10, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "qps_openloop_test", "platforms": [ @@ -2145,6 +2254,7 @@ "cpu_cost": 10, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "qps_test", "platforms": [ @@ -2164,6 +2274,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "secure_auth_context_test", "platforms": [ @@ -2183,6 +2294,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "secure_sync_unary_ping_pong_test", "platforms": [ @@ -2201,6 +2313,7 @@ "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "server_crash_test", "platforms": [ @@ -2220,6 +2333,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "shutdown_test", "platforms": [ @@ -2240,6 +2354,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "status_test", "platforms": [ @@ -2259,6 +2374,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "streaming_throughput_test", "platforms": [ @@ -2277,6 +2393,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "sync_streaming_ping_pong_test", "platforms": [ @@ -2295,6 +2412,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c++", "name": "sync_unary_ping_pong_test", "platforms": [ @@ -2314,6 +2432,7 @@ "cpu_cost": 100, "exclude_configs": [], "flaky": false, + "gtest": true, "language": "c++", "name": "thread_stress_test", "platforms": [ @@ -2334,6 +2453,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c89", "name": "public_headers_must_be_c89", "platforms": [ @@ -2354,6 +2474,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "badreq_bad_client_test", "platforms": [ @@ -2374,6 +2495,7 @@ "cpu_cost": 0.2, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "connection_prefix_bad_client_test", "platforms": [ @@ -2394,6 +2516,7 @@ "cpu_cost": 0.2, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "headers_bad_client_test", "platforms": [ @@ -2414,6 +2537,7 @@ "cpu_cost": 0.2, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "initial_settings_frame_bad_client_test", "platforms": [ @@ -2434,6 +2558,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "server_registered_method_bad_client_test", "platforms": [ @@ -2454,6 +2579,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "simple_request_bad_client_test", "platforms": [ @@ -2474,6 +2600,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "unknown_frame_bad_client_test", "platforms": [ @@ -2494,6 +2621,7 @@ "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "window_overflow_bad_client_test", "platforms": [ @@ -2513,6 +2641,7 @@ "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "bad_ssl_alpn_test", "platforms": [ @@ -2531,6 +2660,7 @@ "cpu_cost": 0.1, "exclude_configs": [], "flaky": false, + "gtest": false, "language": "c", "name": "bad_ssl_cert_test", "platforms": [ From 0488d14c5516c83b56cab29a34f0df00826508ab Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 22:34:48 -0800 Subject: [PATCH 108/236] Add comment --- tools/run_tests/run_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index bdda3686743..49c9b2dc58d 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -161,6 +161,10 @@ class CLanguage(object): binary = 'bins/%s/%s' % (self.config.build_config, target['name']) if os.path.isfile(binary): if 'gtest' in target and target['gtest']: + # here we parse the output of --gtest_list_tests to build up a + # complete list of the tests contained in a binary + # for each test, we then add a job to run, filtering for just that + # test with open(os.devnull, 'w') as fnull: tests = subprocess.check_output([binary, '--gtest_list_tests'], stderr=fnull) From 7204010c05f1e9ac6c053fd2ad03c39302ba8472 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Thu, 25 Feb 2016 02:55:16 +0100 Subject: [PATCH 109/236] Fixing format and copyright. --- package.xml | 4 +-- src/objective-c/tests/GRPCClientTests.m | 2 +- src/objective-c/tests/InteropTestsLocalSSL.m | 2 +- src/objective-c/tests/RxLibraryUnitTests.m | 2 +- test/cpp/util/test_credentials_provider.cc | 4 +-- test/distrib/php/distribtest.php | 32 ++++++++++++++++++++ tools/buildgen/mako_renderer.py | 2 +- 7 files changed, 39 insertions(+), 9 deletions(-) diff --git a/package.xml b/package.xml index 7e987d91c49..ba9f9a09f68 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ grpc-packages@google.com yes - 2016-02-24 + 2016-02-25 0.8.0 @@ -989,7 +989,7 @@ Update to wrap gRPC C Core version 0.10.0 beta beta - 2016-02-24 + 2016-02-25 BSD - Simplify gRPC PHP installation #4517 diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 619f2cf56d5..9a3e5b5009b 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/objective-c/tests/InteropTestsLocalSSL.m b/src/objective-c/tests/InteropTestsLocalSSL.m index f0f4b1d71f0..155e334ecb2 100644 --- a/src/objective-c/tests/InteropTestsLocalSSL.m +++ b/src/objective-c/tests/InteropTestsLocalSSL.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/objective-c/tests/RxLibraryUnitTests.m b/src/objective-c/tests/RxLibraryUnitTests.m index c94dcf533a4..d3426628141 100644 --- a/src/objective-c/tests/RxLibraryUnitTests.m +++ b/src/objective-c/tests/RxLibraryUnitTests.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/cpp/util/test_credentials_provider.cc b/test/cpp/util/test_credentials_provider.cc index cfd3ebbb111..7e1eb0d501c 100644 --- a/test/cpp/util/test_credentials_provider.cc +++ b/test/cpp/util/test_credentials_provider.cc @@ -95,9 +95,7 @@ gpr_once g_once_init_provider_mu = GPR_ONCE_INIT; grpc::mutex* g_provider_mu = nullptr; CredentialsProvider* g_provider = nullptr; -void InitProviderMu() { - g_provider_mu = new grpc::mutex; -} +void InitProviderMu() { g_provider_mu = new grpc::mutex; } grpc::mutex& GetMu() { gpr_once_init(&g_once_init_provider_mu, &InitProviderMu); diff --git a/test/distrib/php/distribtest.php b/test/distrib/php/distribtest.php index 318c6f9cf08..4c34cd674b4 100644 --- a/test/distrib/php/distribtest.php +++ b/test/distrib/php/distribtest.php @@ -1,4 +1,36 @@ Grpc\ChannelCredentials::createInsecure() diff --git a/tools/buildgen/mako_renderer.py b/tools/buildgen/mako_renderer.py index f629e68eb92..5f23f123c2d 100755 --- a/tools/buildgen/mako_renderer.py +++ b/tools/buildgen/mako_renderer.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From 184e423fa4f3fdd00f0e435b731936f6b1d400da Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 24 Feb 2016 22:48:56 -0800 Subject: [PATCH 110/236] Fix bug --- tools/run_tests/run_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 49c9b2dc58d..75de4cb71d6 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -174,11 +174,11 @@ class CLanguage(object): if i >= 0: line = line[:i] if not line: continue if line[0] != ' ': - base = line + base = line.strip() else: assert base is not None assert line[1] == ' ' - test = base + line[2:] + test = base + line.strip() cmdline = [binary] + ['--gtest_filter=%s' % test] out.append(self.config.job_spec(cmdline, [binary], shortname='%s:%s' % (binary, test), From 12e600977c0fac27be664facb98ac81db9300f35 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Thu, 25 Feb 2016 08:01:27 +0100 Subject: [PATCH 111/236] clang-format all the things. --- src/core/support/env_linux.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/support/env_linux.c b/src/core/support/env_linux.c index 5863ad5682f..fe51f846b71 100644 --- a/src/core/support/env_linux.c +++ b/src/core/support/env_linux.c @@ -63,7 +63,9 @@ char *gpr_getenv(const char *name) { for (size_t i = 0; getenv_func == NULL && i < GPR_ARRAY_SIZE(names); i++) { getenv_func = (getenv_type)dlsym(RTLD_DEFAULT, names[i]); if (getenv_func != NULL && strstr(names[i], "secure") == NULL) { - gpr_log(GPR_DEBUG, "Warning: insecure environment read function '%s' used", names[i]); + gpr_log(GPR_DEBUG, + "Warning: insecure environment read function '%s' used", + names[i]); } } char *result = getenv_func(name); @@ -72,7 +74,8 @@ char *gpr_getenv(const char *name) { char *result = secure_getenv(name); return result == NULL ? result : gpr_strdup(result); #else - gpr_log(GPR_DEBUG, "Warning: insecure environment read function '%s' used", "getenv"); + gpr_log(GPR_DEBUG, "Warning: insecure environment read function '%s' used", + "getenv"); char *result = getenv(name); return result == NULL ? result : gpr_strdup(result); #endif From f63c49238e1696d2f74c68d6323e959ceb8b8791 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 24 Feb 2016 17:31:52 -0800 Subject: [PATCH 112/236] Some more cleanup --- tools/gke/run_stress_tests_on_gke.py | 3 ++- tools/run_tests/stress_test/stress_test_utils.py | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/gke/run_stress_tests_on_gke.py b/tools/gke/run_stress_tests_on_gke.py index 9a8a33cac5e..065b11e91c0 100755 --- a/tools/gke/run_stress_tests_on_gke.py +++ b/tools/gke/run_stress_tests_on_gke.py @@ -403,6 +403,7 @@ def run_test(skip_building_image, gcp_project_id, image_name, tag_name, break # Things seem to be running fine. Wait until next poll time to check the # status + print 'Sleeping for %d seconds..' % poll_interval_secs time.sleep(poll_interval_secs) # Print BiqQuery tables @@ -418,7 +419,7 @@ if __name__ == '__main__': gcp_project_id = 'sree-gce' tag_name = 'gcr.io/%s/%s' % (gcp_project_id, image_name) num_client_instances = 3 - poll_interval_secs = 5, + poll_interval_secs = 10 test_duration_secs = 150 run_test(True, gcp_project_id, image_name, tag_name, num_client_instances, poll_interval_secs, test_duration_secs) diff --git a/tools/run_tests/stress_test/stress_test_utils.py b/tools/run_tests/stress_test/stress_test_utils.py index 71f0dcd9211..7adc0068f9a 100755 --- a/tools/run_tests/stress_test/stress_test_utils.py +++ b/tools/run_tests/stress_test/stress_test_utils.py @@ -81,10 +81,9 @@ class BigQueryHelper: 'event_type': event_type, 'details': details } - # Something that uniquely identifies the row (Biquery needs it for duplicate - # detection). + # row_unique_id is something that uniquely identifies the row (BigQuery uses + # it for duplicate detection). row_unique_id = '%s_%s_%s' % (self.run_id, self.pod_name, event_type) - row = bq_utils.make_row(row_unique_id, row_values_dict) return bq_utils.insert_rows(self.bq, self.project_id, self.dataset_id, self.summary_table_id, [row]) @@ -97,6 +96,8 @@ class BigQueryHelper: 'qps': qps } + # row_unique_id is something that uniquely identifies the row (BigQuery uses + # it for duplicate detection). row_unique_id = '%s_%s_%s' % (self.run_id, self.pod_name, recorded_at) row = bq_utils.make_row(row_unique_id, row_values_dict) return bq_utils.insert_rows(self.bq, self.project_id, self.dataset_id, @@ -109,7 +110,6 @@ class BigQueryHelper: query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) page = self.bq.jobs().getQueryResults(**query_job['jobReference']).execute( num_retries=num_query_retries) - print page num_failures = int(page['totalRows']) print 'num rows: ', num_failures return num_failures > 0 @@ -118,7 +118,8 @@ class BigQueryHelper: line = '-' * 120 print line print 'Summary records' - print 'Run Id', self.run_id + print 'Run Id: ', self.run_id + print 'Dataset Id: ', self.dataset_id print line query = ('SELECT pod_name, image_type, event_type, event_date, details' ' FROM %s.%s WHERE run_id = \'%s\' ORDER by event_date;') % ( @@ -147,6 +148,7 @@ class BigQueryHelper: print line print 'QPS Summary' print 'Run Id: ', self.run_id + print 'Dataset Id: ', self.dataset_id print line query = ( 'SELECT pod_name, recorded_at, qps FROM %s.%s WHERE run_id = \'%s\' ' From bdfec2c86ceba99e6cc3e58ee256dffe1632aaf4 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 25 Feb 2016 11:51:21 -0800 Subject: [PATCH 113/236] SimultaneousReadWritesDone test was not observing the contract on the streaming API. In particular, Finish should not be called until the client is sure that there is no more message to be read (as documented in the comments for ClientStreamingInterface::Finish) --- test/cpp/end2end/end2end_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index ce8e4d2a109..42757974b22 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -904,9 +904,9 @@ TEST_P(End2endTest, SimultaneousReadWritesDone) { std::thread reader_thread(ReaderThreadFunc, stream.get(), &ev); gpr_event_wait(&ev, gpr_inf_future(GPR_CLOCK_REALTIME)); stream->WritesDone(); + reader_thread.join(); Status s = stream->Finish(); EXPECT_TRUE(s.ok()); - reader_thread.join(); } TEST_P(End2endTest, ChannelState) { From 5ade2d4a35ba93c56790bc524d35848d20695faf Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 25 Feb 2016 13:44:05 -0800 Subject: [PATCH 114/236] add some CFLAGS to compile on mac --- config.m4 | 3 ++- package.xml | 4 ++-- templates/config.m4.template | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config.m4 b/config.m4 index f09ddcb40ec..8ab45e66037 100644 --- a/config.m4 +++ b/config.m4 @@ -533,7 +533,8 @@ if test "$PHP_GRPC" != "no"; then third_party/boringssl/ssl/t1_enc.c \ third_party/boringssl/ssl/t1_lib.c \ third_party/boringssl/ssl/tls_record.c \ - , $ext_shared, , -Wall -Werror -std=c11 \ + , $ext_shared, , -Wall -Werror \ + -Wno-parentheses-equality -Wno-unused-value -std=c11 \ -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN \ -D_HAS_EXCEPTIONS=0 -DNOMINMAX) diff --git a/package.xml b/package.xml index be4cfc79db7..42a0361df0a 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ grpc-packages@google.com yes - 2016-02-25 + 2016-02-24 0.8.0 @@ -963,7 +963,7 @@ Update to wrap gRPC C Core version 0.10.0 beta beta - 2016-02-25 + 2016-02-24 BSD - Simplify gRPC PHP installation #4517 diff --git a/templates/config.m4.template b/templates/config.m4.template index 5e73901efa9..5847d456f51 100644 --- a/templates/config.m4.template +++ b/templates/config.m4.template @@ -38,7 +38,8 @@ % endfor % endif % endfor - , $ext_shared, , -Wall -Werror -std=c11 ${"\\"} + , $ext_shared, , -Wall -Werror ${"\\"} + -Wno-parentheses-equality -Wno-unused-value -std=c11 ${"\\"} -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN ${"\\"} -D_HAS_EXCEPTIONS=0 -DNOMINMAX) From 85371a2bb09dc955c35e194efb461ee3d374c128 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 25 Feb 2016 13:55:13 -0800 Subject: [PATCH 115/236] Change pollset mutex ownership --- src/core/iomgr/fd_posix.c | 4 +- src/core/iomgr/pollset.h | 2 +- .../iomgr/pollset_multipoller_with_epoll.c | 8 +-- .../pollset_multipoller_with_poll_posix.c | 4 +- src/core/iomgr/pollset_posix.c | 33 +++++------ src/core/iomgr/pollset_posix.h | 2 +- .../security/google_default_credentials.c | 26 +++++---- src/core/surface/completion_queue.c | 47 ++++++++-------- test/core/end2end/fixtures/h2_uchannel.c | 12 ++-- test/core/httpcli/httpcli_test.c | 24 ++++---- test/core/httpcli/httpscli_test.c | 23 ++++---- test/core/iomgr/endpoint_pair_test.c | 6 +- test/core/iomgr/fd_posix_test.c | 56 +++++++++---------- test/core/iomgr/tcp_client_posix_test.c | 44 +++++++-------- test/core/iomgr/tcp_posix_test.c | 56 +++++++++---------- test/core/iomgr/tcp_server_posix_test.c | 16 +++--- test/core/iomgr/workqueue_test.c | 16 +++--- test/core/security/oauth2_utils.c | 10 ++-- .../print_google_default_creds_token.c | 20 +++---- test/core/security/secure_endpoint_test.c | 6 +- test/core/security/verify_jwt.c | 16 +++--- test/core/util/port_posix.c | 25 ++++----- test/core/util/test_tcp_server.c | 6 +- test/core/util/test_tcp_server.h | 2 +- 24 files changed, 220 insertions(+), 244 deletions(-) diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c index 812ff0992e3..4ba7c5df943 100644 --- a/src/core/iomgr/fd_posix.c +++ b/src/core/iomgr/fd_posix.c @@ -177,11 +177,11 @@ int grpc_fd_is_orphaned(grpc_fd *fd) { } static void pollset_kick_locked(grpc_fd_watcher *watcher) { - gpr_mu_lock(watcher->pollset->mu); + gpr_mu_lock(&watcher->pollset->mu); GPR_ASSERT(watcher->worker); grpc_pollset_kick_ext(watcher->pollset, watcher->worker, GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); - gpr_mu_unlock(watcher->pollset->mu); + gpr_mu_unlock(&watcher->pollset->mu); } static void maybe_wake_one_watcher_locked(grpc_fd *fd) { diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index 39e4ff24408..92a0374ddd4 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -53,7 +53,7 @@ typedef struct grpc_pollset grpc_pollset; typedef struct grpc_pollset_worker grpc_pollset_worker; size_t grpc_pollset_size(void); -void grpc_pollset_init(grpc_pollset *pollset, gpr_mu *mu); +void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu); /* Begin shutting down the pollset, and call closure when done. * GRPC_POLLSET_MU(pollset) must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index e1af2b52410..2e0f27fab86 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -149,7 +149,7 @@ static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg, finally_add_fd(exec_ctx, da->pollset, da->fd); } - gpr_mu_lock(da->pollset->mu); + gpr_mu_lock(&da->pollset->mu); da->pollset->in_flight_cbs--; if (da->pollset->shutting_down) { /* We don't care about this pollset anymore. */ @@ -158,7 +158,7 @@ static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg, grpc_exec_ctx_enqueue(exec_ctx, da->pollset->shutdown_done, true, NULL); } } - gpr_mu_unlock(da->pollset->mu); + gpr_mu_unlock(&da->pollset->mu); GRPC_FD_UNREF(da->fd, "delayed_add"); @@ -170,7 +170,7 @@ static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int and_unlock_pollset) { if (and_unlock_pollset) { - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); finally_add_fd(exec_ctx, pollset, fd); } else { delayed_add *da = gpr_malloc(sizeof(*da)); @@ -202,7 +202,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock( * here. */ - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); timeout_ms = grpc_poll_deadline_to_millis_timeout(deadline, now); diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 348d3391048..4dddfff2302 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -80,7 +80,7 @@ static void multipoll_with_poll_pollset_add_fd(grpc_exec_ctx *exec_ctx, GRPC_FD_REF(fd, "multipoller"); exit: if (and_unlock_pollset) { - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); } } @@ -132,7 +132,7 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( } h->del_count = 0; h->fd_count = fd_count; - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); for (i = 2; i < pfd_count; i++) { pfds[i].events = (short)grpc_fd_begin_poll(watchers[i].fd, pollset, worker, diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 63321638bab..e895a778849 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -188,8 +188,9 @@ void grpc_kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); } static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null); -void grpc_pollset_init(grpc_pollset *pollset, gpr_mu *mu) { - pollset->mu = mu; +void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->mu); + *mu = &pollset->mu; pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; pollset->in_flight_cbs = 0; pollset->shutting_down = 0; @@ -228,15 +229,15 @@ void grpc_pollset_reset(grpc_pollset *pollset) { void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { - gpr_mu_lock(pollset->mu); + gpr_mu_lock(&pollset->mu); pollset->vtable->add_fd(exec_ctx, pollset, fd, 1); /* the following (enabled only in debug) will reacquire and then release our lock - meaning that if the unlocking flag passed to add_fd above is not respected, the code will deadlock (in a way that we have a chance of debugging) */ #ifndef NDEBUG - gpr_mu_lock(pollset->mu); - gpr_mu_unlock(pollset->mu); + gpr_mu_lock(&pollset->mu); + gpr_mu_unlock(&pollset->mu); #endif } @@ -285,7 +286,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* Give do_promote priority so we don't starve it out */ if (pollset->in_flight_cbs) { GPR_TIMER_MARK("grpc_pollset_work.in_flight_cbs", 0); - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); locked = 0; goto done; } @@ -319,7 +320,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, done: if (!locked) { queued_work |= grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(pollset->mu); + gpr_mu_lock(&pollset->mu); locked = 1; } /* If we're forced to re-evaluate polling (via grpc_pollset_kick with @@ -349,19 +350,19 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_kick(pollset, NULL); } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) { pollset->called_shutdown = 1; - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); finish_shutdown(exec_ctx, pollset); grpc_exec_ctx_flush(exec_ctx); /* Continuing to access pollset here is safe -- it is the caller's * responsibility to not destroy when it has outstanding calls to * grpc_pollset_work. * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */ - gpr_mu_lock(pollset->mu); + gpr_mu_lock(&pollset->mu); } else if (!grpc_closure_list_empty(pollset->idle_jobs)) { grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(pollset->mu); + gpr_mu_lock(&pollset->mu); } } *worker_hdl = NULL; @@ -429,7 +430,7 @@ static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args, * 4. The pollset may be shutting down. */ - gpr_mu_lock(pollset->mu); + gpr_mu_lock(&pollset->mu); /* First we need to ensure that nobody is polling concurrently */ GPR_ASSERT(!grpc_pollset_has_workers(pollset)); @@ -470,7 +471,7 @@ static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args, } } - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); /* Matching ref in basic_pollset_add_fd */ GRPC_FD_UNREF(fd, "basicpoll_add"); @@ -523,7 +524,7 @@ static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, exit: if (and_unlock_pollset) { - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); } } @@ -559,14 +560,14 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx, pfd[2].fd = fd->fd; pfd[2].revents = 0; GRPC_FD_REF(fd, "basicpoll_begin"); - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); pfd[2].events = (short)grpc_fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, &fd_watcher); if (pfd[2].events != 0) { nfds++; } } else { - gpr_mu_unlock(pollset->mu); + gpr_mu_unlock(&pollset->mu); } /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h index ed7eb5cb7db..bbedb66b007 100644 --- a/src/core/iomgr/pollset_posix.h +++ b/src/core/iomgr/pollset_posix.h @@ -69,7 +69,7 @@ struct grpc_pollset { For example, we may choose a poll() based implementation on linux for few fds, and an epoll() based implementation for many fds */ const grpc_pollset_vtable *vtable; - gpr_mu *mu; + gpr_mu mu; grpc_pollset_worker root_worker; int in_flight_cbs; int shutting_down; diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c index 3b3e38882a6..1f4f3e4aa52 100644 --- a/src/core/security/google_default_credentials.c +++ b/src/core/security/google_default_credentials.c @@ -52,10 +52,11 @@ static grpc_channel_credentials *default_credentials = NULL; static int compute_engine_detection_done = 0; -static gpr_mu g_mu; +static gpr_mu g_state_mu; +static gpr_mu *g_polling_mu; static gpr_once g_once = GPR_ONCE_INIT; -static void init_default_credentials(void) { gpr_mu_init(&g_mu); } +static void init_default_credentials(void) { gpr_mu_init(&g_state_mu); } typedef struct { grpc_pollset *pollset; @@ -80,10 +81,10 @@ static void on_compute_engine_detection_http_response( } } } - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_polling_mu); detector->is_done = 1; grpc_pollset_kick(detector->pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_polling_mu); } static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool s) { @@ -102,7 +103,7 @@ static int is_stack_running_on_compute_engine(void) { gpr_timespec max_detection_delay = gpr_time_from_seconds(1, GPR_TIMESPAN); detector.pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(detector.pollset, &g_mu); + grpc_pollset_init(detector.pollset, &g_polling_mu); detector.is_done = 0; detector.success = 0; @@ -121,19 +122,20 @@ static int is_stack_running_on_compute_engine(void) { /* Block until we get the response. This is not ideal but this should only be called once for the lifetime of the process by the default credentials. */ - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_polling_mu); while (!detector.is_done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, detector.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_polling_mu); grpc_httpcli_context_destroy(&context); - grpc_closure_init(&destroy_closure, destroy_pollset, &detector.pollset); + grpc_closure_init(&destroy_closure, destroy_pollset, detector.pollset); grpc_pollset_shutdown(&exec_ctx, detector.pollset, &destroy_closure); grpc_exec_ctx_finish(&exec_ctx); + g_polling_mu = NULL; gpr_free(detector.pollset); @@ -187,7 +189,7 @@ grpc_channel_credentials *grpc_google_default_credentials_create(void) { gpr_once_init(&g_once, init_default_credentials); - gpr_mu_lock(&g_mu); + gpr_mu_lock(&g_state_mu); if (default_credentials != NULL) { result = grpc_channel_credentials_ref(default_credentials); @@ -233,19 +235,19 @@ end: gpr_log(GPR_ERROR, "Could not create google default credentials."); } } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(&g_state_mu); return result; } void grpc_flush_cached_google_default_credentials(void) { gpr_once_init(&g_once, init_default_credentials); - gpr_mu_lock(&g_mu); + gpr_mu_lock(&g_state_mu); if (default_credentials != NULL) { grpc_channel_credentials_unref(default_credentials); default_credentials = NULL; } compute_engine_detection_done = 0; - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(&g_state_mu); } /* -- Well known credentials path. -- */ diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index 41449fb4a6b..8a9bbace087 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -57,7 +57,8 @@ typedef struct { /* Completion queue structure */ struct grpc_completion_queue { - gpr_mu mu; + /** owned by pollset */ + gpr_mu *mu; /** completed events */ grpc_cq_completion completed_head; grpc_cq_completion *completed_tail; @@ -97,7 +98,6 @@ void grpc_cq_global_shutdown(void) { while (g_freelist) { grpc_completion_queue *next = g_freelist->next_free; grpc_pollset_destroy(POLLSET_FROM_CQ(g_freelist)); - gpr_mu_destroy(&g_freelist->mu); #ifndef NDEBUG gpr_free(g_freelist->outstanding_tags); #endif @@ -128,7 +128,6 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) { gpr_mu_unlock(&g_freelist_mu); cc = gpr_malloc(sizeof(grpc_completion_queue) + grpc_pollset_size()); - gpr_mu_init(&cc->mu); grpc_pollset_init(POLLSET_FROM_CQ(cc), &cc->mu); #ifndef NDEBUG cc->outstanding_tags = NULL; @@ -198,7 +197,7 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) { #ifndef NDEBUG - gpr_mu_lock(&cc->mu); + gpr_mu_lock(cc->mu); GPR_ASSERT(!cc->shutdown_called); if (cc->outstanding_tag_count == cc->outstanding_tag_capacity) { cc->outstanding_tag_capacity = GPR_MAX(4, 2 * cc->outstanding_tag_capacity); @@ -207,7 +206,7 @@ void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) { cc->outstanding_tag_capacity); } cc->outstanding_tags[cc->outstanding_tag_count++] = tag; - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); #endif gpr_ref(&cc->pending_events); } @@ -235,7 +234,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, storage->next = ((uintptr_t)&cc->completed_head) | ((uintptr_t)(success != 0)); - gpr_mu_lock(&cc->mu); + gpr_mu_lock(cc->mu); #ifndef NDEBUG for (i = 0; i < (int)cc->outstanding_tag_count; i++) { if (cc->outstanding_tags[i] == tag) { @@ -261,7 +260,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, } } grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker); - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); } else { cc->completed_tail->next = ((uintptr_t)storage) | (1u & (uintptr_t)cc->completed_tail->next); @@ -271,7 +270,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, cc->shutdown = 1; grpc_pollset_shutdown(exec_ctx, POLLSET_FROM_CQ(cc), &cc->pollset_shutdown_done); - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); } GPR_TIMER_END("grpc_cq_end_op", 0); @@ -299,7 +298,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); GRPC_CQ_INTERNAL_REF(cc, "next"); - gpr_mu_lock(&cc->mu); + gpr_mu_lock(cc->mu); for (;;) { if (cc->completed_tail != &cc->completed_head) { grpc_cq_completion *c = (grpc_cq_completion *)cc->completed_head.next; @@ -307,7 +306,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, if (c == cc->completed_tail) { cc->completed_tail = &cc->completed_head; } - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); ret.type = GRPC_OP_COMPLETE; ret.success = c->next & 1u; ret.tag = c->tag; @@ -315,14 +314,14 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, break; } if (cc->shutdown) { - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_SHUTDOWN; break; } now = gpr_now(GPR_CLOCK_MONOTONIC); if (!first_loop && gpr_time_cmp(now, deadline) >= 0) { - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_TIMEOUT; break; @@ -335,9 +334,9 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_timespec iteration_deadline = deadline; if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { GPR_TIMER_MARK("alarm_triggered", 0); - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(&cc->mu); + gpr_mu_lock(cc->mu); continue; } else { grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, @@ -401,7 +400,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); GRPC_CQ_INTERNAL_REF(cc, "pluck"); - gpr_mu_lock(&cc->mu); + gpr_mu_lock(cc->mu); for (;;) { prev = &cc->completed_head; while ((c = (grpc_cq_completion *)(prev->next & ~(uintptr_t)1)) != @@ -411,7 +410,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, if (c == cc->completed_tail) { cc->completed_tail = prev; } - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); ret.type = GRPC_OP_COMPLETE; ret.success = c->next & 1u; ret.tag = c->tag; @@ -421,7 +420,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, prev = c; } if (cc->shutdown) { - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_SHUTDOWN; break; @@ -431,7 +430,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, "Too many outstanding grpc_completion_queue_pluck calls: maximum " "is %d", GRPC_MAX_COMPLETION_QUEUE_PLUCKERS); - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); /* TODO(ctiller): should we use a different result here */ ret.type = GRPC_QUEUE_TIMEOUT; @@ -440,7 +439,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, now = gpr_now(GPR_CLOCK_MONOTONIC); if (!first_loop && gpr_time_cmp(now, deadline) >= 0) { del_plucker(cc, tag, &worker); - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_TIMEOUT; break; @@ -453,9 +452,9 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, gpr_timespec iteration_deadline = deadline; if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { GPR_TIMER_MARK("alarm_triggered", 0); - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(&cc->mu); + gpr_mu_lock(cc->mu); continue; } else { grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, @@ -479,9 +478,9 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_completion_queue_shutdown", 0); GRPC_API_TRACE("grpc_completion_queue_shutdown(cc=%p)", 1, (cc)); - gpr_mu_lock(&cc->mu); + gpr_mu_lock(cc->mu); if (cc->shutdown_called) { - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); GPR_TIMER_END("grpc_completion_queue_shutdown", 0); return; } @@ -492,7 +491,7 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { grpc_pollset_shutdown(&exec_ctx, POLLSET_FROM_CQ(cc), &cc->pollset_shutdown_done); } - gpr_mu_unlock(&cc->mu); + gpr_mu_unlock(cc->mu); grpc_exec_ctx_finish(&exec_ctx); GPR_TIMER_END("grpc_completion_queue_shutdown", 0); } diff --git a/test/core/end2end/fixtures/h2_uchannel.c b/test/core/end2end/fixtures/h2_uchannel.c index f363b60cba2..46b77a2326f 100644 --- a/test/core/end2end/fixtures/h2_uchannel.c +++ b/test/core/end2end/fixtures/h2_uchannel.c @@ -253,8 +253,7 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *arg, bool success) { } static grpc_connected_subchannel *connect_subchannel(grpc_subchannel *c) { - gpr_mu mu; - gpr_mu_init(&mu); + gpr_mu *mu; grpc_pollset *pollset = gpr_malloc(grpc_pollset_size()); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_init(pollset, &mu); @@ -264,22 +263,21 @@ static grpc_connected_subchannel *connect_subchannel(grpc_subchannel *c) { &g_state, grpc_closure_create(state_changed, c)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(&mu); + gpr_mu_lock(mu); while (g_state != GRPC_CHANNEL_READY) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); - gpr_mu_unlock(&mu); + gpr_mu_unlock(mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(&mu); + gpr_mu_lock(mu); } grpc_pollset_shutdown(&exec_ctx, pollset, grpc_closure_create(destroy_pollset, pollset)); grpc_pollset_set_destroy(&g_interested_parties); - gpr_mu_unlock(&mu); + gpr_mu_unlock(mu); grpc_exec_ctx_finish(&exec_ctx); gpr_free(pollset); - gpr_mu_destroy(&mu); return grpc_subchannel_get_connected_subchannel(c); } diff --git a/test/core/httpcli/httpcli_test.c b/test/core/httpcli/httpcli_test.c index cfa18292149..da1463329d4 100644 --- a/test/core/httpcli/httpcli_test.c +++ b/test/core/httpcli/httpcli_test.c @@ -47,7 +47,7 @@ static int g_done = 0; static grpc_httpcli_context g_context; -static gpr_mu g_mu; +static gpr_mu *g_mu; static grpc_pollset *g_pollset; static gpr_timespec n_seconds_time(int seconds) { @@ -64,10 +64,10 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, GPR_ASSERT(response->status == 200); GPR_ASSERT(response->body_length == strlen(expect)); GPR_ASSERT(0 == memcmp(expect, response->body, response->body_length)); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); g_done = 1; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } static void test_get(int port) { @@ -88,16 +88,16 @@ static void test_get(int port) { grpc_httpcli_get(&exec_ctx, &g_context, g_pollset, &req, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); gpr_free(host); } @@ -119,16 +119,16 @@ static void test_post(int port) { grpc_httpcli_post(&exec_ctx, &g_context, g_pollset, &req, "hello", 5, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); gpr_free(host); } @@ -177,7 +177,6 @@ int main(int argc, char **argv) { grpc_init(); grpc_httpcli_context_init(&g_context); g_pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&g_mu); grpc_pollset_init(g_pollset, &g_mu); test_get(port); @@ -189,7 +188,6 @@ int main(int argc, char **argv) { grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - gpr_mu_destroy(&g_mu); gpr_free(g_pollset); gpr_subprocess_destroy(server); diff --git a/test/core/httpcli/httpscli_test.c b/test/core/httpcli/httpscli_test.c index b12f9472b15..7f765bc614f 100644 --- a/test/core/httpcli/httpscli_test.c +++ b/test/core/httpcli/httpscli_test.c @@ -47,7 +47,7 @@ static int g_done = 0; static grpc_httpcli_context g_context; -static gpr_mu g_mu; +static gpr_mu *g_mu; static grpc_pollset *g_pollset; static gpr_timespec n_seconds_time(int seconds) { @@ -64,10 +64,10 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, GPR_ASSERT(response->status == 200); GPR_ASSERT(response->body_length == strlen(expect)); GPR_ASSERT(0 == memcmp(expect, response->body, response->body_length)); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); g_done = 1; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } static void test_get(int port) { @@ -89,16 +89,16 @@ static void test_get(int port) { grpc_httpcli_get(&exec_ctx, &g_context, g_pollset, &req, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); gpr_free(host); } @@ -121,16 +121,16 @@ static void test_post(int port) { grpc_httpcli_post(&exec_ctx, &g_context, g_pollset, &req, "hello", 5, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); gpr_free(host); } @@ -192,7 +192,6 @@ int main(int argc, char **argv) { grpc_shutdown(); gpr_free(g_pollset); - gpr_mu_destroy(&g_mu); gpr_subprocess_destroy(server); diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c index c02e4149359..c3a91088a57 100644 --- a/test/core/iomgr/endpoint_pair_test.c +++ b/test/core/iomgr/endpoint_pair_test.c @@ -42,7 +42,7 @@ #include "test/core/iomgr/endpoint_tests.h" #include "test/core/util/test_config.h" -static gpr_mu g_mu; +static gpr_mu *g_mu; static grpc_pollset *g_pollset; static void clean_up(void) {} @@ -73,17 +73,15 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) { int main(int argc, char **argv) { grpc_closure destroyed; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - gpr_mu_init(&g_mu); grpc_test_init(argc, argv); grpc_init(); g_pollset = gpr_malloc(grpc_pollset_size()); grpc_pollset_init(g_pollset, &g_mu); - grpc_endpoint_tests(configs[0], g_pollset, &g_mu); + grpc_endpoint_tests(configs[0], g_pollset, g_mu); grpc_closure_init(&destroyed, destroy_pollset, g_pollset); grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - gpr_mu_destroy(&g_mu); gpr_free(g_pollset); return 0; diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index d199e0c272a..99689ebcc3b 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -53,7 +53,7 @@ #include "src/core/iomgr/pollset_posix.h" #include "test/core/util/test_config.h" -static gpr_mu g_mu; +static gpr_mu *g_mu; static grpc_pollset *g_pollset; /* buffer size used to send and receive data. @@ -182,10 +182,10 @@ static void listen_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg /*server */, grpc_fd_orphan(exec_ctx, sv->em_fd, NULL, NULL, "b"); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); sv->done = 1; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } /* Called when a new TCP connection request arrives in the listening port. */ @@ -252,18 +252,18 @@ static int server_start(grpc_exec_ctx *exec_ctx, server *sv) { /* Wait and shutdown a sever. */ static void server_wait_and_shutdown(server *sv) { - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (!sv->done) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } /* ===An upload client to test notify_on_write=== */ @@ -310,9 +310,9 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ ssize_t write_once = 0; if (!success) { - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); client_session_shutdown_cb(exec_ctx, arg, 1); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); return; } @@ -322,7 +322,7 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ } while (write_once > 0); if (errno == EAGAIN) { - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); if (cl->client_write_cnt < CLIENT_TOTAL_WRITE_CNT) { cl->write_closure.cb = client_session_write; cl->write_closure.cb_arg = cl; @@ -331,7 +331,7 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ } else { client_session_shutdown_cb(exec_ctx, arg, 1); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } else { gpr_log(GPR_ERROR, "unknown errno %s", strerror(errno)); abort(); @@ -367,18 +367,18 @@ static void client_start(grpc_exec_ctx *exec_ctx, client *cl, int port) { /* Wait for the signal to shutdown a client. */ static void client_wait_and_shutdown(client *cl) { - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (!cl->done) { grpc_pollset_worker *worker = NULL; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } /* Test grpc_fd. Start an upload server and client, upload a stream of @@ -413,20 +413,20 @@ static void first_read_callback(grpc_exec_ctx *exec_ctx, void *arg /* fd_change_data */, bool success) { fd_change_data *fdc = arg; - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); fdc->cb_that_ran = first_read_callback; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } static void second_read_callback(grpc_exec_ctx *exec_ctx, void *arg /* fd_change_data */, bool success) { fd_change_data *fdc = arg; - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); fdc->cb_that_ran = second_read_callback; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } /* Test that changing the callback we use for notify_on_read actually works. @@ -468,18 +468,18 @@ static void test_grpc_fd_change(void) { GPR_ASSERT(result == 1); /* And now wait for it to run. */ - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (a.cb_that_ran == NULL) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } GPR_ASSERT(a.cb_that_ran == first_read_callback); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); /* And drain the socket so we can generate a new read edge */ result = read(sv[0], &data, 1); @@ -492,19 +492,19 @@ static void test_grpc_fd_change(void) { result = write(sv[1], &data, 1); GPR_ASSERT(result == 1); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (b.cb_that_ran == NULL) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } /* Except now we verify that second_read_callback ran instead */ GPR_ASSERT(b.cb_that_ran == second_read_callback); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_fd_orphan(&exec_ctx, em_fd, NULL, NULL, "d"); grpc_exec_ctx_finish(&exec_ctx); @@ -522,7 +522,6 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_iomgr_init(); - gpr_mu_init(&g_mu); g_pollset = gpr_malloc(grpc_pollset_size()); grpc_pollset_init(g_pollset, &g_mu); test_grpc_fd(); @@ -531,7 +530,6 @@ int main(int argc, char **argv) { grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); gpr_free(g_pollset); - gpr_mu_destroy(&g_mu); grpc_iomgr_shutdown(); return 0; } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 58d6d2cb56e..61023f512ef 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -50,7 +50,7 @@ #include "test/core/util/test_config.h" static grpc_pollset_set g_pollset_set; -static gpr_mu g_mu; +static gpr_mu *g_mu; static grpc_pollset *g_pollset; static int g_connections_complete = 0; static grpc_endpoint *g_connecting = NULL; @@ -60,10 +60,10 @@ static gpr_timespec test_deadline(void) { } static void finish_connection() { - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); g_connections_complete++; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, bool success) { @@ -101,9 +101,9 @@ void test_succeeds(void) { GPR_ASSERT(0 == bind(svr_fd, (struct sockaddr *)&addr, addr_len)); GPR_ASSERT(0 == listen(svr_fd, 1)); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); connections_complete_before = g_connections_complete; - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); /* connect to it */ GPR_ASSERT(getsockname(svr_fd, (struct sockaddr *)&addr, &addr_len) == 0); @@ -120,19 +120,19 @@ void test_succeeds(void) { GPR_ASSERT(r >= 0); close(r); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (g_connections_complete == connections_complete_before) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); } @@ -149,9 +149,9 @@ void test_fails(void) { memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); connections_complete_before = g_connections_complete; - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); /* connect to a broken address */ grpc_closure_init(&done, must_fail, NULL); @@ -159,7 +159,7 @@ void test_fails(void) { (struct sockaddr *)&addr, addr_len, gpr_inf_future(GPR_CLOCK_REALTIME)); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); /* wait for the connection callback to finish */ while (g_connections_complete == connections_complete_before) { @@ -169,12 +169,12 @@ void test_fails(void) { if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { grpc_pollset_work(&exec_ctx, g_pollset, &worker, now, polling_deadline); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); } @@ -219,16 +219,16 @@ void test_times_out(void) { connect_deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); connections_complete_before = g_connections_complete; - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_closure_init(&done, must_fail, NULL); grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, &g_pollset_set, (struct sockaddr *)&addr, addr_len, connect_deadline); /* Make sure the event doesn't trigger early */ - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); for (;;) { grpc_pollset_worker *worker = NULL; gpr_timespec now = gpr_now(connect_deadline.clock_type); @@ -256,11 +256,11 @@ void test_times_out(void) { if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { grpc_pollset_work(&exec_ctx, g_pollset, &worker, now, polling_deadline); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); @@ -281,7 +281,6 @@ int main(int argc, char **argv) { grpc_init(); grpc_pollset_set_init(&g_pollset_set); g_pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&g_mu); grpc_pollset_init(g_pollset, &g_mu); grpc_pollset_set_add_pollset(&exec_ctx, &g_pollset_set, g_pollset); grpc_exec_ctx_finish(&exec_ctx); @@ -295,6 +294,5 @@ int main(int argc, char **argv) { grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); gpr_free(g_pollset); - gpr_mu_destroy(&g_mu); return 0; } diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index d6758f8fe1b..4351642ab6e 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -48,7 +48,7 @@ #include "test/core/iomgr/endpoint_tests.h" #include "test/core/util/test_config.h" -static gpr_mu g_mu; +static gpr_mu *g_mu; static grpc_pollset *g_pollset; /* @@ -146,7 +146,7 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, bool success) { GPR_ASSERT(success); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); current_data = state->read_bytes % 256; read_bytes = count_slices(state->incoming.slices, state->incoming.count, ¤t_data); @@ -154,10 +154,10 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, bool success) { gpr_log(GPR_INFO, "Read %d bytes of %d", read_bytes, state->target_read_bytes); if (state->read_bytes >= state->target_read_bytes) { - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } else { grpc_endpoint_read(exec_ctx, state->ep, &state->incoming, &state->read_cb); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } } @@ -189,17 +189,17 @@ static void read_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); gpr_slice_buffer_destroy(&state.incoming); grpc_endpoint_destroy(&exec_ctx, ep); @@ -235,17 +235,17 @@ static void large_read_test(size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); gpr_slice_buffer_destroy(&state.incoming); grpc_endpoint_destroy(&exec_ctx, ep); @@ -284,11 +284,11 @@ static void write_done(grpc_exec_ctx *exec_ctx, void *user_data /* write_socket_state */, bool success) { struct write_socket_state *state = (struct write_socket_state *)user_data; gpr_log(GPR_INFO, "Write done callback called"); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); gpr_log(GPR_INFO, "Signalling write done"); state->write_done = 1; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { @@ -305,11 +305,11 @@ void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { for (;;) { grpc_pollset_worker *worker = NULL; - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10)); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); do { bytes_read = @@ -364,7 +364,7 @@ static void write_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_write(&exec_ctx, ep, &outgoing, &write_done_closure); drain_socket_blocking(sv[0], num_bytes, num_bytes); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); for (;;) { grpc_pollset_worker *worker = NULL; if (state.write_done) { @@ -372,11 +372,11 @@ static void write_test(size_t num_bytes, size_t slice_size) { } grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); gpr_slice_buffer_destroy(&outgoing); grpc_endpoint_destroy(&exec_ctx, ep); @@ -424,27 +424,27 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); gpr_slice_buffer_destroy(&state.incoming); grpc_tcp_destroy_and_release_fd(&exec_ctx, ep, &fd, &fd_released_cb); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); while (!fd_released_done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); GPR_ASSERT(fd_released_done == 1); GPR_ASSERT(fd == sv[1]); grpc_exec_ctx_finish(&exec_ctx); @@ -514,16 +514,14 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); g_pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&g_mu); grpc_pollset_init(g_pollset, &g_mu); run_tests(); - grpc_endpoint_tests(configs[0], g_pollset, &g_mu); + grpc_endpoint_tests(configs[0], g_pollset, g_mu); grpc_closure_init(&destroyed, destroy_pollset, g_pollset); grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); gpr_free(g_pollset); - gpr_mu_destroy(&g_mu); return 0; } diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index ed627734bfa..7933468355a 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -52,7 +52,7 @@ #define LOG_TEST(x) gpr_log(GPR_INFO, "%s", #x) -static gpr_mu g_mu; +static gpr_mu *g_mu; static grpc_pollset *g_pollset; static int g_nconnects = 0; @@ -117,11 +117,11 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_endpoint_shutdown(exec_ctx, tcp); grpc_endpoint_destroy(exec_ctx, tcp); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); on_connect_result_set(&g_result, acceptor); g_nconnects++; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } static void test_no_op(void) { @@ -178,7 +178,7 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, int clifd = socket(remote->sa_family, SOCK_STREAM, 0); int nconnects_before; - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); nconnects_before = g_nconnects; on_connect_result_init(&g_result); GPR_ASSERT(clifd >= 0); @@ -190,16 +190,16 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, grpc_pollset_worker *worker = NULL; grpc_pollset_work(exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(exec_ctx); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); } gpr_log(GPR_DEBUG, "wait done"); GPR_ASSERT(g_nconnects == nconnects_before + 1); close(clifd); *result = g_result; - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } /* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for @@ -315,7 +315,6 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); g_pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&g_mu); grpc_pollset_init(g_pollset, &g_mu); test_no_op(); @@ -330,6 +329,5 @@ int main(int argc, char **argv) { grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); gpr_free(g_pollset); - gpr_mu_destroy(&g_mu); return 0; } diff --git a/test/core/iomgr/workqueue_test.c b/test/core/iomgr/workqueue_test.c index f557042e39b..8a1faf6303e 100644 --- a/test/core/iomgr/workqueue_test.c +++ b/test/core/iomgr/workqueue_test.c @@ -39,15 +39,15 @@ #include "test/core/util/test_config.h" -static gpr_mu g_mu; +static gpr_mu *g_mu; static grpc_pollset *g_pollset; static void must_succeed(grpc_exec_ctx *exec_ctx, void *p, bool success) { GPR_ASSERT(success == 1); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); *(int *)p = 1; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); } static void test_ref_unref(void) { @@ -71,11 +71,11 @@ static void test_add_closure(void) { grpc_workqueue_push(wq, &c, 1); grpc_workqueue_add_to_pollset(&exec_ctx, wq, g_pollset); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); GPR_ASSERT(!done); grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(deadline.clock_type), deadline); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(done); @@ -96,11 +96,11 @@ static void test_flush(void) { grpc_workqueue_flush(&exec_ctx, wq); grpc_workqueue_add_to_pollset(&exec_ctx, wq, g_pollset); - gpr_mu_lock(&g_mu); + gpr_mu_lock(g_mu); GPR_ASSERT(!done); grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(deadline.clock_type), deadline); - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(done); @@ -117,7 +117,6 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - gpr_mu_init(&g_mu); g_pollset = gpr_malloc(grpc_pollset_size()); grpc_pollset_init(g_pollset, &g_mu); @@ -131,6 +130,5 @@ int main(int argc, char **argv) { grpc_shutdown(); gpr_free(g_pollset); - gpr_mu_destroy(&g_mu); return 0; } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index 3280eac801c..9b70afffe1e 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -45,7 +45,7 @@ #include "src/core/security/credentials.h" typedef struct { - gpr_mu mu; + gpr_mu *mu; grpc_pollset *pollset; int is_done; char *token; @@ -67,11 +67,11 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data, GPR_SLICE_LENGTH(token_slice)); token[GPR_SLICE_LENGTH(token_slice)] = '\0'; } - gpr_mu_lock(&request->mu); + gpr_mu_lock(request->mu); request->is_done = 1; request->token = token; grpc_pollset_kick(request->pollset, NULL); - gpr_mu_unlock(&request->mu); + gpr_mu_unlock(request->mu); } static void do_nothing(grpc_exec_ctx *exec_ctx, void *unused, bool success) {} @@ -95,14 +95,14 @@ char *grpc_test_fetch_oauth2_token_with_credentials( grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&request.mu); + gpr_mu_lock(request.mu); while (!request.is_done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, request.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); } - gpr_mu_unlock(&request.mu); + gpr_mu_unlock(request.mu); grpc_pollset_shutdown(&exec_ctx, request.pollset, &do_nothing_closure); grpc_exec_ctx_finish(&exec_ctx); diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index 2145a79ff97..09673f362cf 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -46,7 +46,7 @@ #include "src/core/support/string.h" typedef struct { - gpr_mu mu; + gpr_mu *mu; grpc_pollset *pollset; int is_done; } synchronizer; @@ -64,10 +64,10 @@ static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *user_data, printf("\nGot token: %s\n\n", token); gpr_free(token); } - gpr_mu_lock(&sync->mu); + gpr_mu_lock(sync->mu); sync->is_done = 1; grpc_pollset_kick(sync->pollset, NULL); - gpr_mu_unlock(&sync->mu); + gpr_mu_unlock(sync->mu); } int main(int argc, char **argv) { @@ -94,7 +94,6 @@ int main(int argc, char **argv) { } sync.pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&sync.mu); grpc_pollset_init(sync.pollset, &sync.mu); sync.is_done = 0; @@ -102,21 +101,22 @@ int main(int argc, char **argv) { &exec_ctx, ((grpc_composite_channel_credentials *)creds)->call_creds, sync.pollset, context, on_metadata_response, &sync); - gpr_mu_lock(&sync.mu); + gpr_mu_lock(sync.mu); while (!sync.is_done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(&sync.mu); - grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&sync.mu); + gpr_mu_unlock(sync.mu); + grpc_exec_ctx_flush(&exec_ctx); + gpr_mu_lock(sync.mu); } - gpr_mu_unlock(&sync.mu); + gpr_mu_unlock(sync.mu); + + grpc_exec_ctx_finish(&exec_ctx); grpc_channel_credentials_release(creds); gpr_free(sync.pollset); - gpr_mu_destroy(&sync.mu); end: gpr_cmdline_destroy(cl); diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index 61543ada43e..0e8c38a53ec 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -45,7 +45,7 @@ #include "src/core/tsi/fake_transport_security.h" #include "test/core/util/test_config.h" -static gpr_mu g_mu; +static gpr_mu *g_mu; static grpc_pollset *g_pollset; static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( @@ -183,9 +183,8 @@ int main(int argc, char **argv) { grpc_init(); g_pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&g_mu); grpc_pollset_init(g_pollset, &g_mu); - grpc_endpoint_tests(configs[0], g_pollset, &g_mu); + grpc_endpoint_tests(configs[0], g_pollset, g_mu); test_leftover(configs[1], 1); grpc_closure_init(&destroyed, destroy_pollset, g_pollset); grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); @@ -193,7 +192,6 @@ int main(int argc, char **argv) { grpc_shutdown(); gpr_free(g_pollset); - gpr_mu_destroy(&g_mu); return 0; } diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index c3cf6bb7399..eb86589681d 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -46,7 +46,7 @@ typedef struct { grpc_pollset *pollset; - gpr_mu mu; + gpr_mu *mu; int is_done; int success; } synchronizer; @@ -79,10 +79,10 @@ static void on_jwt_verification_done(void *user_data, grpc_jwt_verifier_status_to_string(status)); } - gpr_mu_lock(&sync->mu); + gpr_mu_lock(sync->mu); sync->is_done = 1; grpc_pollset_kick(sync->pollset, NULL); - gpr_mu_unlock(&sync->mu); + gpr_mu_unlock(sync->mu); } int main(int argc, char **argv) { @@ -106,26 +106,24 @@ int main(int argc, char **argv) { grpc_init(); sync.pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&sync.mu); grpc_pollset_init(sync.pollset, &sync.mu); sync.is_done = 0; grpc_jwt_verifier_verify(&exec_ctx, verifier, sync.pollset, jwt, aud, on_jwt_verification_done, &sync); - gpr_mu_lock(&sync.mu); + gpr_mu_lock(sync.mu); while (!sync.is_done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(&sync.mu); + gpr_mu_unlock(sync.mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&sync.mu); + gpr_mu_lock(sync.mu); } - gpr_mu_unlock(&sync.mu); + gpr_mu_unlock(sync.mu); - gpr_mu_destroy(&sync.mu); gpr_free(sync.pollset); grpc_jwt_verifier_destroy(verifier); diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c index a04471706ee..8b8514ccc93 100644 --- a/test/core/util/port_posix.c +++ b/test/core/util/port_posix.c @@ -69,7 +69,7 @@ static int has_port_been_chosen(int port) { } typedef struct freereq { - gpr_mu mu; + gpr_mu *mu; grpc_pollset *pollset; int done; } freereq; @@ -83,10 +83,10 @@ static void destroy_pollset_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, const grpc_httpcli_response *response) { freereq *pr = arg; - gpr_mu_lock(&pr->mu); + gpr_mu_lock(pr->mu); pr->done = 1; grpc_pollset_kick(pr->pollset, NULL); - gpr_mu_unlock(&pr->mu); + gpr_mu_unlock(pr->mu); } static void free_port_using_server(char *server, int port) { @@ -103,7 +103,6 @@ static void free_port_using_server(char *server, int port) { memset(&req, 0, sizeof(req)); pr.pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&pr.mu); grpc_pollset_init(pr.pollset, &pr.mu); grpc_closure_init(&shutdown_closure, destroy_pollset_and_shutdown, pr.pollset); @@ -116,21 +115,20 @@ static void free_port_using_server(char *server, int port) { grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), freed_port_from_server, &pr); - gpr_mu_lock(&pr.mu); + gpr_mu_lock(pr.mu); while (!pr.done) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); } - gpr_mu_unlock(&pr.mu); + gpr_mu_unlock(pr.mu); grpc_httpcli_context_destroy(&context); grpc_exec_ctx_finish(&exec_ctx); grpc_pollset_shutdown(&exec_ctx, pr.pollset, &shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); gpr_free(pr.pollset); - gpr_mu_destroy(&pr.mu); gpr_free(path); } @@ -208,7 +206,7 @@ static int is_port_available(int *port, int is_tcp) { } typedef struct portreq { - gpr_mu mu; + gpr_mu *mu; grpc_pollset *pollset; int port; int retries; @@ -253,10 +251,10 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, port = port * 10 + response->body[i] - '0'; } GPR_ASSERT(port > 1024); - gpr_mu_lock(&pr->mu); + gpr_mu_lock(pr->mu); pr->port = port; grpc_pollset_kick(pr->pollset, NULL); - gpr_mu_unlock(&pr->mu); + gpr_mu_unlock(pr->mu); } static int pick_port_using_server(char *server) { @@ -271,7 +269,6 @@ static int pick_port_using_server(char *server) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); pr.pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&pr.mu); grpc_pollset_init(pr.pollset, &pr.mu); grpc_closure_init(&shutdown_closure, destroy_pollset_and_shutdown, pr.pollset); @@ -287,20 +284,20 @@ static int pick_port_using_server(char *server) { GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, &pr); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(&pr.mu); + gpr_mu_lock(pr.mu); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; grpc_pollset_work(&exec_ctx, pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); } - gpr_mu_unlock(&pr.mu); + gpr_mu_unlock(pr.mu); grpc_httpcli_context_destroy(&context); grpc_pollset_shutdown(&exec_ctx, pr.pollset, &shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); gpr_free(pr.pollset); - gpr_mu_destroy(&pr.mu); + gpr_mu_destroy(pr.mu); return pr.port; } diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index 8f1db4e5015..ab379441d8c 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -58,7 +58,6 @@ void test_tcp_server_init(test_tcp_server *server, grpc_closure_init(&server->shutdown_complete, on_server_destroyed, server); server->shutdown = 0; server->pollset = gpr_malloc(grpc_pollset_size()); - gpr_mu_init(&server->mu); grpc_pollset_init(server->pollset, &server->mu); server->on_connect = on_connect; server->cb_data = user_data; @@ -91,10 +90,10 @@ void test_tcp_server_poll(test_tcp_server *server, int seconds) { gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(seconds, GPR_TIMESPAN)); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - gpr_mu_lock(&server->mu); + gpr_mu_lock(server->mu); grpc_pollset_work(&exec_ctx, server->pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(&server->mu); + gpr_mu_unlock(server->mu); grpc_exec_ctx_finish(&exec_ctx); } @@ -116,6 +115,5 @@ void test_tcp_server_destroy(test_tcp_server *server) { grpc_exec_ctx_finish(&exec_ctx); grpc_pollset_destroy(server->pollset); gpr_free(server->pollset); - gpr_mu_destroy(&server->mu); grpc_shutdown(); } diff --git a/test/core/util/test_tcp_server.h b/test/core/util/test_tcp_server.h index ef9dd007c77..15fcb4fb873 100644 --- a/test/core/util/test_tcp_server.h +++ b/test/core/util/test_tcp_server.h @@ -41,7 +41,7 @@ typedef struct test_tcp_server { grpc_tcp_server *tcp_server; grpc_closure shutdown_complete; int shutdown; - gpr_mu mu; + gpr_mu *mu; grpc_pollset *pollset; grpc_tcp_server_cb on_connect; void *cb_data; From 16f47b035e2559f0e47dd879c575b214b0bf73c0 Mon Sep 17 00:00:00 2001 From: Julien Boeuf Date: Thu, 25 Feb 2016 14:00:28 -0800 Subject: [PATCH 116/236] Running project generation. --- package.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.xml b/package.xml index be4cfc79db7..42a0361df0a 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ grpc-packages@google.com yes - 2016-02-25 + 2016-02-24 0.8.0 @@ -963,7 +963,7 @@ Update to wrap gRPC C Core version 0.10.0 beta beta - 2016-02-25 + 2016-02-24 BSD - Simplify gRPC PHP installation #4517 From 1d7704d7a1ac694fd4f3098437b98f2616368010 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 25 Feb 2016 14:07:29 -0800 Subject: [PATCH 117/236] Fix windows --- src/core/httpcli/httpcli.c | 8 +++++--- src/core/iomgr/pollset_windows.c | 7 ++++++- src/core/iomgr/pollset_windows.h | 2 -- test/core/util/port_windows.c | 27 +++++++++++++++------------ 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/core/httpcli/httpcli.c b/src/core/httpcli/httpcli.c index 71237bb6140..2ac93ab6a37 100644 --- a/src/core/httpcli/httpcli.c +++ b/src/core/httpcli/httpcli.c @@ -36,15 +36,17 @@ #include +#include +#include +#include + #include "src/core/iomgr/endpoint.h" +#include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/resolve_address.h" #include "src/core/iomgr/tcp_client.h" #include "src/core/httpcli/format_request.h" #include "src/core/httpcli/parser.h" #include "src/core/support/string.h" -#include -#include -#include typedef struct { gpr_slice request_text; diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index bbce23b46a7..651f8e7334c 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -89,12 +89,17 @@ static void push_front_worker(grpc_pollset_worker *root, worker->links[type].next->links[type].prev = worker; } +size_t grpc_pollset_size(void) { + return sizeof(grpc_pollset); +} + /* There isn't really any such thing as a pollset under Windows, due to the nature of the IO completion ports. We're still going to provide a minimal set of features for the sake of the rest of grpc. But grpc_pollset_work won't actually do any polling, and return as quickly as possible. */ -void grpc_pollset_init(grpc_pollset *pollset) { +void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + *mu = &grpc_polling_mu; memset(pollset, 0, sizeof(*pollset)); pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h index c2f13fdfa8b..dc0b7a4104b 100644 --- a/src/core/iomgr/pollset_windows.h +++ b/src/core/iomgr/pollset_windows.h @@ -72,6 +72,4 @@ struct grpc_pollset { grpc_closure *on_shutdown; }; -extern gpr_mu grpc_polling_mu; - #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/test/core/util/port_windows.c b/test/core/util/port_windows.c index b5bd0168ad2..3b20aeb7182 100644 --- a/test/core/util/port_windows.c +++ b/test/core/util/port_windows.c @@ -129,7 +129,8 @@ static int is_port_available(int *port, int is_tcp) { } typedef struct portreq { - grpc_pollset pollset; + grpc_pollset *pollset; + gpr_mu *mu; int port; } portreq; @@ -145,10 +146,10 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, port = port * 10 + response->body[i] - '0'; } GPR_ASSERT(port > 1024); - gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); + gpr_mu_lock(pr->mu); pr->port = port; - grpc_pollset_kick(&pr->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); + grpc_pollset_kick(pr->pollset, NULL); + gpr_mu_unlock(pr->mu); } static void destroy_pollset_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, @@ -168,32 +169,34 @@ static int pick_port_using_server(char *server) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); - grpc_pollset_init(&pr.pollset); + pr.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(pr.pollset, &pr.mu); pr.port = -1; req.host = server; req.path = "/get"; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, &pr.pollset, &req, + grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, &pr); - gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_lock(pr.mu); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, + grpc_pollset_work(&exec_ctx, pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); - gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_unlock(pr.mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_lock(pr.mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_unlock(pr.mu); grpc_httpcli_context_destroy(&context); grpc_closure_init(&destroy_pollset_closure, destroy_pollset_and_shutdown, &pr.pollset); - grpc_pollset_shutdown(&exec_ctx, &pr.pollset, &destroy_pollset_closure); + grpc_pollset_shutdown(&exec_ctx, pr.pollset, &destroy_pollset_closure); + gpr_free(pr.pollset); grpc_exec_ctx_finish(&exec_ctx); return pr.port; From 188563f474372ca41400d8ff28c44959a1083075 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Thu, 25 Feb 2016 14:27:44 -0800 Subject: [PATCH 118/236] eliminate binary tags --- include/grpc/census.h | 82 +++++++++------------- src/core/census/context.c | 121 ++++++++++---------------------- test/core/census/context_test.c | 121 +++++++++++++++----------------- 3 files changed, 127 insertions(+), 197 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index dfa3bd7e0df..442a754f0a4 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -80,18 +80,18 @@ CENSUSAPI int census_enabled(void); metrics will be recorded. Keys are unique within a context. */ typedef struct census_context census_context; -/* A tag is a key:value pair. The key is a non-empty, printable (UTF-8 - encoded), nil-terminated string. The value is a binary string, that may be - printable. There are limits on the sizes of both keys and values (see - CENSUS_MAX_TAG_KB_LEN definition below), and the number of tags that can be - propagated (CENSUS_MAX_PROPAGATED_TAGS). Users should also remember that - some systems may have limits on, e.g., the number of bytes that can be - transmitted as metadata, and that larger tags means more memory consumed - and time in processing. */ +/* A tag is a key:value pair. Both keys and values are nil-terminated strings, + containing printable ASCII characters (decimal 32-126). Keys must be at + least one character in length. Both keys and values can have at most + CENSUS_MAX_TAG_KB_LEN characters (including the terminating nil). The + maximum number of tags that can be propagated is + CENSUS_MAX_PROPAGATED_TAGS. Users should also remember that some systems + may have limits on, e.g., the number of bytes that can be transmitted as + metadata, and that larger tags means more memory consumed and time in + processing. */ typedef struct { const char *key; const char *value; - size_t value_len; uint8_t flags; } census_tag; @@ -103,28 +103,25 @@ typedef struct { /* Tag flags. */ #define CENSUS_TAG_PROPAGATE 1 /* Tag should be propagated over RPC */ #define CENSUS_TAG_STATS 2 /* Tag will be used for statistics aggregation */ -#define CENSUS_TAG_BINARY 4 /* Tag value is not printable */ -#define CENSUS_TAG_RESERVED 8 /* Reserved for internal use. */ -/* Flag values 8,16,32,64,128 are reserved for future/internal use. Clients +#define CENSUS_TAG_RESERVED 4 /* Reserved for internal use. */ +/* Flag values 4,8,16,32,64,128 are reserved for future/internal use. Clients should not use or rely on their values. */ #define CENSUS_TAG_IS_PROPAGATED(flags) (flags & CENSUS_TAG_PROPAGATE) #define CENSUS_TAG_IS_STATS(flags) (flags & CENSUS_TAG_STATS) -#define CENSUS_TAG_IS_BINARY(flags) (flags & CENSUS_TAG_BINARY) /* An instance of this structure is kept by every context, and records the basic information associated with the creation of that context. */ typedef struct { - int n_propagated_tags; /* number of propagated printable tags */ - int n_propagated_binary_tags; /* number of propagated binary tags */ - int n_local_tags; /* number of non-propagated (local) tags */ - int n_deleted_tags; /* number of tags that were deleted */ - int n_added_tags; /* number of tags that were added */ - int n_modified_tags; /* number of tags that were modified */ - int n_invalid_tags; /* number of tags with bad keys or values (e.g. - longer than CENSUS_MAX_TAG_KV_LEN) */ - int n_ignored_tags; /* number of tags ignored because of - CENSUS_MAX_PROPAGATED_TAGS limit. */ + int n_propagated_tags; /* number of propagated tags */ + int n_local_tags; /* number of non-propagated (local) tags */ + int n_deleted_tags; /* number of tags that were deleted */ + int n_added_tags; /* number of tags that were added */ + int n_modified_tags; /* number of tags that were modified */ + int n_invalid_tags; /* number of tags with bad keys or values (e.g. + longer than CENSUS_MAX_TAG_KV_LEN) */ + int n_ignored_tags; /* number of tags ignored because of + CENSUS_MAX_PROPAGATED_TAGS limit. */ } census_context_status; /* Create a new context, adding and removing tags from an existing context. @@ -132,10 +129,10 @@ typedef struct { to add as many tags in a single operation as is practical for the client. @param base Base context to build upon. Can be NULL. @param tags A set of tags to be added/changed/deleted. Tags with keys that - are in 'tags', but not 'base', are added to the tag set. Keys that are in + are in 'tags', but not 'base', are added to the context. Keys that are in both 'tags' and 'base' will have their value/flags modified. Tags with keys - in both, but with NULL or zero-length values, will be deleted from the tag - set. Tags with invalid (too long or short) keys or values will be ignored. + in both, but with NULL values, will be deleted from the context. Tags with + invalid (too long or short) keys or values will be ignored. If adding a tag will result in more than CENSUS_MAX_PROPAGATED_TAGS in either binary or non-binary tags, they will be ignored, as will deletions of tags that don't exist. @@ -185,32 +182,19 @@ CENSUSAPI int census_context_get_tag(const census_context *context, for use by RPC systems only, for purposes of transmitting/receiving contexts. */ -/* Encode a context into a buffer. The propagated tags are encoded into the - buffer in two regions: one for printable tags, and one for binary tags. +/* Encode a context into a buffer. @param context context to be encoded - @param buffer pointer to buffer. This address will be used to encode the - printable tags. + @param buffer buffer into which the context will be encoded. @param buf_size number of available bytes in buffer. - @param print_buf_size Will be set to the number of bytes consumed by - printable tags. - @param bin_buf_size Will be set to the number of bytes used to encode the - binary tags. - @return A pointer to the binary tag's encoded, or NULL if the buffer was - insufficiently large to hold the encoded tags. Thus, if successful, - printable tags are encoded into - [buffer, buffer + *print_buf_size) and binary tags into - [returned-ptr, returned-ptr + *bin_buf_size) (and the returned - pointer should be buffer + *print_buf_size) */ -CENSUSAPI char *census_context_encode(const census_context *context, - char *buffer, size_t buf_size, - size_t *print_buf_size, - size_t *bin_buf_size); - -/* Decode context buffers encoded with census_context_encode(). Returns NULL + @return The number of buffer bytes consumed for the encoded context, or + zero if the buffer was of insufficient size. */ +CENSUSAPI size_t census_context_encode(const census_context *context, + char *buffer, size_t buf_size); + +/* Decode context buffer encoded with census_context_encode(). Returns NULL if there is an error in parsing either buffer. */ -CENSUSAPI census_context *census_context_decode(const char *buffer, size_t size, - const char *bin_buffer, - size_t bin_size); +CENSUSAPI census_context *census_context_decode(const char *buffer, + size_t size); /* Distributed traces can have a number of options. */ enum census_trace_mask_values { diff --git a/src/core/census/context.c b/src/core/census/context.c index e60330de640..441d3b89a6c 100644 --- a/src/core/census/context.c +++ b/src/core/census/context.c @@ -60,10 +60,6 @@ // limit of 255 for both CENSUS_MAX_TAG_KV_LEN and CENSUS_MAX_PROPAGATED_TAGS. // * Keep all tag information (keys/values/flags) in a single memory buffer, // that can be directly copied to the wire. -// * Binary tags share the same structure as, but are encoded separately from, -// non-binary tags. This is primarily because non-binary tags are far more -// likely to be repeated across multiple RPC calls, so are more efficiently -// cached and compressed in any metadata schemes. // Structure representing a set of tags. Essentially a count of number of tags // present, and pointer to a chunk of memory that contains the per-tag details. @@ -77,7 +73,7 @@ struct tag_set { char *kvm; // key/value memory. Consists of repeated entries of: // Offset Size Description // 0 1 Key length, including trailing 0. (K) - // 1 1 Value length. (V) + // 1 1 Value length, including trailing 0 (V) // 2 1 Flags // 3 K Key bytes // 3 + K V Value bytes @@ -108,19 +104,18 @@ struct raw_tag { #define CENSUS_TAG_DELETED CENSUS_TAG_RESERVED #define CENSUS_TAG_IS_DELETED(flags) (flags & CENSUS_TAG_DELETED) -// Primary (external) representation of a context. Composed of 3 underlying -// tag_set structs, one for each of the binary/printable propagated tags, and -// one for everything else. This is to efficiently support tag -// encoding/decoding. +// Primary representation of a context. Composed of 2 underlying tag_set +// structs, one each for propagated and local (non-propagated) tags. This is +// to efficiently support tag encoding/decoding. +// TODO(aveitch): need to add tracing id's/structure. struct census_context { - struct tag_set tags[3]; + struct tag_set tags[2]; census_context_status status; }; // Indices into the tags member of census_context #define PROPAGATED_TAGS 0 -#define PROPAGATED_BINARY_TAGS 1 -#define LOCAL_TAGS 2 +#define LOCAL_TAGS 1 // Extract a raw tag given a pointer (raw) to the tag header. Allow for some // extra bytes in the tag header (see encode/decode functions for usage: this @@ -166,9 +161,7 @@ static bool context_delete_tag(census_context *context, const census_tag *tag, size_t key_len) { return ( tag_set_delete_tag(&context->tags[LOCAL_TAGS], tag->key, key_len) || - tag_set_delete_tag(&context->tags[PROPAGATED_TAGS], tag->key, key_len) || - tag_set_delete_tag(&context->tags[PROPAGATED_BINARY_TAGS], tag->key, - key_len)); + tag_set_delete_tag(&context->tags[PROPAGATED_TAGS], tag->key, key_len)); } // Add a tag to a tag_set. Return true on success, false if the tag could @@ -176,11 +169,11 @@ static bool context_delete_tag(census_context *context, const census_tag *tag, // not be called if the tag may already exist (in a non-deleted state) in // the tag_set, as that would result in two tags with the same key. static bool tag_set_add_tag(struct tag_set *tags, const census_tag *tag, - size_t key_len) { + size_t key_len, size_t value_len) { if (tags->ntags == CENSUS_MAX_PROPAGATED_TAGS) { return false; } - const size_t tag_size = key_len + tag->value_len + TAG_HEADER_SIZE; + const size_t tag_size = key_len + value_len + TAG_HEADER_SIZE; if (tags->kvm_used + tag_size > tags->kvm_size) { // allocate new memory if needed tags->kvm_size += 2 * CENSUS_MAX_TAG_KV_LEN + TAG_HEADER_SIZE; @@ -191,13 +184,12 @@ static bool tag_set_add_tag(struct tag_set *tags, const census_tag *tag, } char *kvp = tags->kvm + tags->kvm_used; *kvp++ = (char)key_len; - *kvp++ = (char)tag->value_len; + *kvp++ = (char)value_len; // ensure reserved flags are not used. - *kvp++ = (char)(tag->flags & (CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS | - CENSUS_TAG_BINARY)); + *kvp++ = (char)(tag->flags & (CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS)); memcpy(kvp, tag->key, key_len); kvp += key_len; - memcpy(kvp, tag->value, tag->value_len); + memcpy(kvp, tag->value, value_len); tags->kvm_used += tag_size; tags->ntags++; tags->ntags_alloc++; @@ -207,30 +199,20 @@ static bool tag_set_add_tag(struct tag_set *tags, const census_tag *tag, // Add/modify/delete a tag to/in a context. Caller must validate that tag key // etc. are valid. static void context_modify_tag(census_context *context, const census_tag *tag, - size_t key_len) { + size_t key_len, size_t value_len) { // First delete the tag if it is already present. bool deleted = context_delete_tag(context, tag, key_len); - // Determine if we need to add it back. - bool call_add = tag->value != NULL && tag->value_len != 0; bool added = false; - if (call_add) { - if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { - if (CENSUS_TAG_IS_BINARY(tag->flags)) { - added = tag_set_add_tag(&context->tags[PROPAGATED_BINARY_TAGS], tag, - key_len); - } else { - added = tag_set_add_tag(&context->tags[PROPAGATED_TAGS], tag, key_len); - } - } else { - added = tag_set_add_tag(&context->tags[LOCAL_TAGS], tag, key_len); - } + if (CENSUS_TAG_IS_PROPAGATED(tag->flags)) { + added = tag_set_add_tag(&context->tags[PROPAGATED_TAGS], tag, key_len, + value_len); + } else { + added = + tag_set_add_tag(&context->tags[LOCAL_TAGS], tag, key_len, value_len); } + if (deleted) { - if (call_add) { - context->status.n_modified_tags++; - } else { - context->status.n_deleted_tags++; - } + context->status.n_modified_tags++; } else { if (added) { context->status.n_added_tags++; @@ -292,8 +274,6 @@ census_context *census_context_create(const census_context *base, memset(context, 0, sizeof(census_context)); } else { tag_set_copy(&context->tags[PROPAGATED_TAGS], &base->tags[PROPAGATED_TAGS]); - tag_set_copy(&context->tags[PROPAGATED_BINARY_TAGS], - &base->tags[PROPAGATED_BINARY_TAGS]); tag_set_copy(&context->tags[LOCAL_TAGS], &base->tags[LOCAL_TAGS]); memset(&context->status, 0, sizeof(context->status)); } @@ -303,20 +283,27 @@ census_context *census_context_create(const census_context *base, const census_tag *tag = &tags[i]; size_t key_len = strlen(tag->key) + 1; // ignore the tag if it is too long/short. - if (key_len != 1 && key_len <= CENSUS_MAX_TAG_KV_LEN && - tag->value_len <= CENSUS_MAX_TAG_KV_LEN) { - context_modify_tag(context, tag, key_len); + if (key_len != 1 && key_len <= CENSUS_MAX_TAG_KV_LEN) { + if (tag->value != NULL) { + size_t value_len = strlen(tag->value) + 1; + if (value_len <= CENSUS_MAX_TAG_KV_LEN) { + context_modify_tag(context, tag, key_len, value_len); + } else { + context->status.n_invalid_tags++; + } + } else { + if (context_delete_tag(context, tag, key_len)) { + context->status.n_deleted_tags++; + } + } } else { context->status.n_invalid_tags++; } } // Remove any deleted tags, update status if needed, and return. tag_set_flatten(&context->tags[PROPAGATED_TAGS]); - tag_set_flatten(&context->tags[PROPAGATED_BINARY_TAGS]); tag_set_flatten(&context->tags[LOCAL_TAGS]); context->status.n_propagated_tags = context->tags[PROPAGATED_TAGS].ntags; - context->status.n_propagated_binary_tags = - context->tags[PROPAGATED_BINARY_TAGS].ntags; context->status.n_local_tags = context->tags[LOCAL_TAGS].ntags; if (status) { *status = &context->status; @@ -331,7 +318,6 @@ const census_context_status *census_context_get_status( void census_context_destroy(census_context *context) { gpr_free(context->tags[PROPAGATED_TAGS].kvm); - gpr_free(context->tags[PROPAGATED_BINARY_TAGS].kvm); gpr_free(context->tags[LOCAL_TAGS].kvm); gpr_free(context); } @@ -343,9 +329,6 @@ void census_context_initialize_iterator(const census_context *context, if (context->tags[PROPAGATED_TAGS].ntags != 0) { iterator->base = PROPAGATED_TAGS; iterator->kvm = context->tags[PROPAGATED_TAGS].kvm; - } else if (context->tags[PROPAGATED_BINARY_TAGS].ntags != 0) { - iterator->base = PROPAGATED_BINARY_TAGS; - iterator->kvm = context->tags[PROPAGATED_BINARY_TAGS].kvm; } else if (context->tags[LOCAL_TAGS].ntags != 0) { iterator->base = LOCAL_TAGS; iterator->kvm = context->tags[LOCAL_TAGS].kvm; @@ -363,7 +346,6 @@ int census_context_next_tag(census_context_iterator *iterator, iterator->kvm = decode_tag(&raw, iterator->kvm, 0); tag->key = raw.key; tag->value = raw.value; - tag->value_len = raw.value_len; tag->flags = raw.flags; if (++iterator->index == iterator->context->tags[iterator->base].ntags) { do { @@ -388,7 +370,6 @@ static bool tag_set_get_tag(const struct tag_set *tags, const char *key, if (key_len == raw.key_len && memcmp(raw.key, key, key_len) == 0) { tag->key = raw.key; tag->value = raw.value; - tag->value_len = raw.value_len; tag->flags = raw.flags; return true; } @@ -403,8 +384,6 @@ int census_context_get_tag(const census_context *context, const char *key, return 0; } if (tag_set_get_tag(&context->tags[PROPAGATED_TAGS], key, key_len, tag) || - tag_set_get_tag(&context->tags[PROPAGATED_BINARY_TAGS], key, key_len, - tag) || tag_set_get_tag(&context->tags[LOCAL_TAGS], key, key_len, tag)) { return 1; } @@ -447,21 +426,9 @@ static size_t tag_set_encode(const struct tag_set *tags, char *buffer, return ENCODED_HEADER_SIZE + tags->kvm_used; } -char *census_context_encode(const census_context *context, char *buffer, - size_t buf_size, size_t *print_buf_size, - size_t *bin_buf_size) { - *print_buf_size = - tag_set_encode(&context->tags[PROPAGATED_TAGS], buffer, buf_size); - if (*print_buf_size == 0) { - return NULL; - } - char *b_buffer = buffer + *print_buf_size; - *bin_buf_size = tag_set_encode(&context->tags[PROPAGATED_BINARY_TAGS], - b_buffer, buf_size - *print_buf_size); - if (*bin_buf_size == 0) { - return NULL; - } - return b_buffer; +size_t census_context_encode(const census_context *context, char *buffer, + size_t buf_size) { + return tag_set_encode(&context->tags[PROPAGATED_TAGS], buffer, buf_size); } // Decode a tag set. @@ -506,8 +473,7 @@ static void tag_set_decode(struct tag_set *tags, const char *buffer, } } -census_context *census_context_decode(const char *buffer, size_t size, - const char *bin_buffer, size_t bin_size) { +census_context *census_context_decode(const char *buffer, size_t size) { census_context *context = gpr_malloc(sizeof(census_context)); memset(&context->tags[LOCAL_TAGS], 0, sizeof(struct tag_set)); if (buffer == NULL) { @@ -515,16 +481,7 @@ census_context *census_context_decode(const char *buffer, size_t size, } else { tag_set_decode(&context->tags[PROPAGATED_TAGS], buffer, size); } - if (bin_buffer == NULL) { - memset(&context->tags[PROPAGATED_BINARY_TAGS], 0, sizeof(struct tag_set)); - } else { - tag_set_decode(&context->tags[PROPAGATED_BINARY_TAGS], bin_buffer, - bin_size); - } memset(&context->status, 0, sizeof(context->status)); context->status.n_propagated_tags = context->tags[PROPAGATED_TAGS].ntags; - context->status.n_propagated_binary_tags = - context->tags[PROPAGATED_BINARY_TAGS].ntags; - // TODO(aveitch): check that BINARY flag is correct for each type. return context; } diff --git a/test/core/census/context_test.c b/test/core/census/context_test.c index 63e7103ddce..b59ac7c0940 100644 --- a/test/core/census/context_test.c +++ b/test/core/census/context_test.c @@ -42,60 +42,48 @@ #include #include "test/core/util/test_config.h" -static uint8_t one_byte_val = 7; -static uint32_t four_byte_val = 0x12345678; -static uint64_t eight_byte_val = 0x1234567890abcdef; - -// A set of tags Used to create a basic context for testing. Each tag has a -// unique set of flags. Note that replace_add_delete_test() relies on specific -// offsets into this array - if you add or delete entries, you will also need -// to change the test. +// A set of tags Used to create a basic context for testing. Note that +// replace_add_delete_test() relies on specific offsets into this array - if +// you add or delete entries, you will also need to change the test. #define BASIC_TAG_COUNT 8 static census_tag basic_tags[BASIC_TAG_COUNT] = { - /* 0 */ {"key0", "printable", 10, 0}, - /* 1 */ {"k1", "a", 2, CENSUS_TAG_PROPAGATE}, - /* 2 */ {"k2", "longer printable string", 24, CENSUS_TAG_STATS}, - /* 3 */ {"key_three", (char *)&one_byte_val, 1, CENSUS_TAG_BINARY}, - /* 4 */ {"really_long_key_4", "random", 7, + /* 0 */ {"key0", "tag value", 0}, + /* 1 */ {"k1", "a", CENSUS_TAG_PROPAGATE}, + /* 2 */ {"k2", "a longer tag value supercalifragilisticexpialiadocious", + CENSUS_TAG_STATS}, + /* 3 */ {"key_three", "", 0}, + /* 4 */ {"a_really_really_really_really_long_key_4", "random", CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS}, - /* 5 */ {"k5", (char *)&four_byte_val, 4, - CENSUS_TAG_PROPAGATE | CENSUS_TAG_BINARY}, - /* 6 */ {"k6", (char *)&eight_byte_val, 8, - CENSUS_TAG_STATS | CENSUS_TAG_BINARY}, - /* 7 */ {"k7", (char *)&four_byte_val, 4, - CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS | CENSUS_TAG_BINARY}}; + /* 5 */ {"k5", "v5", CENSUS_TAG_PROPAGATE}, + /* 6 */ {"k6", "v6", CENSUS_TAG_STATS}, + /* 7 */ {"k7", "v7", CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS}}; // Set of tags used to modify the basic context. Note that // replace_add_delete_test() relies on specific offsets into this array - if // you add or delete entries, you will also need to change the test. Other // tests that rely on specific instances have XXX_XXX_OFFSET definitions (also // change the defines below if you add/delete entires). -#define MODIFY_TAG_COUNT 11 +#define MODIFY_TAG_COUNT 10 static census_tag modify_tags[MODIFY_TAG_COUNT] = { #define REPLACE_VALUE_OFFSET 0 - /* 0 */ {"key0", "replace printable", 18, 0}, // replaces tag value only + /* 0 */ {"key0", "replace key0", 0}, // replaces tag value only #define ADD_TAG_OFFSET 1 - /* 1 */ {"new_key", "xyzzy", 6, CENSUS_TAG_STATS}, // new tag + /* 1 */ {"new_key", "xyzzy", CENSUS_TAG_STATS}, // new tag #define DELETE_TAG_OFFSET 2 - /* 2 */ {"k5", NULL, 5, - 0}, // should delete tag, despite bogus value length - /* 3 */ {"k6", "foo", 0, 0}, // should delete tag, despite bogus value - /* 4 */ {"k6", "foo", 0, 0}, // try deleting already-deleted tag - /* 5 */ {"non-existent", NULL, 0, 0}, // another non-existent tag -#define REPLACE_FLAG_OFFSET 6 - /* 6 */ {"k1", "a", 2, 0}, // change flags only - /* 7 */ {"k7", "bar", 4, CENSUS_TAG_STATS}, // change flags and value - /* 8 */ {"k2", (char *)&eight_byte_val, 8, - CENSUS_TAG_BINARY | CENSUS_TAG_PROPAGATE}, // more flags change - // non-binary -> binary - /* 9 */ {"k6", "bar", 4, 0}, // add back tag, with different value - /* 10 */ {"foo", "bar", 4, CENSUS_TAG_PROPAGATE}, // another new tag + /* 2 */ {"k5", NULL, 0}, // should delete tag + /* 3 */ {"k5", NULL, 0}, // try deleting already-deleted tag + /* 4 */ {"non-existent", NULL, 0}, // delete non-existent tag +#define REPLACE_FLAG_OFFSET 5 + /* 5 */ {"k1", "a", 0}, // change flags only + /* 6 */ {"k7", "bar", CENSUS_TAG_STATS}, // change flags and value + /* 7 */ {"k2", "", CENSUS_TAG_PROPAGATE}, // more value and flags change + /* 8 */ {"k5", "bar", 0}, // add back tag, with different value + /* 9 */ {"foo", "bar", CENSUS_TAG_PROPAGATE}, // another new tag }; // Utility function to compare tags. Returns true if all fields match. static bool compare_tag(const census_tag *t1, const census_tag *t2) { - return (strcmp(t1->key, t2->key) == 0 && t1->value_len == t2->value_len && - memcmp(t1->value, t2->value, t1->value_len) == 0 && + return (strcmp(t1->key, t2->key) == 0 && strcmp(t1->value, t2->value) == 0 && t1->flags == t2->flags); } @@ -111,7 +99,7 @@ static void empty_test(void) { struct census_context *context = census_context_create(NULL, NULL, 0, NULL); GPR_ASSERT(context != NULL); const census_context_status *status = census_context_get_status(context); - census_context_status expected = {0, 0, 0, 0, 0, 0, 0, 0}; + census_context_status expected = {0, 0, 0, 0, 0, 0, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_context_destroy(context); } @@ -121,7 +109,7 @@ static void basic_test(void) { const census_context_status *status; struct census_context *context = census_context_create(NULL, basic_tags, BASIC_TAG_COUNT, &status); - census_context_status expected = {2, 2, 4, 0, 8, 0, 0, 0}; + census_context_status expected = {4, 4, 0, 8, 0, 0, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_context_iterator it; census_context_initialize_iterator(context, &it); @@ -161,15 +149,18 @@ static void invalid_test(void) { memset(key, 'k', 299); key[299] = 0; char value[300]; - memset(value, 'v', 300); - census_tag tag = {key, value, 3, CENSUS_TAG_BINARY}; + memset(value, 'v', 299); + value[299] = 0; + census_tag tag = {key, value, 0}; // long keys, short value. Key lengths (including terminator) should be // <= 255 (CENSUS_MAX_TAG_KV_LEN) + value[3] = 0; + GPR_ASSERT(strlen(value) == 3); GPR_ASSERT(strlen(key) == 299); const census_context_status *status; struct census_context *context = census_context_create(NULL, &tag, 1, &status); - census_context_status expected = {0, 0, 0, 0, 0, 0, 1, 0}; + census_context_status expected = {0, 0, 0, 0, 0, 1, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_context_destroy(context); key[CENSUS_MAX_TAG_KV_LEN] = 0; @@ -180,24 +171,28 @@ static void invalid_test(void) { key[CENSUS_MAX_TAG_KV_LEN - 1] = 0; GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN - 1); context = census_context_create(NULL, &tag, 1, &status); - census_context_status expected2 = {0, 0, 1, 0, 1, 0, 0, 0}; + census_context_status expected2 = {0, 1, 0, 1, 0, 0, 0}; GPR_ASSERT(memcmp(status, &expected2, sizeof(expected2)) == 0); census_context_destroy(context); // now try with long values - tag.value_len = 300; + value[3] = 'v'; + GPR_ASSERT(strlen(value) == 299); context = census_context_create(NULL, &tag, 1, &status); GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_context_destroy(context); - tag.value_len = CENSUS_MAX_TAG_KV_LEN + 1; + value[CENSUS_MAX_TAG_KV_LEN] = 0; + GPR_ASSERT(strlen(value) == CENSUS_MAX_TAG_KV_LEN); context = census_context_create(NULL, &tag, 1, &status); GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_context_destroy(context); - tag.value_len = CENSUS_MAX_TAG_KV_LEN; + value[CENSUS_MAX_TAG_KV_LEN - 1] = 0; + GPR_ASSERT(strlen(value) == CENSUS_MAX_TAG_KV_LEN - 1); context = census_context_create(NULL, &tag, 1, &status); GPR_ASSERT(memcmp(status, &expected2, sizeof(expected2)) == 0); census_context_destroy(context); // 0 length key. key[0] = 0; + GPR_ASSERT(strlen(key) == 0); context = census_context_create(NULL, &tag, 1, &status); GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_context_destroy(context); @@ -210,7 +205,7 @@ static void copy_test(void) { const census_context_status *status; struct census_context *context2 = census_context_create(context, NULL, 0, &status); - census_context_status expected = {2, 2, 4, 0, 0, 0, 0, 0}; + census_context_status expected = {4, 4, 0, 0, 0, 0, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); for (int i = 0; i < BASIC_TAG_COUNT; i++) { census_tag tag; @@ -228,7 +223,7 @@ static void replace_value_test(void) { const census_context_status *status; struct census_context *context2 = census_context_create( context, modify_tags + REPLACE_VALUE_OFFSET, 1, &status); - census_context_status expected = {2, 2, 4, 0, 0, 1, 0, 0}; + census_context_status expected = {4, 4, 0, 0, 1, 0, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag tag; GPR_ASSERT(census_context_get_tag( @@ -245,7 +240,7 @@ static void replace_flags_test(void) { const census_context_status *status; struct census_context *context2 = census_context_create( context, modify_tags + REPLACE_FLAG_OFFSET, 1, &status); - census_context_status expected = {1, 2, 5, 0, 0, 1, 0, 0}; + census_context_status expected = {3, 5, 0, 0, 1, 0, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag tag; GPR_ASSERT(census_context_get_tag( @@ -262,7 +257,7 @@ static void delete_tag_test(void) { const census_context_status *status; struct census_context *context2 = census_context_create( context, modify_tags + DELETE_TAG_OFFSET, 1, &status); - census_context_status expected = {2, 1, 4, 1, 0, 0, 0, 0}; + census_context_status expected = {3, 4, 1, 0, 0, 0, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag tag; GPR_ASSERT(census_context_get_tag( @@ -278,7 +273,7 @@ static void add_tag_test(void) { const census_context_status *status; struct census_context *context2 = census_context_create(context, modify_tags + ADD_TAG_OFFSET, 1, &status); - census_context_status expected = {2, 2, 5, 0, 1, 0, 0, 0}; + census_context_status expected = {4, 5, 0, 1, 0, 0, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_tag tag; GPR_ASSERT(census_context_get_tag(context2, modify_tags[ADD_TAG_OFFSET].key, @@ -295,24 +290,24 @@ static void replace_add_delete_test(void) { const census_context_status *status; struct census_context *context2 = census_context_create(context, modify_tags, MODIFY_TAG_COUNT, &status); - census_context_status expected = {2, 1, 6, 2, 3, 4, 0, 2}; + census_context_status expected = {3, 7, 1, 3, 4, 0, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); // validate context contents. Use specific indices into the two arrays // holding tag values. GPR_ASSERT(validate_tag(context2, &basic_tags[3])); GPR_ASSERT(validate_tag(context2, &basic_tags[4])); + GPR_ASSERT(validate_tag(context2, &basic_tags[6])); GPR_ASSERT(validate_tag(context2, &modify_tags[0])); GPR_ASSERT(validate_tag(context2, &modify_tags[1])); + GPR_ASSERT(validate_tag(context2, &modify_tags[5])); GPR_ASSERT(validate_tag(context2, &modify_tags[6])); GPR_ASSERT(validate_tag(context2, &modify_tags[7])); GPR_ASSERT(validate_tag(context2, &modify_tags[8])); GPR_ASSERT(validate_tag(context2, &modify_tags[9])); - GPR_ASSERT(validate_tag(context2, &modify_tags[10])); GPR_ASSERT(!validate_tag(context2, &basic_tags[0])); GPR_ASSERT(!validate_tag(context2, &basic_tags[1])); GPR_ASSERT(!validate_tag(context2, &basic_tags[2])); GPR_ASSERT(!validate_tag(context2, &basic_tags[5])); - GPR_ASSERT(!validate_tag(context2, &basic_tags[6])); GPR_ASSERT(!validate_tag(context2, &basic_tags[7])); census_context_destroy(context); census_context_destroy(context2); @@ -325,21 +320,15 @@ static void encode_decode_test(void) { char buffer[BUF_SIZE]; struct census_context *context = census_context_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL); - size_t print_bsize; - size_t bin_bsize; // Test with too small a buffer - GPR_ASSERT(census_context_encode(context, buffer, 2, &print_bsize, - &bin_bsize) == NULL); - char *b_buffer = census_context_encode(context, buffer, BUF_SIZE, - &print_bsize, &bin_bsize); - GPR_ASSERT(b_buffer != NULL && print_bsize > 0 && bin_bsize > 0 && - print_bsize + bin_bsize <= BUF_SIZE && - b_buffer == buffer + print_bsize); - census_context *context2 = - census_context_decode(buffer, print_bsize, b_buffer, bin_bsize); + GPR_ASSERT(census_context_encode(context, buffer, 2) == 0); + // Test with sufficient buffer + size_t buf_used = census_context_encode(context, buffer, BUF_SIZE); + GPR_ASSERT(buf_used != 0); + census_context *context2 = census_context_decode(buffer, buf_used); GPR_ASSERT(context2 != NULL); const census_context_status *status = census_context_get_status(context2); - census_context_status expected = {2, 2, 0, 0, 0, 0, 0, 0}; + census_context_status expected = {4, 0, 0, 0, 0, 0, 0}; GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); for (int i = 0; i < BASIC_TAG_COUNT; i++) { census_tag tag; From 36f8f88529e50a8293197fbe0dc68f1a8bc4f6b1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 25 Feb 2016 16:41:33 -0800 Subject: [PATCH 119/236] Fix crash on mac --- test/core/util/port_posix.c | 1 - 1 file changed, 1 deletion(-) diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c index 8b8514ccc93..ba382d242a3 100644 --- a/test/core/util/port_posix.c +++ b/test/core/util/port_posix.c @@ -297,7 +297,6 @@ static int pick_port_using_server(char *server) { grpc_pollset_shutdown(&exec_ctx, pr.pollset, &shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); gpr_free(pr.pollset); - gpr_mu_destroy(pr.mu); return pr.port; } From efefb6edd8f325594c7a014197c5f6d40a4e74ae Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Feb 2016 15:13:29 -0800 Subject: [PATCH 120/236] update nanopb version --- third_party/nanopb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/nanopb b/third_party/nanopb index 5497a1dfc91..f8ac4637662 160000 --- a/third_party/nanopb +++ b/third_party/nanopb @@ -1 +1 @@ -Subproject commit 5497a1dfc91a86965383cdd1652e348345400435 +Subproject commit f8ac463766281625ad710900479130c7fcb4d63b From b63309f52ed7248da7dcd90aefae8abdbcac1ec0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 25 Feb 2016 16:43:45 -0800 Subject: [PATCH 121/236] Fix nanopb --- third_party/nanopb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/nanopb b/third_party/nanopb index f8ac4637662..5497a1dfc91 160000 --- a/third_party/nanopb +++ b/third_party/nanopb @@ -1 +1 @@ -Subproject commit f8ac463766281625ad710900479130c7fcb4d63b +Subproject commit 5497a1dfc91a86965383cdd1652e348345400435 From 8c13e6645f852d0d0e028e5cb1a89823a99f895e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 25 Feb 2016 16:45:00 -0800 Subject: [PATCH 122/236] Fix nanopb --- third_party/nanopb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/nanopb b/third_party/nanopb index 5497a1dfc91..f8ac4637662 160000 --- a/third_party/nanopb +++ b/third_party/nanopb @@ -1 +1 @@ -Subproject commit 5497a1dfc91a86965383cdd1652e348345400435 +Subproject commit f8ac463766281625ad710900479130c7fcb4d63b From 60ab05ac218f62672f286df33db1855e7aba913b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 25 Feb 2016 17:42:44 -0800 Subject: [PATCH 123/236] allow run_tests.py to pass under python 2.6 --- tools/run_tests/report_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py index 0032a985232..df114e5dae1 100644 --- a/tools/run_tests/report_utils.py +++ b/tools/run_tests/report_utils.py @@ -47,7 +47,7 @@ def _filter_msg(msg, output_format): # that make XML report unparseable. filtered_msg = filter( lambda x: x in string.printable and x != '\f' and x != '\v', - msg.decode(errors='ignore')) + msg.decode('UTF-8', 'ignore')) if output_format == 'HTML': filtered_msg = filtered_msg.replace('"', '"') return filtered_msg From e9ef53645150f7a0400a5f9be770eb2b4ba335b5 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 25 Feb 2016 18:10:24 -0800 Subject: [PATCH 124/236] Revert "Add an implementation firewall against pollset_set" --- src/core/channel/client_channel.c | 45 ++++----- .../client_config/lb_policies/pick_first.c | 31 +++--- .../client_config/lb_policies/round_robin.c | 25 ++--- src/core/client_config/lb_policy.c | 4 +- src/core/client_config/lb_policy.h | 3 +- src/core/client_config/subchannel.c | 30 +++--- src/core/httpcli/httpcli.c | 24 +++-- src/core/httpcli/httpcli.h | 3 +- src/core/iomgr/fd_posix.c | 6 +- src/core/iomgr/pollset.h | 15 +-- .../iomgr/pollset_multipoller_with_epoll.c | 1 - .../pollset_multipoller_with_poll_posix.c | 8 +- src/core/iomgr/pollset_posix.c | 16 ++-- src/core/iomgr/pollset_posix.h | 12 +-- src/core/iomgr/pollset_set.h | 10 +- src/core/iomgr/pollset_set_posix.c | 26 +---- src/core/iomgr/pollset_set_posix.h | 20 +++- src/core/iomgr/pollset_set_windows.c | 4 +- src/core/iomgr/pollset_set_windows.h | 2 +- src/core/iomgr/pollset_windows.c | 7 +- src/core/iomgr/pollset_windows.h | 6 +- src/core/iomgr/tcp_client_posix.c | 12 +-- src/core/iomgr/tcp_posix.c | 10 +- src/core/iomgr/udp_server.h | 1 - src/core/iomgr/workqueue_posix.c | 1 - src/core/iomgr/workqueue_posix.h | 4 +- .../security/google_default_credentials.c | 39 ++++---- src/core/surface/completion_queue.c | 85 ++++++++--------- .../set_initial_connect_string_test.c | 2 +- .../core/end2end/fixtures/h2_full+poll+pipe.c | 18 ++-- test/core/end2end/fixtures/h2_full+poll.c | 16 ++-- test/core/end2end/fixtures/h2_ssl+poll.c | 1 - test/core/end2end/fixtures/h2_uchannel.c | 47 +++++---- test/core/end2end/fixtures/h2_uds+poll.c | 18 ++-- test/core/httpcli/httpcli_test.c | 44 ++++----- test/core/httpcli/httpscli_test.c | 44 ++++----- test/core/iomgr/endpoint_pair_test.c | 19 ++-- test/core/iomgr/endpoint_tests.c | 18 ++-- test/core/iomgr/endpoint_tests.h | 4 +- test/core/iomgr/fd_posix_test.c | 89 ++++++++--------- test/core/iomgr/tcp_client_posix_test.c | 74 +++++++-------- test/core/iomgr/tcp_posix_test.c | 95 +++++++++---------- test/core/iomgr/tcp_server_posix_test.c | 50 +++++----- test/core/iomgr/workqueue_test.c | 39 ++++---- test/core/security/oauth2_utils.c | 25 +++-- .../print_google_default_creds_token.c | 34 +++---- test/core/security/secure_endpoint_test.c | 26 +++-- test/core/security/verify_jwt.c | 29 +++--- test/core/util/port_posix.c | 53 +++++------ test/core/util/port_windows.c | 27 +++--- test/core/util/test_tcp_server.c | 17 ++-- test/core/util/test_tcp_server.h | 4 +- 52 files changed, 563 insertions(+), 680 deletions(-) diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index a96b49ac12f..7176c01b05f 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -78,8 +78,8 @@ typedef struct client_channel_channel_data { int exit_idle_when_lb_policy_arrives; /** owning stack */ grpc_channel_stack *owning_stack; - /** interested parties (owned) */ - grpc_pollset_set *interested_parties; + /** interested parties */ + grpc_pollset_set interested_parties; } channel_data; /** We create one watcher for each new lb_policy that is returned from a @@ -183,8 +183,8 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, chand->incoming_configuration = NULL; if (lb_policy != NULL) { - grpc_pollset_set_add_pollset_set(exec_ctx, lb_policy->interested_parties, - chand->interested_parties); + grpc_pollset_set_add_pollset_set(exec_ctx, &lb_policy->interested_parties, + &chand->interested_parties); } gpr_mu_lock(&chand->mu_config); @@ -231,8 +231,9 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, } if (old_lb_policy != NULL) { - grpc_pollset_set_del_pollset_set( - exec_ctx, old_lb_policy->interested_parties, chand->interested_parties); + grpc_pollset_set_del_pollset_set(exec_ctx, + &old_lb_policy->interested_parties, + &chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, old_lb_policy, "channel"); } @@ -253,7 +254,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, GPR_ASSERT(op->set_accept_stream == NULL); if (op->bind_pollset != NULL) { - grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, + grpc_pollset_set_add_pollset(exec_ctx, &chand->interested_parties, op->bind_pollset); } @@ -283,8 +284,8 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, chand->resolver = NULL; if (chand->lb_policy != NULL) { grpc_pollset_set_del_pollset_set(exec_ctx, - chand->lb_policy->interested_parties, - chand->interested_parties); + &chand->lb_policy->interested_parties, + &chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel"); chand->lb_policy = NULL; } @@ -410,7 +411,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE, "client_channel"); - chand->interested_parties = grpc_pollset_set_create(); + grpc_pollset_set_init(&chand->interested_parties); } /* Destructor for channel_data */ @@ -424,12 +425,12 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, } if (chand->lb_policy != NULL) { grpc_pollset_set_del_pollset_set(exec_ctx, - chand->lb_policy->interested_parties, - chand->interested_parties); + &chand->lb_policy->interested_parties, + &chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel"); } grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker); - grpc_pollset_set_destroy(chand->interested_parties); + grpc_pollset_set_destroy(&chand->interested_parties); gpr_mu_destroy(&chand->mu_config); } @@ -440,17 +441,9 @@ static void cc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, } const grpc_channel_filter grpc_client_channel_filter = { - cc_start_transport_stream_op, - cc_start_transport_op, - sizeof(call_data), - init_call_elem, - cc_set_pollset, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - cc_get_peer, - "client-channel", + cc_start_transport_stream_op, cc_start_transport_op, sizeof(call_data), + init_call_elem, cc_set_pollset, destroy_call_elem, sizeof(channel_data), + init_channel_elem, destroy_channel_elem, cc_get_peer, "client-channel", }; void grpc_client_channel_set_resolver(grpc_exec_ctx *exec_ctx, @@ -508,7 +501,7 @@ static void on_external_watch_complete(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) { external_connectivity_watcher *w = arg; grpc_closure *follow_up = w->on_complete; - grpc_pollset_set_del_pollset(exec_ctx, w->chand->interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, &w->chand->interested_parties, w->pollset); GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "external_connectivity_watcher"); @@ -524,7 +517,7 @@ void grpc_client_channel_watch_connectivity_state( w->chand = chand; w->pollset = pollset; w->on_complete = on_complete; - grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, pollset); + grpc_pollset_set_add_pollset(exec_ctx, &chand->interested_parties, pollset); grpc_closure_init(&w->my_closure, on_external_watch_complete, w); GRPC_CHANNEL_STACK_REF(w->chand->owning_stack, "external_connectivity_watcher"); diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c index 9f38f398d8a..459bbebb68a 100644 --- a/src/core/client_config/lb_policies/pick_first.c +++ b/src/core/client_config/lb_policies/pick_first.c @@ -31,8 +31,8 @@ * */ -#include "src/core/client_config/lb_policies/pick_first.h" #include "src/core/client_config/lb_policy_factory.h" +#include "src/core/client_config/lb_policies/pick_first.h" #include @@ -119,7 +119,7 @@ void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { while (pp != NULL) { pending_pick *next = pp->next; *pp->target = NULL; - grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, pp->pollset); grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL); gpr_free(pp); @@ -137,7 +137,7 @@ static void pf_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, while (pp != NULL) { pending_pick *next = pp->next; if (pp->target == target) { - grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, pp->pollset); *target = NULL; grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL); @@ -158,7 +158,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, pick_first_lb_policy *p) { GRPC_LB_POLICY_WEAK_REF(&p->base, "pick_first_connectivity"); grpc_subchannel_notify_on_state_change( exec_ctx, p->subchannels[p->checking_subchannel], - p->base.interested_parties, &p->checking_connectivity, + &p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); } @@ -195,7 +195,8 @@ int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset, if (!p->started_picking) { start_picking(exec_ctx, p); } - grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset); + grpc_pollset_set_add_pollset(exec_ctx, &p->base.interested_parties, + pollset); pp = gpr_malloc(sizeof(*pp)); pp->next = p->pending_picks; pp->pollset = pollset; @@ -252,7 +253,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, p->checking_connectivity, "selected_changed"); if (p->checking_connectivity != GRPC_CHANNEL_FATAL_FAILURE) { grpc_connected_subchannel_notify_on_state_change( - exec_ctx, selected, p->base.interested_parties, + exec_ctx, selected, &p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); } else { GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "pick_first_connectivity"); @@ -277,13 +278,13 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, while ((pp = p->pending_picks)) { p->pending_picks = pp->next; *pp->target = selected; - grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, pp->pollset); grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL); gpr_free(pp); } grpc_connected_subchannel_notify_on_state_change( - exec_ctx, selected, p->base.interested_parties, + exec_ctx, selected, &p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); break; case GRPC_CHANNEL_TRANSIENT_FAILURE: @@ -297,7 +298,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, if (p->checking_connectivity == GRPC_CHANNEL_TRANSIENT_FAILURE) { grpc_subchannel_notify_on_state_change( exec_ctx, p->subchannels[p->checking_subchannel], - p->base.interested_parties, &p->checking_connectivity, + &p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); } else { goto loop; @@ -310,7 +311,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, "connecting_changed"); grpc_subchannel_notify_on_state_change( exec_ctx, p->subchannels[p->checking_subchannel], - p->base.interested_parties, &p->checking_connectivity, + &p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); break; case GRPC_CHANNEL_FATAL_FAILURE: @@ -378,14 +379,8 @@ void pf_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } static const grpc_lb_policy_vtable pick_first_lb_policy_vtable = { - pf_destroy, - pf_shutdown, - pf_pick, - pf_cancel_pick, - pf_ping_one, - pf_exit_idle, - pf_check_connectivity, - pf_notify_on_state_change}; + pf_destroy, pf_shutdown, pf_pick, pf_cancel_pick, pf_ping_one, pf_exit_idle, + pf_check_connectivity, pf_notify_on_state_change}; static void pick_first_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/lb_policies/round_robin.c b/src/core/client_config/lb_policies/round_robin.c index 114ece6e4d9..b1171c45b00 100644 --- a/src/core/client_config/lb_policies/round_robin.c +++ b/src/core/client_config/lb_policies/round_robin.c @@ -260,7 +260,7 @@ static void rr_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, while (pp != NULL) { pending_pick *next = pp->next; if (pp->target == target) { - grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, pp->pollset); *target = NULL; grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL); @@ -285,7 +285,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) { subchannel_data *sd = p->subchannels[i]; sd->connectivity_state = GRPC_CHANNEL_IDLE; grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, p->base.interested_parties, + exec_ctx, sd->subchannel, &p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); GRPC_LB_POLICY_WEAK_REF(&p->base, "round_robin_connectivity"); } @@ -322,7 +322,8 @@ int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset, if (!p->started_picking) { start_picking(exec_ctx, p); } - grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset); + grpc_pollset_set_add_pollset(exec_ctx, &p->base.interested_parties, + pollset); pp = gpr_malloc(sizeof(*pp)); pp->next = p->pending_picks; pp->pollset = pollset; @@ -373,13 +374,13 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)", selected->subchannel, selected); } - grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, pp->pollset); grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL); gpr_free(pp); } grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, p->base.interested_parties, + exec_ctx, sd->subchannel, &p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); break; case GRPC_CHANNEL_CONNECTING: @@ -388,13 +389,13 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, sd->connectivity_state, "connecting_changed"); grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, p->base.interested_parties, + exec_ctx, sd->subchannel, &p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); break; case GRPC_CHANNEL_TRANSIENT_FAILURE: /* renew state notification */ grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, p->base.interested_parties, + exec_ctx, sd->subchannel, &p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); /* remove from ready list if still present */ @@ -483,14 +484,8 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } static const grpc_lb_policy_vtable round_robin_lb_policy_vtable = { - rr_destroy, - rr_shutdown, - rr_pick, - rr_cancel_pick, - rr_ping_one, - rr_exit_idle, - rr_check_connectivity, - rr_notify_on_state_change}; + rr_destroy, rr_shutdown, rr_pick, rr_cancel_pick, rr_ping_one, rr_exit_idle, + rr_check_connectivity, rr_notify_on_state_change}; static void round_robin_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/lb_policy.c b/src/core/client_config/lb_policy.c index 5ff623e0069..d4672f6b255 100644 --- a/src/core/client_config/lb_policy.c +++ b/src/core/client_config/lb_policy.c @@ -39,7 +39,7 @@ void grpc_lb_policy_init(grpc_lb_policy *policy, const grpc_lb_policy_vtable *vtable) { policy->vtable = vtable; gpr_atm_no_barrier_store(&policy->ref_pair, 1 << WEAK_REF_BITS); - policy->interested_parties = grpc_pollset_set_create(); + grpc_pollset_set_init(&policy->interested_parties); } #ifdef GRPC_LB_POLICY_REFCOUNT_DEBUG @@ -93,7 +93,7 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, gpr_atm old_val = ref_mutate(policy, -(gpr_atm)1, 1 REF_MUTATE_PASS_ARGS("WEAK_UNREF")); if (old_val == 1) { - grpc_pollset_set_destroy(policy->interested_parties); + grpc_pollset_set_destroy(&policy->interested_parties); policy->vtable->destroy(exec_ctx, policy); } } diff --git a/src/core/client_config/lb_policy.h b/src/core/client_config/lb_policy.h index 4fbb12da394..db5238c8ca0 100644 --- a/src/core/client_config/lb_policy.h +++ b/src/core/client_config/lb_policy.h @@ -48,8 +48,7 @@ typedef void (*grpc_lb_completion)(void *cb_arg, grpc_subchannel *subchannel, struct grpc_lb_policy { const grpc_lb_policy_vtable *vtable; gpr_atm ref_pair; - /* owned pointer to interested parties in load balancing decisions */ - grpc_pollset_set *interested_parties; + grpc_pollset_set interested_parties; }; struct grpc_lb_policy_vtable { diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index 291ad3472cc..6599c75dba7 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -108,7 +108,7 @@ struct grpc_subchannel { /** pollset_set tracking who's interested in a connection being setup */ - grpc_pollset_set *pollset_set; + grpc_pollset_set pollset_set; /** active connection, or null; of type grpc_connected_subchannel */ gpr_atm connected_subchannel; @@ -184,8 +184,8 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(c); } -void grpc_connected_subchannel_ref( - grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_connected_subchannel_ref(grpc_connected_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON); } @@ -209,7 +209,7 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_slice_unref(c->initial_connect_string); grpc_connectivity_state_destroy(exec_ctx, &c->state_tracker); grpc_connector_unref(exec_ctx, c->connector); - grpc_pollset_set_destroy(c->pollset_set); + grpc_pollset_set_destroy(&c->pollset_set); grpc_subchannel_key_destroy(exec_ctx, c->key); gpr_free(c); } @@ -226,8 +226,8 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta, return old_val; } -grpc_subchannel *grpc_subchannel_ref( - grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS), 0 REF_MUTATE_PURPOSE("STRONG_REF")); @@ -235,8 +235,8 @@ grpc_subchannel *grpc_subchannel_ref( return c; } -grpc_subchannel *grpc_subchannel_weak_ref( - grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF")); GPR_ASSERT(old_refs != 0); @@ -326,7 +326,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, } c->addr = gpr_malloc(args->addr_len); memcpy(c->addr, args->addr, args->addr_len); - c->pollset_set = grpc_pollset_set_create(); + grpc_pollset_set_init(&c->pollset_set); c->addr_len = args->addr_len; grpc_set_initial_connect_string(&c->addr, &c->addr_len, &c->initial_connect_string); @@ -345,7 +345,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { grpc_connect_in_args args; - args.interested_parties = c->pollset_set; + args.interested_parties = &c->pollset_set; args.addr = c->addr; args.addr_len = c->addr_len; args.deadline = compute_connect_deadline(c); @@ -379,7 +379,7 @@ static void on_external_state_watcher_done(grpc_exec_ctx *exec_ctx, void *arg, external_state_watcher *w = arg; grpc_closure *follow_up = w->notify; if (w->pollset_set != NULL) { - grpc_pollset_set_del_pollset_set(exec_ctx, w->subchannel->pollset_set, + grpc_pollset_set_del_pollset_set(exec_ctx, &w->subchannel->pollset_set, w->pollset_set); } gpr_mu_lock(&w->subchannel->mu); @@ -415,7 +415,7 @@ void grpc_subchannel_notify_on_state_change( w->notify = notify; grpc_closure_init(&w->closure, on_external_state_watcher_done, w); if (interested_parties != NULL) { - grpc_pollset_set_add_pollset_set(exec_ctx, c->pollset_set, + grpc_pollset_set_add_pollset_set(exec_ctx, &c->pollset_set, interested_parties); } GRPC_SUBCHANNEL_WEAK_REF(c, "external_state_watcher"); @@ -573,7 +573,7 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher"); GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); grpc_connected_subchannel_notify_on_state_change( - exec_ctx, con, c->pollset_set, &sw_subchannel->connectivity_state, + exec_ctx, con, &c->pollset_set, &sw_subchannel->connectivity_state, &sw_subchannel->closure); /* signal completion */ @@ -690,8 +690,8 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); } -void grpc_subchannel_call_ref( - grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_subchannel_call_ref(grpc_subchannel_call *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } diff --git a/src/core/httpcli/httpcli.c b/src/core/httpcli/httpcli.c index 1219c444c71..71237bb6140 100644 --- a/src/core/httpcli/httpcli.c +++ b/src/core/httpcli/httpcli.c @@ -31,22 +31,20 @@ * */ -#include "src/core/httpcli/httpcli.h" #include "src/core/iomgr/sockaddr.h" +#include "src/core/httpcli/httpcli.h" #include -#include -#include -#include - -#include "src/core/httpcli/format_request.h" -#include "src/core/httpcli/parser.h" #include "src/core/iomgr/endpoint.h" -#include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/resolve_address.h" #include "src/core/iomgr/tcp_client.h" +#include "src/core/httpcli/format_request.h" +#include "src/core/httpcli/parser.h" #include "src/core/support/string.h" +#include +#include +#include typedef struct { gpr_slice request_text; @@ -86,18 +84,18 @@ const grpc_httpcli_handshaker grpc_httpcli_plaintext = {"http", plaintext_handshake}; void grpc_httpcli_context_init(grpc_httpcli_context *context) { - context->pollset_set = grpc_pollset_set_create(); + grpc_pollset_set_init(&context->pollset_set); } void grpc_httpcli_context_destroy(grpc_httpcli_context *context) { - grpc_pollset_set_destroy(context->pollset_set); + grpc_pollset_set_destroy(&context->pollset_set); } static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req); static void finish(grpc_exec_ctx *exec_ctx, internal_request *req, int success) { - grpc_pollset_set_del_pollset(exec_ctx, req->context->pollset_set, + grpc_pollset_set_del_pollset(exec_ctx, &req->context->pollset_set, req->pollset); req->on_response(exec_ctx, req->user_data, success ? &req->parser.r : NULL); grpc_httpcli_parser_destroy(&req->parser); @@ -199,7 +197,7 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req) { addr = &req->addresses->addrs[req->next_address++]; grpc_closure_init(&req->connected, on_connected, req); grpc_tcp_client_connect( - exec_ctx, &req->connected, &req->ep, req->context->pollset_set, + exec_ctx, &req->connected, &req->ep, &req->context->pollset_set, (struct sockaddr *)&addr->addr, addr->len, req->deadline); } @@ -239,7 +237,7 @@ static void internal_request_begin( req->host = gpr_strdup(request->host); req->ssl_host_override = gpr_strdup(request->ssl_host_override); - grpc_pollset_set_add_pollset(exec_ctx, req->context->pollset_set, + grpc_pollset_set_add_pollset(exec_ctx, &req->context->pollset_set, req->pollset); grpc_resolve_address(request->host, req->handshaker->default_port, on_resolved, req); diff --git a/src/core/httpcli/httpcli.h b/src/core/httpcli/httpcli.h index 86e17c1d697..30875d71f13 100644 --- a/src/core/httpcli/httpcli.h +++ b/src/core/httpcli/httpcli.h @@ -39,7 +39,6 @@ #include #include "src/core/iomgr/endpoint.h" -#include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/pollset_set.h" /* User agent this library reports */ @@ -57,7 +56,7 @@ typedef struct grpc_httpcli_header { TODO(ctiller): allow caching and capturing multiple requests for the same content and combining them */ typedef struct grpc_httpcli_context { - grpc_pollset_set *pollset_set; + grpc_pollset_set pollset_set; } grpc_httpcli_context; typedef struct { diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c index 4ba7c5df943..85eadd754b9 100644 --- a/src/core/iomgr/fd_posix.c +++ b/src/core/iomgr/fd_posix.c @@ -46,8 +46,6 @@ #include #include -#include "src/core/iomgr/pollset_posix.h" - #define CLOSURE_NOT_READY ((grpc_closure *)0) #define CLOSURE_READY ((grpc_closure *)1) @@ -177,11 +175,11 @@ int grpc_fd_is_orphaned(grpc_fd *fd) { } static void pollset_kick_locked(grpc_fd_watcher *watcher) { - gpr_mu_lock(&watcher->pollset->mu); + gpr_mu_lock(GRPC_POLLSET_MU(watcher->pollset)); GPR_ASSERT(watcher->worker); grpc_pollset_kick_ext(watcher->pollset, watcher->worker, GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); - gpr_mu_unlock(&watcher->pollset->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(watcher->pollset)); } static void maybe_wake_one_watcher_locked(grpc_fd *fd) { diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index 92a0374ddd4..6585326f81a 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -35,11 +35,8 @@ #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_H #include -#include #include -#include "src/core/iomgr/exec_ctx.h" - #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) /* A grpc_pollset is a set of file descriptors that a higher level item is @@ -49,11 +46,15 @@ - a completion queue might keep a pollset with an entry for each transport that is servicing a call that it's tracking */ -typedef struct grpc_pollset grpc_pollset; -typedef struct grpc_pollset_worker grpc_pollset_worker; +#ifdef GPR_POSIX_SOCKET +#include "src/core/iomgr/pollset_posix.h" +#endif + +#ifdef GPR_WIN32 +#include "src/core/iomgr/pollset_windows.h" +#endif -size_t grpc_pollset_size(void); -void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu); +void grpc_pollset_init(grpc_pollset *pollset); /* Begin shutting down the pollset, and call closure when done. * GRPC_POLLSET_MU(pollset) must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index 2e0f27fab86..4acae2bb712 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -45,7 +45,6 @@ #include #include #include "src/core/iomgr/fd_posix.h" -#include "src/core/iomgr/pollset_posix.h" #include "src/core/profiling/timers.h" #include "src/core/support/block_annotate.h" diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 4dddfff2302..809f8f39daa 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -42,14 +42,12 @@ #include #include -#include -#include -#include - #include "src/core/iomgr/fd_posix.h" #include "src/core/iomgr/iomgr_internal.h" -#include "src/core/iomgr/pollset_posix.h" #include "src/core/support/block_annotate.h" +#include +#include +#include typedef struct { /* all polled fds */ diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index e895a778849..ee7e9f48f4c 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -42,16 +42,16 @@ #include #include -#include -#include -#include -#include -#include #include "src/core/iomgr/fd_posix.h" #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/socket_utils_posix.h" #include "src/core/profiling/timers.h" #include "src/core/support/block_annotate.h" +#include +#include +#include +#include +#include GPR_TLS_DECL(g_current_thread_poller); GPR_TLS_DECL(g_current_thread_worker); @@ -97,8 +97,6 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { worker->prev->next = worker->next->prev = worker; } -size_t grpc_pollset_size(void) { return sizeof(grpc_pollset); } - void grpc_pollset_kick_ext(grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) { @@ -188,9 +186,8 @@ void grpc_kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); } static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null); -void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { +void grpc_pollset_init(grpc_pollset *pollset) { gpr_mu_init(&pollset->mu); - *mu = &pollset->mu; pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; pollset->in_flight_cbs = 0; pollset->shutting_down = 0; @@ -207,6 +204,7 @@ void grpc_pollset_destroy(grpc_pollset *pollset) { GPR_ASSERT(!grpc_pollset_has_workers(pollset)); GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); pollset->vtable->destroy(pollset); + gpr_mu_destroy(&pollset->mu); while (pollset->local_wakeup_cache) { grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next; grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd); diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h index bbedb66b007..5868b3fa21c 100644 --- a/src/core/iomgr/pollset_posix.h +++ b/src/core/iomgr/pollset_posix.h @@ -37,10 +37,8 @@ #include #include - #include "src/core/iomgr/exec_ctx.h" #include "src/core/iomgr/iomgr.h" -#include "src/core/iomgr/pollset.h" #include "src/core/iomgr/wakeup_fd_posix.h" typedef struct grpc_pollset_vtable grpc_pollset_vtable; @@ -55,15 +53,15 @@ typedef struct grpc_cached_wakeup_fd { struct grpc_cached_wakeup_fd *next; } grpc_cached_wakeup_fd; -struct grpc_pollset_worker { +typedef struct grpc_pollset_worker { grpc_cached_wakeup_fd *wakeup_fd; int reevaluate_polling_on_wakeup; int kicked_specifically; struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; -}; +} grpc_pollset_worker; -struct grpc_pollset { +typedef struct grpc_pollset { /* pollsets under posix can mutate representation as fds are added and removed. For example, we may choose a poll() based implementation on linux for @@ -83,7 +81,7 @@ struct grpc_pollset { } data; /* Local cache of eventfds for workers */ grpc_cached_wakeup_fd *local_wakeup_cache; -}; +} grpc_pollset; struct grpc_pollset_vtable { void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -95,6 +93,8 @@ struct grpc_pollset_vtable { void (*destroy)(grpc_pollset *pollset); }; +#define GRPC_POLLSET_MU(pollset) (&(pollset)->mu) + /* Add an fd to a pollset */ void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, struct grpc_fd *fd); diff --git a/src/core/iomgr/pollset_set.h b/src/core/iomgr/pollset_set.h index 9591bf0d32c..09c04438f71 100644 --- a/src/core/iomgr/pollset_set.h +++ b/src/core/iomgr/pollset_set.h @@ -41,9 +41,15 @@ fd's (etc) that have been registered with the set_set to that pollset. Registering fd's automatically adds them to all current pollsets. */ -typedef struct grpc_pollset_set grpc_pollset_set; +#ifdef GPR_POSIX_SOCKET +#include "src/core/iomgr/pollset_set_posix.h" +#endif -grpc_pollset_set *grpc_pollset_set_create(void); +#ifdef GPR_WIN32 +#include "src/core/iomgr/pollset_set_windows.h" +#endif + +void grpc_pollset_set_init(grpc_pollset_set *pollset_set); void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set); void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/iomgr/pollset_set_posix.c index 9dc9aff4a8d..4ec92202e30 100644 --- a/src/core/iomgr/pollset_set_posix.c +++ b/src/core/iomgr/pollset_set_posix.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2015, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,30 +41,11 @@ #include #include -#include "src/core/iomgr/pollset_posix.h" -#include "src/core/iomgr/pollset_set_posix.h" +#include "src/core/iomgr/pollset_set.h" -struct grpc_pollset_set { - gpr_mu mu; - - size_t pollset_count; - size_t pollset_capacity; - grpc_pollset **pollsets; - - size_t pollset_set_count; - size_t pollset_set_capacity; - struct grpc_pollset_set **pollset_sets; - - size_t fd_count; - size_t fd_capacity; - grpc_fd **fds; -}; - -grpc_pollset_set *grpc_pollset_set_create(void) { - grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set)); +void grpc_pollset_set_init(grpc_pollset_set *pollset_set) { memset(pollset_set, 0, sizeof(*pollset_set)); gpr_mu_init(&pollset_set->mu); - return pollset_set; } void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) { @@ -76,7 +57,6 @@ void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) { gpr_free(pollset_set->pollsets); gpr_free(pollset_set->pollset_sets); gpr_free(pollset_set->fds); - gpr_free(pollset_set); } void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, diff --git a/src/core/iomgr/pollset_set_posix.h b/src/core/iomgr/pollset_set_posix.h index 7d1aaf41817..4820a61e4b2 100644 --- a/src/core/iomgr/pollset_set_posix.h +++ b/src/core/iomgr/pollset_set_posix.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2015, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,7 +35,23 @@ #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_POSIX_H #include "src/core/iomgr/fd_posix.h" -#include "src/core/iomgr/pollset_set.h" +#include "src/core/iomgr/pollset_posix.h" + +typedef struct grpc_pollset_set { + gpr_mu mu; + + size_t pollset_count; + size_t pollset_capacity; + grpc_pollset **pollsets; + + size_t pollset_set_count; + size_t pollset_set_capacity; + struct grpc_pollset_set **pollset_sets; + + size_t fd_count; + size_t fd_capacity; + grpc_fd **fds; +} grpc_pollset_set; void grpc_pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, grpc_fd *fd); diff --git a/src/core/iomgr/pollset_set_windows.c b/src/core/iomgr/pollset_set_windows.c index 9cf8fd4472f..157b46ec32a 100644 --- a/src/core/iomgr/pollset_set_windows.c +++ b/src/core/iomgr/pollset_set_windows.c @@ -35,9 +35,9 @@ #ifdef GPR_WINSOCK_SOCKET -#include "src/core/iomgr/pollset_set_windows.h" +#include "src/core/iomgr/pollset_set.h" -grpc_pollset_set* grpc_pollset_set_create(pollset_set) { return NULL; } +void grpc_pollset_set_init(grpc_pollset_set* pollset_set) {} void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {} diff --git a/src/core/iomgr/pollset_set_windows.h b/src/core/iomgr/pollset_set_windows.h index aa5abe91338..cada0d2b61f 100644 --- a/src/core/iomgr/pollset_set_windows.h +++ b/src/core/iomgr/pollset_set_windows.h @@ -34,6 +34,6 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H -#include "src/core/iomgr/pollset_set.h" +typedef struct grpc_pollset_set { void *unused; } grpc_pollset_set; #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index 651f8e7334c..bbce23b46a7 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -89,17 +89,12 @@ static void push_front_worker(grpc_pollset_worker *root, worker->links[type].next->links[type].prev = worker; } -size_t grpc_pollset_size(void) { - return sizeof(grpc_pollset); -} - /* There isn't really any such thing as a pollset under Windows, due to the nature of the IO completion ports. We're still going to provide a minimal set of features for the sake of the rest of grpc. But grpc_pollset_work won't actually do any polling, and return as quickly as possible. */ -void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - *mu = &grpc_polling_mu; +void grpc_pollset_init(grpc_pollset *pollset) { memset(pollset, 0, sizeof(*pollset)); pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h index dc0b7a4104b..65ba80619b7 100644 --- a/src/core/iomgr/pollset_windows.h +++ b/src/core/iomgr/pollset_windows.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2015, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -72,4 +72,8 @@ struct grpc_pollset { grpc_closure *on_shutdown; }; +extern gpr_mu grpc_polling_mu; + +#define GRPC_POLLSET_MU(pollset) (&grpc_polling_mu) + #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/tcp_client_posix.c b/src/core/iomgr/tcp_client_posix.c index 15727856abf..c76c2e3b0f2 100644 --- a/src/core/iomgr/tcp_client_posix.c +++ b/src/core/iomgr/tcp_client_posix.c @@ -42,19 +42,17 @@ #include #include -#include -#include -#include -#include - +#include "src/core/iomgr/timer.h" #include "src/core/iomgr/iomgr_posix.h" #include "src/core/iomgr/pollset_posix.h" -#include "src/core/iomgr/pollset_set_posix.h" #include "src/core/iomgr/sockaddr_utils.h" #include "src/core/iomgr/socket_utils_posix.h" #include "src/core/iomgr/tcp_posix.h" -#include "src/core/iomgr/timer.h" #include "src/core/support/string.h" +#include +#include +#include +#include extern int grpc_tcp_trace; diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index e8f73811cec..048e9074412 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -40,8 +40,8 @@ #include #include #include -#include #include +#include #include #include @@ -51,11 +51,9 @@ #include #include +#include "src/core/support/string.h" #include "src/core/debug/trace.h" -#include "src/core/iomgr/pollset_posix.h" -#include "src/core/iomgr/pollset_set_posix.h" #include "src/core/profiling/timers.h" -#include "src/core/support/string.h" #ifdef GPR_HAVE_MSG_NOSIGNAL #define SENDMSG_FLAGS MSG_NOSIGNAL @@ -297,7 +295,7 @@ static flush_result tcp_flush(grpc_tcp *tcp) { unwind_slice_idx = tcp->outgoing_slice_idx; unwind_byte_idx = tcp->outgoing_byte_idx; for (iov_size = 0; tcp->outgoing_slice_idx != tcp->outgoing_buffer->count && - iov_size != MAX_WRITE_IOVEC; + iov_size != MAX_WRITE_IOVEC; iov_size++) { iov[iov_size].iov_base = GPR_SLICE_START_PTR( @@ -446,7 +444,7 @@ static char *tcp_get_peer(grpc_endpoint *ep) { } static const grpc_endpoint_vtable vtable = { - tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set, + tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set, tcp_shutdown, tcp_destroy, tcp_get_peer}; grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size, diff --git a/src/core/iomgr/udp_server.h b/src/core/iomgr/udp_server.h index a9d0489edfd..73a21c80ab4 100644 --- a/src/core/iomgr/udp_server.h +++ b/src/core/iomgr/udp_server.h @@ -35,7 +35,6 @@ #define GRPC_INTERNAL_CORE_IOMGR_UDP_SERVER_H #include "src/core/iomgr/endpoint.h" -#include "src/core/iomgr/fd_posix.h" /* Forward decl of grpc_server */ typedef struct grpc_server grpc_server; diff --git a/src/core/iomgr/workqueue_posix.c b/src/core/iomgr/workqueue_posix.c index c096dbfb30c..da11df67efb 100644 --- a/src/core/iomgr/workqueue_posix.c +++ b/src/core/iomgr/workqueue_posix.c @@ -44,7 +44,6 @@ #include #include "src/core/iomgr/fd_posix.h" -#include "src/core/iomgr/pollset_posix.h" static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, bool success); diff --git a/src/core/iomgr/workqueue_posix.h b/src/core/iomgr/workqueue_posix.h index 68f195ee0d2..589034fe1bb 100644 --- a/src/core/iomgr/workqueue_posix.h +++ b/src/core/iomgr/workqueue_posix.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2015, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,8 +34,6 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H #define GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H -#include "src/core/iomgr/wakeup_fd_posix.h" - struct grpc_fd; struct grpc_workqueue { diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c index 1f4f3e4aa52..458d0d3ac32 100644 --- a/src/core/security/google_default_credentials.c +++ b/src/core/security/google_default_credentials.c @@ -52,14 +52,13 @@ static grpc_channel_credentials *default_credentials = NULL; static int compute_engine_detection_done = 0; -static gpr_mu g_state_mu; -static gpr_mu *g_polling_mu; +static gpr_mu g_mu; static gpr_once g_once = GPR_ONCE_INIT; -static void init_default_credentials(void) { gpr_mu_init(&g_state_mu); } +static void init_default_credentials(void) { gpr_mu_init(&g_mu); } typedef struct { - grpc_pollset *pollset; + grpc_pollset pollset; int is_done; int success; } compute_engine_detector; @@ -81,10 +80,10 @@ static void on_compute_engine_detection_http_response( } } } - gpr_mu_lock(g_polling_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&detector->pollset)); detector->is_done = 1; - grpc_pollset_kick(detector->pollset, NULL); - gpr_mu_unlock(g_polling_mu); + grpc_pollset_kick(&detector->pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&detector->pollset)); } static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool s) { @@ -102,8 +101,7 @@ static int is_stack_running_on_compute_engine(void) { on compute engine. */ gpr_timespec max_detection_delay = gpr_time_from_seconds(1, GPR_TIMESPAN); - detector.pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(detector.pollset, &g_polling_mu); + grpc_pollset_init(&detector.pollset); detector.is_done = 0; detector.success = 0; @@ -114,7 +112,7 @@ static int is_stack_running_on_compute_engine(void) { grpc_httpcli_context_init(&context); grpc_httpcli_get( - &exec_ctx, &context, detector.pollset, &request, + &exec_ctx, &context, &detector.pollset, &request, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), max_detection_delay), on_compute_engine_detection_http_response, &detector); @@ -122,22 +120,19 @@ static int is_stack_running_on_compute_engine(void) { /* Block until we get the response. This is not ideal but this should only be called once for the lifetime of the process by the default credentials. */ - gpr_mu_lock(g_polling_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&detector.pollset)); while (!detector.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, detector.pollset, &worker, + grpc_pollset_work(&exec_ctx, &detector.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); } - gpr_mu_unlock(g_polling_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&detector.pollset)); grpc_httpcli_context_destroy(&context); - grpc_closure_init(&destroy_closure, destroy_pollset, detector.pollset); - grpc_pollset_shutdown(&exec_ctx, detector.pollset, &destroy_closure); + grpc_closure_init(&destroy_closure, destroy_pollset, &detector.pollset); + grpc_pollset_shutdown(&exec_ctx, &detector.pollset, &destroy_closure); grpc_exec_ctx_finish(&exec_ctx); - g_polling_mu = NULL; - - gpr_free(detector.pollset); return detector.success; } @@ -189,7 +184,7 @@ grpc_channel_credentials *grpc_google_default_credentials_create(void) { gpr_once_init(&g_once, init_default_credentials); - gpr_mu_lock(&g_state_mu); + gpr_mu_lock(&g_mu); if (default_credentials != NULL) { result = grpc_channel_credentials_ref(default_credentials); @@ -235,19 +230,19 @@ end: gpr_log(GPR_ERROR, "Could not create google default credentials."); } } - gpr_mu_unlock(&g_state_mu); + gpr_mu_unlock(&g_mu); return result; } void grpc_flush_cached_google_default_credentials(void) { gpr_once_init(&g_once, init_default_credentials); - gpr_mu_lock(&g_state_mu); + gpr_mu_lock(&g_mu); if (default_credentials != NULL) { grpc_channel_credentials_unref(default_credentials); default_credentials = NULL; } compute_engine_detection_done = 0; - gpr_mu_unlock(&g_state_mu); + gpr_mu_unlock(&g_mu); } /* -- Well known credentials path. -- */ diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index 8a9bbace087..f9cb852722d 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -36,19 +36,18 @@ #include #include -#include -#include -#include -#include - -#include "src/core/iomgr/pollset.h" #include "src/core/iomgr/timer.h" -#include "src/core/profiling/timers.h" +#include "src/core/iomgr/pollset.h" #include "src/core/support/string.h" #include "src/core/surface/api_trace.h" #include "src/core/surface/call.h" #include "src/core/surface/event_string.h" #include "src/core/surface/surface_trace.h" +#include "src/core/profiling/timers.h" +#include +#include +#include +#include typedef struct { grpc_pollset_worker **worker; @@ -57,8 +56,6 @@ typedef struct { /* Completion queue structure */ struct grpc_completion_queue { - /** owned by pollset */ - gpr_mu *mu; /** completed events */ grpc_cq_completion completed_head; grpc_cq_completion *completed_tail; @@ -66,6 +63,8 @@ struct grpc_completion_queue { gpr_refcount pending_events; /** Once owning_refs drops to zero, we will destroy the cq */ gpr_refcount owning_refs; + /** the set of low level i/o things that concern this cq */ + grpc_pollset pollset; /** 0 initially, 1 once we've begun shutting down */ int shutdown; int shutdown_called; @@ -83,8 +82,6 @@ struct grpc_completion_queue { grpc_completion_queue *next_free; }; -#define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1)) - static gpr_mu g_freelist_mu; grpc_completion_queue *g_freelist; @@ -97,7 +94,7 @@ void grpc_cq_global_shutdown(void) { gpr_mu_destroy(&g_freelist_mu); while (g_freelist) { grpc_completion_queue *next = g_freelist->next_free; - grpc_pollset_destroy(POLLSET_FROM_CQ(g_freelist)); + grpc_pollset_destroy(&g_freelist->pollset); #ifndef NDEBUG gpr_free(g_freelist->outstanding_tags); #endif @@ -127,8 +124,8 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) { if (g_freelist == NULL) { gpr_mu_unlock(&g_freelist_mu); - cc = gpr_malloc(sizeof(grpc_completion_queue) + grpc_pollset_size()); - grpc_pollset_init(POLLSET_FROM_CQ(cc), &cc->mu); + cc = gpr_malloc(sizeof(grpc_completion_queue)); + grpc_pollset_init(&cc->pollset); #ifndef NDEBUG cc->outstanding_tags = NULL; cc->outstanding_tag_capacity = 0; @@ -187,7 +184,7 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { #endif if (gpr_unref(&cc->owning_refs)) { GPR_ASSERT(cc->completed_head.next == (uintptr_t)&cc->completed_head); - grpc_pollset_reset(POLLSET_FROM_CQ(cc)); + grpc_pollset_reset(&cc->pollset); gpr_mu_lock(&g_freelist_mu); cc->next_free = g_freelist; g_freelist = cc; @@ -197,7 +194,7 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) { #ifndef NDEBUG - gpr_mu_lock(cc->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); GPR_ASSERT(!cc->shutdown_called); if (cc->outstanding_tag_count == cc->outstanding_tag_capacity) { cc->outstanding_tag_capacity = GPR_MAX(4, 2 * cc->outstanding_tag_capacity); @@ -206,7 +203,7 @@ void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) { cc->outstanding_tag_capacity); } cc->outstanding_tags[cc->outstanding_tag_count++] = tag; - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); #endif gpr_ref(&cc->pending_events); } @@ -234,7 +231,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, storage->next = ((uintptr_t)&cc->completed_head) | ((uintptr_t)(success != 0)); - gpr_mu_lock(cc->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); #ifndef NDEBUG for (i = 0; i < (int)cc->outstanding_tag_count; i++) { if (cc->outstanding_tags[i] == tag) { @@ -259,8 +256,8 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, break; } } - grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker); - gpr_mu_unlock(cc->mu); + grpc_pollset_kick(&cc->pollset, pluck_worker); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); } else { cc->completed_tail->next = ((uintptr_t)storage) | (1u & (uintptr_t)cc->completed_tail->next); @@ -268,9 +265,8 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, GPR_ASSERT(!cc->shutdown); GPR_ASSERT(cc->shutdown_called); cc->shutdown = 1; - grpc_pollset_shutdown(exec_ctx, POLLSET_FROM_CQ(cc), - &cc->pollset_shutdown_done); - gpr_mu_unlock(cc->mu); + grpc_pollset_shutdown(exec_ctx, &cc->pollset, &cc->pollset_shutdown_done); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); } GPR_TIMER_END("grpc_cq_end_op", 0); @@ -298,7 +294,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); GRPC_CQ_INTERNAL_REF(cc, "next"); - gpr_mu_lock(cc->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); for (;;) { if (cc->completed_tail != &cc->completed_head) { grpc_cq_completion *c = (grpc_cq_completion *)cc->completed_head.next; @@ -306,7 +302,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, if (c == cc->completed_tail) { cc->completed_tail = &cc->completed_head; } - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); ret.type = GRPC_OP_COMPLETE; ret.success = c->next & 1u; ret.tag = c->tag; @@ -314,14 +310,14 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, break; } if (cc->shutdown) { - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_SHUTDOWN; break; } now = gpr_now(GPR_CLOCK_MONOTONIC); if (!first_loop && gpr_time_cmp(now, deadline) >= 0) { - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_TIMEOUT; break; @@ -334,12 +330,11 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_timespec iteration_deadline = deadline; if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { GPR_TIMER_MARK("alarm_triggered", 0); - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(cc->mu); - continue; + gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); } else { - grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, + grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, iteration_deadline); } } @@ -400,7 +395,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); GRPC_CQ_INTERNAL_REF(cc, "pluck"); - gpr_mu_lock(cc->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); for (;;) { prev = &cc->completed_head; while ((c = (grpc_cq_completion *)(prev->next & ~(uintptr_t)1)) != @@ -410,7 +405,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, if (c == cc->completed_tail) { cc->completed_tail = prev; } - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); ret.type = GRPC_OP_COMPLETE; ret.success = c->next & 1u; ret.tag = c->tag; @@ -420,7 +415,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, prev = c; } if (cc->shutdown) { - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_SHUTDOWN; break; @@ -430,7 +425,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, "Too many outstanding grpc_completion_queue_pluck calls: maximum " "is %d", GRPC_MAX_COMPLETION_QUEUE_PLUCKERS); - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); memset(&ret, 0, sizeof(ret)); /* TODO(ctiller): should we use a different result here */ ret.type = GRPC_QUEUE_TIMEOUT; @@ -439,7 +434,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, now = gpr_now(GPR_CLOCK_MONOTONIC); if (!first_loop && gpr_time_cmp(now, deadline) >= 0) { del_plucker(cc, tag, &worker); - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_TIMEOUT; break; @@ -452,12 +447,11 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, gpr_timespec iteration_deadline = deadline; if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { GPR_TIMER_MARK("alarm_triggered", 0); - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(cc->mu); - continue; + gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); } else { - grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, + grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, iteration_deadline); } del_plucker(cc, tag, &worker); @@ -478,9 +472,9 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_completion_queue_shutdown", 0); GRPC_API_TRACE("grpc_completion_queue_shutdown(cc=%p)", 1, (cc)); - gpr_mu_lock(cc->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); if (cc->shutdown_called) { - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); GPR_TIMER_END("grpc_completion_queue_shutdown", 0); return; } @@ -488,10 +482,9 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { if (gpr_unref(&cc->pending_events)) { GPR_ASSERT(!cc->shutdown); cc->shutdown = 1; - grpc_pollset_shutdown(&exec_ctx, POLLSET_FROM_CQ(cc), - &cc->pollset_shutdown_done); + grpc_pollset_shutdown(&exec_ctx, &cc->pollset, &cc->pollset_shutdown_done); } - gpr_mu_unlock(cc->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); grpc_exec_ctx_finish(&exec_ctx); GPR_TIMER_END("grpc_completion_queue_shutdown", 0); } @@ -505,7 +498,7 @@ void grpc_completion_queue_destroy(grpc_completion_queue *cc) { } grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) { - return POLLSET_FROM_CQ(cc); + return &cc->pollset; } void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; } diff --git a/test/core/client_config/set_initial_connect_string_test.c b/test/core/client_config/set_initial_connect_string_test.c index 3cf267fb3bd..bcd1f261235 100644 --- a/test/core/client_config/set_initial_connect_string_test.c +++ b/test/core/client_config/set_initial_connect_string_test.c @@ -85,7 +85,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_slice_buffer_init(&state.incoming_buffer); gpr_slice_buffer_init(&state.temp_incoming_buffer); state.tcp = tcp; - grpc_endpoint_add_to_pollset(exec_ctx, tcp, server->pollset); + grpc_endpoint_add_to_pollset(exec_ctx, tcp, &server->pollset); grpc_endpoint_read(exec_ctx, tcp, &state.temp_incoming_buffer, &on_read); } diff --git a/test/core/end2end/fixtures/h2_full+poll+pipe.c b/test/core/end2end/fixtures/h2_full+poll+pipe.c index 682598fbe2d..d475a7bb552 100644 --- a/test/core/end2end/fixtures/h2_full+poll+pipe.c +++ b/test/core/end2end/fixtures/h2_full+poll+pipe.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2015, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,23 +35,21 @@ #include -#include -#include -#include -#include -#include -#include - #include "src/core/channel/client_channel.h" #include "src/core/channel/connected_channel.h" #include "src/core/channel/http_server_filter.h" -#include "src/core/iomgr/pollset_posix.h" -#include "src/core/iomgr/wakeup_fd_posix.h" #include "src/core/surface/channel.h" #include "src/core/surface/server.h" #include "src/core/transport/chttp2_transport.h" +#include +#include +#include +#include +#include +#include #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#include "src/core/iomgr/wakeup_fd_posix.h" typedef struct fullstack_fixture_data { char *localaddr; diff --git a/test/core/end2end/fixtures/h2_full+poll.c b/test/core/end2end/fixtures/h2_full+poll.c index 5a0b2ef4953..3f5e6096f63 100644 --- a/test/core/end2end/fixtures/h2_full+poll.c +++ b/test/core/end2end/fixtures/h2_full+poll.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2015, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,20 +35,18 @@ #include -#include -#include -#include -#include -#include -#include - #include "src/core/channel/client_channel.h" #include "src/core/channel/connected_channel.h" #include "src/core/channel/http_server_filter.h" -#include "src/core/iomgr/pollset_posix.h" #include "src/core/surface/channel.h" #include "src/core/surface/server.h" #include "src/core/transport/chttp2_transport.h" +#include +#include +#include +#include +#include +#include #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/end2end/fixtures/h2_ssl+poll.c b/test/core/end2end/fixtures/h2_ssl+poll.c index 66268c77d58..0b88876d138 100644 --- a/test/core/end2end/fixtures/h2_ssl+poll.c +++ b/test/core/end2end/fixtures/h2_ssl+poll.c @@ -41,7 +41,6 @@ #include #include "src/core/channel/channel_args.h" -#include "src/core/iomgr/pollset_posix.h" #include "src/core/security/credentials.h" #include "src/core/support/env.h" #include "src/core/support/tmpfile.h" diff --git a/test/core/end2end/fixtures/h2_uchannel.c b/test/core/end2end/fixtures/h2_uchannel.c index 87bbd64d09a..33055561cb3 100644 --- a/test/core/end2end/fixtures/h2_uchannel.c +++ b/test/core/end2end/fixtures/h2_uchannel.c @@ -35,13 +35,6 @@ #include -#include -#include -#include -#include -#include -#include -#include #include "src/core/channel/channel_args.h" #include "src/core/channel/client_channel.h" #include "src/core/channel/client_uchannel.h" @@ -53,6 +46,13 @@ #include "src/core/surface/channel.h" #include "src/core/surface/server.h" #include "src/core/transport/chttp2_transport.h" +#include +#include +#include +#include +#include +#include +#include #include "test/core/util/port.h" #include "test/core/util/test_config.h" @@ -238,12 +238,12 @@ static grpc_end2end_test_fixture chttp2_create_fixture_micro_fullstack( } grpc_connectivity_state g_state = GRPC_CHANNEL_IDLE; -grpc_pollset_set *g_interested_parties; +grpc_pollset_set g_interested_parties; static void state_changed(grpc_exec_ctx *exec_ctx, void *arg, bool success) { if (g_state != GRPC_CHANNEL_READY) { grpc_subchannel_notify_on_state_change( - exec_ctx, arg, g_interested_parties, &g_state, + exec_ctx, arg, &g_interested_parties, &g_state, grpc_closure_create(state_changed, arg)); } } @@ -253,31 +253,30 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *arg, bool success) { } static grpc_connected_subchannel *connect_subchannel(grpc_subchannel *c) { - gpr_mu *mu; - grpc_pollset *pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset pollset; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_pollset_init(pollset, &mu); - g_interested_parties = grpc_pollset_set_create(); - grpc_pollset_set_add_pollset(&exec_ctx, g_interested_parties, pollset); - grpc_subchannel_notify_on_state_change(&exec_ctx, c, g_interested_parties, + grpc_pollset_init(&pollset); + grpc_pollset_set_init(&g_interested_parties); + grpc_pollset_set_add_pollset(&exec_ctx, &g_interested_parties, &pollset); + grpc_subchannel_notify_on_state_change(&exec_ctx, c, &g_interested_parties, &g_state, grpc_closure_create(state_changed, c)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(mu); + gpr_mu_lock(GRPC_POLLSET_MU(&pollset)); while (g_state != GRPC_CHANNEL_READY) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), + grpc_pollset_work(&exec_ctx, &pollset, &worker, + gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); - gpr_mu_unlock(mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&pollset)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(mu); + gpr_mu_lock(GRPC_POLLSET_MU(&pollset)); } - grpc_pollset_shutdown(&exec_ctx, pollset, - grpc_closure_create(destroy_pollset, pollset)); - grpc_pollset_set_destroy(g_interested_parties); - gpr_mu_unlock(mu); + grpc_pollset_shutdown(&exec_ctx, &pollset, + grpc_closure_create(destroy_pollset, &pollset)); + grpc_pollset_set_destroy(&g_interested_parties); + gpr_mu_unlock(GRPC_POLLSET_MU(&pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_free(pollset); return grpc_subchannel_get_connected_subchannel(c); } diff --git a/test/core/end2end/fixtures/h2_uds+poll.c b/test/core/end2end/fixtures/h2_uds+poll.c index c3a855ff883..155017c8875 100644 --- a/test/core/end2end/fixtures/h2_uds+poll.c +++ b/test/core/end2end/fixtures/h2_uds+poll.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2015, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,22 +37,20 @@ #include #include -#include -#include -#include -#include -#include -#include -#include - #include "src/core/channel/client_channel.h" #include "src/core/channel/connected_channel.h" #include "src/core/channel/http_server_filter.h" -#include "src/core/iomgr/pollset_posix.h" #include "src/core/support/string.h" #include "src/core/surface/channel.h" #include "src/core/surface/server.h" #include "src/core/transport/chttp2_transport.h" +#include +#include +#include +#include +#include +#include +#include #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/httpcli/httpcli_test.c b/test/core/httpcli/httpcli_test.c index da1463329d4..e954a920b04 100644 --- a/test/core/httpcli/httpcli_test.c +++ b/test/core/httpcli/httpcli_test.c @@ -36,19 +36,18 @@ #include #include +#include "src/core/iomgr/iomgr.h" #include #include #include #include #include -#include "src/core/iomgr/iomgr.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" static int g_done = 0; static grpc_httpcli_context g_context; -static gpr_mu *g_mu; -static grpc_pollset *g_pollset; +static grpc_pollset g_pollset; static gpr_timespec n_seconds_time(int seconds) { return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(seconds); @@ -64,10 +63,10 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, GPR_ASSERT(response->status == 200); GPR_ASSERT(response->body_length == strlen(expect)); GPR_ASSERT(0 == memcmp(expect, response->body, response->body_length)); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); g_done = 1; - grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + grpc_pollset_kick(&g_pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } static void test_get(int port) { @@ -86,18 +85,18 @@ static void test_get(int port) { req.path = "/get"; req.handshaker = &grpc_httpcli_plaintext; - grpc_httpcli_get(&exec_ctx, &g_context, g_pollset, &req, n_seconds_time(15), + grpc_httpcli_get(&exec_ctx, &g_context, &g_pollset, &req, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); gpr_free(host); } @@ -117,18 +116,18 @@ static void test_post(int port) { req.path = "/post"; req.handshaker = &grpc_httpcli_plaintext; - grpc_httpcli_post(&exec_ctx, &g_context, g_pollset, &req, "hello", 5, + grpc_httpcli_post(&exec_ctx, &g_context, &g_pollset, &req, "hello", 5, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); gpr_free(host); } @@ -176,20 +175,17 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); grpc_httpcli_context_init(&g_context); - g_pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(g_pollset, &g_mu); + grpc_pollset_init(&g_pollset); test_get(port); test_post(port); grpc_httpcli_context_destroy(&g_context); - grpc_closure_init(&destroyed, destroy_pollset, g_pollset); - grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); + grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - gpr_free(g_pollset); - gpr_subprocess_destroy(server); return 0; diff --git a/test/core/httpcli/httpscli_test.c b/test/core/httpcli/httpscli_test.c index 7f765bc614f..9f31eae2782 100644 --- a/test/core/httpcli/httpscli_test.c +++ b/test/core/httpcli/httpscli_test.c @@ -36,19 +36,18 @@ #include #include +#include "src/core/iomgr/iomgr.h" #include #include #include #include #include -#include "src/core/iomgr/iomgr.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" static int g_done = 0; static grpc_httpcli_context g_context; -static gpr_mu *g_mu; -static grpc_pollset *g_pollset; +static grpc_pollset g_pollset; static gpr_timespec n_seconds_time(int seconds) { return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(seconds); @@ -64,10 +63,10 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, GPR_ASSERT(response->status == 200); GPR_ASSERT(response->body_length == strlen(expect)); GPR_ASSERT(0 == memcmp(expect, response->body, response->body_length)); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); g_done = 1; - grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + grpc_pollset_kick(&g_pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } static void test_get(int port) { @@ -87,18 +86,18 @@ static void test_get(int port) { req.path = "/get"; req.handshaker = &grpc_httpcli_ssl; - grpc_httpcli_get(&exec_ctx, &g_context, g_pollset, &req, n_seconds_time(15), + grpc_httpcli_get(&exec_ctx, &g_context, &g_pollset, &req, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); gpr_free(host); } @@ -119,18 +118,18 @@ static void test_post(int port) { req.path = "/post"; req.handshaker = &grpc_httpcli_ssl; - grpc_httpcli_post(&exec_ctx, &g_context, g_pollset, &req, "hello", 5, + grpc_httpcli_post(&exec_ctx, &g_context, &g_pollset, &req, "hello", 5, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); gpr_free(host); } @@ -179,20 +178,17 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); grpc_httpcli_context_init(&g_context); - g_pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(g_pollset, &g_mu); + grpc_pollset_init(&g_pollset); test_get(port); test_post(port); grpc_httpcli_context_destroy(&g_context); - grpc_closure_init(&destroyed, destroy_pollset, g_pollset); - grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); + grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - gpr_free(g_pollset); - gpr_subprocess_destroy(server); return 0; diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c index c3a91088a57..7e266ebfb99 100644 --- a/test/core/iomgr/endpoint_pair_test.c +++ b/test/core/iomgr/endpoint_pair_test.c @@ -39,11 +39,10 @@ #include #include #include "src/core/iomgr/endpoint_pair.h" -#include "test/core/iomgr/endpoint_tests.h" #include "test/core/util/test_config.h" +#include "test/core/iomgr/endpoint_tests.h" -static gpr_mu *g_mu; -static grpc_pollset *g_pollset; +static grpc_pollset g_pollset; static void clean_up(void) {} @@ -55,8 +54,8 @@ static grpc_endpoint_test_fixture create_fixture_endpoint_pair( f.client_ep = p.client; f.server_ep = p.server; - grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset); - grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, &g_pollset); grpc_exec_ctx_finish(&exec_ctx); return f; @@ -75,14 +74,12 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - g_pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(g_pollset, &g_mu); - grpc_endpoint_tests(configs[0], g_pollset, g_mu); - grpc_closure_init(&destroyed, destroy_pollset, g_pollset); - grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); + grpc_pollset_init(&g_pollset); + grpc_endpoint_tests(configs[0], &g_pollset); + grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); + grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - gpr_free(g_pollset); return 0; } diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c index f689e4ba7fb..6ba67df3b10 100644 --- a/test/core/iomgr/endpoint_tests.c +++ b/test/core/iomgr/endpoint_tests.c @@ -36,8 +36,8 @@ #include #include -#include #include +#include #include #include #include "test/core/util/test_config.h" @@ -58,7 +58,6 @@ */ -static gpr_mu *g_mu; static grpc_pollset *g_pollset; size_t count_slices(gpr_slice *slices, size_t nslices, int *current_data) { @@ -135,10 +134,10 @@ static void read_and_write_test_read_handler(grpc_exec_ctx *exec_ctx, state->incoming.slices, state->incoming.count, &state->current_read_data); if (state->bytes_read == state->target_bytes || !success) { gpr_log(GPR_INFO, "Read handler done"); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); state->read_done = 1 + success; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset)); } else if (success) { grpc_endpoint_read(exec_ctx, state->read_ep, &state->incoming, &state->done_read); @@ -170,10 +169,10 @@ static void read_and_write_test_write_handler(grpc_exec_ctx *exec_ctx, } gpr_log(GPR_INFO, "Write handler done"); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); state->write_done = 1 + success; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset)); } /* Do both reading and writing using the grpc_endpoint API. @@ -233,14 +232,14 @@ static void read_and_write_test(grpc_endpoint_test_config config, } grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); while (!state.read_done || !state.write_done) { grpc_pollset_worker *worker = NULL; GPR_ASSERT(gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0); grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset)); grpc_exec_ctx_finish(&exec_ctx); end_test(config); @@ -252,10 +251,9 @@ static void read_and_write_test(grpc_endpoint_test_config config, } void grpc_endpoint_tests(grpc_endpoint_test_config config, - grpc_pollset *pollset, gpr_mu *mu) { + grpc_pollset *pollset) { size_t i; g_pollset = pollset; - g_mu = mu; read_and_write_test(config, 10000000, 100000, 8192, 0); read_and_write_test(config, 1000000, 100000, 1, 0); read_and_write_test(config, 100000000, 100000, 1, 1); diff --git a/test/core/iomgr/endpoint_tests.h b/test/core/iomgr/endpoint_tests.h index 8ea47e345cf..700f854891e 100644 --- a/test/core/iomgr/endpoint_tests.h +++ b/test/core/iomgr/endpoint_tests.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015-2016, Google Inc. + * Copyright 2015, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -53,6 +53,6 @@ struct grpc_endpoint_test_config { }; void grpc_endpoint_tests(grpc_endpoint_test_config config, - grpc_pollset *pollset, gpr_mu *mu); + grpc_pollset *pollset); #endif /* GRPC_TEST_CORE_IOMGR_ENDPOINT_TESTS_H */ diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 99689ebcc3b..07761b6576d 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -49,12 +49,9 @@ #include #include #include - -#include "src/core/iomgr/pollset_posix.h" #include "test/core/util/test_config.h" -static gpr_mu *g_mu; -static grpc_pollset *g_pollset; +static grpc_pollset g_pollset; /* buffer size used to send and receive data. 1024 is the minimal value to set TCP send and receive buffer. */ @@ -182,10 +179,10 @@ static void listen_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg /*server */, grpc_fd_orphan(exec_ctx, sv->em_fd, NULL, NULL, "b"); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); sv->done = 1; - grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + grpc_pollset_kick(&g_pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } /* Called when a new TCP connection request arrives in the listening port. */ @@ -212,7 +209,7 @@ static void listen_cb(grpc_exec_ctx *exec_ctx, void *arg, /*=sv_arg*/ se = gpr_malloc(sizeof(*se)); se->sv = sv; se->em_fd = grpc_fd_create(fd, "listener"); - grpc_pollset_add_fd(exec_ctx, g_pollset, se->em_fd); + grpc_pollset_add_fd(exec_ctx, &g_pollset, se->em_fd); se->session_read_closure.cb = session_read_cb; se->session_read_closure.cb_arg = se; grpc_fd_notify_on_read(exec_ctx, se->em_fd, &se->session_read_closure); @@ -241,7 +238,7 @@ static int server_start(grpc_exec_ctx *exec_ctx, server *sv) { GPR_ASSERT(listen(fd, MAX_NUM_FD) == 0); sv->em_fd = grpc_fd_create(fd, "server"); - grpc_pollset_add_fd(exec_ctx, g_pollset, sv->em_fd); + grpc_pollset_add_fd(exec_ctx, &g_pollset, sv->em_fd); /* Register to be interested in reading from listen_fd. */ sv->listen_closure.cb = listen_cb; sv->listen_closure.cb_arg = sv; @@ -252,18 +249,18 @@ static int server_start(grpc_exec_ctx *exec_ctx, server *sv) { /* Wait and shutdown a sever. */ static void server_wait_and_shutdown(server *sv) { - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!sv->done) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } /* ===An upload client to test notify_on_write=== */ @@ -299,7 +296,7 @@ static void client_session_shutdown_cb(grpc_exec_ctx *exec_ctx, client *cl = arg; grpc_fd_orphan(exec_ctx, cl->em_fd, NULL, NULL, "c"); cl->done = 1; - grpc_pollset_kick(g_pollset, NULL); + grpc_pollset_kick(&g_pollset, NULL); } /* Write as much as possible, then register notify_on_write. */ @@ -310,9 +307,9 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ ssize_t write_once = 0; if (!success) { - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); client_session_shutdown_cb(exec_ctx, arg, 1); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); return; } @@ -322,7 +319,7 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ } while (write_once > 0); if (errno == EAGAIN) { - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); if (cl->client_write_cnt < CLIENT_TOTAL_WRITE_CNT) { cl->write_closure.cb = client_session_write; cl->write_closure.cb_arg = cl; @@ -331,7 +328,7 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ } else { client_session_shutdown_cb(exec_ctx, arg, 1); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } else { gpr_log(GPR_ERROR, "unknown errno %s", strerror(errno)); abort(); @@ -360,25 +357,25 @@ static void client_start(grpc_exec_ctx *exec_ctx, client *cl, int port) { } cl->em_fd = grpc_fd_create(fd, "client"); - grpc_pollset_add_fd(exec_ctx, g_pollset, cl->em_fd); + grpc_pollset_add_fd(exec_ctx, &g_pollset, cl->em_fd); client_session_write(exec_ctx, cl, 1); } /* Wait for the signal to shutdown a client. */ static void client_wait_and_shutdown(client *cl) { - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!cl->done) { grpc_pollset_worker *worker = NULL; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } /* Test grpc_fd. Start an upload server and client, upload a stream of @@ -413,20 +410,20 @@ static void first_read_callback(grpc_exec_ctx *exec_ctx, void *arg /* fd_change_data */, bool success) { fd_change_data *fdc = arg; - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); fdc->cb_that_ran = first_read_callback; - grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + grpc_pollset_kick(&g_pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } static void second_read_callback(grpc_exec_ctx *exec_ctx, void *arg /* fd_change_data */, bool success) { fd_change_data *fdc = arg; - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); fdc->cb_that_ran = second_read_callback; - grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + grpc_pollset_kick(&g_pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } /* Test that changing the callback we use for notify_on_read actually works. @@ -459,7 +456,7 @@ static void test_grpc_fd_change(void) { GPR_ASSERT(fcntl(sv[1], F_SETFL, flags | O_NONBLOCK) == 0); em_fd = grpc_fd_create(sv[0], "test_grpc_fd_change"); - grpc_pollset_add_fd(&exec_ctx, g_pollset, em_fd); + grpc_pollset_add_fd(&exec_ctx, &g_pollset, em_fd); /* Register the first callback, then make its FD readable */ grpc_fd_notify_on_read(&exec_ctx, em_fd, &first_closure); @@ -468,18 +465,18 @@ static void test_grpc_fd_change(void) { GPR_ASSERT(result == 1); /* And now wait for it to run. */ - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (a.cb_that_ran == NULL) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } GPR_ASSERT(a.cb_that_ran == first_read_callback); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); /* And drain the socket so we can generate a new read edge */ result = read(sv[0], &data, 1); @@ -492,19 +489,19 @@ static void test_grpc_fd_change(void) { result = write(sv[1], &data, 1); GPR_ASSERT(result == 1); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (b.cb_that_ran == NULL) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } /* Except now we verify that second_read_callback ran instead */ GPR_ASSERT(b.cb_that_ran == second_read_callback); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_fd_orphan(&exec_ctx, em_fd, NULL, NULL, "d"); grpc_exec_ctx_finish(&exec_ctx); @@ -522,14 +519,12 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_iomgr_init(); - g_pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(g_pollset, &g_mu); + grpc_pollset_init(&g_pollset); test_grpc_fd(); test_grpc_fd_change(); - grpc_closure_init(&destroyed, destroy_pollset, g_pollset); - grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); + grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); - gpr_free(g_pollset); grpc_iomgr_shutdown(); return 0; } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 1e6fa5d45a0..cdd5b096fbc 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -40,7 +40,6 @@ #include #include -#include #include #include @@ -49,9 +48,8 @@ #include "src/core/iomgr/timer.h" #include "test/core/util/test_config.h" -static grpc_pollset_set *g_pollset_set; -static gpr_mu *g_mu; -static grpc_pollset *g_pollset; +static grpc_pollset_set g_pollset_set; +static grpc_pollset g_pollset; static int g_connections_complete = 0; static grpc_endpoint *g_connecting = NULL; @@ -60,10 +58,10 @@ static gpr_timespec test_deadline(void) { } static void finish_connection() { - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); g_connections_complete++; - grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + grpc_pollset_kick(&g_pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, bool success) { @@ -101,14 +99,14 @@ void test_succeeds(void) { GPR_ASSERT(0 == bind(svr_fd, (struct sockaddr *)&addr, addr_len)); GPR_ASSERT(0 == listen(svr_fd, 1)); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); connections_complete_before = g_connections_complete; - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); /* connect to it */ GPR_ASSERT(getsockname(svr_fd, (struct sockaddr *)&addr, &addr_len) == 0); grpc_closure_init(&done, must_succeed, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, &g_pollset_set, (struct sockaddr *)&addr, addr_len, gpr_inf_future(GPR_CLOCK_REALTIME)); @@ -120,19 +118,19 @@ void test_succeeds(void) { GPR_ASSERT(r >= 0); close(r); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (g_connections_complete == connections_complete_before) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); } @@ -149,17 +147,17 @@ void test_fails(void) { memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); connections_complete_before = g_connections_complete; - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); /* connect to a broken address */ grpc_closure_init(&done, must_fail, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, &g_pollset_set, (struct sockaddr *)&addr, addr_len, gpr_inf_future(GPR_CLOCK_REALTIME)); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); /* wait for the connection callback to finish */ while (g_connections_complete == connections_complete_before) { @@ -167,14 +165,14 @@ void test_fails(void) { gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); gpr_timespec polling_deadline = test_deadline(); if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { - grpc_pollset_work(&exec_ctx, g_pollset, &worker, now, polling_deadline); + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, now, polling_deadline); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); } @@ -219,16 +217,16 @@ void test_times_out(void) { connect_deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); connections_complete_before = g_connections_complete; - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_closure_init(&done, must_fail, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, &g_pollset_set, (struct sockaddr *)&addr, addr_len, connect_deadline); /* Make sure the event doesn't trigger early */ - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); for (;;) { grpc_pollset_worker *worker = NULL; gpr_timespec now = gpr_now(connect_deadline.clock_type); @@ -254,13 +252,13 @@ void test_times_out(void) { } gpr_timespec polling_deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10); if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { - grpc_pollset_work(&exec_ctx, g_pollset, &worker, now, polling_deadline); + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, now, polling_deadline); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); @@ -279,20 +277,18 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - g_pollset_set = grpc_pollset_set_create(); - g_pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(g_pollset, &g_mu); - grpc_pollset_set_add_pollset(&exec_ctx, g_pollset_set, g_pollset); + grpc_pollset_set_init(&g_pollset_set); + grpc_pollset_init(&g_pollset); + grpc_pollset_set_add_pollset(&exec_ctx, &g_pollset_set, &g_pollset); grpc_exec_ctx_finish(&exec_ctx); test_succeeds(); gpr_log(GPR_ERROR, "End of first test"); test_fails(); test_times_out(); - grpc_pollset_set_destroy(g_pollset_set); - grpc_closure_init(&destroyed, destroy_pollset, g_pollset); - grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); + grpc_pollset_set_destroy(&g_pollset_set); + grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); + grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - gpr_free(g_pollset); return 0; } diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index 4351642ab6e..20b0b9e13b7 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -36,8 +36,8 @@ #include #include #include -#include #include +#include #include #include @@ -45,11 +45,10 @@ #include #include #include -#include "test/core/iomgr/endpoint_tests.h" #include "test/core/util/test_config.h" +#include "test/core/iomgr/endpoint_tests.h" -static gpr_mu *g_mu; -static grpc_pollset *g_pollset; +static grpc_pollset g_pollset; /* General test notes: @@ -146,7 +145,7 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, bool success) { GPR_ASSERT(success); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); current_data = state->read_bytes % 256; read_bytes = count_slices(state->incoming.slices, state->incoming.count, ¤t_data); @@ -154,10 +153,10 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, bool success) { gpr_log(GPR_INFO, "Read %d bytes of %d", read_bytes, state->target_read_bytes); if (state->read_bytes >= state->target_read_bytes) { - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } else { grpc_endpoint_read(exec_ctx, state->ep, &state->incoming, &state->read_cb); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } } @@ -176,7 +175,7 @@ static void read_test(size_t num_bytes, size_t slice_size) { create_sockets(sv); ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), slice_size, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); gpr_log(GPR_INFO, "Wrote %d bytes", written_bytes); @@ -189,17 +188,17 @@ static void read_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); gpr_slice_buffer_destroy(&state.incoming); grpc_endpoint_destroy(&exec_ctx, ep); @@ -222,7 +221,7 @@ static void large_read_test(size_t slice_size) { ep = grpc_tcp_create(grpc_fd_create(sv[1], "large_read_test"), slice_size, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); written_bytes = fill_socket(sv[0]); gpr_log(GPR_INFO, "Wrote %d bytes", written_bytes); @@ -235,17 +234,17 @@ static void large_read_test(size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); gpr_slice_buffer_destroy(&state.incoming); grpc_endpoint_destroy(&exec_ctx, ep); @@ -284,11 +283,11 @@ static void write_done(grpc_exec_ctx *exec_ctx, void *user_data /* write_socket_state */, bool success) { struct write_socket_state *state = (struct write_socket_state *)user_data; gpr_log(GPR_INFO, "Write done callback called"); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); gpr_log(GPR_INFO, "Signalling write done"); state->write_done = 1; - grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + grpc_pollset_kick(&g_pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { @@ -305,11 +304,11 @@ void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { for (;;) { grpc_pollset_worker *worker = NULL; - gpr_mu_lock(g_mu); - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10)); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); do { bytes_read = @@ -351,7 +350,7 @@ static void write_test(size_t num_bytes, size_t slice_size) { ep = grpc_tcp_create(grpc_fd_create(sv[1], "write_test"), GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); state.ep = ep; state.write_done = 0; @@ -364,19 +363,19 @@ static void write_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_write(&exec_ctx, ep, &outgoing, &write_done_closure); drain_socket_blocking(sv[0], num_bytes, num_bytes); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); for (;;) { grpc_pollset_worker *worker = NULL; if (state.write_done) { break; } - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); gpr_slice_buffer_destroy(&outgoing); grpc_endpoint_destroy(&exec_ctx, ep); @@ -387,7 +386,7 @@ static void write_test(size_t num_bytes, size_t slice_size) { void on_fd_released(grpc_exec_ctx *exec_ctx, void *arg, bool success) { int *done = arg; *done = 1; - grpc_pollset_kick(g_pollset, NULL); + grpc_pollset_kick(&g_pollset, NULL); } /* Do a read_test, then release fd and try to read/write again. Verify that @@ -411,7 +410,7 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), slice_size, "test"); GPR_ASSERT(grpc_tcp_fd(ep) == sv[1] && sv[1] >= 0); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); gpr_log(GPR_INFO, "Wrote %d bytes", written_bytes); @@ -424,27 +423,27 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); gpr_slice_buffer_destroy(&state.incoming); grpc_tcp_destroy_and_release_fd(&exec_ctx, ep, &fd, &fd_released_cb); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); while (!fd_released_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, g_pollset, &worker, + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); } - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); GPR_ASSERT(fd_released_done == 1); GPR_ASSERT(fd == sv[1]); grpc_exec_ctx_finish(&exec_ctx); @@ -492,8 +491,8 @@ static grpc_endpoint_test_fixture create_fixture_tcp_socketpair( slice_size, "test"); f.server_ep = grpc_tcp_create(grpc_fd_create(sv[1], "fixture:server"), slice_size, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset); - grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, &g_pollset); grpc_exec_ctx_finish(&exec_ctx); @@ -513,15 +512,13 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - g_pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(g_pollset, &g_mu); + grpc_pollset_init(&g_pollset); run_tests(); - grpc_endpoint_tests(configs[0], g_pollset, g_mu); - grpc_closure_init(&destroyed, destroy_pollset, g_pollset); - grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); + grpc_endpoint_tests(configs[0], &g_pollset); + grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); + grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - gpr_free(g_pollset); return 0; } diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 7933468355a..43180b16988 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -32,28 +32,24 @@ */ #include "src/core/iomgr/tcp_server.h" - -#include -#include -#include -#include -#include - +#include "src/core/iomgr/iomgr.h" +#include "src/core/iomgr/sockaddr_utils.h" #include -#include #include #include #include - -#include "src/core/iomgr/iomgr.h" -#include "src/core/iomgr/sockaddr_utils.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#include +#include +#include +#include +#include + #define LOG_TEST(x) gpr_log(GPR_INFO, "%s", #x) -static gpr_mu *g_mu; -static grpc_pollset *g_pollset; +static grpc_pollset g_pollset; static int g_nconnects = 0; typedef struct on_connect_result { @@ -117,11 +113,11 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_endpoint_shutdown(exec_ctx, tcp); grpc_endpoint_destroy(exec_ctx, tcp); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); on_connect_result_set(&g_result, acceptor); g_nconnects++; - grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + grpc_pollset_kick(&g_pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } static void test_no_op(void) { @@ -178,7 +174,7 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, int clifd = socket(remote->sa_family, SOCK_STREAM, 0); int nconnects_before; - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); nconnects_before = g_nconnects; on_connect_result_init(&g_result); GPR_ASSERT(clifd >= 0); @@ -188,18 +184,18 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, while (g_nconnects == nconnects_before && gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(exec_ctx, g_pollset, &worker, + grpc_pollset_work(exec_ctx, &g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(exec_ctx); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); } gpr_log(GPR_DEBUG, "wait done"); GPR_ASSERT(g_nconnects == nconnects_before + 1); close(clifd); *result = g_result; - gpr_mu_unlock(g_mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } /* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for @@ -214,6 +210,7 @@ static void test_connect(unsigned n) { unsigned svr1_fd_count; int svr1_port; grpc_tcp_server *s = grpc_tcp_server_create(NULL); + grpc_pollset *pollsets[1]; unsigned i; server_weak_ref weak_ref; server_weak_ref_init(&weak_ref); @@ -262,7 +259,8 @@ static void test_connect(unsigned n) { } } - grpc_tcp_server_start(&exec_ctx, s, &g_pollset, 1, on_connect, NULL); + pollsets[0] = &g_pollset; + grpc_tcp_server_start(&exec_ctx, s, pollsets, 1, on_connect, NULL); for (i = 0; i < n; i++) { on_connect_result result; @@ -314,8 +312,7 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - g_pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(g_pollset, &g_mu); + grpc_pollset_init(&g_pollset); test_no_op(); test_no_op_with_start(); @@ -324,10 +321,9 @@ int main(int argc, char **argv) { test_connect(1); test_connect(10); - grpc_closure_init(&destroyed, destroy_pollset, g_pollset); - grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); + grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - gpr_free(g_pollset); return 0; } diff --git a/test/core/iomgr/workqueue_test.c b/test/core/iomgr/workqueue_test.c index 8a1faf6303e..a0249152566 100644 --- a/test/core/iomgr/workqueue_test.c +++ b/test/core/iomgr/workqueue_test.c @@ -34,20 +34,18 @@ #include "src/core/iomgr/workqueue.h" #include -#include #include #include "test/core/util/test_config.h" -static gpr_mu *g_mu; -static grpc_pollset *g_pollset; +static grpc_pollset g_pollset; static void must_succeed(grpc_exec_ctx *exec_ctx, void *p, bool success) { GPR_ASSERT(success == 1); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); *(int *)p = 1; - grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(g_mu); + grpc_pollset_kick(&g_pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); } static void test_ref_unref(void) { @@ -69,13 +67,13 @@ static void test_add_closure(void) { grpc_closure_init(&c, must_succeed, &done); grpc_workqueue_push(wq, &c, 1); - grpc_workqueue_add_to_pollset(&exec_ctx, wq, g_pollset); + grpc_workqueue_add_to_pollset(&exec_ctx, wq, &g_pollset); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); GPR_ASSERT(!done); - grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(deadline.clock_type), - deadline); - gpr_mu_unlock(g_mu); + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + gpr_now(deadline.clock_type), deadline); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(done); @@ -94,13 +92,13 @@ static void test_flush(void) { grpc_exec_ctx_enqueue(&exec_ctx, &c, true, NULL); grpc_workqueue_flush(&exec_ctx, wq); - grpc_workqueue_add_to_pollset(&exec_ctx, wq, g_pollset); + grpc_workqueue_add_to_pollset(&exec_ctx, wq, &g_pollset); - gpr_mu_lock(g_mu); + gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); GPR_ASSERT(!done); - grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(deadline.clock_type), - deadline); - gpr_mu_unlock(g_mu); + grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + gpr_now(deadline.clock_type), deadline); + gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(done); @@ -117,18 +115,15 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - g_pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(g_pollset, &g_mu); + grpc_pollset_init(&g_pollset); test_ref_unref(); test_add_closure(); test_flush(); - grpc_closure_init(&destroyed, destroy_pollset, g_pollset); - grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); + grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - - gpr_free(g_pollset); return 0; } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index 9b70afffe1e..4dd595df956 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -45,8 +45,7 @@ #include "src/core/security/credentials.h" typedef struct { - gpr_mu *mu; - grpc_pollset *pollset; + grpc_pollset pollset; int is_done; char *token; } oauth2_request; @@ -67,11 +66,11 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data, GPR_SLICE_LENGTH(token_slice)); token[GPR_SLICE_LENGTH(token_slice)] = '\0'; } - gpr_mu_lock(request->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&request->pollset)); request->is_done = 1; request->token = token; - grpc_pollset_kick(request->pollset, NULL); - gpr_mu_unlock(request->mu); + grpc_pollset_kick(&request->pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&request->pollset)); } static void do_nothing(grpc_exec_ctx *exec_ctx, void *unused, bool success) {} @@ -83,30 +82,28 @@ char *grpc_test_fetch_oauth2_token_with_credentials( grpc_closure do_nothing_closure; grpc_auth_metadata_context null_ctx = {"", "", NULL, NULL}; - request.pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(request.pollset, &request.mu); + grpc_pollset_init(&request.pollset); request.is_done = 0; grpc_closure_init(&do_nothing_closure, do_nothing, NULL); - grpc_call_credentials_get_request_metadata(&exec_ctx, creds, request.pollset, + grpc_call_credentials_get_request_metadata(&exec_ctx, creds, &request.pollset, null_ctx, on_oauth2_response, &request); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(request.mu); + gpr_mu_lock(GRPC_POLLSET_MU(&request.pollset)); while (!request.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, request.pollset, &worker, + grpc_pollset_work(&exec_ctx, &request.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); } - gpr_mu_unlock(request.mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&request.pollset)); - grpc_pollset_shutdown(&exec_ctx, request.pollset, &do_nothing_closure); + grpc_pollset_shutdown(&exec_ctx, &request.pollset, &do_nothing_closure); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_destroy(request.pollset); - gpr_free(request.pollset); + grpc_pollset_destroy(&request.pollset); return request.token; } diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index 09673f362cf..6043bf54204 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -34,6 +34,8 @@ #include #include +#include "src/core/security/credentials.h" +#include "src/core/support/string.h" #include #include #include @@ -42,12 +44,8 @@ #include #include -#include "src/core/security/credentials.h" -#include "src/core/support/string.h" - typedef struct { - gpr_mu *mu; - grpc_pollset *pollset; + grpc_pollset pollset; int is_done; } synchronizer; @@ -64,10 +62,10 @@ static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *user_data, printf("\nGot token: %s\n\n", token); gpr_free(token); } - gpr_mu_lock(sync->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&sync->pollset)); sync->is_done = 1; - grpc_pollset_kick(sync->pollset, NULL); - gpr_mu_unlock(sync->mu); + grpc_pollset_kick(&sync->pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&sync->pollset)); } int main(int argc, char **argv) { @@ -93,30 +91,26 @@ int main(int argc, char **argv) { goto end; } - sync.pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(sync.pollset, &sync.mu); + grpc_pollset_init(&sync.pollset); sync.is_done = 0; grpc_call_credentials_get_request_metadata( &exec_ctx, ((grpc_composite_channel_credentials *)creds)->call_creds, - sync.pollset, context, on_metadata_response, &sync); + &sync.pollset, context, on_metadata_response, &sync); - gpr_mu_lock(sync.mu); + gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); while (!sync.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, sync.pollset, &worker, + grpc_pollset_work(&exec_ctx, &sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(sync.mu); - grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(sync.mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); + grpc_exec_ctx_finish(&exec_ctx); + gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); } - gpr_mu_unlock(sync.mu); - - grpc_exec_ctx_finish(&exec_ctx); + gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); grpc_channel_credentials_release(creds); - gpr_free(sync.pollset); end: gpr_cmdline_destroy(cl); diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index 0e8c38a53ec..fb4bd30e2df 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -36,17 +36,16 @@ #include #include +#include "src/core/security/secure_endpoint.h" +#include "src/core/iomgr/endpoint_pair.h" +#include "src/core/iomgr/iomgr.h" #include #include #include -#include "src/core/iomgr/endpoint_pair.h" -#include "src/core/iomgr/iomgr.h" -#include "src/core/security/secure_endpoint.h" -#include "src/core/tsi/fake_transport_security.h" #include "test/core/util/test_config.h" +#include "src/core/tsi/fake_transport_security.h" -static gpr_mu *g_mu; -static grpc_pollset *g_pollset; +static grpc_pollset g_pollset; static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( size_t slice_size, gpr_slice *leftover_slices, size_t leftover_nslices) { @@ -57,8 +56,8 @@ static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( grpc_endpoint_pair tcp; tcp = grpc_iomgr_create_endpoint_pair("fixture", slice_size); - grpc_endpoint_add_to_pollset(&exec_ctx, tcp.client, g_pollset); - grpc_endpoint_add_to_pollset(&exec_ctx, tcp.server, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, tcp.client, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, tcp.server, &g_pollset); if (leftover_nslices == 0) { f.client_ep = @@ -182,16 +181,13 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); - g_pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(g_pollset, &g_mu); - grpc_endpoint_tests(configs[0], g_pollset, g_mu); + grpc_pollset_init(&g_pollset); + grpc_endpoint_tests(configs[0], &g_pollset); test_leftover(configs[1], 1); - grpc_closure_init(&destroyed, destroy_pollset, g_pollset); - grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); + grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); - gpr_free(g_pollset); - return 0; } diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index eb86589681d..5070cf0492c 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -34,6 +34,7 @@ #include #include +#include "src/core/security/jwt_verifier.h" #include #include #include @@ -42,11 +43,8 @@ #include #include -#include "src/core/security/jwt_verifier.h" - typedef struct { - grpc_pollset *pollset; - gpr_mu *mu; + grpc_pollset pollset; int is_done; int success; } synchronizer; @@ -79,10 +77,10 @@ static void on_jwt_verification_done(void *user_data, grpc_jwt_verifier_status_to_string(status)); } - gpr_mu_lock(sync->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&sync->pollset)); sync->is_done = 1; - grpc_pollset_kick(sync->pollset, NULL); - gpr_mu_unlock(sync->mu); + grpc_pollset_kick(&sync->pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&sync->pollset)); } int main(int argc, char **argv) { @@ -105,26 +103,23 @@ int main(int argc, char **argv) { grpc_init(); - sync.pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(sync.pollset, &sync.mu); + grpc_pollset_init(&sync.pollset); sync.is_done = 0; - grpc_jwt_verifier_verify(&exec_ctx, verifier, sync.pollset, jwt, aud, + grpc_jwt_verifier_verify(&exec_ctx, verifier, &sync.pollset, jwt, aud, on_jwt_verification_done, &sync); - gpr_mu_lock(sync.mu); + gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); while (!sync.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, sync.pollset, &worker, + grpc_pollset_work(&exec_ctx, &sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(sync.mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(sync.mu); + gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); } - gpr_mu_unlock(sync.mu); - - gpr_free(sync.pollset); + gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); grpc_jwt_verifier_destroy(verifier); gpr_cmdline_destroy(cl); diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c index ba382d242a3..ad21fe0f536 100644 --- a/test/core/util/port_posix.c +++ b/test/core/util/port_posix.c @@ -69,8 +69,7 @@ static int has_port_been_chosen(int port) { } typedef struct freereq { - gpr_mu *mu; - grpc_pollset *pollset; + grpc_pollset pollset; int done; } freereq; @@ -83,10 +82,10 @@ static void destroy_pollset_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, const grpc_httpcli_response *response) { freereq *pr = arg; - gpr_mu_lock(pr->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); pr->done = 1; - grpc_pollset_kick(pr->pollset, NULL); - gpr_mu_unlock(pr->mu); + grpc_pollset_kick(&pr->pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); } static void free_port_using_server(char *server, int port) { @@ -101,34 +100,31 @@ static void free_port_using_server(char *server, int port) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); - - pr.pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(pr.pollset, &pr.mu); + grpc_pollset_init(&pr.pollset); grpc_closure_init(&shutdown_closure, destroy_pollset_and_shutdown, - pr.pollset); + &pr.pollset); req.host = server; gpr_asprintf(&path, "/drop/%d", port); req.path = path; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, + grpc_httpcli_get(&exec_ctx, &context, &pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), freed_port_from_server, &pr); - gpr_mu_lock(pr.mu); + gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); while (!pr.done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, pr.pollset, &worker, + grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); } - gpr_mu_unlock(pr.mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); grpc_httpcli_context_destroy(&context); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_shutdown(&exec_ctx, pr.pollset, &shutdown_closure); + grpc_pollset_shutdown(&exec_ctx, &pr.pollset, &shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); - gpr_free(pr.pollset); gpr_free(path); } @@ -206,8 +202,7 @@ static int is_port_available(int *port, int is_tcp) { } typedef struct portreq { - gpr_mu *mu; - grpc_pollset *pollset; + grpc_pollset pollset; int port; int retries; char *server; @@ -239,7 +234,7 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->retries++; req.host = pr->server; req.path = "/get"; - grpc_httpcli_get(exec_ctx, pr->ctx, pr->pollset, &req, + grpc_httpcli_get(exec_ctx, pr->ctx, &pr->pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, pr); return; @@ -251,10 +246,10 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, port = port * 10 + response->body[i] - '0'; } GPR_ASSERT(port > 1024); - gpr_mu_lock(pr->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); pr->port = port; - grpc_pollset_kick(pr->pollset, NULL); - gpr_mu_unlock(pr->mu); + grpc_pollset_kick(&pr->pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); } static int pick_port_using_server(char *server) { @@ -268,10 +263,9 @@ static int pick_port_using_server(char *server) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); - pr.pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(pr.pollset, &pr.mu); + grpc_pollset_init(&pr.pollset); grpc_closure_init(&shutdown_closure, destroy_pollset_and_shutdown, - pr.pollset); + &pr.pollset); pr.port = -1; pr.server = server; pr.ctx = &context; @@ -280,23 +274,22 @@ static int pick_port_using_server(char *server) { req.path = "/get"; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, + grpc_httpcli_get(&exec_ctx, &context, &pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, &pr); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(pr.mu); + gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, pr.pollset, &worker, + grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); } - gpr_mu_unlock(pr.mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); grpc_httpcli_context_destroy(&context); - grpc_pollset_shutdown(&exec_ctx, pr.pollset, &shutdown_closure); + grpc_pollset_shutdown(&exec_ctx, &pr.pollset, &shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); - gpr_free(pr.pollset); return pr.port; } diff --git a/test/core/util/port_windows.c b/test/core/util/port_windows.c index 3b20aeb7182..b5bd0168ad2 100644 --- a/test/core/util/port_windows.c +++ b/test/core/util/port_windows.c @@ -129,8 +129,7 @@ static int is_port_available(int *port, int is_tcp) { } typedef struct portreq { - grpc_pollset *pollset; - gpr_mu *mu; + grpc_pollset pollset; int port; } portreq; @@ -146,10 +145,10 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, port = port * 10 + response->body[i] - '0'; } GPR_ASSERT(port > 1024); - gpr_mu_lock(pr->mu); + gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); pr->port = port; - grpc_pollset_kick(pr->pollset, NULL); - gpr_mu_unlock(pr->mu); + grpc_pollset_kick(&pr->pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); } static void destroy_pollset_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, @@ -169,34 +168,32 @@ static int pick_port_using_server(char *server) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); - pr.pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(pr.pollset, &pr.mu); + grpc_pollset_init(&pr.pollset); pr.port = -1; req.host = server; req.path = "/get"; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, + grpc_httpcli_get(&exec_ctx, &context, &pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, &pr); - gpr_mu_lock(pr.mu); + gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, pr.pollset, &worker, + grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); - gpr_mu_unlock(pr.mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(pr.mu); + gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); } - gpr_mu_unlock(pr.mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); grpc_httpcli_context_destroy(&context); grpc_closure_init(&destroy_pollset_closure, destroy_pollset_and_shutdown, &pr.pollset); - grpc_pollset_shutdown(&exec_ctx, pr.pollset, &destroy_pollset_closure); - gpr_free(pr.pollset); + grpc_pollset_shutdown(&exec_ctx, &pr.pollset, &destroy_pollset_closure); grpc_exec_ctx_finish(&exec_ctx); return pr.port; diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index ab379441d8c..e99d5dcffdd 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -57,8 +57,8 @@ void test_tcp_server_init(test_tcp_server *server, server->tcp_server = NULL; grpc_closure_init(&server->shutdown_complete, on_server_destroyed, server); server->shutdown = 0; - server->pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(server->pollset, &server->mu); + grpc_pollset_init(&server->pollset); + server->pollsets[0] = &server->pollset; server->on_connect = on_connect; server->cb_data = user_data; } @@ -77,7 +77,7 @@ void test_tcp_server_start(test_tcp_server *server, int port) { grpc_tcp_server_add_port(server->tcp_server, &addr, sizeof(addr)); GPR_ASSERT(port_added == port); - grpc_tcp_server_start(&exec_ctx, server->tcp_server, &server->pollset, 1, + grpc_tcp_server_start(&exec_ctx, server->tcp_server, server->pollsets, 1, server->on_connect, server->cb_data); gpr_log(GPR_INFO, "test tcp server listening on 0.0.0.0:%d", port); @@ -90,10 +90,10 @@ void test_tcp_server_poll(test_tcp_server *server, int seconds) { gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(seconds, GPR_TIMESPAN)); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - gpr_mu_lock(server->mu); - grpc_pollset_work(&exec_ctx, server->pollset, &worker, + gpr_mu_lock(GRPC_POLLSET_MU(&server->pollset)); + grpc_pollset_work(&exec_ctx, &server->pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(server->mu); + gpr_mu_unlock(GRPC_POLLSET_MU(&server->pollset)); grpc_exec_ctx_finish(&exec_ctx); } @@ -111,9 +111,8 @@ void test_tcp_server_destroy(test_tcp_server *server) { gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), shutdown_deadline) < 0) { test_tcp_server_poll(server, 1); } - grpc_pollset_shutdown(&exec_ctx, server->pollset, &do_nothing_cb); + grpc_pollset_shutdown(&exec_ctx, &server->pollset, &do_nothing_cb); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_destroy(server->pollset); - gpr_free(server->pollset); + grpc_pollset_destroy(&server->pollset); grpc_shutdown(); } diff --git a/test/core/util/test_tcp_server.h b/test/core/util/test_tcp_server.h index 15fcb4fb873..51119cf6c80 100644 --- a/test/core/util/test_tcp_server.h +++ b/test/core/util/test_tcp_server.h @@ -41,8 +41,8 @@ typedef struct test_tcp_server { grpc_tcp_server *tcp_server; grpc_closure shutdown_complete; int shutdown; - gpr_mu *mu; - grpc_pollset *pollset; + grpc_pollset pollset; + grpc_pollset *pollsets[1]; grpc_tcp_server_cb on_connect; void *cb_data; } test_tcp_server; From e5e665564e6080aa81161d059fbe5f1fb862bbce Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 25 Feb 2016 18:19:23 -0800 Subject: [PATCH 125/236] fix python distribtest on fedora21 --- tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile index 1eb4c1e7750..f1b0e2f4b22 100644 --- a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile @@ -35,3 +35,8 @@ FROM fedora:21 RUN yum install -y yum-plugin-ovl RUN yum clean all && yum update -y && yum install -y python python-pip + +# Upgrading six would fail because of docker issue when using overlay. +# Trying twice makes it work fine. +# https://github.com/docker/docker/issues/10180 +RUN pip2 install --upgrade six || pip2 install --upgrade six From 5638cc22f3b8965de1fc36d3c9edce6e712322d6 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 25 Feb 2016 18:31:11 -0800 Subject: [PATCH 126/236] fix ruby distribtest on centos6 --- tools/dockerfile/distribtest/ruby_centos6_x64/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/dockerfile/distribtest/ruby_centos6_x64/Dockerfile b/tools/dockerfile/distribtest/ruby_centos6_x64/Dockerfile index f65b8690d7e..b943b67e8a5 100644 --- a/tools/dockerfile/distribtest/ruby_centos6_x64/Dockerfile +++ b/tools/dockerfile/distribtest/ruby_centos6_x64/Dockerfile @@ -35,7 +35,9 @@ RUN yum install -y tar which # Install rvm RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -RUN \curl -sSL https://get.rvm.io | bash -s stable --ruby +# Running the installation twice to work around docker issue when using overlay. +# https://github.com/docker/docker/issues/10180 +RUN (curl -sSL https://get.rvm.io | bash -s stable --ruby) || (curl -sSL https://get.rvm.io | bash -s stable --ruby) RUN /bin/bash -l -c "echo '. /etc/profile.d/rvm.sh' >> ~/.bashrc" RUN /bin/bash -l -c "gem install --update bundler" From 0222b92cd06332b1d8cb16997e73380b0c78c760 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 25 Feb 2016 18:54:21 -0800 Subject: [PATCH 127/236] fix python distribtest on fedora20 --- tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile index d58036410f6..502c81c7cb0 100644 --- a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile @@ -30,3 +30,8 @@ FROM fedora:20 RUN yum clean all && yum update -y && yum install -y python python-pip + +# Upgrading six would fail because of docker issue when using overlay. +# Trying twice makes it work fine. +# https://github.com/docker/docker/issues/10180 +RUN pip2 install --upgrade six || pip2 install --upgrade six From 69b093b3601bb01bec66391e28cc9f76b7baf303 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 25 Feb 2016 19:04:07 -0800 Subject: [PATCH 128/236] Revert "Revert "Add an implementation firewall against pollset_set"" --- src/core/channel/client_channel.c | 45 +++++---- .../client_config/lb_policies/pick_first.c | 31 +++--- .../client_config/lb_policies/round_robin.c | 25 +++-- src/core/client_config/lb_policy.c | 4 +- src/core/client_config/lb_policy.h | 3 +- src/core/client_config/subchannel.c | 30 +++--- src/core/httpcli/httpcli.c | 24 ++--- src/core/httpcli/httpcli.h | 3 +- src/core/iomgr/fd_posix.c | 6 +- src/core/iomgr/pollset.h | 15 ++- .../iomgr/pollset_multipoller_with_epoll.c | 1 + .../pollset_multipoller_with_poll_posix.c | 8 +- src/core/iomgr/pollset_posix.c | 16 ++-- src/core/iomgr/pollset_posix.h | 12 +-- src/core/iomgr/pollset_set.h | 10 +- src/core/iomgr/pollset_set_posix.c | 26 ++++- src/core/iomgr/pollset_set_posix.h | 20 +--- src/core/iomgr/pollset_set_windows.c | 4 +- src/core/iomgr/pollset_set_windows.h | 2 +- src/core/iomgr/pollset_windows.c | 7 +- src/core/iomgr/pollset_windows.h | 6 +- src/core/iomgr/tcp_client_posix.c | 12 ++- src/core/iomgr/tcp_posix.c | 10 +- src/core/iomgr/udp_server.h | 1 + src/core/iomgr/workqueue_posix.c | 1 + src/core/iomgr/workqueue_posix.h | 4 +- .../security/google_default_credentials.c | 39 ++++---- src/core/surface/completion_queue.c | 85 +++++++++-------- .../set_initial_connect_string_test.c | 2 +- .../core/end2end/fixtures/h2_full+poll+pipe.c | 18 ++-- test/core/end2end/fixtures/h2_full+poll.c | 16 ++-- test/core/end2end/fixtures/h2_ssl+poll.c | 1 + test/core/end2end/fixtures/h2_uchannel.c | 47 ++++----- test/core/end2end/fixtures/h2_uds+poll.c | 18 ++-- test/core/httpcli/httpcli_test.c | 44 +++++---- test/core/httpcli/httpscli_test.c | 44 +++++---- test/core/iomgr/endpoint_pair_test.c | 19 ++-- test/core/iomgr/endpoint_tests.c | 18 ++-- test/core/iomgr/endpoint_tests.h | 4 +- test/core/iomgr/fd_posix_test.c | 89 +++++++++-------- test/core/iomgr/tcp_client_posix_test.c | 74 ++++++++------- test/core/iomgr/tcp_posix_test.c | 95 ++++++++++--------- test/core/iomgr/tcp_server_posix_test.c | 50 +++++----- test/core/iomgr/workqueue_test.c | 39 ++++---- test/core/security/oauth2_utils.c | 25 ++--- .../print_google_default_creds_token.c | 34 ++++--- test/core/security/secure_endpoint_test.c | 26 ++--- test/core/security/verify_jwt.c | 29 +++--- test/core/util/port_posix.c | 53 ++++++----- test/core/util/port_windows.c | 27 +++--- test/core/util/test_tcp_server.c | 17 ++-- test/core/util/test_tcp_server.h | 4 +- 52 files changed, 680 insertions(+), 563 deletions(-) diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index 7176c01b05f..a96b49ac12f 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -78,8 +78,8 @@ typedef struct client_channel_channel_data { int exit_idle_when_lb_policy_arrives; /** owning stack */ grpc_channel_stack *owning_stack; - /** interested parties */ - grpc_pollset_set interested_parties; + /** interested parties (owned) */ + grpc_pollset_set *interested_parties; } channel_data; /** We create one watcher for each new lb_policy that is returned from a @@ -183,8 +183,8 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, chand->incoming_configuration = NULL; if (lb_policy != NULL) { - grpc_pollset_set_add_pollset_set(exec_ctx, &lb_policy->interested_parties, - &chand->interested_parties); + grpc_pollset_set_add_pollset_set(exec_ctx, lb_policy->interested_parties, + chand->interested_parties); } gpr_mu_lock(&chand->mu_config); @@ -231,9 +231,8 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, } if (old_lb_policy != NULL) { - grpc_pollset_set_del_pollset_set(exec_ctx, - &old_lb_policy->interested_parties, - &chand->interested_parties); + grpc_pollset_set_del_pollset_set( + exec_ctx, old_lb_policy->interested_parties, chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, old_lb_policy, "channel"); } @@ -254,7 +253,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, GPR_ASSERT(op->set_accept_stream == NULL); if (op->bind_pollset != NULL) { - grpc_pollset_set_add_pollset(exec_ctx, &chand->interested_parties, + grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, op->bind_pollset); } @@ -284,8 +283,8 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, chand->resolver = NULL; if (chand->lb_policy != NULL) { grpc_pollset_set_del_pollset_set(exec_ctx, - &chand->lb_policy->interested_parties, - &chand->interested_parties); + chand->lb_policy->interested_parties, + chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel"); chand->lb_policy = NULL; } @@ -411,7 +410,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE, "client_channel"); - grpc_pollset_set_init(&chand->interested_parties); + chand->interested_parties = grpc_pollset_set_create(); } /* Destructor for channel_data */ @@ -425,12 +424,12 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, } if (chand->lb_policy != NULL) { grpc_pollset_set_del_pollset_set(exec_ctx, - &chand->lb_policy->interested_parties, - &chand->interested_parties); + chand->lb_policy->interested_parties, + chand->interested_parties); GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "channel"); } grpc_connectivity_state_destroy(exec_ctx, &chand->state_tracker); - grpc_pollset_set_destroy(&chand->interested_parties); + grpc_pollset_set_destroy(chand->interested_parties); gpr_mu_destroy(&chand->mu_config); } @@ -441,9 +440,17 @@ static void cc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, } const grpc_channel_filter grpc_client_channel_filter = { - cc_start_transport_stream_op, cc_start_transport_op, sizeof(call_data), - init_call_elem, cc_set_pollset, destroy_call_elem, sizeof(channel_data), - init_channel_elem, destroy_channel_elem, cc_get_peer, "client-channel", + cc_start_transport_stream_op, + cc_start_transport_op, + sizeof(call_data), + init_call_elem, + cc_set_pollset, + destroy_call_elem, + sizeof(channel_data), + init_channel_elem, + destroy_channel_elem, + cc_get_peer, + "client-channel", }; void grpc_client_channel_set_resolver(grpc_exec_ctx *exec_ctx, @@ -501,7 +508,7 @@ static void on_external_watch_complete(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) { external_connectivity_watcher *w = arg; grpc_closure *follow_up = w->on_complete; - grpc_pollset_set_del_pollset(exec_ctx, &w->chand->interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, w->chand->interested_parties, w->pollset); GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "external_connectivity_watcher"); @@ -517,7 +524,7 @@ void grpc_client_channel_watch_connectivity_state( w->chand = chand; w->pollset = pollset; w->on_complete = on_complete; - grpc_pollset_set_add_pollset(exec_ctx, &chand->interested_parties, pollset); + grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, pollset); grpc_closure_init(&w->my_closure, on_external_watch_complete, w); GRPC_CHANNEL_STACK_REF(w->chand->owning_stack, "external_connectivity_watcher"); diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c index 459bbebb68a..9f38f398d8a 100644 --- a/src/core/client_config/lb_policies/pick_first.c +++ b/src/core/client_config/lb_policies/pick_first.c @@ -31,8 +31,8 @@ * */ -#include "src/core/client_config/lb_policy_factory.h" #include "src/core/client_config/lb_policies/pick_first.h" +#include "src/core/client_config/lb_policy_factory.h" #include @@ -119,7 +119,7 @@ void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { while (pp != NULL) { pending_pick *next = pp->next; *pp->target = NULL; - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL); gpr_free(pp); @@ -137,7 +137,7 @@ static void pf_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, while (pp != NULL) { pending_pick *next = pp->next; if (pp->target == target) { - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); *target = NULL; grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL); @@ -158,7 +158,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, pick_first_lb_policy *p) { GRPC_LB_POLICY_WEAK_REF(&p->base, "pick_first_connectivity"); grpc_subchannel_notify_on_state_change( exec_ctx, p->subchannels[p->checking_subchannel], - &p->base.interested_parties, &p->checking_connectivity, + p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); } @@ -195,8 +195,7 @@ int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset, if (!p->started_picking) { start_picking(exec_ctx, p); } - grpc_pollset_set_add_pollset(exec_ctx, &p->base.interested_parties, - pollset); + grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset); pp = gpr_malloc(sizeof(*pp)); pp->next = p->pending_picks; pp->pollset = pollset; @@ -253,7 +252,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, p->checking_connectivity, "selected_changed"); if (p->checking_connectivity != GRPC_CHANNEL_FATAL_FAILURE) { grpc_connected_subchannel_notify_on_state_change( - exec_ctx, selected, &p->base.interested_parties, + exec_ctx, selected, p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); } else { GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "pick_first_connectivity"); @@ -278,13 +277,13 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, while ((pp = p->pending_picks)) { p->pending_picks = pp->next; *pp->target = selected; - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL); gpr_free(pp); } grpc_connected_subchannel_notify_on_state_change( - exec_ctx, selected, &p->base.interested_parties, + exec_ctx, selected, p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); break; case GRPC_CHANNEL_TRANSIENT_FAILURE: @@ -298,7 +297,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, if (p->checking_connectivity == GRPC_CHANNEL_TRANSIENT_FAILURE) { grpc_subchannel_notify_on_state_change( exec_ctx, p->subchannels[p->checking_subchannel], - &p->base.interested_parties, &p->checking_connectivity, + p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); } else { goto loop; @@ -311,7 +310,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, "connecting_changed"); grpc_subchannel_notify_on_state_change( exec_ctx, p->subchannels[p->checking_subchannel], - &p->base.interested_parties, &p->checking_connectivity, + p->base.interested_parties, &p->checking_connectivity, &p->connectivity_changed); break; case GRPC_CHANNEL_FATAL_FAILURE: @@ -379,8 +378,14 @@ void pf_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } static const grpc_lb_policy_vtable pick_first_lb_policy_vtable = { - pf_destroy, pf_shutdown, pf_pick, pf_cancel_pick, pf_ping_one, pf_exit_idle, - pf_check_connectivity, pf_notify_on_state_change}; + pf_destroy, + pf_shutdown, + pf_pick, + pf_cancel_pick, + pf_ping_one, + pf_exit_idle, + pf_check_connectivity, + pf_notify_on_state_change}; static void pick_first_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/lb_policies/round_robin.c b/src/core/client_config/lb_policies/round_robin.c index b1171c45b00..114ece6e4d9 100644 --- a/src/core/client_config/lb_policies/round_robin.c +++ b/src/core/client_config/lb_policies/round_robin.c @@ -260,7 +260,7 @@ static void rr_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, while (pp != NULL) { pending_pick *next = pp->next; if (pp->target == target) { - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); *target = NULL; grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, false, NULL); @@ -285,7 +285,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) { subchannel_data *sd = p->subchannels[i]; sd->connectivity_state = GRPC_CHANNEL_IDLE; grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, &p->base.interested_parties, + exec_ctx, sd->subchannel, p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); GRPC_LB_POLICY_WEAK_REF(&p->base, "round_robin_connectivity"); } @@ -322,8 +322,7 @@ int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset, if (!p->started_picking) { start_picking(exec_ctx, p); } - grpc_pollset_set_add_pollset(exec_ctx, &p->base.interested_parties, - pollset); + grpc_pollset_set_add_pollset(exec_ctx, p->base.interested_parties, pollset); pp = gpr_malloc(sizeof(*pp)); pp->next = p->pending_picks; pp->pollset = pollset; @@ -374,13 +373,13 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)", selected->subchannel, selected); } - grpc_pollset_set_del_pollset(exec_ctx, &p->base.interested_parties, + grpc_pollset_set_del_pollset(exec_ctx, p->base.interested_parties, pp->pollset); grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, true, NULL); gpr_free(pp); } grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, &p->base.interested_parties, + exec_ctx, sd->subchannel, p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); break; case GRPC_CHANNEL_CONNECTING: @@ -389,13 +388,13 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, sd->connectivity_state, "connecting_changed"); grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, &p->base.interested_parties, + exec_ctx, sd->subchannel, p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); break; case GRPC_CHANNEL_TRANSIENT_FAILURE: /* renew state notification */ grpc_subchannel_notify_on_state_change( - exec_ctx, sd->subchannel, &p->base.interested_parties, + exec_ctx, sd->subchannel, p->base.interested_parties, &sd->connectivity_state, &sd->connectivity_changed_closure); /* remove from ready list if still present */ @@ -484,8 +483,14 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } static const grpc_lb_policy_vtable round_robin_lb_policy_vtable = { - rr_destroy, rr_shutdown, rr_pick, rr_cancel_pick, rr_ping_one, rr_exit_idle, - rr_check_connectivity, rr_notify_on_state_change}; + rr_destroy, + rr_shutdown, + rr_pick, + rr_cancel_pick, + rr_ping_one, + rr_exit_idle, + rr_check_connectivity, + rr_notify_on_state_change}; static void round_robin_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/lb_policy.c b/src/core/client_config/lb_policy.c index d4672f6b255..5ff623e0069 100644 --- a/src/core/client_config/lb_policy.c +++ b/src/core/client_config/lb_policy.c @@ -39,7 +39,7 @@ void grpc_lb_policy_init(grpc_lb_policy *policy, const grpc_lb_policy_vtable *vtable) { policy->vtable = vtable; gpr_atm_no_barrier_store(&policy->ref_pair, 1 << WEAK_REF_BITS); - grpc_pollset_set_init(&policy->interested_parties); + policy->interested_parties = grpc_pollset_set_create(); } #ifdef GRPC_LB_POLICY_REFCOUNT_DEBUG @@ -93,7 +93,7 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, gpr_atm old_val = ref_mutate(policy, -(gpr_atm)1, 1 REF_MUTATE_PASS_ARGS("WEAK_UNREF")); if (old_val == 1) { - grpc_pollset_set_destroy(&policy->interested_parties); + grpc_pollset_set_destroy(policy->interested_parties); policy->vtable->destroy(exec_ctx, policy); } } diff --git a/src/core/client_config/lb_policy.h b/src/core/client_config/lb_policy.h index db5238c8ca0..4fbb12da394 100644 --- a/src/core/client_config/lb_policy.h +++ b/src/core/client_config/lb_policy.h @@ -48,7 +48,8 @@ typedef void (*grpc_lb_completion)(void *cb_arg, grpc_subchannel *subchannel, struct grpc_lb_policy { const grpc_lb_policy_vtable *vtable; gpr_atm ref_pair; - grpc_pollset_set interested_parties; + /* owned pointer to interested parties in load balancing decisions */ + grpc_pollset_set *interested_parties; }; struct grpc_lb_policy_vtable { diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index 6599c75dba7..291ad3472cc 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -108,7 +108,7 @@ struct grpc_subchannel { /** pollset_set tracking who's interested in a connection being setup */ - grpc_pollset_set pollset_set; + grpc_pollset_set *pollset_set; /** active connection, or null; of type grpc_connected_subchannel */ gpr_atm connected_subchannel; @@ -184,8 +184,8 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(c); } -void grpc_connected_subchannel_ref(grpc_connected_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_connected_subchannel_ref( + grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON); } @@ -209,7 +209,7 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_slice_unref(c->initial_connect_string); grpc_connectivity_state_destroy(exec_ctx, &c->state_tracker); grpc_connector_unref(exec_ctx, c->connector); - grpc_pollset_set_destroy(&c->pollset_set); + grpc_pollset_set_destroy(c->pollset_set); grpc_subchannel_key_destroy(exec_ctx, c->key); gpr_free(c); } @@ -226,8 +226,8 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta, return old_val; } -grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_ref( + grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS), 0 REF_MUTATE_PURPOSE("STRONG_REF")); @@ -235,8 +235,8 @@ grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c return c; } -grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_weak_ref( + grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF")); GPR_ASSERT(old_refs != 0); @@ -326,7 +326,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, } c->addr = gpr_malloc(args->addr_len); memcpy(c->addr, args->addr, args->addr_len); - grpc_pollset_set_init(&c->pollset_set); + c->pollset_set = grpc_pollset_set_create(); c->addr_len = args->addr_len; grpc_set_initial_connect_string(&c->addr, &c->addr_len, &c->initial_connect_string); @@ -345,7 +345,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { grpc_connect_in_args args; - args.interested_parties = &c->pollset_set; + args.interested_parties = c->pollset_set; args.addr = c->addr; args.addr_len = c->addr_len; args.deadline = compute_connect_deadline(c); @@ -379,7 +379,7 @@ static void on_external_state_watcher_done(grpc_exec_ctx *exec_ctx, void *arg, external_state_watcher *w = arg; grpc_closure *follow_up = w->notify; if (w->pollset_set != NULL) { - grpc_pollset_set_del_pollset_set(exec_ctx, &w->subchannel->pollset_set, + grpc_pollset_set_del_pollset_set(exec_ctx, w->subchannel->pollset_set, w->pollset_set); } gpr_mu_lock(&w->subchannel->mu); @@ -415,7 +415,7 @@ void grpc_subchannel_notify_on_state_change( w->notify = notify; grpc_closure_init(&w->closure, on_external_state_watcher_done, w); if (interested_parties != NULL) { - grpc_pollset_set_add_pollset_set(exec_ctx, &c->pollset_set, + grpc_pollset_set_add_pollset_set(exec_ctx, c->pollset_set, interested_parties); } GRPC_SUBCHANNEL_WEAK_REF(c, "external_state_watcher"); @@ -573,7 +573,7 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher"); GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); grpc_connected_subchannel_notify_on_state_change( - exec_ctx, con, &c->pollset_set, &sw_subchannel->connectivity_state, + exec_ctx, con, c->pollset_set, &sw_subchannel->connectivity_state, &sw_subchannel->closure); /* signal completion */ @@ -690,8 +690,8 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); } -void grpc_subchannel_call_ref(grpc_subchannel_call *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_subchannel_call_ref( + grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } diff --git a/src/core/httpcli/httpcli.c b/src/core/httpcli/httpcli.c index 71237bb6140..1219c444c71 100644 --- a/src/core/httpcli/httpcli.c +++ b/src/core/httpcli/httpcli.c @@ -31,20 +31,22 @@ * */ -#include "src/core/iomgr/sockaddr.h" #include "src/core/httpcli/httpcli.h" +#include "src/core/iomgr/sockaddr.h" #include +#include +#include +#include + +#include "src/core/httpcli/format_request.h" +#include "src/core/httpcli/parser.h" #include "src/core/iomgr/endpoint.h" +#include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/resolve_address.h" #include "src/core/iomgr/tcp_client.h" -#include "src/core/httpcli/format_request.h" -#include "src/core/httpcli/parser.h" #include "src/core/support/string.h" -#include -#include -#include typedef struct { gpr_slice request_text; @@ -84,18 +86,18 @@ const grpc_httpcli_handshaker grpc_httpcli_plaintext = {"http", plaintext_handshake}; void grpc_httpcli_context_init(grpc_httpcli_context *context) { - grpc_pollset_set_init(&context->pollset_set); + context->pollset_set = grpc_pollset_set_create(); } void grpc_httpcli_context_destroy(grpc_httpcli_context *context) { - grpc_pollset_set_destroy(&context->pollset_set); + grpc_pollset_set_destroy(context->pollset_set); } static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req); static void finish(grpc_exec_ctx *exec_ctx, internal_request *req, int success) { - grpc_pollset_set_del_pollset(exec_ctx, &req->context->pollset_set, + grpc_pollset_set_del_pollset(exec_ctx, req->context->pollset_set, req->pollset); req->on_response(exec_ctx, req->user_data, success ? &req->parser.r : NULL); grpc_httpcli_parser_destroy(&req->parser); @@ -197,7 +199,7 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req) { addr = &req->addresses->addrs[req->next_address++]; grpc_closure_init(&req->connected, on_connected, req); grpc_tcp_client_connect( - exec_ctx, &req->connected, &req->ep, &req->context->pollset_set, + exec_ctx, &req->connected, &req->ep, req->context->pollset_set, (struct sockaddr *)&addr->addr, addr->len, req->deadline); } @@ -237,7 +239,7 @@ static void internal_request_begin( req->host = gpr_strdup(request->host); req->ssl_host_override = gpr_strdup(request->ssl_host_override); - grpc_pollset_set_add_pollset(exec_ctx, &req->context->pollset_set, + grpc_pollset_set_add_pollset(exec_ctx, req->context->pollset_set, req->pollset); grpc_resolve_address(request->host, req->handshaker->default_port, on_resolved, req); diff --git a/src/core/httpcli/httpcli.h b/src/core/httpcli/httpcli.h index 30875d71f13..86e17c1d697 100644 --- a/src/core/httpcli/httpcli.h +++ b/src/core/httpcli/httpcli.h @@ -39,6 +39,7 @@ #include #include "src/core/iomgr/endpoint.h" +#include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/pollset_set.h" /* User agent this library reports */ @@ -56,7 +57,7 @@ typedef struct grpc_httpcli_header { TODO(ctiller): allow caching and capturing multiple requests for the same content and combining them */ typedef struct grpc_httpcli_context { - grpc_pollset_set pollset_set; + grpc_pollset_set *pollset_set; } grpc_httpcli_context; typedef struct { diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c index 85eadd754b9..4ba7c5df943 100644 --- a/src/core/iomgr/fd_posix.c +++ b/src/core/iomgr/fd_posix.c @@ -46,6 +46,8 @@ #include #include +#include "src/core/iomgr/pollset_posix.h" + #define CLOSURE_NOT_READY ((grpc_closure *)0) #define CLOSURE_READY ((grpc_closure *)1) @@ -175,11 +177,11 @@ int grpc_fd_is_orphaned(grpc_fd *fd) { } static void pollset_kick_locked(grpc_fd_watcher *watcher) { - gpr_mu_lock(GRPC_POLLSET_MU(watcher->pollset)); + gpr_mu_lock(&watcher->pollset->mu); GPR_ASSERT(watcher->worker); grpc_pollset_kick_ext(watcher->pollset, watcher->worker, GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); - gpr_mu_unlock(GRPC_POLLSET_MU(watcher->pollset)); + gpr_mu_unlock(&watcher->pollset->mu); } static void maybe_wake_one_watcher_locked(grpc_fd *fd) { diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index 6585326f81a..92a0374ddd4 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -35,8 +35,11 @@ #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_H #include +#include #include +#include "src/core/iomgr/exec_ctx.h" + #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) /* A grpc_pollset is a set of file descriptors that a higher level item is @@ -46,15 +49,11 @@ - a completion queue might keep a pollset with an entry for each transport that is servicing a call that it's tracking */ -#ifdef GPR_POSIX_SOCKET -#include "src/core/iomgr/pollset_posix.h" -#endif - -#ifdef GPR_WIN32 -#include "src/core/iomgr/pollset_windows.h" -#endif +typedef struct grpc_pollset grpc_pollset; +typedef struct grpc_pollset_worker grpc_pollset_worker; -void grpc_pollset_init(grpc_pollset *pollset); +size_t grpc_pollset_size(void); +void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu); /* Begin shutting down the pollset, and call closure when done. * GRPC_POLLSET_MU(pollset) must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index 4acae2bb712..2e0f27fab86 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -45,6 +45,7 @@ #include #include #include "src/core/iomgr/fd_posix.h" +#include "src/core/iomgr/pollset_posix.h" #include "src/core/profiling/timers.h" #include "src/core/support/block_annotate.h" diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 809f8f39daa..4dddfff2302 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -42,13 +42,15 @@ #include #include -#include "src/core/iomgr/fd_posix.h" -#include "src/core/iomgr/iomgr_internal.h" -#include "src/core/support/block_annotate.h" #include #include #include +#include "src/core/iomgr/fd_posix.h" +#include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/support/block_annotate.h" + typedef struct { /* all polled fds */ size_t fd_count; diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index ee7e9f48f4c..e895a778849 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -42,16 +42,16 @@ #include #include -#include "src/core/iomgr/fd_posix.h" -#include "src/core/iomgr/iomgr_internal.h" -#include "src/core/iomgr/socket_utils_posix.h" -#include "src/core/profiling/timers.h" -#include "src/core/support/block_annotate.h" #include #include #include #include #include +#include "src/core/iomgr/fd_posix.h" +#include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/socket_utils_posix.h" +#include "src/core/profiling/timers.h" +#include "src/core/support/block_annotate.h" GPR_TLS_DECL(g_current_thread_poller); GPR_TLS_DECL(g_current_thread_worker); @@ -97,6 +97,8 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { worker->prev->next = worker->next->prev = worker; } +size_t grpc_pollset_size(void) { return sizeof(grpc_pollset); } + void grpc_pollset_kick_ext(grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) { @@ -186,8 +188,9 @@ void grpc_kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); } static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null); -void grpc_pollset_init(grpc_pollset *pollset) { +void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); + *mu = &pollset->mu; pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; pollset->in_flight_cbs = 0; pollset->shutting_down = 0; @@ -204,7 +207,6 @@ void grpc_pollset_destroy(grpc_pollset *pollset) { GPR_ASSERT(!grpc_pollset_has_workers(pollset)); GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); pollset->vtable->destroy(pollset); - gpr_mu_destroy(&pollset->mu); while (pollset->local_wakeup_cache) { grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next; grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd); diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h index 5868b3fa21c..bbedb66b007 100644 --- a/src/core/iomgr/pollset_posix.h +++ b/src/core/iomgr/pollset_posix.h @@ -37,8 +37,10 @@ #include #include + #include "src/core/iomgr/exec_ctx.h" #include "src/core/iomgr/iomgr.h" +#include "src/core/iomgr/pollset.h" #include "src/core/iomgr/wakeup_fd_posix.h" typedef struct grpc_pollset_vtable grpc_pollset_vtable; @@ -53,15 +55,15 @@ typedef struct grpc_cached_wakeup_fd { struct grpc_cached_wakeup_fd *next; } grpc_cached_wakeup_fd; -typedef struct grpc_pollset_worker { +struct grpc_pollset_worker { grpc_cached_wakeup_fd *wakeup_fd; int reevaluate_polling_on_wakeup; int kicked_specifically; struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; -} grpc_pollset_worker; +}; -typedef struct grpc_pollset { +struct grpc_pollset { /* pollsets under posix can mutate representation as fds are added and removed. For example, we may choose a poll() based implementation on linux for @@ -81,7 +83,7 @@ typedef struct grpc_pollset { } data; /* Local cache of eventfds for workers */ grpc_cached_wakeup_fd *local_wakeup_cache; -} grpc_pollset; +}; struct grpc_pollset_vtable { void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -93,8 +95,6 @@ struct grpc_pollset_vtable { void (*destroy)(grpc_pollset *pollset); }; -#define GRPC_POLLSET_MU(pollset) (&(pollset)->mu) - /* Add an fd to a pollset */ void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, struct grpc_fd *fd); diff --git a/src/core/iomgr/pollset_set.h b/src/core/iomgr/pollset_set.h index 09c04438f71..9591bf0d32c 100644 --- a/src/core/iomgr/pollset_set.h +++ b/src/core/iomgr/pollset_set.h @@ -41,15 +41,9 @@ fd's (etc) that have been registered with the set_set to that pollset. Registering fd's automatically adds them to all current pollsets. */ -#ifdef GPR_POSIX_SOCKET -#include "src/core/iomgr/pollset_set_posix.h" -#endif +typedef struct grpc_pollset_set grpc_pollset_set; -#ifdef GPR_WIN32 -#include "src/core/iomgr/pollset_set_windows.h" -#endif - -void grpc_pollset_set_init(grpc_pollset_set *pollset_set); +grpc_pollset_set *grpc_pollset_set_create(void); void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set); void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/iomgr/pollset_set_posix.c index 4ec92202e30..9dc9aff4a8d 100644 --- a/src/core/iomgr/pollset_set_posix.c +++ b/src/core/iomgr/pollset_set_posix.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,11 +41,30 @@ #include #include -#include "src/core/iomgr/pollset_set.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/iomgr/pollset_set_posix.h" -void grpc_pollset_set_init(grpc_pollset_set *pollset_set) { +struct grpc_pollset_set { + gpr_mu mu; + + size_t pollset_count; + size_t pollset_capacity; + grpc_pollset **pollsets; + + size_t pollset_set_count; + size_t pollset_set_capacity; + struct grpc_pollset_set **pollset_sets; + + size_t fd_count; + size_t fd_capacity; + grpc_fd **fds; +}; + +grpc_pollset_set *grpc_pollset_set_create(void) { + grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set)); memset(pollset_set, 0, sizeof(*pollset_set)); gpr_mu_init(&pollset_set->mu); + return pollset_set; } void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) { @@ -57,6 +76,7 @@ void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) { gpr_free(pollset_set->pollsets); gpr_free(pollset_set->pollset_sets); gpr_free(pollset_set->fds); + gpr_free(pollset_set); } void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, diff --git a/src/core/iomgr/pollset_set_posix.h b/src/core/iomgr/pollset_set_posix.h index 4820a61e4b2..7d1aaf41817 100644 --- a/src/core/iomgr/pollset_set_posix.h +++ b/src/core/iomgr/pollset_set_posix.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,23 +35,7 @@ #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_POSIX_H #include "src/core/iomgr/fd_posix.h" -#include "src/core/iomgr/pollset_posix.h" - -typedef struct grpc_pollset_set { - gpr_mu mu; - - size_t pollset_count; - size_t pollset_capacity; - grpc_pollset **pollsets; - - size_t pollset_set_count; - size_t pollset_set_capacity; - struct grpc_pollset_set **pollset_sets; - - size_t fd_count; - size_t fd_capacity; - grpc_fd **fds; -} grpc_pollset_set; +#include "src/core/iomgr/pollset_set.h" void grpc_pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, grpc_fd *fd); diff --git a/src/core/iomgr/pollset_set_windows.c b/src/core/iomgr/pollset_set_windows.c index 157b46ec32a..9cf8fd4472f 100644 --- a/src/core/iomgr/pollset_set_windows.c +++ b/src/core/iomgr/pollset_set_windows.c @@ -35,9 +35,9 @@ #ifdef GPR_WINSOCK_SOCKET -#include "src/core/iomgr/pollset_set.h" +#include "src/core/iomgr/pollset_set_windows.h" -void grpc_pollset_set_init(grpc_pollset_set* pollset_set) {} +grpc_pollset_set* grpc_pollset_set_create(pollset_set) { return NULL; } void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {} diff --git a/src/core/iomgr/pollset_set_windows.h b/src/core/iomgr/pollset_set_windows.h index cada0d2b61f..aa5abe91338 100644 --- a/src/core/iomgr/pollset_set_windows.h +++ b/src/core/iomgr/pollset_set_windows.h @@ -34,6 +34,6 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H #define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H -typedef struct grpc_pollset_set { void *unused; } grpc_pollset_set; +#include "src/core/iomgr/pollset_set.h" #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index bbce23b46a7..651f8e7334c 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -89,12 +89,17 @@ static void push_front_worker(grpc_pollset_worker *root, worker->links[type].next->links[type].prev = worker; } +size_t grpc_pollset_size(void) { + return sizeof(grpc_pollset); +} + /* There isn't really any such thing as a pollset under Windows, due to the nature of the IO completion ports. We're still going to provide a minimal set of features for the sake of the rest of grpc. But grpc_pollset_work won't actually do any polling, and return as quickly as possible. */ -void grpc_pollset_init(grpc_pollset *pollset) { +void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + *mu = &grpc_polling_mu; memset(pollset, 0, sizeof(*pollset)); pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h index 65ba80619b7..dc0b7a4104b 100644 --- a/src/core/iomgr/pollset_windows.h +++ b/src/core/iomgr/pollset_windows.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -72,8 +72,4 @@ struct grpc_pollset { grpc_closure *on_shutdown; }; -extern gpr_mu grpc_polling_mu; - -#define GRPC_POLLSET_MU(pollset) (&grpc_polling_mu) - #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/tcp_client_posix.c b/src/core/iomgr/tcp_client_posix.c index c76c2e3b0f2..15727856abf 100644 --- a/src/core/iomgr/tcp_client_posix.c +++ b/src/core/iomgr/tcp_client_posix.c @@ -42,17 +42,19 @@ #include #include -#include "src/core/iomgr/timer.h" +#include +#include +#include +#include + #include "src/core/iomgr/iomgr_posix.h" #include "src/core/iomgr/pollset_posix.h" +#include "src/core/iomgr/pollset_set_posix.h" #include "src/core/iomgr/sockaddr_utils.h" #include "src/core/iomgr/socket_utils_posix.h" #include "src/core/iomgr/tcp_posix.h" +#include "src/core/iomgr/timer.h" #include "src/core/support/string.h" -#include -#include -#include -#include extern int grpc_tcp_trace; diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index 048e9074412..e8f73811cec 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -40,8 +40,8 @@ #include #include #include -#include #include +#include #include #include @@ -51,9 +51,11 @@ #include #include -#include "src/core/support/string.h" #include "src/core/debug/trace.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/iomgr/pollset_set_posix.h" #include "src/core/profiling/timers.h" +#include "src/core/support/string.h" #ifdef GPR_HAVE_MSG_NOSIGNAL #define SENDMSG_FLAGS MSG_NOSIGNAL @@ -295,7 +297,7 @@ static flush_result tcp_flush(grpc_tcp *tcp) { unwind_slice_idx = tcp->outgoing_slice_idx; unwind_byte_idx = tcp->outgoing_byte_idx; for (iov_size = 0; tcp->outgoing_slice_idx != tcp->outgoing_buffer->count && - iov_size != MAX_WRITE_IOVEC; + iov_size != MAX_WRITE_IOVEC; iov_size++) { iov[iov_size].iov_base = GPR_SLICE_START_PTR( @@ -444,7 +446,7 @@ static char *tcp_get_peer(grpc_endpoint *ep) { } static const grpc_endpoint_vtable vtable = { - tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set, + tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set, tcp_shutdown, tcp_destroy, tcp_get_peer}; grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size, diff --git a/src/core/iomgr/udp_server.h b/src/core/iomgr/udp_server.h index 73a21c80ab4..a9d0489edfd 100644 --- a/src/core/iomgr/udp_server.h +++ b/src/core/iomgr/udp_server.h @@ -35,6 +35,7 @@ #define GRPC_INTERNAL_CORE_IOMGR_UDP_SERVER_H #include "src/core/iomgr/endpoint.h" +#include "src/core/iomgr/fd_posix.h" /* Forward decl of grpc_server */ typedef struct grpc_server grpc_server; diff --git a/src/core/iomgr/workqueue_posix.c b/src/core/iomgr/workqueue_posix.c index da11df67efb..c096dbfb30c 100644 --- a/src/core/iomgr/workqueue_posix.c +++ b/src/core/iomgr/workqueue_posix.c @@ -44,6 +44,7 @@ #include #include "src/core/iomgr/fd_posix.h" +#include "src/core/iomgr/pollset_posix.h" static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, bool success); diff --git a/src/core/iomgr/workqueue_posix.h b/src/core/iomgr/workqueue_posix.h index 589034fe1bb..68f195ee0d2 100644 --- a/src/core/iomgr/workqueue_posix.h +++ b/src/core/iomgr/workqueue_posix.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,6 +34,8 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H #define GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H +#include "src/core/iomgr/wakeup_fd_posix.h" + struct grpc_fd; struct grpc_workqueue { diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c index 458d0d3ac32..1f4f3e4aa52 100644 --- a/src/core/security/google_default_credentials.c +++ b/src/core/security/google_default_credentials.c @@ -52,13 +52,14 @@ static grpc_channel_credentials *default_credentials = NULL; static int compute_engine_detection_done = 0; -static gpr_mu g_mu; +static gpr_mu g_state_mu; +static gpr_mu *g_polling_mu; static gpr_once g_once = GPR_ONCE_INIT; -static void init_default_credentials(void) { gpr_mu_init(&g_mu); } +static void init_default_credentials(void) { gpr_mu_init(&g_state_mu); } typedef struct { - grpc_pollset pollset; + grpc_pollset *pollset; int is_done; int success; } compute_engine_detector; @@ -80,10 +81,10 @@ static void on_compute_engine_detection_http_response( } } } - gpr_mu_lock(GRPC_POLLSET_MU(&detector->pollset)); + gpr_mu_lock(g_polling_mu); detector->is_done = 1; - grpc_pollset_kick(&detector->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&detector->pollset)); + grpc_pollset_kick(detector->pollset, NULL); + gpr_mu_unlock(g_polling_mu); } static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool s) { @@ -101,7 +102,8 @@ static int is_stack_running_on_compute_engine(void) { on compute engine. */ gpr_timespec max_detection_delay = gpr_time_from_seconds(1, GPR_TIMESPAN); - grpc_pollset_init(&detector.pollset); + detector.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(detector.pollset, &g_polling_mu); detector.is_done = 0; detector.success = 0; @@ -112,7 +114,7 @@ static int is_stack_running_on_compute_engine(void) { grpc_httpcli_context_init(&context); grpc_httpcli_get( - &exec_ctx, &context, &detector.pollset, &request, + &exec_ctx, &context, detector.pollset, &request, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), max_detection_delay), on_compute_engine_detection_http_response, &detector); @@ -120,19 +122,22 @@ static int is_stack_running_on_compute_engine(void) { /* Block until we get the response. This is not ideal but this should only be called once for the lifetime of the process by the default credentials. */ - gpr_mu_lock(GRPC_POLLSET_MU(&detector.pollset)); + gpr_mu_lock(g_polling_mu); while (!detector.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &detector.pollset, &worker, + grpc_pollset_work(&exec_ctx, detector.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); } - gpr_mu_unlock(GRPC_POLLSET_MU(&detector.pollset)); + gpr_mu_unlock(g_polling_mu); grpc_httpcli_context_destroy(&context); - grpc_closure_init(&destroy_closure, destroy_pollset, &detector.pollset); - grpc_pollset_shutdown(&exec_ctx, &detector.pollset, &destroy_closure); + grpc_closure_init(&destroy_closure, destroy_pollset, detector.pollset); + grpc_pollset_shutdown(&exec_ctx, detector.pollset, &destroy_closure); grpc_exec_ctx_finish(&exec_ctx); + g_polling_mu = NULL; + + gpr_free(detector.pollset); return detector.success; } @@ -184,7 +189,7 @@ grpc_channel_credentials *grpc_google_default_credentials_create(void) { gpr_once_init(&g_once, init_default_credentials); - gpr_mu_lock(&g_mu); + gpr_mu_lock(&g_state_mu); if (default_credentials != NULL) { result = grpc_channel_credentials_ref(default_credentials); @@ -230,19 +235,19 @@ end: gpr_log(GPR_ERROR, "Could not create google default credentials."); } } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(&g_state_mu); return result; } void grpc_flush_cached_google_default_credentials(void) { gpr_once_init(&g_once, init_default_credentials); - gpr_mu_lock(&g_mu); + gpr_mu_lock(&g_state_mu); if (default_credentials != NULL) { grpc_channel_credentials_unref(default_credentials); default_credentials = NULL; } compute_engine_detection_done = 0; - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(&g_state_mu); } /* -- Well known credentials path. -- */ diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index f9cb852722d..8a9bbace087 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -36,18 +36,19 @@ #include #include -#include "src/core/iomgr/timer.h" +#include +#include +#include +#include + #include "src/core/iomgr/pollset.h" +#include "src/core/iomgr/timer.h" +#include "src/core/profiling/timers.h" #include "src/core/support/string.h" #include "src/core/surface/api_trace.h" #include "src/core/surface/call.h" #include "src/core/surface/event_string.h" #include "src/core/surface/surface_trace.h" -#include "src/core/profiling/timers.h" -#include -#include -#include -#include typedef struct { grpc_pollset_worker **worker; @@ -56,6 +57,8 @@ typedef struct { /* Completion queue structure */ struct grpc_completion_queue { + /** owned by pollset */ + gpr_mu *mu; /** completed events */ grpc_cq_completion completed_head; grpc_cq_completion *completed_tail; @@ -63,8 +66,6 @@ struct grpc_completion_queue { gpr_refcount pending_events; /** Once owning_refs drops to zero, we will destroy the cq */ gpr_refcount owning_refs; - /** the set of low level i/o things that concern this cq */ - grpc_pollset pollset; /** 0 initially, 1 once we've begun shutting down */ int shutdown; int shutdown_called; @@ -82,6 +83,8 @@ struct grpc_completion_queue { grpc_completion_queue *next_free; }; +#define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1)) + static gpr_mu g_freelist_mu; grpc_completion_queue *g_freelist; @@ -94,7 +97,7 @@ void grpc_cq_global_shutdown(void) { gpr_mu_destroy(&g_freelist_mu); while (g_freelist) { grpc_completion_queue *next = g_freelist->next_free; - grpc_pollset_destroy(&g_freelist->pollset); + grpc_pollset_destroy(POLLSET_FROM_CQ(g_freelist)); #ifndef NDEBUG gpr_free(g_freelist->outstanding_tags); #endif @@ -124,8 +127,8 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) { if (g_freelist == NULL) { gpr_mu_unlock(&g_freelist_mu); - cc = gpr_malloc(sizeof(grpc_completion_queue)); - grpc_pollset_init(&cc->pollset); + cc = gpr_malloc(sizeof(grpc_completion_queue) + grpc_pollset_size()); + grpc_pollset_init(POLLSET_FROM_CQ(cc), &cc->mu); #ifndef NDEBUG cc->outstanding_tags = NULL; cc->outstanding_tag_capacity = 0; @@ -184,7 +187,7 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { #endif if (gpr_unref(&cc->owning_refs)) { GPR_ASSERT(cc->completed_head.next == (uintptr_t)&cc->completed_head); - grpc_pollset_reset(&cc->pollset); + grpc_pollset_reset(POLLSET_FROM_CQ(cc)); gpr_mu_lock(&g_freelist_mu); cc->next_free = g_freelist; g_freelist = cc; @@ -194,7 +197,7 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) { #ifndef NDEBUG - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(cc->mu); GPR_ASSERT(!cc->shutdown_called); if (cc->outstanding_tag_count == cc->outstanding_tag_capacity) { cc->outstanding_tag_capacity = GPR_MAX(4, 2 * cc->outstanding_tag_capacity); @@ -203,7 +206,7 @@ void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) { cc->outstanding_tag_capacity); } cc->outstanding_tags[cc->outstanding_tag_count++] = tag; - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); #endif gpr_ref(&cc->pending_events); } @@ -231,7 +234,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, storage->next = ((uintptr_t)&cc->completed_head) | ((uintptr_t)(success != 0)); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(cc->mu); #ifndef NDEBUG for (i = 0; i < (int)cc->outstanding_tag_count; i++) { if (cc->outstanding_tags[i] == tag) { @@ -256,8 +259,8 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, break; } } - grpc_pollset_kick(&cc->pollset, pluck_worker); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker); + gpr_mu_unlock(cc->mu); } else { cc->completed_tail->next = ((uintptr_t)storage) | (1u & (uintptr_t)cc->completed_tail->next); @@ -265,8 +268,9 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, GPR_ASSERT(!cc->shutdown); GPR_ASSERT(cc->shutdown_called); cc->shutdown = 1; - grpc_pollset_shutdown(exec_ctx, &cc->pollset, &cc->pollset_shutdown_done); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + grpc_pollset_shutdown(exec_ctx, POLLSET_FROM_CQ(cc), + &cc->pollset_shutdown_done); + gpr_mu_unlock(cc->mu); } GPR_TIMER_END("grpc_cq_end_op", 0); @@ -294,7 +298,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); GRPC_CQ_INTERNAL_REF(cc, "next"); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(cc->mu); for (;;) { if (cc->completed_tail != &cc->completed_head) { grpc_cq_completion *c = (grpc_cq_completion *)cc->completed_head.next; @@ -302,7 +306,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, if (c == cc->completed_tail) { cc->completed_tail = &cc->completed_head; } - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); ret.type = GRPC_OP_COMPLETE; ret.success = c->next & 1u; ret.tag = c->tag; @@ -310,14 +314,14 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, break; } if (cc->shutdown) { - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_SHUTDOWN; break; } now = gpr_now(GPR_CLOCK_MONOTONIC); if (!first_loop && gpr_time_cmp(now, deadline) >= 0) { - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_TIMEOUT; break; @@ -330,11 +334,12 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_timespec iteration_deadline = deadline; if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { GPR_TIMER_MARK("alarm_triggered", 0); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(cc->mu); + continue; } else { - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, + grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, iteration_deadline); } } @@ -395,7 +400,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); GRPC_CQ_INTERNAL_REF(cc, "pluck"); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(cc->mu); for (;;) { prev = &cc->completed_head; while ((c = (grpc_cq_completion *)(prev->next & ~(uintptr_t)1)) != @@ -405,7 +410,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, if (c == cc->completed_tail) { cc->completed_tail = prev; } - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); ret.type = GRPC_OP_COMPLETE; ret.success = c->next & 1u; ret.tag = c->tag; @@ -415,7 +420,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, prev = c; } if (cc->shutdown) { - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_SHUTDOWN; break; @@ -425,7 +430,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, "Too many outstanding grpc_completion_queue_pluck calls: maximum " "is %d", GRPC_MAX_COMPLETION_QUEUE_PLUCKERS); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); /* TODO(ctiller): should we use a different result here */ ret.type = GRPC_QUEUE_TIMEOUT; @@ -434,7 +439,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, now = gpr_now(GPR_CLOCK_MONOTONIC); if (!first_loop && gpr_time_cmp(now, deadline) >= 0) { del_plucker(cc, tag, &worker); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); memset(&ret, 0, sizeof(ret)); ret.type = GRPC_QUEUE_TIMEOUT; break; @@ -447,11 +452,12 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, gpr_timespec iteration_deadline = deadline; if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { GPR_TIMER_MARK("alarm_triggered", 0); - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(cc->mu); + continue; } else { - grpc_pollset_work(&exec_ctx, &cc->pollset, &worker, now, + grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, iteration_deadline); } del_plucker(cc, tag, &worker); @@ -472,9 +478,9 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_completion_queue_shutdown", 0); GRPC_API_TRACE("grpc_completion_queue_shutdown(cc=%p)", 1, (cc)); - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_lock(cc->mu); if (cc->shutdown_called) { - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); GPR_TIMER_END("grpc_completion_queue_shutdown", 0); return; } @@ -482,9 +488,10 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { if (gpr_unref(&cc->pending_events)) { GPR_ASSERT(!cc->shutdown); cc->shutdown = 1; - grpc_pollset_shutdown(&exec_ctx, &cc->pollset, &cc->pollset_shutdown_done); + grpc_pollset_shutdown(&exec_ctx, POLLSET_FROM_CQ(cc), + &cc->pollset_shutdown_done); } - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + gpr_mu_unlock(cc->mu); grpc_exec_ctx_finish(&exec_ctx); GPR_TIMER_END("grpc_completion_queue_shutdown", 0); } @@ -498,7 +505,7 @@ void grpc_completion_queue_destroy(grpc_completion_queue *cc) { } grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) { - return &cc->pollset; + return POLLSET_FROM_CQ(cc); } void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; } diff --git a/test/core/client_config/set_initial_connect_string_test.c b/test/core/client_config/set_initial_connect_string_test.c index bcd1f261235..3cf267fb3bd 100644 --- a/test/core/client_config/set_initial_connect_string_test.c +++ b/test/core/client_config/set_initial_connect_string_test.c @@ -85,7 +85,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_slice_buffer_init(&state.incoming_buffer); gpr_slice_buffer_init(&state.temp_incoming_buffer); state.tcp = tcp; - grpc_endpoint_add_to_pollset(exec_ctx, tcp, &server->pollset); + grpc_endpoint_add_to_pollset(exec_ctx, tcp, server->pollset); grpc_endpoint_read(exec_ctx, tcp, &state.temp_incoming_buffer, &on_read); } diff --git a/test/core/end2end/fixtures/h2_full+poll+pipe.c b/test/core/end2end/fixtures/h2_full+poll+pipe.c index d475a7bb552..682598fbe2d 100644 --- a/test/core/end2end/fixtures/h2_full+poll+pipe.c +++ b/test/core/end2end/fixtures/h2_full+poll+pipe.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,21 +35,23 @@ #include -#include "src/core/channel/client_channel.h" -#include "src/core/channel/connected_channel.h" -#include "src/core/channel/http_server_filter.h" -#include "src/core/surface/channel.h" -#include "src/core/surface/server.h" -#include "src/core/transport/chttp2_transport.h" #include #include #include #include #include #include + +#include "src/core/channel/client_channel.h" +#include "src/core/channel/connected_channel.h" +#include "src/core/channel/http_server_filter.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/iomgr/wakeup_fd_posix.h" +#include "src/core/surface/channel.h" +#include "src/core/surface/server.h" +#include "src/core/transport/chttp2_transport.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" -#include "src/core/iomgr/wakeup_fd_posix.h" typedef struct fullstack_fixture_data { char *localaddr; diff --git a/test/core/end2end/fixtures/h2_full+poll.c b/test/core/end2end/fixtures/h2_full+poll.c index 3f5e6096f63..5a0b2ef4953 100644 --- a/test/core/end2end/fixtures/h2_full+poll.c +++ b/test/core/end2end/fixtures/h2_full+poll.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,18 +35,20 @@ #include -#include "src/core/channel/client_channel.h" -#include "src/core/channel/connected_channel.h" -#include "src/core/channel/http_server_filter.h" -#include "src/core/surface/channel.h" -#include "src/core/surface/server.h" -#include "src/core/transport/chttp2_transport.h" #include #include #include #include #include #include + +#include "src/core/channel/client_channel.h" +#include "src/core/channel/connected_channel.h" +#include "src/core/channel/http_server_filter.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/surface/channel.h" +#include "src/core/surface/server.h" +#include "src/core/transport/chttp2_transport.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/end2end/fixtures/h2_ssl+poll.c b/test/core/end2end/fixtures/h2_ssl+poll.c index 0b88876d138..66268c77d58 100644 --- a/test/core/end2end/fixtures/h2_ssl+poll.c +++ b/test/core/end2end/fixtures/h2_ssl+poll.c @@ -41,6 +41,7 @@ #include #include "src/core/channel/channel_args.h" +#include "src/core/iomgr/pollset_posix.h" #include "src/core/security/credentials.h" #include "src/core/support/env.h" #include "src/core/support/tmpfile.h" diff --git a/test/core/end2end/fixtures/h2_uchannel.c b/test/core/end2end/fixtures/h2_uchannel.c index 33055561cb3..87bbd64d09a 100644 --- a/test/core/end2end/fixtures/h2_uchannel.c +++ b/test/core/end2end/fixtures/h2_uchannel.c @@ -35,6 +35,13 @@ #include +#include +#include +#include +#include +#include +#include +#include #include "src/core/channel/channel_args.h" #include "src/core/channel/client_channel.h" #include "src/core/channel/client_uchannel.h" @@ -46,13 +53,6 @@ #include "src/core/surface/channel.h" #include "src/core/surface/server.h" #include "src/core/transport/chttp2_transport.h" -#include -#include -#include -#include -#include -#include -#include #include "test/core/util/port.h" #include "test/core/util/test_config.h" @@ -238,12 +238,12 @@ static grpc_end2end_test_fixture chttp2_create_fixture_micro_fullstack( } grpc_connectivity_state g_state = GRPC_CHANNEL_IDLE; -grpc_pollset_set g_interested_parties; +grpc_pollset_set *g_interested_parties; static void state_changed(grpc_exec_ctx *exec_ctx, void *arg, bool success) { if (g_state != GRPC_CHANNEL_READY) { grpc_subchannel_notify_on_state_change( - exec_ctx, arg, &g_interested_parties, &g_state, + exec_ctx, arg, g_interested_parties, &g_state, grpc_closure_create(state_changed, arg)); } } @@ -253,30 +253,31 @@ static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *arg, bool success) { } static grpc_connected_subchannel *connect_subchannel(grpc_subchannel *c) { - grpc_pollset pollset; + gpr_mu *mu; + grpc_pollset *pollset = gpr_malloc(grpc_pollset_size()); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_pollset_init(&pollset); - grpc_pollset_set_init(&g_interested_parties); - grpc_pollset_set_add_pollset(&exec_ctx, &g_interested_parties, &pollset); - grpc_subchannel_notify_on_state_change(&exec_ctx, c, &g_interested_parties, + grpc_pollset_init(pollset, &mu); + g_interested_parties = grpc_pollset_set_create(); + grpc_pollset_set_add_pollset(&exec_ctx, g_interested_parties, pollset); + grpc_subchannel_notify_on_state_change(&exec_ctx, c, g_interested_parties, &g_state, grpc_closure_create(state_changed, c)); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&pollset)); + gpr_mu_lock(mu); while (g_state != GRPC_CHANNEL_READY) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &pollset, &worker, - gpr_now(GPR_CLOCK_MONOTONIC), + grpc_pollset_work(&exec_ctx, pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); - gpr_mu_unlock(GRPC_POLLSET_MU(&pollset)); + gpr_mu_unlock(mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&pollset)); + gpr_mu_lock(mu); } - grpc_pollset_shutdown(&exec_ctx, &pollset, - grpc_closure_create(destroy_pollset, &pollset)); - grpc_pollset_set_destroy(&g_interested_parties); - gpr_mu_unlock(GRPC_POLLSET_MU(&pollset)); + grpc_pollset_shutdown(&exec_ctx, pollset, + grpc_closure_create(destroy_pollset, pollset)); + grpc_pollset_set_destroy(g_interested_parties); + gpr_mu_unlock(mu); grpc_exec_ctx_finish(&exec_ctx); + gpr_free(pollset); return grpc_subchannel_get_connected_subchannel(c); } diff --git a/test/core/end2end/fixtures/h2_uds+poll.c b/test/core/end2end/fixtures/h2_uds+poll.c index 155017c8875..c3a855ff883 100644 --- a/test/core/end2end/fixtures/h2_uds+poll.c +++ b/test/core/end2end/fixtures/h2_uds+poll.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,13 +37,6 @@ #include #include -#include "src/core/channel/client_channel.h" -#include "src/core/channel/connected_channel.h" -#include "src/core/channel/http_server_filter.h" -#include "src/core/support/string.h" -#include "src/core/surface/channel.h" -#include "src/core/surface/server.h" -#include "src/core/transport/chttp2_transport.h" #include #include #include @@ -51,6 +44,15 @@ #include #include #include + +#include "src/core/channel/client_channel.h" +#include "src/core/channel/connected_channel.h" +#include "src/core/channel/http_server_filter.h" +#include "src/core/iomgr/pollset_posix.h" +#include "src/core/support/string.h" +#include "src/core/surface/channel.h" +#include "src/core/surface/server.h" +#include "src/core/transport/chttp2_transport.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/httpcli/httpcli_test.c b/test/core/httpcli/httpcli_test.c index e954a920b04..da1463329d4 100644 --- a/test/core/httpcli/httpcli_test.c +++ b/test/core/httpcli/httpcli_test.c @@ -36,18 +36,19 @@ #include #include -#include "src/core/iomgr/iomgr.h" #include #include #include #include #include +#include "src/core/iomgr/iomgr.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" static int g_done = 0; static grpc_httpcli_context g_context; -static grpc_pollset g_pollset; +static gpr_mu *g_mu; +static grpc_pollset *g_pollset; static gpr_timespec n_seconds_time(int seconds) { return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(seconds); @@ -63,10 +64,10 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, GPR_ASSERT(response->status == 200); GPR_ASSERT(response->body_length == strlen(expect)); GPR_ASSERT(0 == memcmp(expect, response->body, response->body_length)); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); g_done = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(g_mu); } static void test_get(int port) { @@ -85,18 +86,18 @@ static void test_get(int port) { req.path = "/get"; req.handshaker = &grpc_httpcli_plaintext; - grpc_httpcli_get(&exec_ctx, &g_context, &g_pollset, &req, n_seconds_time(15), + grpc_httpcli_get(&exec_ctx, &g_context, g_pollset, &req, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); gpr_free(host); } @@ -116,18 +117,18 @@ static void test_post(int port) { req.path = "/post"; req.handshaker = &grpc_httpcli_plaintext; - grpc_httpcli_post(&exec_ctx, &g_context, &g_pollset, &req, "hello", 5, + grpc_httpcli_post(&exec_ctx, &g_context, g_pollset, &req, "hello", 5, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); gpr_free(host); } @@ -175,17 +176,20 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); grpc_httpcli_context_init(&g_context); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); test_get(port); test_post(port); grpc_httpcli_context_destroy(&g_context); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); + gpr_subprocess_destroy(server); return 0; diff --git a/test/core/httpcli/httpscli_test.c b/test/core/httpcli/httpscli_test.c index 9f31eae2782..7f765bc614f 100644 --- a/test/core/httpcli/httpscli_test.c +++ b/test/core/httpcli/httpscli_test.c @@ -36,18 +36,19 @@ #include #include -#include "src/core/iomgr/iomgr.h" #include #include #include #include #include +#include "src/core/iomgr/iomgr.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" static int g_done = 0; static grpc_httpcli_context g_context; -static grpc_pollset g_pollset; +static gpr_mu *g_mu; +static grpc_pollset *g_pollset; static gpr_timespec n_seconds_time(int seconds) { return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(seconds); @@ -63,10 +64,10 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, GPR_ASSERT(response->status == 200); GPR_ASSERT(response->body_length == strlen(expect)); GPR_ASSERT(0 == memcmp(expect, response->body, response->body_length)); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); g_done = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(g_mu); } static void test_get(int port) { @@ -86,18 +87,18 @@ static void test_get(int port) { req.path = "/get"; req.handshaker = &grpc_httpcli_ssl; - grpc_httpcli_get(&exec_ctx, &g_context, &g_pollset, &req, n_seconds_time(15), + grpc_httpcli_get(&exec_ctx, &g_context, g_pollset, &req, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); gpr_free(host); } @@ -118,18 +119,18 @@ static void test_post(int port) { req.path = "/post"; req.handshaker = &grpc_httpcli_ssl; - grpc_httpcli_post(&exec_ctx, &g_context, &g_pollset, &req, "hello", 5, + grpc_httpcli_post(&exec_ctx, &g_context, g_pollset, &req, "hello", 5, n_seconds_time(15), on_finish, (void *)42); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (!g_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), n_seconds_time(20)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); gpr_free(host); } @@ -178,17 +179,20 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); grpc_httpcli_context_init(&g_context); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); test_get(port); test_post(port); grpc_httpcli_context_destroy(&g_context); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); + gpr_subprocess_destroy(server); return 0; diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c index 7e266ebfb99..c3a91088a57 100644 --- a/test/core/iomgr/endpoint_pair_test.c +++ b/test/core/iomgr/endpoint_pair_test.c @@ -39,10 +39,11 @@ #include #include #include "src/core/iomgr/endpoint_pair.h" -#include "test/core/util/test_config.h" #include "test/core/iomgr/endpoint_tests.h" +#include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu *g_mu; +static grpc_pollset *g_pollset; static void clean_up(void) {} @@ -54,8 +55,8 @@ static grpc_endpoint_test_fixture create_fixture_endpoint_pair( f.client_ep = p.client; f.server_ep = p.server; - grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, &g_pollset); - grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, g_pollset); grpc_exec_ctx_finish(&exec_ctx); return f; @@ -74,12 +75,14 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); - grpc_endpoint_tests(configs[0], &g_pollset); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); + grpc_endpoint_tests(configs[0], g_pollset, g_mu); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); return 0; } diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c index 6ba67df3b10..f689e4ba7fb 100644 --- a/test/core/iomgr/endpoint_tests.c +++ b/test/core/iomgr/endpoint_tests.c @@ -36,8 +36,8 @@ #include #include -#include #include +#include #include #include #include "test/core/util/test_config.h" @@ -58,6 +58,7 @@ */ +static gpr_mu *g_mu; static grpc_pollset *g_pollset; size_t count_slices(gpr_slice *slices, size_t nslices, int *current_data) { @@ -134,10 +135,10 @@ static void read_and_write_test_read_handler(grpc_exec_ctx *exec_ctx, state->incoming.slices, state->incoming.count, &state->current_read_data); if (state->bytes_read == state->target_bytes || !success) { gpr_log(GPR_INFO, "Read handler done"); - gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_lock(g_mu); state->read_done = 1 + success; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_unlock(g_mu); } else if (success) { grpc_endpoint_read(exec_ctx, state->read_ep, &state->incoming, &state->done_read); @@ -169,10 +170,10 @@ static void read_and_write_test_write_handler(grpc_exec_ctx *exec_ctx, } gpr_log(GPR_INFO, "Write handler done"); - gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_lock(g_mu); state->write_done = 1 + success; grpc_pollset_kick(g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_unlock(g_mu); } /* Do both reading and writing using the grpc_endpoint API. @@ -232,14 +233,14 @@ static void read_and_write_test(grpc_endpoint_test_config config, } grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_lock(g_mu); while (!state.read_done || !state.write_done) { grpc_pollset_worker *worker = NULL; GPR_ASSERT(gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0); grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); } - gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); end_test(config); @@ -251,9 +252,10 @@ static void read_and_write_test(grpc_endpoint_test_config config, } void grpc_endpoint_tests(grpc_endpoint_test_config config, - grpc_pollset *pollset) { + grpc_pollset *pollset, gpr_mu *mu) { size_t i; g_pollset = pollset; + g_mu = mu; read_and_write_test(config, 10000000, 100000, 8192, 0); read_and_write_test(config, 1000000, 100000, 1, 0); read_and_write_test(config, 100000000, 100000, 1, 1); diff --git a/test/core/iomgr/endpoint_tests.h b/test/core/iomgr/endpoint_tests.h index 700f854891e..8ea47e345cf 100644 --- a/test/core/iomgr/endpoint_tests.h +++ b/test/core/iomgr/endpoint_tests.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -53,6 +53,6 @@ struct grpc_endpoint_test_config { }; void grpc_endpoint_tests(grpc_endpoint_test_config config, - grpc_pollset *pollset); + grpc_pollset *pollset, gpr_mu *mu); #endif /* GRPC_TEST_CORE_IOMGR_ENDPOINT_TESTS_H */ diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 07761b6576d..99689ebcc3b 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -49,9 +49,12 @@ #include #include #include + +#include "src/core/iomgr/pollset_posix.h" #include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu *g_mu; +static grpc_pollset *g_pollset; /* buffer size used to send and receive data. 1024 is the minimal value to set TCP send and receive buffer. */ @@ -179,10 +182,10 @@ static void listen_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg /*server */, grpc_fd_orphan(exec_ctx, sv->em_fd, NULL, NULL, "b"); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); sv->done = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(g_mu); } /* Called when a new TCP connection request arrives in the listening port. */ @@ -209,7 +212,7 @@ static void listen_cb(grpc_exec_ctx *exec_ctx, void *arg, /*=sv_arg*/ se = gpr_malloc(sizeof(*se)); se->sv = sv; se->em_fd = grpc_fd_create(fd, "listener"); - grpc_pollset_add_fd(exec_ctx, &g_pollset, se->em_fd); + grpc_pollset_add_fd(exec_ctx, g_pollset, se->em_fd); se->session_read_closure.cb = session_read_cb; se->session_read_closure.cb_arg = se; grpc_fd_notify_on_read(exec_ctx, se->em_fd, &se->session_read_closure); @@ -238,7 +241,7 @@ static int server_start(grpc_exec_ctx *exec_ctx, server *sv) { GPR_ASSERT(listen(fd, MAX_NUM_FD) == 0); sv->em_fd = grpc_fd_create(fd, "server"); - grpc_pollset_add_fd(exec_ctx, &g_pollset, sv->em_fd); + grpc_pollset_add_fd(exec_ctx, g_pollset, sv->em_fd); /* Register to be interested in reading from listen_fd. */ sv->listen_closure.cb = listen_cb; sv->listen_closure.cb_arg = sv; @@ -249,18 +252,18 @@ static int server_start(grpc_exec_ctx *exec_ctx, server *sv) { /* Wait and shutdown a sever. */ static void server_wait_and_shutdown(server *sv) { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (!sv->done) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); } /* ===An upload client to test notify_on_write=== */ @@ -296,7 +299,7 @@ static void client_session_shutdown_cb(grpc_exec_ctx *exec_ctx, client *cl = arg; grpc_fd_orphan(exec_ctx, cl->em_fd, NULL, NULL, "c"); cl->done = 1; - grpc_pollset_kick(&g_pollset, NULL); + grpc_pollset_kick(g_pollset, NULL); } /* Write as much as possible, then register notify_on_write. */ @@ -307,9 +310,9 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ ssize_t write_once = 0; if (!success) { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); client_session_shutdown_cb(exec_ctx, arg, 1); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); return; } @@ -319,7 +322,7 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ } while (write_once > 0); if (errno == EAGAIN) { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); if (cl->client_write_cnt < CLIENT_TOTAL_WRITE_CNT) { cl->write_closure.cb = client_session_write; cl->write_closure.cb_arg = cl; @@ -328,7 +331,7 @@ static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */ } else { client_session_shutdown_cb(exec_ctx, arg, 1); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); } else { gpr_log(GPR_ERROR, "unknown errno %s", strerror(errno)); abort(); @@ -357,25 +360,25 @@ static void client_start(grpc_exec_ctx *exec_ctx, client *cl, int port) { } cl->em_fd = grpc_fd_create(fd, "client"); - grpc_pollset_add_fd(exec_ctx, &g_pollset, cl->em_fd); + grpc_pollset_add_fd(exec_ctx, g_pollset, cl->em_fd); client_session_write(exec_ctx, cl, 1); } /* Wait for the signal to shutdown a client. */ static void client_wait_and_shutdown(client *cl) { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (!cl->done) { grpc_pollset_worker *worker = NULL; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); } /* Test grpc_fd. Start an upload server and client, upload a stream of @@ -410,20 +413,20 @@ static void first_read_callback(grpc_exec_ctx *exec_ctx, void *arg /* fd_change_data */, bool success) { fd_change_data *fdc = arg; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); fdc->cb_that_ran = first_read_callback; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(g_mu); } static void second_read_callback(grpc_exec_ctx *exec_ctx, void *arg /* fd_change_data */, bool success) { fd_change_data *fdc = arg; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); fdc->cb_that_ran = second_read_callback; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(g_mu); } /* Test that changing the callback we use for notify_on_read actually works. @@ -456,7 +459,7 @@ static void test_grpc_fd_change(void) { GPR_ASSERT(fcntl(sv[1], F_SETFL, flags | O_NONBLOCK) == 0); em_fd = grpc_fd_create(sv[0], "test_grpc_fd_change"); - grpc_pollset_add_fd(&exec_ctx, &g_pollset, em_fd); + grpc_pollset_add_fd(&exec_ctx, g_pollset, em_fd); /* Register the first callback, then make its FD readable */ grpc_fd_notify_on_read(&exec_ctx, em_fd, &first_closure); @@ -465,18 +468,18 @@ static void test_grpc_fd_change(void) { GPR_ASSERT(result == 1); /* And now wait for it to run. */ - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (a.cb_that_ran == NULL) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } GPR_ASSERT(a.cb_that_ran == first_read_callback); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); /* And drain the socket so we can generate a new read edge */ result = read(sv[0], &data, 1); @@ -489,19 +492,19 @@ static void test_grpc_fd_change(void) { result = write(sv[1], &data, 1); GPR_ASSERT(result == 1); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (b.cb_that_ran == NULL) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } /* Except now we verify that second_read_callback ran instead */ GPR_ASSERT(b.cb_that_ran == second_read_callback); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_fd_orphan(&exec_ctx, em_fd, NULL, NULL, "d"); grpc_exec_ctx_finish(&exec_ctx); @@ -519,12 +522,14 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_iomgr_init(); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); test_grpc_fd(); test_grpc_fd_change(); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); + gpr_free(g_pollset); grpc_iomgr_shutdown(); return 0; } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index cdd5b096fbc..1e6fa5d45a0 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -40,6 +40,7 @@ #include #include +#include #include #include @@ -48,8 +49,9 @@ #include "src/core/iomgr/timer.h" #include "test/core/util/test_config.h" -static grpc_pollset_set g_pollset_set; -static grpc_pollset g_pollset; +static grpc_pollset_set *g_pollset_set; +static gpr_mu *g_mu; +static grpc_pollset *g_pollset; static int g_connections_complete = 0; static grpc_endpoint *g_connecting = NULL; @@ -58,10 +60,10 @@ static gpr_timespec test_deadline(void) { } static void finish_connection() { - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); g_connections_complete++; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(g_mu); } static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, bool success) { @@ -99,14 +101,14 @@ void test_succeeds(void) { GPR_ASSERT(0 == bind(svr_fd, (struct sockaddr *)&addr, addr_len)); GPR_ASSERT(0 == listen(svr_fd, 1)); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); connections_complete_before = g_connections_complete; - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); /* connect to it */ GPR_ASSERT(getsockname(svr_fd, (struct sockaddr *)&addr, &addr_len) == 0); grpc_closure_init(&done, must_succeed, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, &g_pollset_set, + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, (struct sockaddr *)&addr, addr_len, gpr_inf_future(GPR_CLOCK_REALTIME)); @@ -118,19 +120,19 @@ void test_succeeds(void) { GPR_ASSERT(r >= 0); close(r); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (g_connections_complete == connections_complete_before) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); } @@ -147,17 +149,17 @@ void test_fails(void) { memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); connections_complete_before = g_connections_complete; - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); /* connect to a broken address */ grpc_closure_init(&done, must_fail, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, &g_pollset_set, + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, (struct sockaddr *)&addr, addr_len, gpr_inf_future(GPR_CLOCK_REALTIME)); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); /* wait for the connection callback to finish */ while (g_connections_complete == connections_complete_before) { @@ -165,14 +167,14 @@ void test_fails(void) { gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); gpr_timespec polling_deadline = test_deadline(); if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, now, polling_deadline); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, now, polling_deadline); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); } @@ -217,16 +219,16 @@ void test_times_out(void) { connect_deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); connections_complete_before = g_connections_complete; - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_closure_init(&done, must_fail, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, &g_pollset_set, + grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, (struct sockaddr *)&addr, addr_len, connect_deadline); /* Make sure the event doesn't trigger early */ - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); for (;;) { grpc_pollset_worker *worker = NULL; gpr_timespec now = gpr_now(connect_deadline.clock_type); @@ -252,13 +254,13 @@ void test_times_out(void) { } gpr_timespec polling_deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10); if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, now, polling_deadline); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, now, polling_deadline); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); @@ -277,18 +279,20 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_set_init(&g_pollset_set); - grpc_pollset_init(&g_pollset); - grpc_pollset_set_add_pollset(&exec_ctx, &g_pollset_set, &g_pollset); + g_pollset_set = grpc_pollset_set_create(); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); + grpc_pollset_set_add_pollset(&exec_ctx, g_pollset_set, g_pollset); grpc_exec_ctx_finish(&exec_ctx); test_succeeds(); gpr_log(GPR_ERROR, "End of first test"); test_fails(); test_times_out(); - grpc_pollset_set_destroy(&g_pollset_set); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_pollset_set_destroy(g_pollset_set); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); return 0; } diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index 20b0b9e13b7..4351642ab6e 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -36,8 +36,8 @@ #include #include #include -#include #include +#include #include #include @@ -45,10 +45,11 @@ #include #include #include -#include "test/core/util/test_config.h" #include "test/core/iomgr/endpoint_tests.h" +#include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu *g_mu; +static grpc_pollset *g_pollset; /* General test notes: @@ -145,7 +146,7 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, bool success) { GPR_ASSERT(success); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); current_data = state->read_bytes % 256; read_bytes = count_slices(state->incoming.slices, state->incoming.count, ¤t_data); @@ -153,10 +154,10 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, bool success) { gpr_log(GPR_INFO, "Read %d bytes of %d", read_bytes, state->target_read_bytes); if (state->read_bytes >= state->target_read_bytes) { - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); } else { grpc_endpoint_read(exec_ctx, state->ep, &state->incoming, &state->read_cb); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); } } @@ -175,7 +176,7 @@ static void read_test(size_t num_bytes, size_t slice_size) { create_sockets(sv); ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), slice_size, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); gpr_log(GPR_INFO, "Wrote %d bytes", written_bytes); @@ -188,17 +189,17 @@ static void read_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); gpr_slice_buffer_destroy(&state.incoming); grpc_endpoint_destroy(&exec_ctx, ep); @@ -221,7 +222,7 @@ static void large_read_test(size_t slice_size) { ep = grpc_tcp_create(grpc_fd_create(sv[1], "large_read_test"), slice_size, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket(sv[0]); gpr_log(GPR_INFO, "Wrote %d bytes", written_bytes); @@ -234,17 +235,17 @@ static void large_read_test(size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); gpr_slice_buffer_destroy(&state.incoming); grpc_endpoint_destroy(&exec_ctx, ep); @@ -283,11 +284,11 @@ static void write_done(grpc_exec_ctx *exec_ctx, void *user_data /* write_socket_state */, bool success) { struct write_socket_state *state = (struct write_socket_state *)user_data; gpr_log(GPR_INFO, "Write done callback called"); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); gpr_log(GPR_INFO, "Signalling write done"); state->write_done = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(g_mu); } void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { @@ -304,11 +305,11 @@ void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { for (;;) { grpc_pollset_worker *worker = NULL; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + gpr_mu_lock(g_mu); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10)); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); do { bytes_read = @@ -350,7 +351,7 @@ static void write_test(size_t num_bytes, size_t slice_size) { ep = grpc_tcp_create(grpc_fd_create(sv[1], "write_test"), GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); state.ep = ep; state.write_done = 0; @@ -363,19 +364,19 @@ static void write_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_write(&exec_ctx, ep, &outgoing, &write_done_closure); drain_socket_blocking(sv[0], num_bytes, num_bytes); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); for (;;) { grpc_pollset_worker *worker = NULL; if (state.write_done) { break; } - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); gpr_slice_buffer_destroy(&outgoing); grpc_endpoint_destroy(&exec_ctx, ep); @@ -386,7 +387,7 @@ static void write_test(size_t num_bytes, size_t slice_size) { void on_fd_released(grpc_exec_ctx *exec_ctx, void *arg, bool success) { int *done = arg; *done = 1; - grpc_pollset_kick(&g_pollset, NULL); + grpc_pollset_kick(g_pollset, NULL); } /* Do a read_test, then release fd and try to read/write again. Verify that @@ -410,7 +411,7 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), slice_size, "test"); GPR_ASSERT(grpc_tcp_fd(ep) == sv[1] && sv[1] >= 0); - grpc_endpoint_add_to_pollset(&exec_ctx, ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); gpr_log(GPR_INFO, "Wrote %d bytes", written_bytes); @@ -423,27 +424,27 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { grpc_endpoint_read(&exec_ctx, ep, &state.incoming, &state.read_cb); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (state.read_bytes < state.target_read_bytes) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } GPR_ASSERT(state.read_bytes == state.target_read_bytes); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); gpr_slice_buffer_destroy(&state.incoming); grpc_tcp_destroy_and_release_fd(&exec_ctx, ep, &fd, &fd_released_cb); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); while (!fd_released_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); } - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); GPR_ASSERT(fd_released_done == 1); GPR_ASSERT(fd == sv[1]); grpc_exec_ctx_finish(&exec_ctx); @@ -491,8 +492,8 @@ static grpc_endpoint_test_fixture create_fixture_tcp_socketpair( slice_size, "test"); f.server_ep = grpc_tcp_create(grpc_fd_create(sv[1], "fixture:server"), slice_size, "test"); - grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, &g_pollset); - grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, g_pollset); grpc_exec_ctx_finish(&exec_ctx); @@ -512,13 +513,15 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); run_tests(); - grpc_endpoint_tests(configs[0], &g_pollset); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_endpoint_tests(configs[0], g_pollset, g_mu); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); return 0; } diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 43180b16988..7933468355a 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -32,24 +32,28 @@ */ #include "src/core/iomgr/tcp_server.h" -#include "src/core/iomgr/iomgr.h" -#include "src/core/iomgr/sockaddr_utils.h" + +#include +#include +#include +#include +#include + #include +#include #include #include #include + +#include "src/core/iomgr/iomgr.h" +#include "src/core/iomgr/sockaddr_utils.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" -#include -#include -#include -#include -#include - #define LOG_TEST(x) gpr_log(GPR_INFO, "%s", #x) -static grpc_pollset g_pollset; +static gpr_mu *g_mu; +static grpc_pollset *g_pollset; static int g_nconnects = 0; typedef struct on_connect_result { @@ -113,11 +117,11 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_endpoint_shutdown(exec_ctx, tcp); grpc_endpoint_destroy(exec_ctx, tcp); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); on_connect_result_set(&g_result, acceptor); g_nconnects++; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(g_mu); } static void test_no_op(void) { @@ -174,7 +178,7 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, int clifd = socket(remote->sa_family, SOCK_STREAM, 0); int nconnects_before; - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); nconnects_before = g_nconnects; on_connect_result_init(&g_result); GPR_ASSERT(clifd >= 0); @@ -184,18 +188,18 @@ static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, while (g_nconnects == nconnects_before && gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(exec_ctx, &g_pollset, &worker, + grpc_pollset_work(exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); } gpr_log(GPR_DEBUG, "wait done"); GPR_ASSERT(g_nconnects == nconnects_before + 1); close(clifd); *result = g_result; - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_unlock(g_mu); } /* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for @@ -210,7 +214,6 @@ static void test_connect(unsigned n) { unsigned svr1_fd_count; int svr1_port; grpc_tcp_server *s = grpc_tcp_server_create(NULL); - grpc_pollset *pollsets[1]; unsigned i; server_weak_ref weak_ref; server_weak_ref_init(&weak_ref); @@ -259,8 +262,7 @@ static void test_connect(unsigned n) { } } - pollsets[0] = &g_pollset; - grpc_tcp_server_start(&exec_ctx, s, pollsets, 1, on_connect, NULL); + grpc_tcp_server_start(&exec_ctx, s, &g_pollset, 1, on_connect, NULL); for (i = 0; i < n; i++) { on_connect_result result; @@ -312,7 +314,8 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); test_no_op(); test_no_op_with_start(); @@ -321,9 +324,10 @@ int main(int argc, char **argv) { test_connect(1); test_connect(10); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); return 0; } diff --git a/test/core/iomgr/workqueue_test.c b/test/core/iomgr/workqueue_test.c index a0249152566..8a1faf6303e 100644 --- a/test/core/iomgr/workqueue_test.c +++ b/test/core/iomgr/workqueue_test.c @@ -34,18 +34,20 @@ #include "src/core/iomgr/workqueue.h" #include +#include #include #include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu *g_mu; +static grpc_pollset *g_pollset; static void must_succeed(grpc_exec_ctx *exec_ctx, void *p, bool success) { GPR_ASSERT(success == 1); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); *(int *)p = 1; - grpc_pollset_kick(&g_pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_kick(g_pollset, NULL); + gpr_mu_unlock(g_mu); } static void test_ref_unref(void) { @@ -67,13 +69,13 @@ static void test_add_closure(void) { grpc_closure_init(&c, must_succeed, &done); grpc_workqueue_push(wq, &c, 1); - grpc_workqueue_add_to_pollset(&exec_ctx, wq, &g_pollset); + grpc_workqueue_add_to_pollset(&exec_ctx, wq, g_pollset); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); GPR_ASSERT(!done); - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, - gpr_now(deadline.clock_type), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(deadline.clock_type), + deadline); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(done); @@ -92,13 +94,13 @@ static void test_flush(void) { grpc_exec_ctx_enqueue(&exec_ctx, &c, true, NULL); grpc_workqueue_flush(&exec_ctx, wq); - grpc_workqueue_add_to_pollset(&exec_ctx, wq, &g_pollset); + grpc_workqueue_add_to_pollset(&exec_ctx, wq, g_pollset); - gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); + gpr_mu_lock(g_mu); GPR_ASSERT(!done); - grpc_pollset_work(&exec_ctx, &g_pollset, &worker, - gpr_now(deadline.clock_type), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset)); + grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(deadline.clock_type), + deadline); + gpr_mu_unlock(g_mu); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(done); @@ -115,15 +117,18 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); test_ref_unref(); test_add_closure(); test_flush(); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + + gpr_free(g_pollset); return 0; } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index 4dd595df956..9b70afffe1e 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -45,7 +45,8 @@ #include "src/core/security/credentials.h" typedef struct { - grpc_pollset pollset; + gpr_mu *mu; + grpc_pollset *pollset; int is_done; char *token; } oauth2_request; @@ -66,11 +67,11 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data, GPR_SLICE_LENGTH(token_slice)); token[GPR_SLICE_LENGTH(token_slice)] = '\0'; } - gpr_mu_lock(GRPC_POLLSET_MU(&request->pollset)); + gpr_mu_lock(request->mu); request->is_done = 1; request->token = token; - grpc_pollset_kick(&request->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&request->pollset)); + grpc_pollset_kick(request->pollset, NULL); + gpr_mu_unlock(request->mu); } static void do_nothing(grpc_exec_ctx *exec_ctx, void *unused, bool success) {} @@ -82,28 +83,30 @@ char *grpc_test_fetch_oauth2_token_with_credentials( grpc_closure do_nothing_closure; grpc_auth_metadata_context null_ctx = {"", "", NULL, NULL}; - grpc_pollset_init(&request.pollset); + request.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(request.pollset, &request.mu); request.is_done = 0; grpc_closure_init(&do_nothing_closure, do_nothing, NULL); - grpc_call_credentials_get_request_metadata(&exec_ctx, creds, &request.pollset, + grpc_call_credentials_get_request_metadata(&exec_ctx, creds, request.pollset, null_ctx, on_oauth2_response, &request); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&request.pollset)); + gpr_mu_lock(request.mu); while (!request.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &request.pollset, &worker, + grpc_pollset_work(&exec_ctx, request.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); } - gpr_mu_unlock(GRPC_POLLSET_MU(&request.pollset)); + gpr_mu_unlock(request.mu); - grpc_pollset_shutdown(&exec_ctx, &request.pollset, &do_nothing_closure); + grpc_pollset_shutdown(&exec_ctx, request.pollset, &do_nothing_closure); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_destroy(&request.pollset); + grpc_pollset_destroy(request.pollset); + gpr_free(request.pollset); return request.token; } diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index 6043bf54204..09673f362cf 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -34,8 +34,6 @@ #include #include -#include "src/core/security/credentials.h" -#include "src/core/support/string.h" #include #include #include @@ -44,8 +42,12 @@ #include #include +#include "src/core/security/credentials.h" +#include "src/core/support/string.h" + typedef struct { - grpc_pollset pollset; + gpr_mu *mu; + grpc_pollset *pollset; int is_done; } synchronizer; @@ -62,10 +64,10 @@ static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *user_data, printf("\nGot token: %s\n\n", token); gpr_free(token); } - gpr_mu_lock(GRPC_POLLSET_MU(&sync->pollset)); + gpr_mu_lock(sync->mu); sync->is_done = 1; - grpc_pollset_kick(&sync->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&sync->pollset)); + grpc_pollset_kick(sync->pollset, NULL); + gpr_mu_unlock(sync->mu); } int main(int argc, char **argv) { @@ -91,26 +93,30 @@ int main(int argc, char **argv) { goto end; } - grpc_pollset_init(&sync.pollset); + sync.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(sync.pollset, &sync.mu); sync.is_done = 0; grpc_call_credentials_get_request_metadata( &exec_ctx, ((grpc_composite_channel_credentials *)creds)->call_creds, - &sync.pollset, context, on_metadata_response, &sync); + sync.pollset, context, on_metadata_response, &sync); - gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_lock(sync.mu); while (!sync.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &sync.pollset, &worker, + grpc_pollset_work(&exec_ctx, sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); - grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_unlock(sync.mu); + grpc_exec_ctx_flush(&exec_ctx); + gpr_mu_lock(sync.mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_unlock(sync.mu); + + grpc_exec_ctx_finish(&exec_ctx); grpc_channel_credentials_release(creds); + gpr_free(sync.pollset); end: gpr_cmdline_destroy(cl); diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index fb4bd30e2df..0e8c38a53ec 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -36,16 +36,17 @@ #include #include -#include "src/core/security/secure_endpoint.h" -#include "src/core/iomgr/endpoint_pair.h" -#include "src/core/iomgr/iomgr.h" #include #include #include -#include "test/core/util/test_config.h" +#include "src/core/iomgr/endpoint_pair.h" +#include "src/core/iomgr/iomgr.h" +#include "src/core/security/secure_endpoint.h" #include "src/core/tsi/fake_transport_security.h" +#include "test/core/util/test_config.h" -static grpc_pollset g_pollset; +static gpr_mu *g_mu; +static grpc_pollset *g_pollset; static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( size_t slice_size, gpr_slice *leftover_slices, size_t leftover_nslices) { @@ -56,8 +57,8 @@ static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( grpc_endpoint_pair tcp; tcp = grpc_iomgr_create_endpoint_pair("fixture", slice_size); - grpc_endpoint_add_to_pollset(&exec_ctx, tcp.client, &g_pollset); - grpc_endpoint_add_to_pollset(&exec_ctx, tcp.server, &g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, tcp.client, g_pollset); + grpc_endpoint_add_to_pollset(&exec_ctx, tcp.server, g_pollset); if (leftover_nslices == 0) { f.client_ep = @@ -181,13 +182,16 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); - grpc_pollset_init(&g_pollset); - grpc_endpoint_tests(configs[0], &g_pollset); + g_pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(g_pollset, &g_mu); + grpc_endpoint_tests(configs[0], g_pollset, g_mu); test_leftover(configs[1], 1); - grpc_closure_init(&destroyed, destroy_pollset, &g_pollset); - grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed); + grpc_closure_init(&destroyed, destroy_pollset, g_pollset); + grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(g_pollset); + return 0; } diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index 5070cf0492c..eb86589681d 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -34,7 +34,6 @@ #include #include -#include "src/core/security/jwt_verifier.h" #include #include #include @@ -43,8 +42,11 @@ #include #include +#include "src/core/security/jwt_verifier.h" + typedef struct { - grpc_pollset pollset; + grpc_pollset *pollset; + gpr_mu *mu; int is_done; int success; } synchronizer; @@ -77,10 +79,10 @@ static void on_jwt_verification_done(void *user_data, grpc_jwt_verifier_status_to_string(status)); } - gpr_mu_lock(GRPC_POLLSET_MU(&sync->pollset)); + gpr_mu_lock(sync->mu); sync->is_done = 1; - grpc_pollset_kick(&sync->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&sync->pollset)); + grpc_pollset_kick(sync->pollset, NULL); + gpr_mu_unlock(sync->mu); } int main(int argc, char **argv) { @@ -103,23 +105,26 @@ int main(int argc, char **argv) { grpc_init(); - grpc_pollset_init(&sync.pollset); + sync.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(sync.pollset, &sync.mu); sync.is_done = 0; - grpc_jwt_verifier_verify(&exec_ctx, verifier, &sync.pollset, jwt, aud, + grpc_jwt_verifier_verify(&exec_ctx, verifier, sync.pollset, jwt, aud, on_jwt_verification_done, &sync); - gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_lock(sync.mu); while (!sync.is_done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &sync.pollset, &worker, + grpc_pollset_work(&exec_ctx, sync.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_unlock(sync.mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_lock(sync.mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset)); + gpr_mu_unlock(sync.mu); + + gpr_free(sync.pollset); grpc_jwt_verifier_destroy(verifier); gpr_cmdline_destroy(cl); diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c index ad21fe0f536..ba382d242a3 100644 --- a/test/core/util/port_posix.c +++ b/test/core/util/port_posix.c @@ -69,7 +69,8 @@ static int has_port_been_chosen(int port) { } typedef struct freereq { - grpc_pollset pollset; + gpr_mu *mu; + grpc_pollset *pollset; int done; } freereq; @@ -82,10 +83,10 @@ static void destroy_pollset_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, const grpc_httpcli_response *response) { freereq *pr = arg; - gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); + gpr_mu_lock(pr->mu); pr->done = 1; - grpc_pollset_kick(&pr->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); + grpc_pollset_kick(pr->pollset, NULL); + gpr_mu_unlock(pr->mu); } static void free_port_using_server(char *server, int port) { @@ -100,31 +101,34 @@ static void free_port_using_server(char *server, int port) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); - grpc_pollset_init(&pr.pollset); + + pr.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(pr.pollset, &pr.mu); grpc_closure_init(&shutdown_closure, destroy_pollset_and_shutdown, - &pr.pollset); + pr.pollset); req.host = server; gpr_asprintf(&path, "/drop/%d", port); req.path = path; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, &pr.pollset, &req, + grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), freed_port_from_server, &pr); - gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_lock(pr.mu); while (!pr.done) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, + grpc_pollset_work(&exec_ctx, pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); } - gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_unlock(pr.mu); grpc_httpcli_context_destroy(&context); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_shutdown(&exec_ctx, &pr.pollset, &shutdown_closure); + grpc_pollset_shutdown(&exec_ctx, pr.pollset, &shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); + gpr_free(pr.pollset); gpr_free(path); } @@ -202,7 +206,8 @@ static int is_port_available(int *port, int is_tcp) { } typedef struct portreq { - grpc_pollset pollset; + gpr_mu *mu; + grpc_pollset *pollset; int port; int retries; char *server; @@ -234,7 +239,7 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->retries++; req.host = pr->server; req.path = "/get"; - grpc_httpcli_get(exec_ctx, pr->ctx, &pr->pollset, &req, + grpc_httpcli_get(exec_ctx, pr->ctx, pr->pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, pr); return; @@ -246,10 +251,10 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, port = port * 10 + response->body[i] - '0'; } GPR_ASSERT(port > 1024); - gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); + gpr_mu_lock(pr->mu); pr->port = port; - grpc_pollset_kick(&pr->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); + grpc_pollset_kick(pr->pollset, NULL); + gpr_mu_unlock(pr->mu); } static int pick_port_using_server(char *server) { @@ -263,9 +268,10 @@ static int pick_port_using_server(char *server) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); - grpc_pollset_init(&pr.pollset); + pr.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(pr.pollset, &pr.mu); grpc_closure_init(&shutdown_closure, destroy_pollset_and_shutdown, - &pr.pollset); + pr.pollset); pr.port = -1; pr.server = server; pr.ctx = &context; @@ -274,22 +280,23 @@ static int pick_port_using_server(char *server) { req.path = "/get"; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, &pr.pollset, &req, + grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, &pr); grpc_exec_ctx_finish(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_lock(pr.mu); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, + grpc_pollset_work(&exec_ctx, pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); } - gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_unlock(pr.mu); grpc_httpcli_context_destroy(&context); - grpc_pollset_shutdown(&exec_ctx, &pr.pollset, &shutdown_closure); + grpc_pollset_shutdown(&exec_ctx, pr.pollset, &shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); + gpr_free(pr.pollset); return pr.port; } diff --git a/test/core/util/port_windows.c b/test/core/util/port_windows.c index b5bd0168ad2..3b20aeb7182 100644 --- a/test/core/util/port_windows.c +++ b/test/core/util/port_windows.c @@ -129,7 +129,8 @@ static int is_port_available(int *port, int is_tcp) { } typedef struct portreq { - grpc_pollset pollset; + grpc_pollset *pollset; + gpr_mu *mu; int port; } portreq; @@ -145,10 +146,10 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, port = port * 10 + response->body[i] - '0'; } GPR_ASSERT(port > 1024); - gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); + gpr_mu_lock(pr->mu); pr->port = port; - grpc_pollset_kick(&pr->pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); + grpc_pollset_kick(pr->pollset, NULL); + gpr_mu_unlock(pr->mu); } static void destroy_pollset_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, @@ -168,32 +169,34 @@ static int pick_port_using_server(char *server) { memset(&pr, 0, sizeof(pr)); memset(&req, 0, sizeof(req)); - grpc_pollset_init(&pr.pollset); + pr.pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(pr.pollset, &pr.mu); pr.port = -1; req.host = server; req.path = "/get"; grpc_httpcli_context_init(&context); - grpc_httpcli_get(&exec_ctx, &context, &pr.pollset, &req, + grpc_httpcli_get(&exec_ctx, &context, pr.pollset, &req, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, &pr); - gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_lock(pr.mu); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; - grpc_pollset_work(&exec_ctx, &pr.pollset, &worker, + grpc_pollset_work(&exec_ctx, pr.pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); - gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_unlock(pr.mu); grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_lock(pr.mu); } - gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); + gpr_mu_unlock(pr.mu); grpc_httpcli_context_destroy(&context); grpc_closure_init(&destroy_pollset_closure, destroy_pollset_and_shutdown, &pr.pollset); - grpc_pollset_shutdown(&exec_ctx, &pr.pollset, &destroy_pollset_closure); + grpc_pollset_shutdown(&exec_ctx, pr.pollset, &destroy_pollset_closure); + gpr_free(pr.pollset); grpc_exec_ctx_finish(&exec_ctx); return pr.port; diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index e99d5dcffdd..ab379441d8c 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -57,8 +57,8 @@ void test_tcp_server_init(test_tcp_server *server, server->tcp_server = NULL; grpc_closure_init(&server->shutdown_complete, on_server_destroyed, server); server->shutdown = 0; - grpc_pollset_init(&server->pollset); - server->pollsets[0] = &server->pollset; + server->pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(server->pollset, &server->mu); server->on_connect = on_connect; server->cb_data = user_data; } @@ -77,7 +77,7 @@ void test_tcp_server_start(test_tcp_server *server, int port) { grpc_tcp_server_add_port(server->tcp_server, &addr, sizeof(addr)); GPR_ASSERT(port_added == port); - grpc_tcp_server_start(&exec_ctx, server->tcp_server, server->pollsets, 1, + grpc_tcp_server_start(&exec_ctx, server->tcp_server, &server->pollset, 1, server->on_connect, server->cb_data); gpr_log(GPR_INFO, "test tcp server listening on 0.0.0.0:%d", port); @@ -90,10 +90,10 @@ void test_tcp_server_poll(test_tcp_server *server, int seconds) { gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(seconds, GPR_TIMESPAN)); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - gpr_mu_lock(GRPC_POLLSET_MU(&server->pollset)); - grpc_pollset_work(&exec_ctx, &server->pollset, &worker, + gpr_mu_lock(server->mu); + grpc_pollset_work(&exec_ctx, server->pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), deadline); - gpr_mu_unlock(GRPC_POLLSET_MU(&server->pollset)); + gpr_mu_unlock(server->mu); grpc_exec_ctx_finish(&exec_ctx); } @@ -111,8 +111,9 @@ void test_tcp_server_destroy(test_tcp_server *server) { gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), shutdown_deadline) < 0) { test_tcp_server_poll(server, 1); } - grpc_pollset_shutdown(&exec_ctx, &server->pollset, &do_nothing_cb); + grpc_pollset_shutdown(&exec_ctx, server->pollset, &do_nothing_cb); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_destroy(&server->pollset); + grpc_pollset_destroy(server->pollset); + gpr_free(server->pollset); grpc_shutdown(); } diff --git a/test/core/util/test_tcp_server.h b/test/core/util/test_tcp_server.h index 51119cf6c80..15fcb4fb873 100644 --- a/test/core/util/test_tcp_server.h +++ b/test/core/util/test_tcp_server.h @@ -41,8 +41,8 @@ typedef struct test_tcp_server { grpc_tcp_server *tcp_server; grpc_closure shutdown_complete; int shutdown; - grpc_pollset pollset; - grpc_pollset *pollsets[1]; + gpr_mu *mu; + grpc_pollset *pollset; grpc_tcp_server_cb on_connect; void *cb_data; } test_tcp_server; From 4a76dcd4f91d81c0200c1339903f0bc5dbcabd15 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 25 Feb 2016 19:39:27 -0800 Subject: [PATCH 129/236] Fix merge error made at some point --- src/core/surface/completion_queue.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index 8a9bbace087..f6a95ebbd3f 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -455,7 +455,6 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, gpr_mu_unlock(cc->mu); grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(cc->mu); - continue; } else { grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, iteration_deadline); From 9d84fa8ca62cab74fab5ffdfe07ebbf608674a7a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 25 Feb 2016 20:08:36 -0800 Subject: [PATCH 130/236] Fix copyrights --- src/core/client_config/lb_policy.c | 2 +- src/core/client_config/lb_policy.h | 2 +- src/core/httpcli/httpcli.h | 2 +- src/core/iomgr/pollset_set.h | 2 +- src/core/iomgr/pollset_set_windows.c | 2 +- src/core/iomgr/pollset_set_windows.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/client_config/lb_policy.c b/src/core/client_config/lb_policy.c index 5ff623e0069..0d8b0073369 100644 --- a/src/core/client_config/lb_policy.c +++ b/src/core/client_config/lb_policy.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/client_config/lb_policy.h b/src/core/client_config/lb_policy.h index 4fbb12da394..34573906068 100644 --- a/src/core/client_config/lb_policy.h +++ b/src/core/client_config/lb_policy.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/httpcli/httpcli.h b/src/core/httpcli/httpcli.h index 86e17c1d697..c9cd987c794 100644 --- a/src/core/httpcli/httpcli.h +++ b/src/core/httpcli/httpcli.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/pollset_set.h b/src/core/iomgr/pollset_set.h index 9591bf0d32c..dddcd8313f2 100644 --- a/src/core/iomgr/pollset_set.h +++ b/src/core/iomgr/pollset_set.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/pollset_set_windows.c b/src/core/iomgr/pollset_set_windows.c index 9cf8fd4472f..3b8eca28e61 100644 --- a/src/core/iomgr/pollset_set_windows.c +++ b/src/core/iomgr/pollset_set_windows.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/iomgr/pollset_set_windows.h b/src/core/iomgr/pollset_set_windows.h index aa5abe91338..9661cd2c398 100644 --- a/src/core/iomgr/pollset_set_windows.h +++ b/src/core/iomgr/pollset_set_windows.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 814169681d97885149482ceba9d2bab4f20983c2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 25 Feb 2016 20:22:11 -0800 Subject: [PATCH 131/236] clang-format --- src/core/channel/client_channel.c | 14 +++----------- src/core/client_config/lb_policies/pick_first.c | 10 ++-------- src/core/client_config/lb_policies/round_robin.c | 10 ++-------- src/core/client_config/subchannel.c | 16 ++++++++-------- src/core/iomgr/pollset_windows.c | 4 +--- src/core/iomgr/tcp_posix.c | 4 ++-- 6 files changed, 18 insertions(+), 40 deletions(-) diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index a96b49ac12f..eeac3c146c3 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -440,17 +440,9 @@ static void cc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, } const grpc_channel_filter grpc_client_channel_filter = { - cc_start_transport_stream_op, - cc_start_transport_op, - sizeof(call_data), - init_call_elem, - cc_set_pollset, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - cc_get_peer, - "client-channel", + cc_start_transport_stream_op, cc_start_transport_op, sizeof(call_data), + init_call_elem, cc_set_pollset, destroy_call_elem, sizeof(channel_data), + init_channel_elem, destroy_channel_elem, cc_get_peer, "client-channel", }; void grpc_client_channel_set_resolver(grpc_exec_ctx *exec_ctx, diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c index 9f38f398d8a..81167b31c8c 100644 --- a/src/core/client_config/lb_policies/pick_first.c +++ b/src/core/client_config/lb_policies/pick_first.c @@ -378,14 +378,8 @@ void pf_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } static const grpc_lb_policy_vtable pick_first_lb_policy_vtable = { - pf_destroy, - pf_shutdown, - pf_pick, - pf_cancel_pick, - pf_ping_one, - pf_exit_idle, - pf_check_connectivity, - pf_notify_on_state_change}; + pf_destroy, pf_shutdown, pf_pick, pf_cancel_pick, pf_ping_one, pf_exit_idle, + pf_check_connectivity, pf_notify_on_state_change}; static void pick_first_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/lb_policies/round_robin.c b/src/core/client_config/lb_policies/round_robin.c index 114ece6e4d9..98d9acc75bf 100644 --- a/src/core/client_config/lb_policies/round_robin.c +++ b/src/core/client_config/lb_policies/round_robin.c @@ -483,14 +483,8 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } static const grpc_lb_policy_vtable round_robin_lb_policy_vtable = { - rr_destroy, - rr_shutdown, - rr_pick, - rr_cancel_pick, - rr_ping_one, - rr_exit_idle, - rr_check_connectivity, - rr_notify_on_state_change}; + rr_destroy, rr_shutdown, rr_pick, rr_cancel_pick, rr_ping_one, rr_exit_idle, + rr_check_connectivity, rr_notify_on_state_change}; static void round_robin_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index 291ad3472cc..bec06bf4144 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -184,8 +184,8 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(c); } -void grpc_connected_subchannel_ref( - grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_connected_subchannel_ref(grpc_connected_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON); } @@ -226,8 +226,8 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta, return old_val; } -grpc_subchannel *grpc_subchannel_ref( - grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS), 0 REF_MUTATE_PURPOSE("STRONG_REF")); @@ -235,8 +235,8 @@ grpc_subchannel *grpc_subchannel_ref( return c; } -grpc_subchannel *grpc_subchannel_weak_ref( - grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF")); GPR_ASSERT(old_refs != 0); @@ -690,8 +690,8 @@ static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); } -void grpc_subchannel_call_ref( - grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +void grpc_subchannel_call_ref(grpc_subchannel_call *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { GRPC_CALL_STACK_REF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index 651f8e7334c..c7f30f435fa 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -89,9 +89,7 @@ static void push_front_worker(grpc_pollset_worker *root, worker->links[type].next->links[type].prev = worker; } -size_t grpc_pollset_size(void) { - return sizeof(grpc_pollset); -} +size_t grpc_pollset_size(void) { return sizeof(grpc_pollset); } /* There isn't really any such thing as a pollset under Windows, due to the nature of the IO completion ports. We're still going to provide a minimal diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index e8f73811cec..f74eb3fe511 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -297,7 +297,7 @@ static flush_result tcp_flush(grpc_tcp *tcp) { unwind_slice_idx = tcp->outgoing_slice_idx; unwind_byte_idx = tcp->outgoing_byte_idx; for (iov_size = 0; tcp->outgoing_slice_idx != tcp->outgoing_buffer->count && - iov_size != MAX_WRITE_IOVEC; + iov_size != MAX_WRITE_IOVEC; iov_size++) { iov[iov_size].iov_base = GPR_SLICE_START_PTR( @@ -446,7 +446,7 @@ static char *tcp_get_peer(grpc_endpoint *ep) { } static const grpc_endpoint_vtable vtable = { - tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set, + tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set, tcp_shutdown, tcp_destroy, tcp_get_peer}; grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size, From 75138b93e1d2db53b3c331f386e040bb0becef28 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 26 Feb 2016 08:50:46 -0800 Subject: [PATCH 132/236] Fix fedora20 python distribtest --- tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile index 502c81c7cb0..5e6cb157629 100644 --- a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile @@ -34,4 +34,4 @@ RUN yum clean all && yum update -y && yum install -y python python-pip # Upgrading six would fail because of docker issue when using overlay. # Trying twice makes it work fine. # https://github.com/docker/docker/issues/10180 -RUN pip2 install --upgrade six || pip2 install --upgrade six +RUN pip install --upgrade six || pip install --upgrade six From 600e993e7a6cfa1b1c4d5b2d6e248cc6bc9ad0e0 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Fri, 26 Feb 2016 09:04:19 -0800 Subject: [PATCH 133/236] add checking of character values --- src/core/census/context.c | 36 ++++++++++++++++++++++++++------- test/core/census/context_test.c | 16 +++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/core/census/context.c b/src/core/census/context.c index 441d3b89a6c..89b8ee0b399 100644 --- a/src/core/census/context.c +++ b/src/core/census/context.c @@ -61,6 +61,10 @@ // * Keep all tag information (keys/values/flags) in a single memory buffer, // that can be directly copied to the wire. +// min and max valid chars in tag keys and values. All printable ASCII is OK. +#define MIN_VALID_TAG_CHAR 32 // ' ' +#define MAX_VALID_TAG_CHAR 126 // '~' + // Structure representing a set of tags. Essentially a count of number of tags // present, and pointer to a chunk of memory that contains the per-tag details. struct tag_set { @@ -117,6 +121,24 @@ struct census_context { #define PROPAGATED_TAGS 0 #define LOCAL_TAGS 1 +// Validate (check all characters are in range and size is less than limit) a +// key or value string. Returns 0 if the string is invalid, or the length +// (including terminator) if valid. +static size_t validate_tag(const char *kv) { + size_t len = 1; + char ch; + while ((ch = *kv++) != 0) { + if (ch < MIN_VALID_TAG_CHAR || ch > MAX_VALID_TAG_CHAR) { + return 0; + } + len++; + } + if (len > CENSUS_MAX_TAG_KV_LEN) { + return 0; + } + return len; +} + // Extract a raw tag given a pointer (raw) to the tag header. Allow for some // extra bytes in the tag header (see encode/decode functions for usage: this // allows for future expansion of the tag header). @@ -281,12 +303,14 @@ census_context *census_context_create(const census_context *base, // the context to add/replace/delete as required. for (int i = 0; i < ntags; i++) { const census_tag *tag = &tags[i]; - size_t key_len = strlen(tag->key) + 1; - // ignore the tag if it is too long/short. - if (key_len != 1 && key_len <= CENSUS_MAX_TAG_KV_LEN) { + size_t key_len = validate_tag(tag->key); + // ignore the tag if it is invalid or too short. + if (key_len <= 1) { + context->status.n_invalid_tags++; + } else { if (tag->value != NULL) { - size_t value_len = strlen(tag->value) + 1; - if (value_len <= CENSUS_MAX_TAG_KV_LEN) { + size_t value_len = validate_tag(tag->value); + if (value_len != 0) { context_modify_tag(context, tag, key_len, value_len); } else { context->status.n_invalid_tags++; @@ -296,8 +320,6 @@ census_context *census_context_create(const census_context *base, context->status.n_deleted_tags++; } } - } else { - context->status.n_invalid_tags++; } } // Remove any deleted tags, update status if needed, and return. diff --git a/test/core/census/context_test.c b/test/core/census/context_test.c index b59ac7c0940..ad4c337465c 100644 --- a/test/core/census/context_test.c +++ b/test/core/census/context_test.c @@ -196,6 +196,22 @@ static void invalid_test(void) { context = census_context_create(NULL, &tag, 1, &status); GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); census_context_destroy(context); + // invalid key character + key[0] = 31; // 32 (' ') is the first valid character value + key[1] = 0; + GPR_ASSERT(strlen(key) == 1); + context = census_context_create(NULL, &tag, 1, &status); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); + census_context_destroy(context); + // invalid value character + key[0] = ' '; + value[5] = 127; // 127 (DEL) is ('~' + 1) + value[8] = 0; + GPR_ASSERT(strlen(key) == 1); + GPR_ASSERT(strlen(value) == 8); + context = census_context_create(NULL, &tag, 1, &status); + GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0); + census_context_destroy(context); } // Make a copy of a context From 0cc824275d578d4f859dc5c6a0446cd4d1ed7099 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Fri, 26 Feb 2016 10:54:20 -0800 Subject: [PATCH 134/236] regenerate imports --- src/python/grpcio/grpc/_cython/imports.generated.h | 4 ++-- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h index ca30742abcb..a395dce7d6a 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.h +++ b/src/python/grpcio/grpc/_cython/imports.generated.h @@ -91,10 +91,10 @@ extern census_context_next_tag_type census_context_next_tag_import; typedef int(*census_context_get_tag_type)(const census_context *context, const char *key, census_tag *tag); extern census_context_get_tag_type census_context_get_tag_import; #define census_context_get_tag census_context_get_tag_import -typedef char *(*census_context_encode_type)(const census_context *context, char *buffer, size_t buf_size, size_t *print_buf_size, size_t *bin_buf_size); +typedef size_t(*census_context_encode_type)(const census_context *context, char *buffer, size_t buf_size); extern census_context_encode_type census_context_encode_import; #define census_context_encode census_context_encode_import -typedef census_context *(*census_context_decode_type)(const char *buffer, size_t size, const char *bin_buffer, size_t bin_size); +typedef census_context *(*census_context_decode_type)(const char *buffer, size_t size); extern census_context_decode_type census_context_decode_import; #define census_context_decode census_context_decode_import typedef int(*census_trace_mask_type)(const census_context *context); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index b61c5282b6d..38aabfaca8f 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -91,10 +91,10 @@ extern census_context_next_tag_type census_context_next_tag_import; typedef int(*census_context_get_tag_type)(const census_context *context, const char *key, census_tag *tag); extern census_context_get_tag_type census_context_get_tag_import; #define census_context_get_tag census_context_get_tag_import -typedef char *(*census_context_encode_type)(const census_context *context, char *buffer, size_t buf_size, size_t *print_buf_size, size_t *bin_buf_size); +typedef size_t(*census_context_encode_type)(const census_context *context, char *buffer, size_t buf_size); extern census_context_encode_type census_context_encode_import; #define census_context_encode census_context_encode_import -typedef census_context *(*census_context_decode_type)(const char *buffer, size_t size, const char *bin_buffer, size_t bin_size); +typedef census_context *(*census_context_decode_type)(const char *buffer, size_t size); extern census_context_decode_type census_context_decode_import; #define census_context_decode census_context_decode_import typedef int(*census_trace_mask_type)(const census_context *context); From 61c134f5f83a1897264016ef765e82b61ebd3992 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 26 Feb 2016 11:03:29 -0800 Subject: [PATCH 135/236] Significantly rewrite tools/gke/run_stress_tests_on_gke.py and make everything configurable --- tools/gke/kubernetes_api.py | 47 ++- tools/gke/run_stress_tests_on_gke.py | 541 +++++++++++++++++---------- 2 files changed, 387 insertions(+), 201 deletions(-) diff --git a/tools/gke/kubernetes_api.py b/tools/gke/kubernetes_api.py index d14c26ad6ac..e1017e9da63 100755 --- a/tools/gke/kubernetes_api.py +++ b/tools/gke/kubernetes_api.py @@ -50,7 +50,8 @@ def _make_pod_config(pod_name, image_name, container_port_list, cmd_list, 'name': pod_name, 'image': image_name, 'ports': [{'containerPort': port, - 'protocol': 'TCP'} for port in container_port_list], + 'protocol': 'TCP'} + for port in container_port_list], 'imagePullPolicy': 'Always' } ] @@ -222,3 +223,47 @@ def delete_pod(kube_host, kube_port, namespace, pod_name): del_url = 'http://%s:%d/api/v1/namespaces/%s/pods/%s' % (kube_host, kube_port, namespace, pod_name) return _do_delete(del_url, 'Delete Pod') + + +def create_pod_and_service(kube_host, kube_port, namespace, pod_name, + image_name, container_port_list, cmd_list, arg_list, + env_dict, is_headless_service): + """A simple helper function that creates a pod and a service (if pod creation was successful).""" + is_success = create_pod(kube_host, kube_port, namespace, pod_name, image_name, + container_port_list, cmd_list, arg_list, env_dict) + if not is_success: + print 'Error in creating Pod' + return False + + is_success = create_service( + kube_host, + kube_port, + namespace, + pod_name, # Use pod_name for service + pod_name, + container_port_list, # Service port list same as container port list + container_port_list, + is_headless_service) + if not is_success: + print 'Error in creating Service' + return False + + print 'Successfully created the pod/service %s' % pod_name + return True + + +def delete_pod_and_service(kube_host, kube_port, namespace, pod_name): + """ A simple helper function that calls delete_pod and delete_service """ + is_success = delete_pod(kube_host, kube_port, namespace, pod_name) + if not is_success: + print 'Error in deleting pod %s' % pod_name + return False + + # Note: service name assumed to the the same as pod name + is_success = delete_service(kube_host, kube_port, namespace, pod_name) + if not is_success: + print 'Error in deleting service %s' % pod_name + return False + + print 'Successfully deleted the Pod/Service: %s' % pod_name + return True diff --git a/tools/gke/run_stress_tests_on_gke.py b/tools/gke/run_stress_tests_on_gke.py index 065b11e91c0..cf0ef595a6e 100755 --- a/tools/gke/run_stress_tests_on_gke.py +++ b/tools/gke/run_stress_tests_on_gke.py @@ -27,6 +27,7 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import argparse import datetime import os import subprocess @@ -40,17 +41,52 @@ from stress_test_utils import BigQueryHelper import kubernetes_api -GRPC_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) -os.chdir(GRPC_ROOT) +_GRPC_ROOT = os.path.abspath(os.path.join( + os.path.dirname(sys.argv[0]), '../..')) +os.chdir(_GRPC_ROOT) +# num of seconds to wait for the GKE image to start and warmup +_GKE_IMAGE_WARMUP_WAIT_SECS = 60 -class BigQuerySettings: +_SERVER_POD_NAME = 'stress-server' +_CLIENT_POD_NAME_PREFIX = 'stress-client' +_DATASET_ID_PREFIX = 'stress_test' +_SUMMARY_TABLE_ID = 'summary' +_QPS_TABLE_ID = 'qps' - def __init__(self, run_id, dataset_id, summary_table_id, qps_table_id): - self.run_id = run_id - self.dataset_id = dataset_id - self.summary_table_id = summary_table_id - self.qps_table_id = qps_table_id +_DEFAULT_DOCKER_IMAGE_NAME = 'grpc_stress_test' + +# The default port on which the kubernetes proxy server is started on localhost +# (i.e kubectl proxy --port=) +_DEFAULT_KUBERNETES_PROXY_PORT = 8001 + +# How frequently should the stress client wrapper script (running inside a GKE +# container) poll the health of the stress client (also running inside the GKE +# container) and upload metrics to BigQuery +_DEFAULT_STRESS_CLIENT_POLL_INTERVAL_SECS = 60 + +# The default setting for stress test server and client +_DEFAULT_STRESS_SERVER_PORT = 8080 +_DEFAULT_METRICS_PORT = 8081 +_DEFAULT_TEST_CASES_STR = 'empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1' +_DEFAULT_NUM_CHANNELS_PER_SERVER = 5 +_DEFAULT_NUM_STUBS_PER_CHANNEL = 10 +_DEFAULT_METRICS_COLLECTION_INTERVAL_SECS = 30 + +# Number of stress client instances to launch +_DEFAULT_NUM_CLIENTS = 3 + +# How frequently should this test monitor the health of Stress clients and +# Servers running in GKE +_DEFAULT_TEST_POLL_INTERVAL_SECS = 60 + +# Default run time for this test (2 hour) +_DEFAULT_TEST_DURATION_SECS = 7200 + +# The number of seconds it would take a GKE pod to warm up (i.e get to 'Running' +# state from the time of creation). Ideally this is something the test should +# automatically determine by using Kubernetes API to poll the pods status. +_DEFAULT_GKE_WARMUP_SECS = 60 class KubernetesProxy: @@ -76,11 +112,74 @@ class KubernetesProxy: def __del__(self): if self.p is not None: + print 'Shutting down Kubernetes proxy..' self.p.kill() +class TestSettings: + + def __init__(self, build_docker_image, test_poll_interval_secs, + test_duration_secs, kubernetes_proxy_port): + self.build_docker_image = build_docker_image + self.test_poll_interval_secs = test_poll_interval_secs + self.test_duration_secs = test_duration_secs + self.kubernetes_proxy_port = kubernetes_proxy_port + + +class GkeSettings: + + def __init__(self, project_id, docker_image_name): + self.project_id = project_id + self.docker_image_name = docker_image_name + self.tag_name = 'gcr.io/%s/%s' % (project_id, docker_image_name) + + +class BigQuerySettings: + + def __init__(self, run_id, dataset_id, summary_table_id, qps_table_id): + self.run_id = run_id + self.dataset_id = dataset_id + self.summary_table_id = summary_table_id + self.qps_table_id = qps_table_id + + +class StressServerSettings: + + def __init__(self, server_pod_name, server_port): + self.server_pod_name = server_pod_name + self.server_port = server_port + + +class StressClientSettings: + + def __init__(self, num_clients, client_pod_name_prefix, server_pod_name, + server_port, metrics_port, metrics_collection_interval_secs, + stress_client_poll_interval_secs, num_channels_per_server, + num_stubs_per_channel, test_cases_str): + self.num_clients = num_clients + self.client_pod_name_prefix = client_pod_name_prefix + self.server_pod_name = server_pod_name + self.server_port = server_port + self.metrics_port = metrics_port + self.metrics_collection_interval_secs = metrics_collection_interval_secs + self.stress_client_poll_interval_secs = stress_client_poll_interval_secs + self.num_channels_per_server = num_channels_per_server + self.num_stubs_per_channel = num_stubs_per_channel + self.test_cases_str = test_cases_str + + # == Derived properties == + # Note: Client can accept a list of server addresses (a comma separated list + # of 'server_name:server_port'). In this case, we only have one server + # address to pass + self.server_addresses = '%s.default.svc.cluster.local:%d' % ( + server_pod_name, server_port) + self.client_pod_names_list = ['%s-%d' % (client_pod_name_prefix, i) + for i in range(1, num_clients + 1)] + + def _build_docker_image(image_name, tag_name): """ Build the docker image and add a tag """ + print 'Building docker image: %s' % image_name os.environ['INTEROP_IMAGE'] = image_name # Note that 'BASE_NAME' HAS to be 'grpc_interop_stress_cxx' since the script # build_interop_stress_image.sh invokes the following script: @@ -93,6 +192,7 @@ def _build_docker_image(image_name, tag_name): print 'Error in building docker image' return False + print 'Adding an additional tag %s to the image %s' % (tag_name, image_name) cmd = ['docker', 'tag', '-f', image_name, tag_name] p = subprocess.Popen(args=cmd) retcode = p.wait() @@ -115,144 +215,86 @@ def _push_docker_image_to_gke_registry(docker_tag_name): return True -def _launch_image_on_gke(kubernetes_api_server, kubernetes_api_port, namespace, - pod_name, image_name, port_list, cmd_list, arg_list, - env_dict, is_headless_service): - """Creates a GKE Pod and a Service object for a given image by calling Kubernetes API""" - is_success = kubernetes_api.create_pod( - kubernetes_api_server, - kubernetes_api_port, - namespace, - pod_name, - image_name, - port_list, # The ports to be exposed on this container/pod - cmd_list, # The command that launches the stress server - arg_list, - env_dict # Environment variables to be passed to the pod - ) - if not is_success: - print 'Error in creating Pod' - return False - - is_success = kubernetes_api.create_service( - kubernetes_api_server, - kubernetes_api_port, - namespace, - pod_name, # Use the pod name for service name as well - pod_name, - port_list, # Service port list - port_list, # Container port list (same as service port list) - is_headless_service) - if not is_success: - print 'Error in creating Service' - return False - - print 'Successfully created the pod/service %s' % pod_name - return True - - -def _delete_image_on_gke(kubernetes_proxy, pod_name_list): - """Deletes a GKE Pod and Service object for given list of Pods by calling Kubernetes API""" - if not kubernetes_proxy.is_started: - print 'Kubernetes proxy must be started before calling this function' - return False - - is_success = True - for pod_name in pod_name_list: - is_success = kubernetes_api.delete_pod( - 'localhost', kubernetes_proxy.get_port(), 'default', pod_name) - if not is_success: - print 'Error in deleting pod %s' % pod_name - break - - is_success = kubernetes_api.delete_service( - 'localhost', kubernetes_proxy.get_port(), 'default', - pod_name) # service name same as pod name - if not is_success: - print 'Error in deleting service %s' % pod_name - break - - if is_success: - print 'Successfully deleted the Pods/Services: %s' % ','.join(pod_name_list) - - return is_success - - -def _launch_server(gcp_project_id, docker_image_name, bq_settings, - kubernetes_proxy, server_pod_name, server_port): +def _launch_server(gke_settings, stress_server_settings, bq_settings, + kubernetes_proxy): """ Launches a stress test server instance in GKE cluster """ if not kubernetes_proxy.is_started: print 'Kubernetes proxy must be started before calling this function' return False + # This is the wrapper script that is run in the container. This script runs + # the actual stress test server server_cmd_list = [ '/var/local/git/grpc/tools/run_tests/stress_test/run_server.py' - ] # Process that is launched - server_arg_list = [] # run_server.py does not take any args (for now) + ] - # == Parameters to the server process launched in GKE == + # run_server.py does not take any args from the command line. The args are + # instead passed via environment variables (see server_env below) + server_arg_list = [] + + # The parameters to the script run_server.py are injected into the container + # via environment variables server_env = { 'STRESS_TEST_IMAGE_TYPE': 'SERVER', 'STRESS_TEST_IMAGE': '/var/local/git/grpc/bins/opt/interop_server', - 'STRESS_TEST_ARGS_STR': '--port=%s' % server_port, + 'STRESS_TEST_ARGS_STR': '--port=%s' % stress_server_settings.server_port, 'RUN_ID': bq_settings.run_id, - 'POD_NAME': server_pod_name, - 'GCP_PROJECT_ID': gcp_project_id, + 'POD_NAME': stress_server_settings.server_pod_name, + 'GCP_PROJECT_ID': gke_settings.project_id, 'DATASET_ID': bq_settings.dataset_id, 'SUMMARY_TABLE_ID': bq_settings.summary_table_id, 'QPS_TABLE_ID': bq_settings.qps_table_id } # Launch Server - is_success = _launch_image_on_gke( + is_success = kubernetes_api.create_pod_and_service( 'localhost', kubernetes_proxy.get_port(), - 'default', - server_pod_name, - docker_image_name, - [server_port], # Port that should be exposed on the container + 'default', # Use 'default' namespace + stress_server_settings.server_pod_name, + gke_settings.tag_name, + [stress_server_settings.server_port], # Port that should be exposed server_cmd_list, server_arg_list, server_env, - True # Headless = True for server. Since we want DNS records to be greated by GKE + True # Headless = True for server. Since we want DNS records to be created by GKE ) return is_success -def _launch_client(gcp_project_id, docker_image_name, bq_settings, - kubernetes_proxy, num_instances, client_pod_name_prefix, - server_pod_name, server_port): +def _launch_client(gke_settings, stress_server_settings, stress_client_settings, + bq_settings, kubernetes_proxy): """ Launches a configurable number of stress test clients on GKE cluster """ if not kubernetes_proxy.is_started: print 'Kubernetes proxy must be started before calling this function' return False - server_address = '%s.default.svc.cluster.local:%d' % (server_pod_name, - server_port) - #TODO(sree) Make the whole client args configurable - test_cases_str = 'empty_unary:1,large_unary:1' stress_client_arg_list = [ - '--server_addresses=%s' % server_address, - '--test_cases=%s' % test_cases_str, '--num_stubs_per_channel=10' + '--server_addresses=%s' % stress_client_settings.server_addresses, + '--test_cases=%s' % stress_client_settings.test_cases_str, + '--num_stubs_per_channel=%d' % + stress_client_settings.num_stubs_per_channel ] + # This is the wrapper script that is run in the container. This script runs + # the actual stress client client_cmd_list = [ '/var/local/git/grpc/tools/run_tests/stress_test/run_client.py' ] - # run_client.py takes no args. All args are passed as env variables - client_arg_list = [] - # TODO(sree) Make this configurable (and also less frequent) - poll_interval_secs = 30 + # run_client.py takes no args. All args are passed as env variables (see + # client_env) + client_arg_list = [] - metrics_port = 8081 - metrics_server_address = 'localhost:%d' % metrics_port + metrics_server_address = 'localhost:%d' % stress_client_settings.metrics_port metrics_client_arg_list = [ '--metrics_server_address=%s' % metrics_server_address, '--total_only=true' ] + # The parameters to the script run_client.py are injected into the container + # via environment variables client_env = { 'STRESS_TEST_IMAGE_TYPE': 'CLIENT', 'STRESS_TEST_IMAGE': '/var/local/git/grpc/bins/opt/stress_test', @@ -260,27 +302,28 @@ def _launch_client(gcp_project_id, docker_image_name, bq_settings, 'METRICS_CLIENT_IMAGE': '/var/local/git/grpc/bins/opt/metrics_client', 'METRICS_CLIENT_ARGS_STR': ' '.join(metrics_client_arg_list), 'RUN_ID': bq_settings.run_id, - 'POLL_INTERVAL_SECS': str(poll_interval_secs), - 'GCP_PROJECT_ID': gcp_project_id, + 'POLL_INTERVAL_SECS': + str(stress_client_settings.stress_client_poll_interval_secs), + 'GCP_PROJECT_ID': gke_settings.project_id, 'DATASET_ID': bq_settings.dataset_id, 'SUMMARY_TABLE_ID': bq_settings.summary_table_id, 'QPS_TABLE_ID': bq_settings.qps_table_id } - for i in range(1, num_instances + 1): - pod_name = '%s-%d' % (client_pod_name_prefix, i) + for pod_name in stress_client_settings.client_pod_names_list: client_env['POD_NAME'] = pod_name - is_success = _launch_image_on_gke( - 'localhost', + is_success = kubernetes_api.create_pod_and_service( + 'localhost', # Since proxy is running on localhost kubernetes_proxy.get_port(), - 'default', + 'default', # default namespace pod_name, - docker_image_name, - [metrics_port], # Client pods expose metrics port + gke_settings.tag_name, + [stress_client_settings.metrics_port + ], # Client pods expose metrics port client_cmd_list, client_arg_list, client_env, - False # Client is not a headless service. + False # Client is not a headless service ) if not is_success: print 'Error in launching client %s' % pod_name @@ -289,20 +332,17 @@ def _launch_client(gcp_project_id, docker_image_name, bq_settings, return True -def _launch_server_and_client(bq_settings, gcp_project_id, docker_image_name, - num_client_instances): +def _launch_server_and_client(gke_settings, stress_server_settings, + stress_client_settings, bq_settings, + kubernetes_proxy_port): # Start kubernetes proxy - kubernetes_api_port = 9001 - kubernetes_proxy = KubernetesProxy(kubernetes_api_port) + print 'Kubernetes proxy' + kubernetes_proxy = KubernetesProxy(kubernetes_proxy_port) kubernetes_proxy.start() - # num of seconds to wait for the GKE image to start and warmup - image_warmp_secs = 60 - - server_pod_name = 'stress-server' - server_port = 8080 - is_success = _launch_server(gcp_project_id, docker_image_name, bq_settings, - kubernetes_proxy, server_pod_name, server_port) + print 'Launching server..' + is_success = _launch_server(gke_settings, stress_server_settings, bq_settings, + kubernetes_proxy) if not is_success: print 'Error in launching server' return False @@ -310,116 +350,217 @@ def _launch_server_and_client(bq_settings, gcp_project_id, docker_image_name, # Server takes a while to start. # TODO(sree) Use Kubernetes API to query the status of the server instead of # sleeping - print 'Waiting for %s seconds for the server to start...' % image_warmp_secs - time.sleep(image_warmp_secs) + print 'Waiting for %s seconds for the server to start...' % _GKE_IMAGE_WARMUP_WAIT_SECS + time.sleep(_GKE_IMAGE_WARMUP_WAIT_SECS) # Launch client - server_address = '%s.default.svc.cluster.local:%d' % (server_pod_name, - server_port) client_pod_name_prefix = 'stress-client' - is_success = _launch_client(gcp_project_id, docker_image_name, bq_settings, - kubernetes_proxy, num_client_instances, - client_pod_name_prefix, server_pod_name, - server_port) + is_success = _launch_client(gke_settings, stress_server_settings, + stress_client_settings, bq_settings, + kubernetes_proxy) + if not is_success: print 'Error in launching client(s)' return False - print 'Waiting for %s seconds for the client images to start...' % image_warmp_secs - time.sleep(image_warmp_secs) + print 'Waiting for %s seconds for the client images to start...' % _GKE_IMAGE_WARMUP_WAIT_SECS + time.sleep(_GKE_IMAGE_WARMUP_WAIT_SECS) return True -def _delete_server_and_client(num_client_instances): - kubernetes_api_port = 9001 - kubernetes_proxy = KubernetesProxy(kubernetes_api_port) +def _delete_server_and_client(stress_server_settings, stress_client_settings, + kubernetes_proxy_port): + kubernetes_proxy = KubernetesProxy(kubernetes_proxy_port) kubernetes_proxy.start() # Delete clients first - client_pod_names = ['stress-client-%d' % i - for i in range(1, num_client_instances + 1)] - - is_success = _delete_image_on_gke(kubernetes_proxy, client_pod_names) - if not is_success: - return False + is_success = True + for pod_name in stress_client_settings.client_pod_names_list: + is_success = kubernetes_api.delete_pod_and_service( + 'localhost', kubernetes_proxy_port, 'default', pod_name) + if not is_success: + return False # Delete server - server_pod_name = 'stress-server' - return _delete_image_on_gke(kubernetes_proxy, [server_pod_name]) + is_success = kubernetes_api.delete_pod_and_service( + 'localhost', kubernetes_proxy_port, 'default', + stress_server_settings.server_pod_name) + return is_success -def _build_and_push_docker_image(gcp_project_id, docker_image_name, tag_name): - is_success = _build_docker_image(docker_image_name, tag_name) - if not is_success: - return False - return _push_docker_image_to_gke_registry(tag_name) - +def run_test_main(test_settings, gke_settings, stress_server_settings, + stress_client_clients): + is_success = True -# TODO(sree): This is just to test the above APIs. Rewrite this to make -# everything configurable (like image names / number of instances etc) -def run_test(skip_building_image, gcp_project_id, image_name, tag_name, - num_client_instances, poll_interval_secs, total_duration_secs): - if not skip_building_image: - is_success = _build_docker_image(image_name, tag_name) + if test_settings.build_docker_image: + is_success = _build_docker_image(gke_settings.docker_image_name, + gke_settings.tag_name) if not is_success: return False - is_success = _push_docker_image_to_gke_registry(tag_name) + is_success = _push_docker_image_to_gke_registry(gke_settings.tag_name) if not is_success: return False - # == Big Query tables related settings (Common for both server and client) == - # Create a unique id for this run (Note: Using timestamp instead of UUID to # make it easier to deduce the date/time of the run just by looking at the run # run id. This is useful in debugging when looking at records in Biq query) run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') - dataset_id = 'stress_test_%s' % run_id - summary_table_id = 'summary' - qps_table_id = 'qps' - bq_settings = BigQuerySettings(run_id, dataset_id, summary_table_id, - qps_table_id) - - bq_helper = BigQueryHelper(run_id, '', '', gcp_project_id, dataset_id, - summary_table_id, qps_table_id) + dataset_id = '%s_%s' % (_DATASET_ID_PREFIX, run_id) + + # Big Query settings (common for both Stress Server and Client) + bq_settings = BigQuerySettings(run_id, dataset_id, _SUMMARY_TABLE_ID, + _QPS_TABLE_ID) + + bq_helper = BigQueryHelper(run_id, '', '', args.project_id, dataset_id, + _SUMMARY_TABLE_ID, _QPS_TABLE_ID) bq_helper.initialize() - is_success = _launch_server_and_client(bq_settings, gcp_project_id, tag_name, - num_client_instances) - if not is_success: - return False - start_time = datetime.datetime.now() - end_time = start_time + datetime.timedelta(seconds=total_duration_secs) - - while True: - if datetime.datetime.now() > end_time: - print 'Test was run for %d seconds' % total_duration_secs - break - - # Check if either stress server or clients have failed - if bq_helper.check_if_any_tests_failed(): - is_success = False - print 'Some tests failed.' - break - # Things seem to be running fine. Wait until next poll time to check the - # status - print 'Sleeping for %d seconds..' % poll_interval_secs - time.sleep(poll_interval_secs) - - # Print BiqQuery tables - bq_helper.print_summary_records() - bq_helper.print_qps_records() - - _delete_server_and_client(num_client_instances) + try: + is_success = _launch_server_and_client(gke_settings, stress_server_settings, + stress_client_settings, bq_settings, + test_settings.kubernetes_proxy_port) + if not is_success: + return False + + start_time = datetime.datetime.now() + end_time = start_time + datetime.timedelta( + seconds=test_settings.test_duration_secs) + print 'Running the test until %s' % end_time.isoformat() + + while True: + if datetime.datetime.now() > end_time: + print 'Test was run for %d seconds' % test_settings.test_duration_secs + break + + # Check if either stress server or clients have failed + if bq_helper.check_if_any_tests_failed(): + is_success = False + print 'Some tests failed.' + break + + # Things seem to be running fine. Wait until next poll time to check the + # status + print 'Sleeping for %d seconds..' % test_settings.test_poll_interval_secs + time.sleep(test_settings.test_poll_interval_secs) + + # Print BiqQuery tables + bq_helper.print_summary_records() + bq_helper.print_qps_records() + + finally: + # If is_success is False at this point, it means that the stress tests were + # started successfully but failed while running the tests. In this case we + # do should not delete the pods (since they contain all the failure + # information) + if is_success: + _delete_server_and_client(stress_server_settings, stress_client_settings, + test_settings.kubernetes_proxy_port) + return is_success +argp = argparse.ArgumentParser( + description='Launch stress tests in GKE', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +argp.add_argument('--project_id', + required=True, + help='The Google Cloud Platform Project Id') +argp.add_argument('--num_clients', + default=1, + type=int, + help='Number of client instances to start') +argp.add_argument('--docker_image_name', + default=_DEFAULT_DOCKER_IMAGE_NAME, + help='The name of the docker image containing stress client ' + 'and stress servers') +argp.add_argument('--build_docker_image', + dest='build_docker_image', + action='store_true', + help='Build a docker image and push to Google Container ' + 'Registry') +argp.add_argument('--do_not_build_docker_image', + dest='build_docker_image', + action='store_false', + help='Do not build and push docker image to Google Container ' + 'Registry') +argp.set_defaults(build_docker_image=True) + +argp.add_argument('--test_poll_interval_secs', + default=_DEFAULT_TEST_POLL_INTERVAL_SECS, + type=int, + help='How frequently should this script should monitor the ' + 'health of stress clients and servers running in the GKE ' + 'cluster') +argp.add_argument('--test_duration_secs', + default=_DEFAULT_TEST_DURATION_SECS, + type=int, + help='How long should this test be run') +argp.add_argument('--kubernetes_proxy_port', + default=_DEFAULT_KUBERNETES_PROXY_PORT, + type=int, + help='The port on which the kubernetes proxy (on localhost)' + ' is started') +argp.add_argument('--stress_server_port', + default=_DEFAULT_STRESS_SERVER_PORT, + type=int, + help='The port on which the stress server (in GKE ' + 'containers) listens') +argp.add_argument('--stress_client_metrics_port', + default=_DEFAULT_METRICS_PORT, + type=int, + help='The port on which the stress clients (in GKE ' + 'containers) expose metrics') +argp.add_argument('--stress_client_poll_interval_secs', + default=_DEFAULT_STRESS_CLIENT_POLL_INTERVAL_SECS, + type=int, + help='How frequently should the stress client wrapper script' + ' running inside GKE should monitor health of the actual ' + ' stress client process and upload the metrics to BigQuery') +argp.add_argument('--stress_client_metrics_collection_interval_secs', + default=_DEFAULT_METRICS_COLLECTION_INTERVAL_SECS, + type=int, + help='How frequently should metrics be collected in-memory on' + ' the stress clients (running inside GKE containers). Note ' + 'that this is NOT the same as the upload-to-BigQuery ' + 'frequency. The metrics upload frequency is controlled by the' + ' --stress_client_poll_interval_secs flag') +argp.add_argument('--stress_client_num_channels_per_server', + default=_DEFAULT_NUM_CHANNELS_PER_SERVER, + type=int, + help='The number of channels created to each server from a ' + 'stress client') +argp.add_argument('--stress_client_num_stubs_per_channel', + default=_DEFAULT_NUM_STUBS_PER_CHANNEL, + type=int, + help='The number of stubs created per channel. This number ' + 'indicates the max number of RPCs that can be made in ' + 'parallel on each channel at any given time') +argp.add_argument('--stress_client_test_cases', + default=_DEFAULT_TEST_CASES_STR, + help='List of test cases (with weights) to be executed by the' + ' stress test client. The list is in the following format:\n' + ' ..\n' + ' (Note: The weights do not have to add up to 100)') + if __name__ == '__main__': - image_name = 'grpc_stress_test' - gcp_project_id = 'sree-gce' - tag_name = 'gcr.io/%s/%s' % (gcp_project_id, image_name) - num_client_instances = 3 - poll_interval_secs = 10 - test_duration_secs = 150 - run_test(True, gcp_project_id, image_name, tag_name, num_client_instances, - poll_interval_secs, test_duration_secs) + args = argp.parse_args() + + test_settings = TestSettings( + args.build_docker_image, args.test_poll_interval_secs, + args.test_duration_secs, args.kubernetes_proxy_port) + + gke_settings = GkeSettings(args.project_id, args.docker_image_name) + + stress_server_settings = StressServerSettings(_SERVER_POD_NAME, + args.stress_server_port) + stress_client_settings = StressClientSettings( + args.num_clients, _CLIENT_POD_NAME_PREFIX, _SERVER_POD_NAME, + args.stress_server_port, args.stress_client_metrics_port, + args.stress_client_metrics_collection_interval_secs, + args.stress_client_poll_interval_secs, + args.stress_client_num_channels_per_server, + args.stress_client_num_stubs_per_channel, args.stress_client_test_cases) + + run_test_main(test_settings, gke_settings, stress_server_settings, + stress_client_settings) From 606d6bfd12905c4cc3a6a16e79dd6d82bab12ed2 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 26 Feb 2016 13:31:31 -0800 Subject: [PATCH 136/236] Explicitly delete node_modules directory after running tests on Windows --- tools/run_tests/post_test_node.bat | 30 ++++++++++++++++++++++++++++++ tools/run_tests/run_tests.py | 5 ++++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tools/run_tests/post_test_node.bat diff --git a/tools/run_tests/post_test_node.bat b/tools/run_tests/post_test_node.bat new file mode 100644 index 00000000000..1a2a5491fa7 --- /dev/null +++ b/tools/run_tests/post_test_node.bat @@ -0,0 +1,30 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +rmdir node_modules /S /Q \ No newline at end of file diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 75de4cb71d6..08a5ff0e8fa 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -290,7 +290,10 @@ class NodeLanguage(object): return [['tools/run_tests/build_node.sh', self.node_version]] def post_tests_steps(self): - return [] + if self.platform == 'windows': + return [['tools\\run_tests\\post_test_node.bat']] + else: + return [] def makefile_name(self): return 'Makefile' From da25fdb8826d59c30a851e4e936b09e450247301 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 26 Feb 2016 13:41:06 -0800 Subject: [PATCH 137/236] Address code review comments --- tools/gke/kubernetes_api.py | 4 ++-- tools/gke/run_stress_tests_on_gke.py | 18 ++++-------------- tools/jenkins/build_interop_stress_image.sh | 3 +++ tools/run_tests/stress_test/run_client.py | 1 - 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/tools/gke/kubernetes_api.py b/tools/gke/kubernetes_api.py index e1017e9da63..2d3f771e93f 100755 --- a/tools/gke/kubernetes_api.py +++ b/tools/gke/kubernetes_api.py @@ -228,7 +228,7 @@ def delete_pod(kube_host, kube_port, namespace, pod_name): def create_pod_and_service(kube_host, kube_port, namespace, pod_name, image_name, container_port_list, cmd_list, arg_list, env_dict, is_headless_service): - """A simple helper function that creates a pod and a service (if pod creation was successful).""" + """A helper function that creates a pod and a service (if pod creation was successful).""" is_success = create_pod(kube_host, kube_port, namespace, pod_name, image_name, container_port_list, cmd_list, arg_list, env_dict) if not is_success: @@ -253,7 +253,7 @@ def create_pod_and_service(kube_host, kube_port, namespace, pod_name, def delete_pod_and_service(kube_host, kube_port, namespace, pod_name): - """ A simple helper function that calls delete_pod and delete_service """ + """ A helper function that calls delete_pod and delete_service """ is_success = delete_pod(kube_host, kube_port, namespace, pod_name) if not is_success: print 'Error in deleting pod %s' % pod_name diff --git a/tools/gke/run_stress_tests_on_gke.py b/tools/gke/run_stress_tests_on_gke.py index cf0ef595a6e..d126e3db439 100755 --- a/tools/gke/run_stress_tests_on_gke.py +++ b/tools/gke/run_stress_tests_on_gke.py @@ -178,28 +178,19 @@ class StressClientSettings: def _build_docker_image(image_name, tag_name): - """ Build the docker image and add a tag """ + """ Build the docker image and add tag it to the GKE repository """ print 'Building docker image: %s' % image_name os.environ['INTEROP_IMAGE'] = image_name + os.environ['INTEROP_IMAGE_REPOSITORY_TAG'] = tag_name # Note that 'BASE_NAME' HAS to be 'grpc_interop_stress_cxx' since the script # build_interop_stress_image.sh invokes the following script: # tools/dockerfile/$BASE_NAME/build_interop_stress.sh os.environ['BASE_NAME'] = 'grpc_interop_stress_cxx' cmd = ['tools/jenkins/build_interop_stress_image.sh'] - p = subprocess.Popen(args=cmd) - retcode = p.wait() + retcode = subprocess.call(args=cmd) if retcode != 0: print 'Error in building docker image' return False - - print 'Adding an additional tag %s to the image %s' % (tag_name, image_name) - cmd = ['docker', 'tag', '-f', image_name, tag_name] - p = subprocess.Popen(args=cmd) - retcode = p.wait() - if retcode != 0: - print 'Error in creating the tag %s for %s' % (tag_name, image_name) - return False - return True @@ -207,8 +198,7 @@ def _push_docker_image_to_gke_registry(docker_tag_name): """Executes 'gcloud docker push ' to push the image to GKE registry""" cmd = ['gcloud', 'docker', 'push', docker_tag_name] print 'Pushing %s to GKE registry..' % docker_tag_name - p = subprocess.Popen(args=cmd) - retcode = p.wait() + retcode = subprocess.call(args=cmd) if retcode != 0: print 'Error in pushing docker image %s to the GKE registry' % docker_tag_name return False diff --git a/tools/jenkins/build_interop_stress_image.sh b/tools/jenkins/build_interop_stress_image.sh index 92f2dab5e34..4c8e998a8a8 100755 --- a/tools/jenkins/build_interop_stress_image.sh +++ b/tools/jenkins/build_interop_stress_image.sh @@ -35,6 +35,8 @@ set -x # Params: # INTEROP_IMAGE - name of tag of the final interop image +# INTEROP_IMAGE_TAG - Optional. If set, the created image will be tagged using +# the command: 'docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG' # BASE_NAME - base name used to locate the base Dockerfile and build script # TTY_FLAG - optional -t flag to make docker allocate tty # BUILD_INTEROP_DOCKER_EXTRA_ARGS - optional args to be passed to the @@ -77,6 +79,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" $BASE_IMAGE \ bash -l /var/local/jenkins/grpc/tools/dockerfile/$BASE_NAME/build_interop_stress.sh \ && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ + && ( if [ -n $INTEROP_IMAGE_REPOSITORY_TAG ]; then docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG ; fi ) \ && echo "Successfully built image $INTEROP_IMAGE") EXITCODE=$? diff --git a/tools/run_tests/stress_test/run_client.py b/tools/run_tests/stress_test/run_client.py index 33958bce496..0fa1bf1cb97 100755 --- a/tools/run_tests/stress_test/run_client.py +++ b/tools/run_tests/stress_test/run_client.py @@ -142,7 +142,6 @@ def run_client(): # Check if stress_client is still running. If so, collect metrics and upload # to BigQuery status table if stress_p.poll() is not None: - # TODO(sree) Upload completion status to BigQuery end_time = datetime.datetime.now().isoformat() event_type = EventType.SUCCESS details = 'End time: %s' % end_time From c2b3490fc5af4a9acd029014055b9614300a6408 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 26 Feb 2016 16:38:17 -0800 Subject: [PATCH 138/236] Added a test to verify user agent prefix can be set correctly. --- src/objective-c/tests/GRPCClientTests.m | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 9a3e5b5009b..d052f31a3d5 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -103,6 +103,8 @@ static ProtoMethod *kUnaryCallMethod; @implementation GRPCClientTests - (void)setUp { + // Add a custom user agent prefix that will be used in test + [GRPCCall setUserAgentPrefix:@"Foo" forHost:kHostAddress]; // Register test server as non-SSL. [GRPCCall useInsecureConnectionsForHost:kHostAddress]; @@ -257,6 +259,36 @@ static ProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:8 handler:nil]; } +- (void)testUserAgentPrefix { + __weak XCTestExpectation *response = [self expectationWithDescription:@"Empty response received."]; + __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; + + GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress + path:kEmptyCallMethod.HTTPPath + requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; + // Setting this special key in the header will cause the interop server to echo back the + // user-agent value, which we confirm. + call.requestHeaders[@"x-grpc-test-echo-useragent"] = @""; + + id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + XCTAssertNotNil(value, @"nil value received as response."); + XCTAssertEqual([value length], 0, @"Non-empty response received: %@", value); + XCTAssertEqualObjects(call.responseHeaders[@"x-grpc-test-echo-useragent"], + @"Foo grpc-objc/0.13.0 grpc-c/0.14.0-dev (ios)", + @"Did not receive expected user agent %@", + call.responseHeaders[@"x-grpc-test-echo-useragent"]); + + [response fulfill]; + } completionHandler:^(NSError *errorOrNil) { + XCTAssertNil(errorOrNil, @"Finished with unexpected error: %@", errorOrNil); + [completion fulfill]; + }]; + + [call startWithWriteable:responsesWriteable]; + + [self waitForExpectationsWithTimeout:8 handler:nil]; +} + // TODO(makarandd): Move to a different file that contains only unit tests - (void)testExceptions { // Try to set userAgentPrefix for host that is nil. This should cause From 8ed902e6f7458b92929258f3e3d9f03fe71f9709 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 26 Feb 2016 16:47:47 -0800 Subject: [PATCH 139/236] Change directory structure for the scripts (remove tools/big_query and tools/gke directies and instead create tools/gcp). Move scripts around in to the appropriate directories --- .../{run_tests => gcp}/stress_test/run_client.py | 0 .../{run_tests => gcp}/stress_test/run_server.py | 0 .../stress_test/stress_test_utils.py | 2 +- .../{big_query => gcp/utils}/big_query_utils.py | 0 tools/{gke => gcp/utils}/kubernetes_api.py | 0 tools/jenkins/build_interop_stress_image.sh | 2 +- .../stress_test}/run_stress_tests_on_gke.py | 16 ++++++++-------- 7 files changed, 10 insertions(+), 10 deletions(-) rename tools/{run_tests => gcp}/stress_test/run_client.py (100%) rename tools/{run_tests => gcp}/stress_test/run_server.py (100%) rename tools/{run_tests => gcp}/stress_test/stress_test_utils.py (99%) rename tools/{big_query => gcp/utils}/big_query_utils.py (100%) rename tools/{gke => gcp/utils}/kubernetes_api.py (100%) rename tools/{gke => run_tests/stress_test}/run_stress_tests_on_gke.py (98%) diff --git a/tools/run_tests/stress_test/run_client.py b/tools/gcp/stress_test/run_client.py similarity index 100% rename from tools/run_tests/stress_test/run_client.py rename to tools/gcp/stress_test/run_client.py diff --git a/tools/run_tests/stress_test/run_server.py b/tools/gcp/stress_test/run_server.py similarity index 100% rename from tools/run_tests/stress_test/run_server.py rename to tools/gcp/stress_test/run_server.py diff --git a/tools/run_tests/stress_test/stress_test_utils.py b/tools/gcp/stress_test/stress_test_utils.py similarity index 99% rename from tools/run_tests/stress_test/stress_test_utils.py rename to tools/gcp/stress_test/stress_test_utils.py index 7adc0068f9a..c4b437e3459 100755 --- a/tools/run_tests/stress_test/stress_test_utils.py +++ b/tools/gcp/stress_test/stress_test_utils.py @@ -39,7 +39,7 @@ import time # Import big_query_utils module bq_utils_dir = os.path.abspath(os.path.join( - os.path.dirname(__file__), '../../big_query')) + os.path.dirname(__file__), '../utils')) sys.path.append(bq_utils_dir) import big_query_utils as bq_utils diff --git a/tools/big_query/big_query_utils.py b/tools/gcp/utils/big_query_utils.py similarity index 100% rename from tools/big_query/big_query_utils.py rename to tools/gcp/utils/big_query_utils.py diff --git a/tools/gke/kubernetes_api.py b/tools/gcp/utils/kubernetes_api.py similarity index 100% rename from tools/gke/kubernetes_api.py rename to tools/gcp/utils/kubernetes_api.py diff --git a/tools/jenkins/build_interop_stress_image.sh b/tools/jenkins/build_interop_stress_image.sh index 4c8e998a8a8..41f6358df64 100755 --- a/tools/jenkins/build_interop_stress_image.sh +++ b/tools/jenkins/build_interop_stress_image.sh @@ -79,7 +79,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" $BASE_IMAGE \ bash -l /var/local/jenkins/grpc/tools/dockerfile/$BASE_NAME/build_interop_stress.sh \ && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ - && ( if [ -n $INTEROP_IMAGE_REPOSITORY_TAG ]; then docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG ; fi ) \ + && ( if [ -n $INTEROP_IMAGE_REPOSITORY_TAG ]; then docker tag -f $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG ; fi ) \ && echo "Successfully built image $INTEROP_IMAGE") EXITCODE=$? diff --git a/tools/gke/run_stress_tests_on_gke.py b/tools/run_tests/stress_test/run_stress_tests_on_gke.py similarity index 98% rename from tools/gke/run_stress_tests_on_gke.py rename to tools/run_tests/stress_test/run_stress_tests_on_gke.py index d126e3db439..634eb1aca53 100755 --- a/tools/gke/run_stress_tests_on_gke.py +++ b/tools/run_tests/stress_test/run_stress_tests_on_gke.py @@ -35,14 +35,18 @@ import sys import time stress_test_utils_dir = os.path.abspath(os.path.join( - os.path.dirname(__file__), '../run_tests/stress_test')) + os.path.dirname(__file__), '../../gcp/stress_test')) sys.path.append(stress_test_utils_dir) from stress_test_utils import BigQueryHelper +kubernetes_api_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../../gcp/utils')) +sys.path.append(kubernetes_api_dir) + import kubernetes_api _GRPC_ROOT = os.path.abspath(os.path.join( - os.path.dirname(sys.argv[0]), '../..')) + os.path.dirname(sys.argv[0]), '../../..')) os.chdir(_GRPC_ROOT) # num of seconds to wait for the GKE image to start and warmup @@ -214,9 +218,7 @@ def _launch_server(gke_settings, stress_server_settings, bq_settings, # This is the wrapper script that is run in the container. This script runs # the actual stress test server - server_cmd_list = [ - '/var/local/git/grpc/tools/run_tests/stress_test/run_server.py' - ] + server_cmd_list = ['/var/local/git/grpc/tools/gcp/stress_test/run_server.py'] # run_server.py does not take any args from the command line. The args are # instead passed via environment variables (see server_env below) @@ -269,9 +271,7 @@ def _launch_client(gke_settings, stress_server_settings, stress_client_settings, # This is the wrapper script that is run in the container. This script runs # the actual stress client - client_cmd_list = [ - '/var/local/git/grpc/tools/run_tests/stress_test/run_client.py' - ] + client_cmd_list = ['/var/local/git/grpc/tools/gcp/stress_test/run_client.py'] # run_client.py takes no args. All args are passed as env variables (see # client_env) From c9dbf645e43515269b61b7337692afc037dbea5a Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 26 Feb 2016 16:59:08 -0800 Subject: [PATCH 140/236] fixed indentation and removed unnecessary empty line --- src/objective-c/tests/GRPCClientTests.m | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index d052f31a3d5..d6f1b8f232a 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -277,8 +277,7 @@ static ProtoMethod *kUnaryCallMethod; @"Foo grpc-objc/0.13.0 grpc-c/0.14.0-dev (ios)", @"Did not receive expected user agent %@", call.responseHeaders[@"x-grpc-test-echo-useragent"]); - - [response fulfill]; + [response fulfill]; } completionHandler:^(NSError *errorOrNil) { XCTAssertNil(errorOrNil, @"Finished with unexpected error: %@", errorOrNil); [completion fulfill]; From f1588b5154c01b0282f36cde07f0a8fcdb84d2e2 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 26 Feb 2016 17:05:47 -0800 Subject: [PATCH 141/236] Another nit fix.. --- src/objective-c/tests/GRPCClientTests.m | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index d6f1b8f232a..624958f4b97 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -274,9 +274,9 @@ static ProtoMethod *kUnaryCallMethod; XCTAssertNotNil(value, @"nil value received as response."); XCTAssertEqual([value length], 0, @"Non-empty response received: %@", value); XCTAssertEqualObjects(call.responseHeaders[@"x-grpc-test-echo-useragent"], - @"Foo grpc-objc/0.13.0 grpc-c/0.14.0-dev (ios)", - @"Did not receive expected user agent %@", - call.responseHeaders[@"x-grpc-test-echo-useragent"]); + @"Foo grpc-objc/0.13.0 grpc-c/0.14.0-dev (ios)", + @"Did not receive expected user agent %@", + call.responseHeaders[@"x-grpc-test-echo-useragent"]); [response fulfill]; } completionHandler:^(NSError *errorOrNil) { XCTAssertNil(errorOrNil, @"Finished with unexpected error: %@", errorOrNil); From aadb910524a62a816f334e1d5b1c74ee5d7f974c Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 26 Feb 2016 16:47:47 -0800 Subject: [PATCH 142/236] Change directory structure for the scripts (remove tools/big_query and tools/gke directies and instead create tools/gcp). Move scripts around in to the appropriate directories --- .../{run_tests => gcp}/stress_test/run_client.py | 0 .../{run_tests => gcp}/stress_test/run_server.py | 0 .../stress_test/stress_test_utils.py | 2 +- .../{big_query => gcp/utils}/big_query_utils.py | 0 tools/{gke => gcp/utils}/kubernetes_api.py | 0 tools/jenkins/build_interop_stress_image.sh | 2 +- .../stress_test}/run_stress_tests_on_gke.py | 16 ++++++++-------- 7 files changed, 10 insertions(+), 10 deletions(-) rename tools/{run_tests => gcp}/stress_test/run_client.py (100%) rename tools/{run_tests => gcp}/stress_test/run_server.py (100%) rename tools/{run_tests => gcp}/stress_test/stress_test_utils.py (99%) rename tools/{big_query => gcp/utils}/big_query_utils.py (100%) rename tools/{gke => gcp/utils}/kubernetes_api.py (100%) rename tools/{gke => run_tests/stress_test}/run_stress_tests_on_gke.py (98%) diff --git a/tools/run_tests/stress_test/run_client.py b/tools/gcp/stress_test/run_client.py similarity index 100% rename from tools/run_tests/stress_test/run_client.py rename to tools/gcp/stress_test/run_client.py diff --git a/tools/run_tests/stress_test/run_server.py b/tools/gcp/stress_test/run_server.py similarity index 100% rename from tools/run_tests/stress_test/run_server.py rename to tools/gcp/stress_test/run_server.py diff --git a/tools/run_tests/stress_test/stress_test_utils.py b/tools/gcp/stress_test/stress_test_utils.py similarity index 99% rename from tools/run_tests/stress_test/stress_test_utils.py rename to tools/gcp/stress_test/stress_test_utils.py index 7adc0068f9a..c4b437e3459 100755 --- a/tools/run_tests/stress_test/stress_test_utils.py +++ b/tools/gcp/stress_test/stress_test_utils.py @@ -39,7 +39,7 @@ import time # Import big_query_utils module bq_utils_dir = os.path.abspath(os.path.join( - os.path.dirname(__file__), '../../big_query')) + os.path.dirname(__file__), '../utils')) sys.path.append(bq_utils_dir) import big_query_utils as bq_utils diff --git a/tools/big_query/big_query_utils.py b/tools/gcp/utils/big_query_utils.py similarity index 100% rename from tools/big_query/big_query_utils.py rename to tools/gcp/utils/big_query_utils.py diff --git a/tools/gke/kubernetes_api.py b/tools/gcp/utils/kubernetes_api.py similarity index 100% rename from tools/gke/kubernetes_api.py rename to tools/gcp/utils/kubernetes_api.py diff --git a/tools/jenkins/build_interop_stress_image.sh b/tools/jenkins/build_interop_stress_image.sh index 4c8e998a8a8..501dc5b7ca4 100755 --- a/tools/jenkins/build_interop_stress_image.sh +++ b/tools/jenkins/build_interop_stress_image.sh @@ -79,7 +79,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" $BASE_IMAGE \ bash -l /var/local/jenkins/grpc/tools/dockerfile/$BASE_NAME/build_interop_stress.sh \ && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ - && ( if [ -n $INTEROP_IMAGE_REPOSITORY_TAG ]; then docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG ; fi ) \ + && ( if [ -n "$INTEROP_IMAGE_REPOSITORY_TAG" ]; then docker tag -f $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG ; fi ) \ && echo "Successfully built image $INTEROP_IMAGE") EXITCODE=$? diff --git a/tools/gke/run_stress_tests_on_gke.py b/tools/run_tests/stress_test/run_stress_tests_on_gke.py similarity index 98% rename from tools/gke/run_stress_tests_on_gke.py rename to tools/run_tests/stress_test/run_stress_tests_on_gke.py index d126e3db439..634eb1aca53 100755 --- a/tools/gke/run_stress_tests_on_gke.py +++ b/tools/run_tests/stress_test/run_stress_tests_on_gke.py @@ -35,14 +35,18 @@ import sys import time stress_test_utils_dir = os.path.abspath(os.path.join( - os.path.dirname(__file__), '../run_tests/stress_test')) + os.path.dirname(__file__), '../../gcp/stress_test')) sys.path.append(stress_test_utils_dir) from stress_test_utils import BigQueryHelper +kubernetes_api_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../../gcp/utils')) +sys.path.append(kubernetes_api_dir) + import kubernetes_api _GRPC_ROOT = os.path.abspath(os.path.join( - os.path.dirname(sys.argv[0]), '../..')) + os.path.dirname(sys.argv[0]), '../../..')) os.chdir(_GRPC_ROOT) # num of seconds to wait for the GKE image to start and warmup @@ -214,9 +218,7 @@ def _launch_server(gke_settings, stress_server_settings, bq_settings, # This is the wrapper script that is run in the container. This script runs # the actual stress test server - server_cmd_list = [ - '/var/local/git/grpc/tools/run_tests/stress_test/run_server.py' - ] + server_cmd_list = ['/var/local/git/grpc/tools/gcp/stress_test/run_server.py'] # run_server.py does not take any args from the command line. The args are # instead passed via environment variables (see server_env below) @@ -269,9 +271,7 @@ def _launch_client(gke_settings, stress_server_settings, stress_client_settings, # This is the wrapper script that is run in the container. This script runs # the actual stress client - client_cmd_list = [ - '/var/local/git/grpc/tools/run_tests/stress_test/run_client.py' - ] + client_cmd_list = ['/var/local/git/grpc/tools/gcp/stress_test/run_client.py'] # run_client.py takes no args. All args are passed as env variables (see # client_env) From 2463010ed258210351a13e97ee5cb85f6bae104c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 26 Feb 2016 16:13:34 -0800 Subject: [PATCH 143/236] package protoc artifacts for all platforms in Grpc.Tools --- src/csharp/Grpc.Tools.nuspec | 17 ++++++++++++++--- src/csharp/build_packages.bat | 17 ++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/csharp/Grpc.Tools.nuspec b/src/csharp/Grpc.Tools.nuspec index 48a7b1f3af2..31d1bed6477 100644 --- a/src/csharp/Grpc.Tools.nuspec +++ b/src/csharp/Grpc.Tools.nuspec @@ -4,18 +4,29 @@ Grpc.Tools gRPC C# Tools Tools for C# implementation of gRPC - an RPC library and framework - Precompiled Windows binary for generating gRPC client/server code + Precompiled protobuf compiler and gRPC protobuf compiler plugin for generating gRPC client/server C# code. Binaries are available for Windows, Linux and MacOS. $version$ Google Inc. grpc-packages https://github.com/grpc/grpc/blob/master/LICENSE https://github.com/grpc/grpc false - grpc_csharp_plugin.exe - gRPC C# protoc plugin version $version$ + Release $version$ Copyright 2015, Google Inc. gRPC RPC Protocol HTTP/2 - + + + + + + + + + + + + diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat index b7768f78214..7c42a6d3fc4 100644 --- a/src/csharp/build_packages.bat +++ b/src/csharp/build_packages.bat @@ -19,6 +19,14 @@ xcopy /Y /I ..\..\architecture=x64,language=csharp,platform=linux\artifacts\* gr xcopy /Y /I ..\..\architecture=x86,language=csharp,platform=macos\artifacts\* grpc.native.csharp\macosx_x86\ xcopy /Y /I ..\..\architecture=x64,language=csharp,platform=macos\artifacts\* grpc.native.csharp\macosx_x64\ +@rem Collect protoc artifacts built by the previous build step +xcopy /Y /I ..\..\architecture=x86,language=protoc,platform=windows\artifacts\* protoc_plugins\windows_x86\ +xcopy /Y /I ..\..\architecture=x64,language=protoc,platform=windows\artifacts\* protoc_plugins\windows_x64\ +xcopy /Y /I ..\..\architecture=x86,language=protoc,platform=linux\artifacts\* protoc_plugins\linux_x86\ +xcopy /Y /I ..\..\architecture=x64,language=protoc,platform=linux\artifacts\* protoc_plugins\linux_x64\ +xcopy /Y /I ..\..\architecture=x86,language=protoc,platform=macos\artifacts\* protoc_plugins\macosx_x86\ +xcopy /Y /I ..\..\architecture=x64,language=protoc,platform=macos\artifacts\* protoc_plugins\macosx_x64\ + @rem Fetch all dependencies %NUGET% restore ..\..\vsprojects\grpc_csharp_ext.sln || goto :error %NUGET% restore Grpc.sln || goto :error @@ -27,24 +35,19 @@ setlocal @call "%VS120COMNTOOLS%\..\..\vc\vcvarsall.bat" x86 -@rem We won't use the native libraries from this step, but without this Grpc.sln will fail. +@rem We won't use the native libraries from this step, but without this Grpc.sln will fail. msbuild ..\..\vsprojects\grpc_csharp_ext.sln /p:Configuration=Release /p:PlatformToolset=v120 || goto :error msbuild Grpc.sln /p:Configuration=ReleaseSigned || goto :error endlocal -@rem TODO(jtattermusch): re-enable protoc plugin building -@rem @call ..\..\vsprojects\build_plugins.bat || goto :error - %NUGET% pack grpc.native.csharp\grpc.native.csharp.nuspec -Version %VERSION% || goto :error %NUGET% pack Grpc.Auth\Grpc.Auth.nuspec -Symbols -Version %VERSION% || goto :error %NUGET% pack Grpc.Core\Grpc.Core.nuspec -Symbols -Version %VERSION% || goto :error %NUGET% pack Grpc.HealthCheck\Grpc.HealthCheck.nuspec -Symbols -Version %VERSION_WITH_BETA% -Properties ProtobufVersion=%PROTOBUF_VERSION% || goto :error %NUGET% pack Grpc.nuspec -Version %VERSION% || goto :error - -@rem TODO(jtattermusch): re-enable building Grpc.Tools package -@rem %NUGET% pack Grpc.Tools.nuspec -Version %VERSION% || goto :error +%NUGET% pack Grpc.Tools.nuspec -Version %VERSION% || goto :error @rem copy resulting nuget packages to artifacts directory xcopy /Y /I *.nupkg ..\..\artifacts\ From fa4b163feac008bf7147cdad4f2dd88ef778ad2c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 26 Feb 2016 17:14:01 -0800 Subject: [PATCH 144/236] windows C# distribtest --- test/distrib/csharp/DistribTest.sln | 6 ++++++ .../csharp/DistribTest/DistribTest.csproj | 20 ++++++++++++++++++ test/distrib/csharp/run_distrib_test.bat | 21 +++++++++++++++++++ test/distrib/csharp/run_distrib_test.sh | 4 +--- test/distrib/csharp/update_version.sh | 10 ++++++++- tools/run_tests/distribtest_targets.py | 11 +++++++++- 6 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 test/distrib/csharp/run_distrib_test.bat diff --git a/test/distrib/csharp/DistribTest.sln b/test/distrib/csharp/DistribTest.sln index 0eca35c30fb..78d5397ca97 100644 --- a/test/distrib/csharp/DistribTest.sln +++ b/test/distrib/csharp/DistribTest.sln @@ -8,13 +8,19 @@ EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {A3E61CC3-3710-49A3-A830-A0066EDBCE2F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A3E61CC3-3710-49A3-A830-A0066EDBCE2F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A3E61CC3-3710-49A3-A830-A0066EDBCE2F}.Debug|x64.ActiveCfg = Debug|x64 + {A3E61CC3-3710-49A3-A830-A0066EDBCE2F}.Debug|x64.Build.0 = Debug|x64 {A3E61CC3-3710-49A3-A830-A0066EDBCE2F}.Release|Any CPU.ActiveCfg = Release|Any CPU {A3E61CC3-3710-49A3-A830-A0066EDBCE2F}.Release|Any CPU.Build.0 = Release|Any CPU + {A3E61CC3-3710-49A3-A830-A0066EDBCE2F}.Release|x64.ActiveCfg = Release|x64 + {A3E61CC3-3710-49A3-A830-A0066EDBCE2F}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/test/distrib/csharp/DistribTest/DistribTest.csproj b/test/distrib/csharp/DistribTest/DistribTest.csproj index 124fc1bdf0f..7605495f0f6 100644 --- a/test/distrib/csharp/DistribTest/DistribTest.csproj +++ b/test/distrib/csharp/DistribTest/DistribTest.csproj @@ -32,6 +32,26 @@ prompt 4 + + true + bin\x64\Debug\ + DEBUG;TRACE + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + bin\x64\Release\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll diff --git a/test/distrib/csharp/run_distrib_test.bat b/test/distrib/csharp/run_distrib_test.bat new file mode 100644 index 00000000000..950894f6681 --- /dev/null +++ b/test/distrib/csharp/run_distrib_test.bat @@ -0,0 +1,21 @@ + +@rem enter this directory +cd /d %~dp0 + +@rem extract input artifacts +powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('../../../input_artifacts/csharp_nugets.zip', 'TestNugetFeed');" + +update_version.sh auto + +set NUGET=C:\nuget\nuget.exe +%NUGET% restore || goto :error + +@call build_vs2015.bat DistribTest.sln %MSBUILD_EXTRA_ARGS% || goto :error + +%DISTRIBTEST_OUTPATH%\DistribTest.exe || goto :error + +goto :EOF + +:error +echo Failed! +exit /b %errorlevel% diff --git a/test/distrib/csharp/run_distrib_test.sh b/test/distrib/csharp/run_distrib_test.sh index 1de62041b3f..934174a9a4e 100755 --- a/test/distrib/csharp/run_distrib_test.sh +++ b/test/distrib/csharp/run_distrib_test.sh @@ -34,9 +34,7 @@ cd $(dirname $0) unzip -o "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets.zip" -d TestNugetFeed -# Extract the version number from Grpc nuget package name. -CSHARP_VERSION=$(ls TestNugetFeed | grep '^Grpc\.[0-9].*\.nupkg$' | sed s/^Grpc\.// | sed s/\.nupkg$//) -./update_version.sh $CSHARP_VERSION +./update_version.sh auto nuget restore diff --git a/test/distrib/csharp/update_version.sh b/test/distrib/csharp/update_version.sh index f2554e89981..b0d07721f6c 100755 --- a/test/distrib/csharp/update_version.sh +++ b/test/distrib/csharp/update_version.sh @@ -32,5 +32,13 @@ set -e cd $(dirname $0) +CSHARP_VERSION="$1" +if [ "$CSHARP_VERSION" == "auto" ] +then + # autodetect C# version + CSHARP_VERSION=$(ls TestNugetFeed | grep '^Grpc\.[0-9].*\.nupkg$' | sed s/^Grpc\.// | sed s/\.nupkg$//) + echo "Autodetected nuget ${CSHARP_VERSION}" +fi + # Replaces version placeholder with value provided as first argument. -sed -ibak "s/__GRPC_NUGET_VERSION__/$1/g" DistribTest/packages.config DistribTest/DistribTest.csproj +sed -ibak "s/__GRPC_NUGET_VERSION__/${CSHARP_VERSION}/g" DistribTest/packages.config DistribTest/DistribTest.csproj diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index 933103f0a05..fb951b68f95 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -97,7 +97,14 @@ class CSharpDistribTest(object): ['test/distrib/csharp/run_distrib_test.sh'], environ={'EXTERNAL_GIT_ROOT': '../../..'}) else: - raise Exception("Not supported yet.") + if self.arch == 'x64': + environ={'MSBUILD_EXTRA_ARGS': '/p:Platform=x64', + 'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'} + else: + environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\\Debug'} + return create_jobspec(self.name, + ['test\\distrib\\csharp\\run_distrib_test.bat'], + environ=environ) def __str__(self): return self.name @@ -240,6 +247,8 @@ def targets(): CSharpDistribTest('linux', 'x64', 'ubuntu1510'), CSharpDistribTest('linux', 'x64', 'ubuntu1604'), CSharpDistribTest('macos', 'x86'), + CSharpDistribTest('windows', 'x86'), + CSharpDistribTest('windows', 'x64'), PythonDistribTest('linux', 'x64', 'wheezy'), PythonDistribTest('linux', 'x64', 'jessie'), PythonDistribTest('linux', 'x86', 'jessie'), From e1dd18a945ea11d1eba412ea0483b3996a222c3b Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 29 Feb 2016 13:28:55 -0800 Subject: [PATCH 145/236] Fix copyright --- test/cpp/interop/metrics_client.cc | 2 +- tools/gcp/utils/big_query_utils.py | 2 +- tools/gcp/utils/kubernetes_api.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/cpp/interop/metrics_client.cc b/test/cpp/interop/metrics_client.cc index cc304f2e895..bd48c7d4ef2 100644 --- a/test/cpp/interop/metrics_client.cc +++ b/test/cpp/interop/metrics_client.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/tools/gcp/utils/big_query_utils.py b/tools/gcp/utils/big_query_utils.py index e2379fd1aa7..7bb1e143549 100755 --- a/tools/gcp/utils/big_query_utils.py +++ b/tools/gcp/utils/big_query_utils.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015-2016 Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/gcp/utils/kubernetes_api.py b/tools/gcp/utils/kubernetes_api.py index 2d3f771e93f..e8ddd2f1b35 100755 --- a/tools/gcp/utils/kubernetes_api.py +++ b/tools/gcp/utils/kubernetes_api.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015-2016 Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From 334e0ee37012752a9747594dbdbde9f3f0dbc8d5 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 29 Feb 2016 14:41:42 -0800 Subject: [PATCH 146/236] Address some memory hazards in Cython code Some __dealloc__ methods were calling Python methods, and some references were being dropped on the floor instead of threaded through gRPC core. --- .../grpc/_cython/_cygrpc/channel.pyx.pxi | 7 ++++--- .../_cython/_cygrpc/completion_queue.pyx.pxi | 10 ++++++--- .../grpc/_cython/_cygrpc/server.pxd.pxi | 1 + .../grpc/_cython/_cygrpc/server.pyx.pxi | 21 +++++++++++-------- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index ac67f32d923..f68dfd1b245 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -89,12 +89,13 @@ cdef class Channel: def check_connectivity_state(self, bint try_to_connect): return grpc_channel_check_connectivity_state(self.c_channel, - try_to_connect) + try_to_connect) def watch_connectivity_state( - self, last_observed_state, Timespec deadline not None, - CompletionQueue queue not None, tag): + self, grpc_connectivity_state last_observed_state, + Timespec deadline not None, CompletionQueue queue not None, tag): cdef OperationTag operation_tag = OperationTag(tag) + operation_tag.references = [self, queue] cpython.Py_INCREF(operation_tag) grpc_channel_watch_connectivity_state( self.c_channel, last_observed_state, deadline.c_time, diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index bbf84132993..59cfc1f452e 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -137,10 +137,14 @@ cdef class CompletionQueue: pass def __dealloc__(self): + cdef gpr_timespec c_deadline = gpr_inf_future(GPR_CLOCK_REALTIME) if self.c_completion_queue != NULL: - # Ensure shutdown, pump the queue + # Ensure shutdown if not self.is_shutting_down: - self.shutdown() + grpc_completion_queue_shutdown(self.c_completion_queue) + # Pump the queue while not self.is_shutdown: - self.poll() + event = grpc_completion_queue_next( + self.c_completion_queue, c_deadline, NULL) + self._interpret_event(event) grpc_completion_queue_destroy(self.c_completion_queue) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pxd.pxi index 9db49e4d307..a35eb5ea771 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pxd.pxi @@ -39,4 +39,5 @@ cdef class Server: cdef list references cdef list registered_completion_queues + cdef _c_shutdown(self, CompletionQueue queue, tag) cdef notify_shutdown_complete(self) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index 8b65935c3b9..60db4477988 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -102,6 +102,16 @@ cdef class Server: else: return grpc_server_add_insecure_http2_port(self.c_server, address) + cdef _c_shutdown(self, CompletionQueue queue, tag): + self.is_shutting_down = True + operation_tag = OperationTag(tag) + operation_tag.shutting_down_server = self + operation_tag.references.extend([self, queue]) + cpython.Py_INCREF(operation_tag) + grpc_server_shutdown_and_notify( + self.c_server, queue.c_completion_queue, + operation_tag) + def shutdown(self, CompletionQueue queue not None, tag): cdef OperationTag operation_tag if queue.is_shutting_down: @@ -113,14 +123,7 @@ cdef class Server: elif queue not in self.registered_completion_queues: raise ValueError("expected registered completion queue") else: - self.is_shutting_down = True - operation_tag = OperationTag(tag) - operation_tag.shutting_down_server = self - operation_tag.references.extend([self, queue]) - cpython.Py_INCREF(operation_tag) - grpc_server_shutdown_and_notify( - self.c_server, queue.c_completion_queue, - operation_tag) + self._c_shutdown(queue, tag) cdef notify_shutdown_complete(self): # called only by a completion queue on receiving our shutdown operation tag @@ -142,7 +145,7 @@ cdef class Server: pass elif not self.is_shutting_down: # the user didn't call shutdown - use our backup queue - self.shutdown(self.backup_shutdown_queue, None) + self._c_shutdown(self.backup_shutdown_queue, None) # and now we wait while not self.is_shutdown: self.backup_shutdown_queue.poll() From 3f1aa9b99ae6752cc22cab2707b1d8c3846f21be Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 29 Feb 2016 15:15:23 -0800 Subject: [PATCH 147/236] add copyright and cleanup python code --- test/distrib/csharp/run_distrib_test.bat | 28 ++++++++++++++++++++++++ tools/run_tests/distribtest_targets.py | 4 +++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/test/distrib/csharp/run_distrib_test.bat b/test/distrib/csharp/run_distrib_test.bat index 950894f6681..67bfc58ac8e 100644 --- a/test/distrib/csharp/run_distrib_test.bat +++ b/test/distrib/csharp/run_distrib_test.bat @@ -1,3 +1,31 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @rem enter this directory cd /d %~dp0 diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index fb951b68f95..34cc1cd7101 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -96,7 +96,7 @@ class CSharpDistribTest(object): return create_jobspec(self.name, ['test/distrib/csharp/run_distrib_test.sh'], environ={'EXTERNAL_GIT_ROOT': '../../..'}) - else: + elif self.platform == 'windows': if self.arch == 'x64': environ={'MSBUILD_EXTRA_ARGS': '/p:Platform=x64', 'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'} @@ -105,6 +105,8 @@ class CSharpDistribTest(object): return create_jobspec(self.name, ['test\\distrib\\csharp\\run_distrib_test.bat'], environ=environ) + else: + raise Exception("Not supported yet.") def __str__(self): return self.name From 3742703f5fa12ef68531504ea7035b597ffa2990 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 29 Feb 2016 15:51:22 -0800 Subject: [PATCH 148/236] fix sanity test --- .../src/csharp/build_packages.bat.template | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/templates/src/csharp/build_packages.bat.template b/templates/src/csharp/build_packages.bat.template index b855126ae60..32455683be8 100644 --- a/templates/src/csharp/build_packages.bat.template +++ b/templates/src/csharp/build_packages.bat.template @@ -21,6 +21,14 @@ xcopy /Y /I ..\..\architecture=x86,language=csharp,platform=macos\artifacts\* grpc.native.csharp\macosx_x86${"\\"} xcopy /Y /I ..\..\architecture=x64,language=csharp,platform=macos\artifacts\* grpc.native.csharp\macosx_x64${"\\"} + @rem Collect protoc artifacts built by the previous build step + xcopy /Y /I ..\..\architecture=x86,language=protoc,platform=windows\artifacts\* protoc_plugins\windows_x86${"\\"} + xcopy /Y /I ..\..\architecture=x64,language=protoc,platform=windows\artifacts\* protoc_plugins\windows_x64${"\\"} + xcopy /Y /I ..\..\architecture=x86,language=protoc,platform=linux\artifacts\* protoc_plugins\linux_x86${"\\"} + xcopy /Y /I ..\..\architecture=x64,language=protoc,platform=linux\artifacts\* protoc_plugins\linux_x64${"\\"} + xcopy /Y /I ..\..\architecture=x86,language=protoc,platform=macos\artifacts\* protoc_plugins\macosx_x86${"\\"} + xcopy /Y /I ..\..\architecture=x64,language=protoc,platform=macos\artifacts\* protoc_plugins\macosx_x64${"\\"} + @rem Fetch all dependencies %%NUGET% restore ..\..\vsprojects\grpc_csharp_ext.sln || goto :error %%NUGET% restore Grpc.sln || goto :error @@ -29,24 +37,19 @@ @call "%VS120COMNTOOLS%\..\..\vc\vcvarsall.bat" x86 - @rem We won't use the native libraries from this step, but without this Grpc.sln will fail. + @rem We won't use the native libraries from this step, but without this Grpc.sln will fail. msbuild ..\..\vsprojects\grpc_csharp_ext.sln /p:Configuration=Release /p:PlatformToolset=v120 || goto :error msbuild Grpc.sln /p:Configuration=ReleaseSigned || goto :error endlocal - @rem TODO(jtattermusch): re-enable protoc plugin building - @rem @call ..\..\vsprojects\build_plugins.bat || goto :error - %%NUGET% pack grpc.native.csharp\grpc.native.csharp.nuspec -Version %VERSION% || goto :error %%NUGET% pack Grpc.Auth\Grpc.Auth.nuspec -Symbols -Version %VERSION% || goto :error %%NUGET% pack Grpc.Core\Grpc.Core.nuspec -Symbols -Version %VERSION% || goto :error %%NUGET% pack Grpc.HealthCheck\Grpc.HealthCheck.nuspec -Symbols -Version %VERSION_WITH_BETA% -Properties ProtobufVersion=%PROTOBUF_VERSION% || goto :error %%NUGET% pack Grpc.nuspec -Version %VERSION% || goto :error - - @rem TODO(jtattermusch): re-enable building Grpc.Tools package - @rem %NUGET% pack Grpc.Tools.nuspec -Version %VERSION% || goto :error + %%NUGET% pack Grpc.Tools.nuspec -Version %VERSION% || goto :error @rem copy resulting nuget packages to artifacts directory xcopy /Y /I *.nupkg ..\..\artifacts${"\\"} From df1e05ad244efa8a7912aa164d72c132c0e3f442 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 29 Feb 2016 16:26:57 -0800 Subject: [PATCH 149/236] Simplify PHP installation --- package.xml | 23 ++++++++++++++++++---- src/php/README.md | 36 ---------------------------------- templates/package.xml.template | 23 ++++++++++++++++++---- 3 files changed, 38 insertions(+), 44 deletions(-) diff --git a/package.xml b/package.xml index 42a0361df0a..b6a59040188 100644 --- a/package.xml +++ b/package.xml @@ -10,11 +10,11 @@ grpc-packages@google.com yes - 2016-02-24 + 2016-03-01 - 0.8.0 - 0.8.0 + 0.9.0 + 0.9.0 beta @@ -22,7 +22,7 @@ BSD -- Simplify gRPC PHP installation #4517 +- Increase unit test code coverage #5225 @@ -969,5 +969,20 @@ Update to wrap gRPC C Core version 0.10.0 - Simplify gRPC PHP installation #4517 + + + 0.9.0 + 0.9.0 + + + beta + beta + + 2016-03-01 + BSD + +- Increase unit test code coverage #5225 + + diff --git a/src/php/README.md b/src/php/README.md index b1823b92261..b368482f068 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -33,45 +33,12 @@ $ sudo mv phpunit.phar /usr/local/bin/phpunit ## Quick Install -**Linux (Debian):** - -Add [Debian jessie-backports][] to your `sources.list` file. Example: - -```sh -echo "deb http://http.debian.net/debian jessie-backports main" | \ -sudo tee -a /etc/apt/sources.list -``` - -Install the gRPC Debian package - -```sh -sudo apt-get update -sudo apt-get install libgrpc-dev -``` - Install the gRPC PHP extension ```sh sudo pecl install grpc-beta ``` -**Mac OS X:** - -Install [homebrew][]. Example: - -```sh -ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -``` - -Install the gRPC core library and the PHP extension in one step - -```sh -$ curl -fsSL https://goo.gl/getgrpc | bash -s php -``` - -This will download and run the [gRPC install script][] and compile the gRPC PHP extension. - - ## Build from Source Clone this repository @@ -297,7 +264,4 @@ Connect to `localhost/math_client.php` in your browser, or run this from command $ curl localhost/math_client.php ``` -[homebrew]:http://brew.sh -[gRPC install script]:https://raw.githubusercontent.com/grpc/homebrew-grpc/master/scripts/install [Node]:https://github.com/grpc/grpc/tree/master/src/node/examples -[Debian jessie-backports]:http://backports.debian.org/Instructions/ diff --git a/templates/package.xml.template b/templates/package.xml.template index 067c8839d5a..bca20a31426 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -12,11 +12,11 @@ grpc-packages@google.com yes - 2016-02-24 + 2016-03-01 - 0.8.0 - 0.8.0 + 0.9.0 + 0.9.0 beta @@ -24,7 +24,7 @@ BSD - - Simplify gRPC PHP installation #4517 + - Increase unit test code coverage #5225 @@ -155,5 +155,20 @@ - Simplify gRPC PHP installation #4517 + + + 0.9.0 + 0.9.0 + + + beta + beta + + 2016-03-01 + BSD + + - Increase unit test code coverage #5225 + + From 68291709f407d9c9698e11b3f00be31e86b02324 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Tue, 1 Mar 2016 02:55:18 +0100 Subject: [PATCH 150/236] Fixing copyrights. --- src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi | 2 +- src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi | 2 +- src/python/grpcio/grpc/_cython/_cygrpc/server.pxd.pxi | 2 +- src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index f68dfd1b245..1f1833d5ecc 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index 59cfc1f452e..b299dfee22f 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pxd.pxi index a35eb5ea771..a344230be40 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pxd.pxi @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index 60db4477988..fe93da6c124 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From 8d543e8e309fb95cc423c19724197ffde2f5dd28 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 29 Feb 2016 18:22:25 -0800 Subject: [PATCH 151/236] Fix ResponseStreamServerCancelAfter test flake --- test/cpp/end2end/end2end_test.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index 42757974b22..dc2c4f6426f 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -437,9 +437,10 @@ class End2endServerTryCancelTest : public End2endTest { break; case CANCEL_AFTER_PROCESSING: - // Server cancelled after writing all messages. Client must have read - // all messages - EXPECT_EQ(num_msgs_read, kNumResponseStreamsMsgs); + // Even though the Server cancelled after writing all messages, the RPC + // may be cancelled before the Client got a chance to read all the + // messages. + EXPECT_LE(num_msgs_read, kNumResponseStreamsMsgs); break; default: { @@ -519,7 +520,11 @@ class End2endServerTryCancelTest : public End2endTest { case CANCEL_AFTER_PROCESSING: EXPECT_EQ(num_msgs_sent, num_messages); - EXPECT_EQ(num_msgs_read, num_msgs_sent); + + // The Server cancelled after reading the last message and after writing + // the message to the client. However, the RPC cancellation might have + // taken effect before the client actually read the response. + EXPECT_LE(num_msgs_read, num_msgs_sent); break; default: From 7fe08a23f17701266e69a3d3d3cab642482e7cdf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 29 Feb 2016 20:17:48 -0800 Subject: [PATCH 152/236] clang-format --- test/cpp/util/test_credentials_provider.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/cpp/util/test_credentials_provider.cc b/test/cpp/util/test_credentials_provider.cc index 65d32057673..e314fd6d75a 100644 --- a/test/cpp/util/test_credentials_provider.cc +++ b/test/cpp/util/test_credentials_provider.cc @@ -139,9 +139,7 @@ class DefaultCredentialsProvider : public CredentialsProvider { gpr_once g_once_init_provider = GPR_ONCE_INIT; CredentialsProvider* g_provider = nullptr; -void CreateDefaultProvider() { - g_provider = new DefaultCredentialsProvider; -} +void CreateDefaultProvider() { g_provider = new DefaultCredentialsProvider; } CredentialsProvider* GetProvider() { gpr_once_init(&g_once_init_provider, &CreateDefaultProvider); From 7c075b39538e047f4dd4d114b5cb7647ef4ce01a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 29 Feb 2016 21:46:06 -0800 Subject: [PATCH 153/236] Remove broken test This test is inherently flaky and I don't see any way to make it not so. Historically this test has not (in my memory) given any signal that something is actually broken. Let's save maintenance and just nuke it. --- test/core/iomgr/tcp_client_posix_test.c | 93 ------------------------- 1 file changed, 93 deletions(-) diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 1e6fa5d45a0..746dfd85be6 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -178,98 +178,6 @@ void test_fails(void) { grpc_exec_ctx_finish(&exec_ctx); } -void test_times_out(void) { - struct sockaddr_in addr; - socklen_t addr_len = sizeof(addr); - int svr_fd; -#define NUM_CLIENT_CONNECTS 100 - int client_fd[NUM_CLIENT_CONNECTS]; - int i; - int r; - int connections_complete_before; - gpr_timespec connect_deadline; - grpc_closure done; - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - - gpr_log(GPR_DEBUG, "test_times_out"); - - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - - /* create a dummy server */ - svr_fd = socket(AF_INET, SOCK_STREAM, 0); - GPR_ASSERT(svr_fd >= 0); - GPR_ASSERT(0 == bind(svr_fd, (struct sockaddr *)&addr, addr_len)); - GPR_ASSERT(0 == listen(svr_fd, 1)); - /* Get its address */ - GPR_ASSERT(getsockname(svr_fd, (struct sockaddr *)&addr, &addr_len) == 0); - - /* tie up the listen buffer, which is somewhat arbitrarily sized. */ - for (i = 0; i < NUM_CLIENT_CONNECTS; ++i) { - client_fd[i] = socket(AF_INET, SOCK_STREAM, 0); - grpc_set_socket_nonblocking(client_fd[i], 1); - do { - r = connect(client_fd[i], (struct sockaddr *)&addr, addr_len); - } while (r == -1 && errno == EINTR); - GPR_ASSERT(r < 0); - GPR_ASSERT(errno == EWOULDBLOCK || errno == EINPROGRESS); - } - - /* connect to dummy server address */ - - connect_deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1); - - gpr_mu_lock(g_mu); - connections_complete_before = g_connections_complete; - gpr_mu_unlock(g_mu); - - grpc_closure_init(&done, must_fail, NULL); - grpc_tcp_client_connect(&exec_ctx, &done, &g_connecting, g_pollset_set, - (struct sockaddr *)&addr, addr_len, connect_deadline); - - /* Make sure the event doesn't trigger early */ - gpr_mu_lock(g_mu); - for (;;) { - grpc_pollset_worker *worker = NULL; - gpr_timespec now = gpr_now(connect_deadline.clock_type); - gpr_timespec continue_verifying_time = - gpr_time_from_seconds(5, GPR_TIMESPAN); - gpr_timespec grace_time = gpr_time_from_seconds(3, GPR_TIMESPAN); - gpr_timespec finish_time = - gpr_time_add(connect_deadline, continue_verifying_time); - gpr_timespec restart_verifying_time = - gpr_time_add(connect_deadline, grace_time); - int is_after_deadline = gpr_time_cmp(now, connect_deadline) > 0; - if (gpr_time_cmp(now, finish_time) > 0) { - break; - } - gpr_log(GPR_DEBUG, "now=%lld.%09d connect_deadline=%lld.%09d", - (long long)now.tv_sec, (int)now.tv_nsec, - (long long)connect_deadline.tv_sec, (int)connect_deadline.tv_nsec); - if (is_after_deadline && gpr_time_cmp(now, restart_verifying_time) <= 0) { - /* allow some slack before insisting that things be done */ - } else { - GPR_ASSERT(g_connections_complete == - connections_complete_before + is_after_deadline); - } - gpr_timespec polling_deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10); - if (!grpc_timer_check(&exec_ctx, now, &polling_deadline)) { - grpc_pollset_work(&exec_ctx, g_pollset, &worker, now, polling_deadline); - } - gpr_mu_unlock(g_mu); - grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(g_mu); - } - gpr_mu_unlock(g_mu); - - grpc_exec_ctx_finish(&exec_ctx); - - close(svr_fd); - for (i = 0; i < NUM_CLIENT_CONNECTS; ++i) { - close(client_fd[i]); - } -} - static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) { grpc_pollset_destroy(p); } @@ -287,7 +195,6 @@ int main(int argc, char **argv) { test_succeeds(); gpr_log(GPR_ERROR, "End of first test"); test_fails(); - test_times_out(); grpc_pollset_set_destroy(g_pollset_set); grpc_closure_init(&destroyed, destroy_pollset, g_pollset); grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); From 9e5a05af8a322991ce4f909ce84a58889b1f151b Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 1 Mar 2016 09:40:12 -0800 Subject: [PATCH 154/236] Fix clang format issue --- test/cpp/util/test_credentials_provider.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/cpp/util/test_credentials_provider.cc b/test/cpp/util/test_credentials_provider.cc index 65d32057673..e314fd6d75a 100644 --- a/test/cpp/util/test_credentials_provider.cc +++ b/test/cpp/util/test_credentials_provider.cc @@ -139,9 +139,7 @@ class DefaultCredentialsProvider : public CredentialsProvider { gpr_once g_once_init_provider = GPR_ONCE_INIT; CredentialsProvider* g_provider = nullptr; -void CreateDefaultProvider() { - g_provider = new DefaultCredentialsProvider; -} +void CreateDefaultProvider() { g_provider = new DefaultCredentialsProvider; } CredentialsProvider* GetProvider() { gpr_once_init(&g_once_init_provider, &CreateDefaultProvider); From a91a5db3630e0bdb673a5888de043cd56007a0ee Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 1 Mar 2016 11:19:16 -0800 Subject: [PATCH 155/236] sync php with core version; --- package.xml | 8 ++++---- templates/package.xml.template | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.xml b/package.xml index b6a59040188..e65ab73b54a 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2016-03-01 - 0.9.0 - 0.9.0 + 0.14.0 + 0.14.0 beta @@ -971,8 +971,8 @@ Update to wrap gRPC C Core version 0.10.0 - 0.9.0 - 0.9.0 + 0.14.0 + 0.14.0 beta diff --git a/templates/package.xml.template b/templates/package.xml.template index bca20a31426..d309bfddbc3 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -15,8 +15,8 @@ 2016-03-01 - 0.9.0 - 0.9.0 + 0.14.0 + 0.14.0 beta @@ -157,8 +157,8 @@ - 0.9.0 - 0.9.0 + 0.14.0 + 0.14.0 beta From 494f3128331adcf601903b00ea62b767d99f84c2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 1 Mar 2016 13:55:42 -0800 Subject: [PATCH 156/236] fix reporting for multiple test runs --- tools/run_tests/jobset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index adf178bb3c3..a3b246dc084 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -384,7 +384,8 @@ class Jobset(object): self._travis, self._add_env) self._running.add(job) - self.resultset[job.GetSpec().shortname] = [] + if not self.resultset.has_key(job.GetSpec().shortname): + self.resultset[job.GetSpec().shortname] = [] return True def reap(self): From 7d91dc3181e94bef7cf85bf8885cb8c3f3d0ff62 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 1 Mar 2016 12:33:20 -0800 Subject: [PATCH 157/236] fix #4427 --- src/csharp/Grpc.Core/Internal/AsyncCallServer.cs | 10 ---------- .../Grpc.IntegrationTesting/InteropClientServerTest.cs | 2 -- 2 files changed, 12 deletions(-) diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs index b72cbd795f6..9380c0d0ea9 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs @@ -193,16 +193,6 @@ namespace Grpc.Core.Internal lock (myLock) { finished = true; - - if (cancelled) - { - // Once we cancel, we don't have to care that much - // about reads and writes. - - // TODO(jtattermusch): is this still necessary? - Cancel(); - } - ReleaseResourcesIfPossible(); } // TODO(jtattermusch): handle error diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs index 18168f99704..5facb87971e 100644 --- a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs @@ -140,14 +140,12 @@ namespace Grpc.IntegrationTesting } [Test] - [Ignore("TODO: see #4427")] public async Task StatusCodeAndMessage() { await InteropClient.RunStatusCodeAndMessageAsync(client); } [Test] - [Ignore("TODO: see #4427")] public void UnimplementedMethod() { InteropClient.RunUnimplementedMethod(UnimplementedService.NewClient(channel)); From 06c98fa49a6a92d230932b79a153de03a2f16db9 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 1 Mar 2016 14:45:03 -0800 Subject: [PATCH 158/236] fix copyright --- src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs index 5facb87971e..0d12c4168c9 100644 --- a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs @@ -1,6 +1,6 @@ #region Copyright notice and license -// Copyright 2015, Google Inc. +// Copyright 2015-2016, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without From 6c016efa34113916810cf16a0a0981c637204f50 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 1 Mar 2016 16:27:53 -0800 Subject: [PATCH 159/236] ServerTryCancel was not actually respecting the API since it could be an arbitrary amount of time between when the cancel is tried and actually observable. --- test/cpp/end2end/test_service_impl.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc index 66d11d0dfce..7c3e514effa 100644 --- a/test/cpp/end2end/test_service_impl.cc +++ b/test/cpp/end2end/test_service_impl.cc @@ -326,7 +326,11 @@ void TestServiceImpl::ServerTryCancel(ServerContext* context) { EXPECT_FALSE(context->IsCancelled()); context->TryCancel(); gpr_log(GPR_INFO, "Server called TryCancel() to cancel the request"); - EXPECT_TRUE(context->IsCancelled()); + // Now wait until it's really canceled + while (!context->IsCancelled()) { + gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_micros(1000, GPR_TIMESPAN))); + } } } // namespace testing From 9c691c59da630832e8b5962ff2e51e8e890e359e Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 1 Mar 2016 16:45:40 -0800 Subject: [PATCH 160/236] updated templates/README.md --- templates/README.md | 103 ++++++++++++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/templates/README.md b/templates/README.md index 6740972cfb6..a27c44da3c1 100644 --- a/templates/README.md +++ b/templates/README.md @@ -6,12 +6,12 @@ was going to single handedly cover all of our usage cases. So instead we decided to work the following way: -* A build.json file at the root is the source of truth for listing all of the -target and files needed to build grpc and its tests, as well as basic system -dependencies description. +* A `build.yaml` file at the root is the source of truth for listing all the +targets and files needed to build grpc and its tests, as well as a basic system +for dependency description. * Each project file (Makefile, Visual Studio project files, Bazel's BUILD) is -a plain-text template that uses the build.json file to generate the final +a plain-text template that uses the `build.yaml` file to generate the final output file. This way we can maintain as many project system as we see fit, without having @@ -20,64 +20,77 @@ Only the structure of the project file is relevant to the template. The actual list of source code and targets isn't. We currently have template files for GNU Make, Visual Studio 2010 to 2015, -and Bazel. In the future, we would like to expand to generating gyp or cmake +and [Bazel](http://bazel.io). In the future, we would like to expand to +generating [gyp](https://gyp.gsrc.io/) or [cmake](https://cmake.org) project files (or potentially both), XCode project files, and an Android.mk file to be able to compile gRPC using Android's NDK. We'll gladly accept contribution that'd create additional project files using that system. -# Structure of build.json +# Structure of `build.yaml` -The build.json file has the following structure: +The `build.yaml` file has the following structure: ``` -{ - "settings": { ... }, # global settings, such as version number - "filegroups": [ ... ], # groups of file that is automatically expanded - "libs": [ ... ], # list of libraries to build - "targets": [ ... ], # list of targets to build -} +settings: # global settings, such as version number + ... +filegroups: # groups of files that are automatically expanded + ... +libs: # list of libraries to build + ... +target: # list of targets to build + ... ``` The `filegroups` are helpful to re-use a subset of files in multiple targets. One `filegroups` entry has the following structure: ``` -{ - "name": "arbitrary string", # the name of the filegroup - "public_headers": [ ... ], # list of public headers defined in that filegroup - "headers": [ ... ], # list of headers defined in that filegroup - "src": [ ... ], # list of source files defined in that filegroup -} +- name: "arbitrary string", # the name of the filegroup + public_headers: # list of public headers defined in that filegroup + - ... + headers: # list of headers defined in that filegroup + - ... + src: # list of source files defined in that filegroup + - ... ``` -The `libs` array contains the list of all the libraries we describe. Some may be +The `libs` collection contains the list of all the libraries we describe. Some may be helper libraries for the tests. Some may be installable libraries. Some may be helper libraries for installable binaries. The `targets` array contains the list of all the binary targets we describe. Some may be installable binaries. -One `libs` or `targets` entry has the following structure: +One `libs` or `targets` entry has the following structure (see below for +details): ``` -{ - "name": "arbitrary string", # the name of the library - "build": "build type", # in which situation we want that library to be - # built and potentially installed - "language": "...", # the language tag; "c" or "c++" - "public_headers": [ ... ], # list of public headers to install - "headers": [ ... ], # list of headers used by that target - "src": [ ... ], # list of files to compile - "secure": "...", # "yes", "no" or "check" - "baselib": boolean, # this is a low level library that has system - # dependencies - "vs_project_guid: "...", # Visual Studio's unique guid for that project - "filegroups": [ ... ], # list of filegroups to merge to that project - # note that this will be expanded automatically - "deps": [ ... ], # list of libraries this target depends on -} +name: "arbitrary string", # the name of the library +build: "build type", # in which situation we want that library to be + # built and potentially installed (see below). +language: "...", # the language tag; "c" or "c++" +public_headers: # list of public headers to install +headers: # list of headers used by that target +src: # list of files to compile +secure: boolean, # see below +baselib: boolean, # this is a low level library that has system + # dependencies +vs_project_guid: '{...}', # Visual Studio's unique guid for that project +filegroups: # list of filegroups to merge to that project + # note that this will be expanded automatically +deps: # list of libraries this target depends on +deps_linkage: "..." # "static" or "dynamic". Used by the Makefile only to + # determine the way dependencies are linkned. Defaults + # to "dynamic". +dll: "..." # see below. +dll_def: "..." # Visual Studio's dll definition file. +vs_props: # List of property sheets to attach to that project. +vs_config_type: "..." # DynamicLibrary/StaticLibrary. Used only when + # creating a library. Specifies if we're building a + # static library or a dll. Use in conjunction with `dll_def`. +vs_packages: # List of nuget packages this project depends on. ``` ## The `"build"` tag @@ -86,8 +99,9 @@ Currently, the "`build`" tag have these meanings: * `"all"`: library to build on `"make all"`, and install on the system. * `"protoc"`: a protoc plugin to build on `"make all"` and install on the system. -* `"priviate"`: a library to only build for tests. +* `"private"`: a library to only build for tests. * `"test"`: a test binary to run on `"make test"`. +* `"tool"`: a binary to be built upon `"make tools"`. All of the targets should always be present in the generated project file, if possible and applicable. But the build tag is what should group the targets @@ -111,6 +125,18 @@ should merge OpenSSL, protobuf or zlib inside that library. That effect depends on the `"language"` tag. OpenSSL and zlib are for `"c"` libraries, while protobuf is for `"c++"` ones. +## The `"dll"` tag + +Used only by Visual Studio's project files. "true" means the project will be +built with both static and dynamic runtimes. "false" means it'll only be built +with static runtime. "only" means it'll only be built with the dll runtime. + +## The `"dll_def"` tag + +Specifies the visual studio's dll definition file. When creating a DLL, you +sometimes (not always) need a def file (see grpc.def). + + # The template system We're currently using the [mako templates](http://www.makotemplates.org/) @@ -137,3 +163,4 @@ The structure of a plugin is simple. The plugin must defined the function `mako_plugin` that takes a Python dictionary. That dictionary represents the current state of the build.json contents. The plugin can alter it to whatever feature it needs to add. + From 3db3c63a41ec8245f8b4fcd274086eae6db04543 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 1 Mar 2016 17:03:18 -0800 Subject: [PATCH 161/236] Addressed comments --- templates/README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/templates/README.md b/templates/README.md index a27c44da3c1..283a0f5e7be 100644 --- a/templates/README.md +++ b/templates/README.md @@ -11,19 +11,19 @@ targets and files needed to build grpc and its tests, as well as a basic system for dependency description. * Each project file (Makefile, Visual Studio project files, Bazel's BUILD) is -a plain-text template that uses the `build.yaml` file to generate the final -output file. +a [YAML](http://yaml.org) file used by the `build.yaml` file to generate the +final output file. This way we can maintain as many project system as we see fit, without having to manually maintain them when we add or remove new code to the repository. Only the structure of the project file is relevant to the template. The actual list of source code and targets isn't. -We currently have template files for GNU Make, Visual Studio 2010 to 2015, -and [Bazel](http://bazel.io). In the future, we would like to expand to -generating [gyp](https://gyp.gsrc.io/) or [cmake](https://cmake.org) -project files (or potentially both), XCode project files, and an Android.mk -file to be able to compile gRPC using Android's NDK. +We currently have template files for GNU Make, Visual Studio 2013, +[Bazel](http://bazel.io) and [gyp](https://gyp.gsrc.io/). In the future, we +would like to expand to also generate [cmake](https://cmake.org) +project files, XCode project files, and an Android.mk file allowing to compile +gRPC using Android's NDK. We'll gladly accept contribution that'd create additional project files using that system. @@ -163,4 +163,3 @@ The structure of a plugin is simple. The plugin must defined the function `mako_plugin` that takes a Python dictionary. That dictionary represents the current state of the build.json contents. The plugin can alter it to whatever feature it needs to add. - From a66225957532c1847bc215aa9dbca537ae943526 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 1 Mar 2016 17:16:25 -0800 Subject: [PATCH 162/236] Added comment for gyp --- templates/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/README.md b/templates/README.md index 283a0f5e7be..eedc6e9c09f 100644 --- a/templates/README.md +++ b/templates/README.md @@ -20,7 +20,8 @@ Only the structure of the project file is relevant to the template. The actual list of source code and targets isn't. We currently have template files for GNU Make, Visual Studio 2013, -[Bazel](http://bazel.io) and [gyp](https://gyp.gsrc.io/). In the future, we +[Bazel](http://bazel.io) and [gyp](https://gyp.gsrc.io/) (albeit only for +Node.js). In the future, we would like to expand to also generate [cmake](https://cmake.org) project files, XCode project files, and an Android.mk file allowing to compile gRPC using Android's NDK. From 8e19f61d6230bfb938a82532d5da4c79824501df Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 1 Mar 2016 21:41:13 -0800 Subject: [PATCH 163/236] Fix esan detected race in subchannel state --- src/core/client_config/subchannel.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index bec06bf4144..d91dd116b8c 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -395,7 +395,6 @@ void grpc_subchannel_notify_on_state_change( grpc_exec_ctx *exec_ctx, grpc_subchannel *c, grpc_pollset_set *interested_parties, grpc_connectivity_state *state, grpc_closure *notify) { - int do_connect = 0; external_state_watcher *w; if (state == NULL) { @@ -425,17 +424,13 @@ void grpc_subchannel_notify_on_state_change( w->next->prev = w->prev->next = w; if (grpc_connectivity_state_notify_on_state_change( exec_ctx, &c->state_tracker, state, &w->closure)) { - do_connect = 1; c->connecting = 1; /* released by connection */ GRPC_SUBCHANNEL_WEAK_REF(c, "connecting"); + start_connect(exec_ctx, c); } gpr_mu_unlock(&c->mu); } - - if (do_connect) { - start_connect(exec_ctx, c); - } } void grpc_connected_subchannel_process_transport_op( @@ -635,11 +630,12 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, bool iomgr_success) { if (c->disconnected) { iomgr_success = 0; } - gpr_mu_unlock(&c->mu); if (iomgr_success) { update_reconnect_parameters(c); continue_connect(exec_ctx, c); + gpr_mu_unlock(&c->mu); } else { + gpr_mu_unlock(&c->mu); GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting"); } } From 18720fff2f2bd79542753837401fd24359c3d5f8 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 2 Mar 2016 11:59:42 -0800 Subject: [PATCH 164/236] Maintain correct queue invariants against core --- .../_cython/_cygrpc/completion_queue.pxd.pxi | 7 ++-- .../_cython/_cygrpc/completion_queue.pyx.pxi | 40 +++++++++---------- .../grpcio/grpc/_cython/_cygrpc/grpc.pxi | 2 + 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi index 757f1245e85..305475c0060 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -31,8 +31,9 @@ cdef class CompletionQueue: cdef grpc_completion_queue *c_completion_queue - cdef object poll_condition - cdef bint is_polling + cdef object pluck_condition + cdef int num_plucking + cdef int num_polling cdef bint is_shutting_down cdef bint is_shutdown diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index b299dfee22f..c139147114b 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -39,8 +39,9 @@ cdef class CompletionQueue: self.c_completion_queue = grpc_completion_queue_create(NULL) self.is_shutting_down = False self.is_shutdown = False - self.poll_condition = threading.Condition() - self.is_polling = False + self.pluck_condition = threading.Condition() + self.num_plucking = 0 + self.num_polling = 0 cdef _interpret_event(self, grpc_event event): cdef OperationTag tag = None @@ -87,19 +88,15 @@ cdef class CompletionQueue: c_deadline = deadline.c_time cdef grpc_event event - # Poll within a critical section - # TODO(atash) consider making queue polling contention a hard error to - # enable easier bug discovery - with self.poll_condition: - while self.is_polling: - self.poll_condition.wait(float(deadline) - time.time()) - self.is_polling = True + # Poll within a critical section to detect contention + with self.pluck_condition: + assert self.num_plucking == 0, 'cannot simultaneously pluck and poll' + self.num_polling += 1 with nogil: event = grpc_completion_queue_next( self.c_completion_queue, c_deadline, NULL) - with self.poll_condition: - self.is_polling = False - self.poll_condition.notify() + with self.pluck_condition: + self.num_polling -= 1 return self._interpret_event(event) def pluck(self, OperationTag tag, Timespec deadline=None): @@ -111,19 +108,18 @@ cdef class CompletionQueue: c_deadline = deadline.c_time cdef grpc_event event - # Poll within a critical section - # TODO(atash) consider making queue polling contention a hard error to - # enable easier bug discovery - with self.poll_condition: - while self.is_polling: - self.poll_condition.wait(float(deadline) - time.time()) - self.is_polling = True + # Pluck within a critical section to detect contention + with self.pluck_condition: + assert self.num_polling == 0, 'cannot simultaneously pluck and poll' + assert self.num_plucking < GRPC_MAX_COMPLETION_QUEUE_PLUCKERS, ( + 'cannot pluck more than {} times simultaneously'.format( + GRPC_MAX_COMPLETION_QUEUE_PLUCKERS)) + self.num_plucking += 1 with nogil: event = grpc_completion_queue_pluck( self.c_completion_queue, tag, c_deadline, NULL) - with self.poll_condition: - self.is_polling = False - self.poll_condition.notify() + with self.pluck_condition: + self.num_plucking -= 1 return self._interpret_event(event) def shutdown(self): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 800d0ea2f6f..dbf0045710e 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -138,6 +138,8 @@ cdef extern from "grpc/_cython/loader.h": const int GRPC_WRITE_NO_COMPRESS const int GRPC_WRITE_USED_MASK + const int GRPC_MAX_COMPLETION_QUEUE_PLUCKERS + ctypedef struct grpc_completion_queue: # We don't care about the internals (and in fact don't know them) pass From bd50f305a3af3c45aadb2b7eb7f4752475f6d820 Mon Sep 17 00:00:00 2001 From: makdharma Date: Wed, 2 Mar 2016 13:49:51 -0800 Subject: [PATCH 165/236] Update reconnect_interop_client.cc --- test/cpp/interop/reconnect_interop_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1f6b352db17..3ad733e1114 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -30,7 +30,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ - +// Test description at doc/connection-backoff-interop-test-description.md #include #include From 38a560b6ba45bb8db16fac609a72a6c235469841 Mon Sep 17 00:00:00 2001 From: makdharma Date: Wed, 2 Mar 2016 13:50:58 -0800 Subject: [PATCH 166/236] Update reconnect_interop_server.cc --- test/cpp/interop/reconnect_interop_server.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 3602b8c2b05..785f9c7ad5f 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -30,6 +30,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ + +// Test description at doc/connection-backoff-interop-test-description.md #include #include From 7bee07555af388cd080fd8c15e7c8ee4725974f9 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 2 Mar 2016 14:38:07 -0800 Subject: [PATCH 167/236] Add troubleshooting section to package description --- src/python/grpcio/README.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index 698c760ebe2..f3e962c1971 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -38,3 +38,17 @@ package named `python-dev`). Note that `$REPO_ROOT` can be assigned to whatever directory name floats your fancy. + +Troubleshooting +~~~~~~~~~~~~~~~ + +Help, I ... + +* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install + grpc!** + + This is likely because :code:`pip` doesn't own the offending dependency, + which in turn is likely because your operating system's package manager owns + it. You'll need to force the installation of the dependency: + + :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY` From 42dab364a337666003a17d72b4dcad0c4568587a Mon Sep 17 00:00:00 2001 From: Greg Haines Date: Wed, 2 Mar 2016 15:04:41 -0800 Subject: [PATCH 168/236] Pass a non-infinite deadline to grpc_completion_queue_next() to prevent queues from blocking indefinitely in poll(). --- .../GRPCClient/private/GRPCCompletionQueue.h | 7 ++++++ .../GRPCClient/private/GRPCCompletionQueue.m | 25 +++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h index fe3b8f39d12..03fd2e0d955 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h @@ -36,6 +36,8 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); +extern const int64_t kGRPCCompletionQueueDefaultTimeoutSecs; + /** * This class lets one more easily use |grpc_completion_queue|. To use it, pass the value of the * |unmanagedQueue| property of an instance of this class to |grpc_channel_create_call|. Then for @@ -49,6 +51,11 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); */ @interface GRPCCompletionQueue : NSObject @property(nonatomic, readonly) grpc_completion_queue *unmanagedQueue; +@property(nonatomic, readonly) int64_t timeoutSecs; + (instancetype)completionQueue; + +- (instancetype)init; +- (instancetype)initWithTimeout:(int64_t)timeoutSecs NS_DESIGNATED_INITIALIZER; + @end diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index ea2b01ee1d7..8163236cc4a 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -35,6 +35,9 @@ #import + +const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; + @implementation GRPCCompletionQueue + (instancetype)completionQueue { @@ -42,8 +45,13 @@ } - (instancetype)init { + return [self initWithTimeout:kGRPCCompletionQueueDefaultTimeoutSecs]; +} + +- (instancetype)initWithTimeout:(int64_t)timeoutSecs { if ((self = [super init])) { _unmanagedQueue = grpc_completion_queue_create(NULL); + _timeoutSecs = timeoutSecs; // This is for the following block to capture the pointer by value (instead // of retaining self and doing self->_unmanagedQueue). This is essential @@ -52,6 +60,7 @@ // anymore (i.e. on self dealloc). So the block would never end if it // retained self. grpc_completion_queue *unmanagedQueue = _unmanagedQueue; + int64_t lTimeoutSecs = _timeoutSecs; // Start a loop on a concurrent queue to read events from the completion // queue and dispatch each. @@ -61,22 +70,24 @@ gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ + gpr_timespec deadline = gpr_time_from_seconds(lTimeoutSecs, GPR_CLOCK_REALTIME); while (YES) { - // The following call blocks until an event is available. - grpc_event event = grpc_completion_queue_next(unmanagedQueue, - gpr_inf_future(GPR_CLOCK_REALTIME), - NULL); + // The following call blocks until an event is available or the deadline elapses. + grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); GRPCQueueCompletionHandler handler; switch (event.type) { - case GRPC_OP_COMPLETE: + case GRPC_OP_COMPLETE: // Falling through deliberately + case GRPC_QUEUE_TIMEOUT: handler = (__bridge_transfer GRPCQueueCompletionHandler)event.tag; - handler(event.success); + if (handler) { + handler(event.success); + } break; case GRPC_QUEUE_SHUTDOWN: grpc_completion_queue_destroy(unmanagedQueue); return; default: - [NSException raise:@"Unrecognized completion type" format:@""]; + [NSException raise:@"Unrecognized completion type" format:@"type=%d", event.type]; } }; }); From c6611efb67e8f5eae0451963e98df1890dcdb3d5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 2 Mar 2016 17:43:09 -0800 Subject: [PATCH 169/236] Revert "Update reconnect_interop_client.cc" --- test/cpp/interop/reconnect_interop_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 3ad733e1114..1f6b352db17 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -30,7 +30,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ -// Test description at doc/connection-backoff-interop-test-description.md + #include #include From 072ebaa1537673e100b0a027d3935252da14fccb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 1 Mar 2016 18:33:12 -0800 Subject: [PATCH 170/236] make python test suites run in parallel --- src/python/grpcio/tests/_runner.py | 11 +++- src/python/grpcio/tests/tests.json | 62 +++++++++++++++++++ .../grpcio/tests/unit/_sanity/__init__.py | 30 +++++++++ .../grpcio/tests/unit/_sanity/_sanity_test.py | 53 ++++++++++++++++ tools/run_tests/build_python.sh | 2 + tools/run_tests/run_python.sh | 7 ++- tools/run_tests/run_tests.py | 26 +++++--- 7 files changed, 181 insertions(+), 10 deletions(-) create mode 100644 src/python/grpcio/tests/tests.json create mode 100644 src/python/grpcio/tests/unit/_sanity/__init__.py create mode 100644 src/python/grpcio/tests/unit/_sanity/_sanity_test.py diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py index 4f1ddb57fcf..38a5432e791 100644 --- a/src/python/grpcio/tests/_runner.py +++ b/src/python/grpcio/tests/_runner.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -143,10 +143,17 @@ class Runner(object): def run(self, suite): """See setuptools' test_runner setup argument for information.""" + # only run test cases with id starting with given prefix + testcase_filter = os.getenv('GPRC_PYTHON_TESTRUNNER_FILTER') + filtered_cases = [] + for case in _loader.iterate_suite_cases(suite): + if not testcase_filter or case.id().startswith(testcase_filter): + filtered_cases.append(case) + # Ensure that every test case has no collision with any other test case in # the augmented results. augmented_cases = [AugmentedCase(case, uuid.uuid4()) - for case in _loader.iterate_suite_cases(suite)] + for case in filtered_cases] case_id_by_case = dict((augmented_case.case, augmented_case.id) for augmented_case in augmented_cases) result_out = StringIO.StringIO() diff --git a/src/python/grpcio/tests/tests.json b/src/python/grpcio/tests/tests.json new file mode 100644 index 00000000000..388d040d5ca --- /dev/null +++ b/src/python/grpcio/tests/tests.json @@ -0,0 +1,62 @@ +[ + "_base_interface_test.AsyncEasyTest", + "_base_interface_test.AsyncPeasyTest", + "_base_interface_test.SyncEasyTest", + "_base_interface_test.SyncPeasyTest", + "_beta_features_test.BetaFeaturesTest", + "_beta_features_test.ContextManagementAndLifecycleTest", + "_channel_test.ChannelTest", + "_connectivity_channel_test.ChannelConnectivityTest", + "_core_over_links_base_interface_test.AsyncEasyTest", + "_core_over_links_base_interface_test.AsyncPeasyTest", + "_core_over_links_base_interface_test.SyncEasyTest", + "_core_over_links_base_interface_test.SyncPeasyTest", + "_crust_over_core_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_over_links_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_over_links_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", + "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", + "_crust_over_core_over_links_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", + "_face_interface_test.DynamicInvokerBlockingInvocationInlineServiceTest", + "_face_interface_test.DynamicInvokerEventInvocationSynchronousEventServiceTest", + "_face_interface_test.DynamicInvokerFutureInvocationAsynchronousEventServiceTest", + "_face_interface_test.GenericInvokerBlockingInvocationInlineServiceTest", + "_face_interface_test.GenericInvokerEventInvocationSynchronousEventServiceTest", + "_face_interface_test.GenericInvokerFutureInvocationAsynchronousEventServiceTest", + "_face_interface_test.MultiCallableInvokerBlockingInvocationInlineServiceTest", + "_face_interface_test.MultiCallableInvokerEventInvocationSynchronousEventServiceTest", + "_face_interface_test.MultiCallableInvokerFutureInvocationAsynchronousEventServiceTest", + "_implementations_test.ChannelCredentialsTest", + "_insecure_interop_test.InsecureInteropTest", + "_intermediary_low_test.CancellationTest", + "_intermediary_low_test.EchoTest", + "_intermediary_low_test.ExpirationTest", + "_intermediary_low_test.LonelyClientTest", + "_later_test.LaterTest", + "_logging_pool_test.LoggingPoolTest", + "_lonely_invocation_link_test.LonelyInvocationLinkTest", + "_low_test.HangingServerShutdown", + "_low_test.InsecureServerInsecureClient", + "_not_found_test.NotFoundTest", + "_sanity_test.Sanity", + "_secure_interop_test.SecureInteropTest", + "_transmission_test.RoundTripTest", + "_transmission_test.TransmissionTest", + "_utilities_test.ChannelConnectivityTest", + "beta_python_plugin_test.PythonPluginTest", + "cygrpc_test.InsecureServerInsecureClient", + "cygrpc_test.SecureServerSecureClient", + "cygrpc_test.TypeSmokeTest" +] \ No newline at end of file diff --git a/src/python/grpcio/tests/unit/_sanity/__init__.py b/src/python/grpcio/tests/unit/_sanity/__init__.py new file mode 100644 index 00000000000..2f88fa04122 --- /dev/null +++ b/src/python/grpcio/tests/unit/_sanity/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + diff --git a/src/python/grpcio/tests/unit/_sanity/_sanity_test.py b/src/python/grpcio/tests/unit/_sanity/_sanity_test.py new file mode 100644 index 00000000000..0a5a715c0e1 --- /dev/null +++ b/src/python/grpcio/tests/unit/_sanity/_sanity_test.py @@ -0,0 +1,53 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +import unittest + +import tests + + +class Sanity(unittest.TestCase): + + def testTestsJsonUpToDate(self): + """Autodiscovers all test suites and checks that tests.json is up to date""" + loader = tests.Loader() + loader.loadTestsFromNames(['tests']) + test_suite_names = [ + test_case_class.id().rsplit('.', 1)[0] + for test_case_class in tests._loader.iterate_suite_cases(loader.suite)] + test_suite_names = sorted(set(test_suite_names)) + + with open('src/python/grpcio/tests/tests.json') as tests_json_file: + tests_json = json.load(tests_json_file) + self.assertListEqual(test_suite_names, tests_json) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index e0fcbb602d4..365abb4d863 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -45,3 +45,5 @@ export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 tox --notest $ROOT/.tox/py27/bin/python $ROOT/setup.py build +$ROOT/.tox/py27/bin/python $ROOT/setup.py build_py +$ROOT/.tox/py27/bin/python $ROOT/setup.py gather --test diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index ffe9c12af1a..beb747a6169 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -42,7 +42,12 @@ export LDFLAGS="-L$ROOT/libs/$CONFIG" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 -tox +if [ "$CONFIG" = "gcov" ] +then + tox +else + $ROOT/.tox/py27/bin/python $ROOT/setup.py test +fi mkdir -p $ROOT/reports rm -rf $ROOT/reports/python-coverage diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 08a5ff0e8fa..c55d1fbe633 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -353,15 +353,27 @@ class PythonLanguage(object): _check_compiler(self.args.compiler, ['default']) def test_specs(self): + # load list of known test suites + with open('src/python/grpcio/tests/tests.json') as tests_json_file: + tests_json = json.load(tests_json_file) environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) environment['PYVER'] = '2.7' - return [self.config.job_spec( - ['tools/run_tests/run_python.sh'], - None, - environ=environment, - shortname='py.test', - timeout_seconds=15*60 - )] + if self.config.build_config != 'gcov': + return [self.config.job_spec( + ['tools/run_tests/run_python.sh'], + None, + environ=dict(environment.items() + + [('GPRC_PYTHON_TESTRUNNER_FILTER', suite_name)]), + shortname='py.test.%s' % suite_name, + timeout_seconds=5*60) + for suite_name in tests_json] + else: + return [self.config.job_spec(['tools/run_tests/run_python.sh'], + None, + environ=environment, + shortname='py.test.coverage', + timeout_seconds=15*60)] + def pre_build_steps(self): return [] From 7d757ca29dc413777a9648cc2db9246235438d43 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 2 Mar 2016 19:44:16 -0800 Subject: [PATCH 171/236] Ensure that no #includes are inside of a namespace. --- test/cpp/qps/limit_cores.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/limit_cores.cc b/test/cpp/qps/limit_cores.cc index fad9a323afd..28264d031a8 100644 --- a/test/cpp/qps/limit_cores.cc +++ b/test/cpp/qps/limit_cores.cc @@ -37,14 +37,16 @@ #include #include -namespace grpc { -namespace testing { #ifdef GPR_CPU_LINUX #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include + +namespace grpc { +namespace testing { + int LimitCores(const int* cores, int cores_size) { const int num_cores = gpr_cpu_num_cores(); int cores_set = 0; From 2a8c28037061b94c02eb05fb532e268709af88e5 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 2 Mar 2016 20:46:54 -0800 Subject: [PATCH 172/236] sanity --- test/cpp/interop/reconnect_interop_client.cc | 12 ++++++------ test/cpp/interop/reconnect_interop_server.cc | 12 ++++++------ test/cpp/qps/limit_cores.cc | 1 - 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1f6b352db17..c668edaceb0 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,16 +34,16 @@ #include #include -#include -#include #include #include #include -#include "test/cpp/util/create_test_channel.h" -#include "test/cpp/util/test_config.h" -#include "src/proto/grpc/testing/test.grpc.pb.h" +#include +#include #include "src/proto/grpc/testing/empty.grpc.pb.h" #include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/test.grpc.pb.h" +#include "test/cpp/util/create_test_channel.h" +#include "test/cpp/util/test_config.h" DEFINE_int32(server_control_port, 0, "Server port for control rpcs."); DEFINE_int32(server_retry_port, 0, "Server port for testing reconnection."); diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 785f9c7ad5f..1f9147d0efa 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -30,7 +30,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ - + // Test description at doc/connection-backoff-interop-test-description.md #include @@ -42,17 +42,17 @@ #include #include -#include -#include #include #include #include +#include +#include -#include "test/core/util/reconnect_server.h" -#include "test/cpp/util/test_config.h" -#include "src/proto/grpc/testing/test.grpc.pb.h" #include "src/proto/grpc/testing/empty.grpc.pb.h" #include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/test.grpc.pb.h" +#include "test/core/util/reconnect_server.h" +#include "test/cpp/util/test_config.h" DEFINE_int32(control_port, 0, "Server port for controlling the server."); DEFINE_int32(retry_port, 0, diff --git a/test/cpp/qps/limit_cores.cc b/test/cpp/qps/limit_cores.cc index 28264d031a8..1fb2d628f6d 100644 --- a/test/cpp/qps/limit_cores.cc +++ b/test/cpp/qps/limit_cores.cc @@ -37,7 +37,6 @@ #include #include - #ifdef GPR_CPU_LINUX #ifndef _GNU_SOURCE #define _GNU_SOURCE From 98990726a02829c29674f143ab6e02df5415514d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 2 Mar 2016 22:04:00 -0800 Subject: [PATCH 173/236] Revert "Update reconnect_interop_server.cc" --- test/cpp/interop/reconnect_interop_server.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 785f9c7ad5f..3602b8c2b05 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -30,8 +30,6 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ - -// Test description at doc/connection-backoff-interop-test-description.md #include #include From 0cb803d9ca4286601e9e6a3240cfa3488b662b7c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 2 Mar 2016 22:17:24 -0800 Subject: [PATCH 174/236] Always ref writable streams We suffered a bug whereby doing a follow-up write to another write could resurrect a deleted stream, causing all sorts of crash. Fix: when a stream becomes writable (vs when we start writing) take a ref on the stream, and only relinquish it once we're done writing. --- grpc.def | 1 + include/grpc/impl/codegen/sync.h | 4 ++ src/core/iomgr/iomgr.c | 16 +++++++ src/core/iomgr/iomgr_internal.h | 6 ++- src/core/support/sync.c | 7 ++- src/core/transport/chttp2/internal.h | 18 +++++--- src/core/transport/chttp2/parsing.c | 6 +-- src/core/transport/chttp2/stream_lists.c | 38 ++++++++-------- src/core/transport/chttp2/writing.c | 37 ++++++---------- src/core/transport/chttp2_transport.c | 43 +++++++++++-------- src/core/transport/metadata.c | 8 ++++ src/core/transport/transport.c | 2 +- .../grpcio/grpc/_cython/imports.generated.c | 2 + .../grpcio/grpc/_cython/imports.generated.h | 3 ++ src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 + src/ruby/ext/grpc/rb_grpc_imports.generated.h | 3 ++ test/cpp/interop/reconnect_interop_client.cc | 2 +- 17 files changed, 127 insertions(+), 71 deletions(-) diff --git a/grpc.def b/grpc.def index bd0bc85a7c3..f81aa1b05a6 100644 --- a/grpc.def +++ b/grpc.def @@ -182,6 +182,7 @@ EXPORTS gpr_event_wait gpr_ref_init gpr_ref + gpr_ref_non_zero gpr_refn gpr_unref gpr_stats_init diff --git a/include/grpc/impl/codegen/sync.h b/include/grpc/impl/codegen/sync.h index d2f19d37d66..6fd7d64b299 100644 --- a/include/grpc/impl/codegen/sync.h +++ b/include/grpc/impl/codegen/sync.h @@ -182,6 +182,10 @@ GPRAPI void gpr_ref_init(gpr_refcount *r, int n); /* Increment the reference count *r. Requires *r initialized. */ GPRAPI void gpr_ref(gpr_refcount *r); +/* Increment the reference count *r. Requires *r initialized. + Crashes if refcount is zero */ +GPRAPI void gpr_ref_non_zero(gpr_refcount *r); + /* Increment the reference count *r by n. Requires *r initialized, n > 0. */ GPRAPI void gpr_refn(gpr_refcount *r, int n); diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c index 04580150f3a..9c89c2c08a4 100644 --- a/src/core/iomgr/iomgr.c +++ b/src/core/iomgr/iomgr.c @@ -41,9 +41,11 @@ #include #include #include +#include #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/timer.h" +#include "src/core/support/env.h" #include "src/core/support/string.h" static gpr_mu g_mu; @@ -116,6 +118,9 @@ void grpc_iomgr_shutdown(void) { "memory leaks are likely", count_objects()); dump_objects("LEAKED"); + if (grpc_iomgr_abort_on_leaks()) { + abort(); + } } break; } @@ -154,3 +159,14 @@ void grpc_iomgr_unregister_object(grpc_iomgr_object *obj) { gpr_mu_unlock(&g_mu); gpr_free(obj->name); } + +bool grpc_iomgr_abort_on_leaks(void) { + char *env = gpr_getenv("GRPC_ABORT_ON_LEAKS"); + if (env == NULL) return false; + static const char *truthy[] = {"yes", "Yes", "YES", "true", + "True", "TRUE", "1"}; + for (size_t i = 0; i < GPR_ARRAY_SIZE(truthy); i++) { + if (0 == strcmp(env, truthy[i])) return true; + } + return false; +} diff --git a/src/core/iomgr/iomgr_internal.h b/src/core/iomgr/iomgr_internal.h index e372c18e8a0..ac2c46ebe62 100644 --- a/src/core/iomgr/iomgr_internal.h +++ b/src/core/iomgr/iomgr_internal.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,6 +34,8 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H #define GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H +#include + #include "src/core/iomgr/iomgr.h" #include @@ -55,4 +57,6 @@ void grpc_iomgr_platform_flush(void); /** tear down all platform specific global iomgr structures */ void grpc_iomgr_platform_shutdown(void); +bool grpc_iomgr_abort_on_leaks(void); + #endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H */ diff --git a/src/core/support/sync.c b/src/core/support/sync.c index d368422d9ea..69e3e39c5ce 100644 --- a/src/core/support/sync.c +++ b/src/core/support/sync.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -98,6 +98,11 @@ void gpr_ref_init(gpr_refcount *r, int n) { gpr_atm_rel_store(&r->count, n); } void gpr_ref(gpr_refcount *r) { gpr_atm_no_barrier_fetch_add(&r->count, 1); } +void gpr_ref_non_zero(gpr_refcount *r) { + gpr_atm prior = gpr_atm_no_barrier_fetch_add(&r->count, 1); + GPR_ASSERT(prior > 0); +} + void gpr_refn(gpr_refcount *r, int n) { gpr_atm_no_barrier_fetch_add(&r->count, n); } diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h index d76d31be23f..891aad6ef28 100644 --- a/src/core/transport/chttp2/internal.h +++ b/src/core/transport/chttp2/internal.h @@ -417,7 +417,7 @@ typedef struct { /** HTTP2 stream id for this stream, or zero if one has not been assigned */ uint32_t id; uint8_t fetching; - uint8_t sent_initial_metadata; + bool sent_initial_metadata; uint8_t sent_message; uint8_t sent_trailing_metadata; uint8_t read_closed; @@ -509,7 +509,7 @@ void grpc_chttp2_publish_reads(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *global, grpc_chttp2_transport_parsing *parsing); -void grpc_chttp2_list_add_writable_stream( +bool grpc_chttp2_list_add_writable_stream( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global); /** Get a writable stream @@ -519,14 +519,13 @@ int grpc_chttp2_list_pop_writable_stream( grpc_chttp2_transport_writing *transport_writing, grpc_chttp2_stream_global **stream_global, grpc_chttp2_stream_writing **stream_writing); -void grpc_chttp2_list_remove_writable_stream( +bool grpc_chttp2_list_remove_writable_stream( grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); + grpc_chttp2_stream_global *stream_global) GRPC_MUST_USE_RESULT; -/* returns 1 if stream added, 0 if it was already present */ -int grpc_chttp2_list_add_writing_stream( +void grpc_chttp2_list_add_writing_stream( grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing) GRPC_MUST_USE_RESULT; + grpc_chttp2_stream_writing *stream_writing); int grpc_chttp2_list_have_writing_streams( grpc_chttp2_transport_writing *transport_writing); int grpc_chttp2_list_pop_writing_stream( @@ -770,4 +769,9 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *parsing, const uint8_t *opaque_8bytes); +/** add a ref to the stream and add it to the writable list; + ref will be dropped in writing.c */ +void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global); + #endif diff --git a/src/core/transport/chttp2/parsing.c b/src/core/transport/chttp2/parsing.c index 8fdebd7f139..0516f39fa9e 100644 --- a/src/core/transport/chttp2/parsing.c +++ b/src/core/transport/chttp2/parsing.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -149,7 +149,7 @@ void grpc_chttp2_publish_reads( if (was_zero && !is_zero) { while (grpc_chttp2_list_pop_stalled_by_transport(transport_global, &stream_global)) { - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -178,7 +178,7 @@ void grpc_chttp2_publish_reads( outgoing_window); is_zero = stream_global->outgoing_window <= 0; if (was_zero && !is_zero) { - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } stream_global->max_recv_bytes -= (uint32_t)GPR_MIN( diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/transport/chttp2/stream_lists.c index b284c788183..60fe735cfce 100644 --- a/src/core/transport/chttp2/stream_lists.c +++ b/src/core/transport/chttp2/stream_lists.c @@ -100,11 +100,14 @@ static void stream_list_remove(grpc_chttp2_transport *t, grpc_chttp2_stream *s, } } -static void stream_list_maybe_remove(grpc_chttp2_transport *t, +static bool stream_list_maybe_remove(grpc_chttp2_transport *t, grpc_chttp2_stream *s, grpc_chttp2_stream_list_id id) { if (s->included[id]) { stream_list_remove(t, s, id); + return true; + } else { + return false; } } @@ -125,23 +128,24 @@ static void stream_list_add_tail(grpc_chttp2_transport *t, s->included[id] = 1; } -static int stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, - grpc_chttp2_stream_list_id id) { +static bool stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, + grpc_chttp2_stream_list_id id) { if (s->included[id]) { - return 0; + return false; } stream_list_add_tail(t, s, id); - return 1; + return true; } /* wrappers for specializations */ -void grpc_chttp2_list_add_writable_stream( +bool grpc_chttp2_list_add_writable_stream( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { GPR_ASSERT(stream_global->id != 0); - stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), GRPC_CHTTP2_LIST_WRITABLE); + return stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), + STREAM_FROM_GLOBAL(stream_global), + GRPC_CHTTP2_LIST_WRITABLE); } int grpc_chttp2_list_pop_writable_stream( @@ -159,20 +163,20 @@ int grpc_chttp2_list_pop_writable_stream( return r; } -void grpc_chttp2_list_remove_writable_stream( +bool grpc_chttp2_list_remove_writable_stream( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { - stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_WRITABLE); + return stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global), + STREAM_FROM_GLOBAL(stream_global), + GRPC_CHTTP2_LIST_WRITABLE); } -int grpc_chttp2_list_add_writing_stream( +void grpc_chttp2_list_add_writing_stream( grpc_chttp2_transport_writing *transport_writing, grpc_chttp2_stream_writing *stream_writing) { - return stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), - STREAM_FROM_WRITING(stream_writing), - GRPC_CHTTP2_LIST_WRITING); + GPR_ASSERT(stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), + STREAM_FROM_WRITING(stream_writing), + GRPC_CHTTP2_LIST_WRITING)); } int grpc_chttp2_list_have_writing_streams( @@ -332,7 +336,7 @@ void grpc_chttp2_list_flush_writing_stalled_by_transport( while (stream_list_pop(transport, &stream, GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT)) { if (is_window_available) { - grpc_chttp2_list_add_writable_stream(&transport->global, &stream->global); + grpc_chttp2_become_writable(&transport->global, &stream->global); } else { grpc_chttp2_list_add_stalled_by_transport(transport_writing, &stream->writing); diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c index 356fd8174a7..107725cbc79 100644 --- a/src/core/transport/chttp2/writing.c +++ b/src/core/transport/chttp2/writing.c @@ -83,7 +83,8 @@ int grpc_chttp2_unlocking_check_writes( (according to available window sizes) and add to the output buffer */ while (grpc_chttp2_list_pop_writable_stream( transport_global, transport_writing, &stream_global, &stream_writing)) { - uint8_t sent_initial_metadata; + bool sent_initial_metadata = stream_writing->sent_initial_metadata; + bool become_writable = false; stream_writing->id = stream_global->id; stream_writing->read_closed = stream_global->read_closed; @@ -92,16 +93,12 @@ int grpc_chttp2_unlocking_check_writes( outgoing_window, stream_global, outgoing_window); - sent_initial_metadata = stream_writing->sent_initial_metadata; if (!sent_initial_metadata && stream_global->send_initial_metadata) { stream_writing->send_initial_metadata = stream_global->send_initial_metadata; stream_global->send_initial_metadata = NULL; - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); - } - sent_initial_metadata = 1; + become_writable = true; + sent_initial_metadata = true; } if (sent_initial_metadata) { if (stream_global->send_message != NULL) { @@ -128,10 +125,7 @@ int grpc_chttp2_unlocking_check_writes( stream_writing->flow_controlled_buffer.length > 0) && stream_writing->outgoing_window > 0) { if (transport_writing->outgoing_window > 0) { - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); - } + become_writable = true; } else { grpc_chttp2_list_add_stalled_by_transport(transport_writing, stream_writing); @@ -141,10 +135,7 @@ int grpc_chttp2_unlocking_check_writes( stream_writing->send_trailing_metadata = stream_global->send_trailing_metadata; stream_global->send_trailing_metadata = NULL; - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); - } + become_writable = true; } } @@ -153,10 +144,13 @@ int grpc_chttp2_unlocking_check_writes( GRPC_CHTTP2_FLOW_MOVE_STREAM("write", transport_global, stream_writing, announce_window, stream_global, unannounced_incoming_window_for_writing); - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); - } + become_writable = true; + } + + if (become_writable) { + grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing); + } else { + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing"); } } @@ -310,10 +304,7 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx, (stream_writing->send_message && !stream_writing->fetching)) && stream_writing->outgoing_window > 0) { if (transport_writing->outgoing_window > 0) { - if (grpc_chttp2_list_add_writing_stream(transport_writing, - stream_writing)) { - /* do nothing - already reffed */ - } + grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing); } else { grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing, stream_writing); diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index b9f511e9460..a1ca78d4c4f 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -142,7 +142,7 @@ static void incoming_byte_stream_update_flow_control( static void fail_pending_writes(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream_global *stream_global); -/* +/******************************************************************************* * CONSTRUCTION/DESTRUCTION/REFCOUNTING */ @@ -521,7 +521,6 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->global.id) == NULL); } - grpc_chttp2_list_remove_writable_stream(&t->global, &s->global); grpc_chttp2_list_remove_unannounced_incoming_window_available(&t->global, &s->global); grpc_chttp2_list_remove_stalled_by_transport(&t->global, &s->global); @@ -583,7 +582,7 @@ grpc_chttp2_stream_parsing *grpc_chttp2_parsing_accept_stream( return &accepting->parsing; } -/* +/******************************************************************************* * LOCK MANAGEMENT */ @@ -611,10 +610,18 @@ static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { GPR_TIMER_END("unlock", 0); } -/* +/******************************************************************************* * OUTPUT PROCESSING */ +void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global) { + if (!TRANSPORT_FROM_GLOBAL(transport_global)->closed && + grpc_chttp2_list_add_writable_stream(transport_global, stream_global)) { + GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); + } +} + static void push_setting(grpc_chttp2_transport *t, grpc_chttp2_setting_id id, uint32_t value) { const grpc_chttp2_setting_parameters *sp = @@ -732,7 +739,7 @@ static void maybe_start_some_streams( stream_global->id, STREAM_FROM_GLOBAL(stream_global)); stream_global->in_stream_map = 1; transport_global->concurrent_stream_count++; - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } /* cancel out streams that will never be started */ while (transport_global->next_stream_id >= MAX_CLIENT_STREAM_ID && @@ -821,7 +828,7 @@ static void perform_stream_op_locked( maybe_start_some_streams(exec_ctx, transport_global); } else { GPR_ASSERT(stream_global->id != 0); - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } else { grpc_chttp2_complete_closure_step( @@ -838,7 +845,7 @@ static void perform_stream_op_locked( exec_ctx, &stream_global->send_message_finished, 0); } else if (stream_global->id != 0) { stream_global->send_message = op->send_message; - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -858,7 +865,7 @@ static void perform_stream_op_locked( } else if (stream_global->id != 0) { /* TODO(ctiller): check if there's flow control for any outstanding bytes before going writable */ - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -999,7 +1006,7 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, } } -/* +/******************************************************************************* * INPUT PROCESSING */ @@ -1064,7 +1071,6 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (!s) { s = grpc_chttp2_stream_map_delete(&t->new_stream_map, id); } - grpc_chttp2_list_remove_writable_stream(&t->global, &s->global); GPR_ASSERT(s); s->global.in_stream_map = 0; if (t->parsing.incoming_stream == &s->parsing) { @@ -1080,6 +1086,9 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (grpc_chttp2_unregister_stream(t, s) && t->global.sent_goaway) { close_transport_locked(exec_ctx, t); } + if (grpc_chttp2_list_remove_writable_stream(&t->global, &s->global)) { + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &s->global, "chttp2_writing"); + } new_stream_count = grpc_chttp2_stream_map_size(&t->parsing_stream_map) + grpc_chttp2_stream_map_size(&t->new_stream_map); @@ -1331,7 +1340,7 @@ static void update_global_window(void *args, uint32_t id, void *stream) { is_zero = stream_global->outgoing_window <= 0; if (was_zero && !is_zero) { - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -1426,7 +1435,7 @@ static void recv_data(grpc_exec_ctx *exec_ctx, void *tp, bool success) { GPR_TIMER_END("recv_data", 0); } -/* +/******************************************************************************* * CALLBACK LOOP */ @@ -1440,7 +1449,7 @@ static void connectivity_state_set( state, reason); } -/* +/******************************************************************************* * POLLSET STUFF */ @@ -1468,7 +1477,7 @@ static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt, unlock(exec_ctx, t); } -/* +/******************************************************************************* * BYTE STREAM */ @@ -1508,7 +1517,7 @@ static void incoming_byte_stream_update_flow_control( add_max_recv_bytes); grpc_chttp2_list_add_unannounced_incoming_window_available(transport_global, stream_global); - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + grpc_chttp2_become_writable(transport_global, stream_global); } } @@ -1623,7 +1632,7 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( return incoming_byte_stream; } -/* +/******************************************************************************* * TRACING */ @@ -1709,7 +1718,7 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase, gpr_free(prefix); } -/* +/******************************************************************************* * INTEGRATION GLUE */ diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c index 14912af7df1..807ae071a30 100644 --- a/src/core/transport/metadata.c +++ b/src/core/transport/metadata.c @@ -43,11 +43,13 @@ #include #include #include + #include "src/core/profiling/timers.h" #include "src/core/support/murmur_hash.h" #include "src/core/support/string.h" #include "src/core/transport/chttp2/bin_encoder.h" #include "src/core/transport/static_metadata.h" +#include "src/core/iomgr/iomgr_internal.h" /* There are two kinds of mdelem and mdstr instances. * Static instances are declared in static_metadata.{h,c} and @@ -227,6 +229,9 @@ void grpc_mdctx_global_shutdown(void) { if (shard->count != 0) { gpr_log(GPR_DEBUG, "WARNING: %d metadata elements were leaked", shard->count); + if (grpc_iomgr_abort_on_leaks()) { + abort(); + } } gpr_free(shard->elems); } @@ -237,6 +242,9 @@ void grpc_mdctx_global_shutdown(void) { if (shard->count != 0) { gpr_log(GPR_DEBUG, "WARNING: %d metadata strings were leaked", shard->count); + if (grpc_iomgr_abort_on_leaks()) { + abort(); + } } gpr_free(shard->strs); } diff --git a/src/core/transport/transport.c b/src/core/transport/transport.c index 6e154b629ab..3b555fa9338 100644 --- a/src/core/transport/transport.c +++ b/src/core/transport/transport.c @@ -45,7 +45,7 @@ void grpc_stream_ref(grpc_stream_refcount *refcount, const char *reason) { #else void grpc_stream_ref(grpc_stream_refcount *refcount) { #endif - gpr_ref(&refcount->refs); + gpr_ref_non_zero(&refcount->refs); } #ifdef GRPC_STREAM_REFCOUNT_DEBUG diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c index 4b1860ce8c9..8bd6ae6372b 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.c +++ b/src/python/grpcio/grpc/_cython/imports.generated.c @@ -220,6 +220,7 @@ gpr_event_get_type gpr_event_get_import; gpr_event_wait_type gpr_event_wait_import; gpr_ref_init_type gpr_ref_init_import; gpr_ref_type gpr_ref_import; +gpr_ref_non_zero_type gpr_ref_non_zero_import; gpr_refn_type gpr_refn_import; gpr_unref_type gpr_unref_import; gpr_stats_init_type gpr_stats_init_import; @@ -485,6 +486,7 @@ void pygrpc_load_imports(HMODULE library) { gpr_event_wait_import = (gpr_event_wait_type) GetProcAddress(library, "gpr_event_wait"); gpr_ref_init_import = (gpr_ref_init_type) GetProcAddress(library, "gpr_ref_init"); gpr_ref_import = (gpr_ref_type) GetProcAddress(library, "gpr_ref"); + gpr_ref_non_zero_import = (gpr_ref_non_zero_type) GetProcAddress(library, "gpr_ref_non_zero"); gpr_refn_import = (gpr_refn_type) GetProcAddress(library, "gpr_refn"); gpr_unref_import = (gpr_unref_type) GetProcAddress(library, "gpr_unref"); gpr_stats_init_import = (gpr_stats_init_type) GetProcAddress(library, "gpr_stats_init"); diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h index a395dce7d6a..b70dcccd17a 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.h +++ b/src/python/grpcio/grpc/_cython/imports.generated.h @@ -610,6 +610,9 @@ extern gpr_ref_init_type gpr_ref_init_import; typedef void(*gpr_ref_type)(gpr_refcount *r); extern gpr_ref_type gpr_ref_import; #define gpr_ref gpr_ref_import +typedef void(*gpr_ref_non_zero_type)(gpr_refcount *r); +extern gpr_ref_non_zero_type gpr_ref_non_zero_import; +#define gpr_ref_non_zero gpr_ref_non_zero_import typedef void(*gpr_refn_type)(gpr_refcount *r, int n); extern gpr_refn_type gpr_refn_import; #define gpr_refn gpr_refn_import diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 1af34d97fbc..56db4ec686b 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -220,6 +220,7 @@ gpr_event_get_type gpr_event_get_import; gpr_event_wait_type gpr_event_wait_import; gpr_ref_init_type gpr_ref_init_import; gpr_ref_type gpr_ref_import; +gpr_ref_non_zero_type gpr_ref_non_zero_import; gpr_refn_type gpr_refn_import; gpr_unref_type gpr_unref_import; gpr_stats_init_type gpr_stats_init_import; @@ -481,6 +482,7 @@ void grpc_rb_load_imports(HMODULE library) { gpr_event_wait_import = (gpr_event_wait_type) GetProcAddress(library, "gpr_event_wait"); gpr_ref_init_import = (gpr_ref_init_type) GetProcAddress(library, "gpr_ref_init"); gpr_ref_import = (gpr_ref_type) GetProcAddress(library, "gpr_ref"); + gpr_ref_non_zero_import = (gpr_ref_non_zero_type) GetProcAddress(library, "gpr_ref_non_zero"); gpr_refn_import = (gpr_refn_type) GetProcAddress(library, "gpr_refn"); gpr_unref_import = (gpr_unref_type) GetProcAddress(library, "gpr_unref"); gpr_stats_init_import = (gpr_stats_init_type) GetProcAddress(library, "gpr_stats_init"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 38aabfaca8f..b972f60fc35 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -610,6 +610,9 @@ extern gpr_ref_init_type gpr_ref_init_import; typedef void(*gpr_ref_type)(gpr_refcount *r); extern gpr_ref_type gpr_ref_import; #define gpr_ref gpr_ref_import +typedef void(*gpr_ref_non_zero_type)(gpr_refcount *r); +extern gpr_ref_non_zero_type gpr_ref_non_zero_import; +#define gpr_ref_non_zero gpr_ref_non_zero_import typedef void(*gpr_refn_type)(gpr_refcount *r, int n); extern gpr_refn_type gpr_refn_import; #define gpr_refn gpr_refn_import diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1f6b352db17..79a60cc8602 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 91bd627de4fb1ab1b8c31a915d4a91ccdcf72bbe Mon Sep 17 00:00:00 2001 From: Greg Haines Date: Wed, 2 Mar 2016 23:05:46 -0800 Subject: [PATCH 175/236] Feedback from @jcanizales and @vjpai --- .../GRPCClient/private/GRPCCompletionQueue.m | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index 8163236cc4a..ffbb14374d1 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -60,7 +60,6 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; // anymore (i.e. on self dealloc). So the block would never end if it // retained self. grpc_completion_queue *unmanagedQueue = _unmanagedQueue; - int64_t lTimeoutSecs = _timeoutSecs; // Start a loop on a concurrent queue to read events from the completion // queue and dispatch each. @@ -70,18 +69,18 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ - gpr_timespec deadline = gpr_time_from_seconds(lTimeoutSecs, GPR_CLOCK_REALTIME); + gpr_timespec deadline = gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); while (YES) { // The following call blocks until an event is available or the deadline elapses. grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); GRPCQueueCompletionHandler handler; switch (event.type) { - case GRPC_OP_COMPLETE: // Falling through deliberately - case GRPC_QUEUE_TIMEOUT: + case GRPC_OP_COMPLETE: handler = (__bridge_transfer GRPCQueueCompletionHandler)event.tag; - if (handler) { - handler(event.success); - } + handler(event.success); + break; + case GRPC_QUEUE_TIMEOUT: + // Nothing to do here break; case GRPC_QUEUE_SHUTDOWN: grpc_completion_queue_destroy(unmanagedQueue); From d7f12e30a0050b3fecb72fe7ad558b4d837e140f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 3 Mar 2016 10:08:31 -0800 Subject: [PATCH 176/236] Fix accept_stream being called post-channel deletion - Have the server clear the accept_stream callback prior to destroying the channel (required a small transport op protocol change) - Have the transport not enact transport ops until parsing is completed (prevents accept_stream from disappearing mid-parse) --- src/core/channel/client_channel.c | 2 +- src/core/channel/client_uchannel.c | 2 +- src/core/surface/server.c | 14 +++++++-- src/core/transport/chttp2/internal.h | 3 ++ src/core/transport/chttp2_transport.c | 43 +++++++++++++++++++-------- src/core/transport/transport.h | 6 ++-- 6 files changed, 51 insertions(+), 19 deletions(-) diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index eeac3c146c3..d4ba9508185 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -251,7 +251,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL); - GPR_ASSERT(op->set_accept_stream == NULL); + GPR_ASSERT(op->set_accept_stream == false); if (op->bind_pollset != NULL) { grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, op->bind_pollset); diff --git a/src/core/channel/client_uchannel.c b/src/core/channel/client_uchannel.c index b1e7155773f..83fcc3a87f5 100644 --- a/src/core/channel/client_uchannel.c +++ b/src/core/channel/client_uchannel.c @@ -107,7 +107,7 @@ static void cuc_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL); - GPR_ASSERT(op->set_accept_stream == NULL); + GPR_ASSERT(op->set_accept_stream == false); GPR_ASSERT(op->bind_pollset == NULL); if (op->on_connectivity_state_change != NULL) { diff --git a/src/core/surface/server.c b/src/core/surface/server.c index fb5e0d4b9e7..5b13d4ba526 100644 --- a/src/core/surface/server.c +++ b/src/core/surface/server.c @@ -407,8 +407,15 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand) { maybe_finish_shutdown(exec_ctx, chand->server); chand->finish_destroy_channel_closure.cb = finish_destroy_channel; chand->finish_destroy_channel_closure.cb_arg = chand; - grpc_exec_ctx_enqueue(exec_ctx, &chand->finish_destroy_channel_closure, true, - NULL); + + grpc_transport_op op; + memset(&op, 0, sizeof(op)); + op.set_accept_stream = true; + op.on_consumed = &chand->finish_destroy_channel_closure; + grpc_channel_next_op(exec_ctx, + grpc_channel_stack_element( + grpc_channel_get_channel_stack(chand->channel), 0), + &op); } static void finish_start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_server *server, @@ -971,7 +978,8 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s, GRPC_CHANNEL_INTERNAL_REF(channel, "connectivity"); memset(&op, 0, sizeof(op)); - op.set_accept_stream = accept_stream; + op.set_accept_stream = true; + op.set_accept_stream_fn = accept_stream; op.set_accept_stream_user_data = chand; op.on_connectivity_state_change = &chand->channel_connectivity_changed; op.connectivity_state = &chand->connectivity_state; diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h index 891aad6ef28..b720d1ab3e5 100644 --- a/src/core/transport/chttp2/internal.h +++ b/src/core/transport/chttp2/internal.h @@ -358,6 +358,9 @@ struct grpc_chttp2_transport { /** connectivity tracking */ grpc_connectivity_state_tracker state_tracker; } channel_callback; + + /** Transport op to be applied post-parsing */ + grpc_transport_op *post_parsing_op; }; typedef struct { diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index a1ca78d4c4f..369ff0ad7f6 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -951,12 +951,10 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, unlock(exec_ctx, t); } -static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_transport_op *op) { - grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; - int close_transport = 0; - - lock(t); +static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_transport_op *op) { + bool close_transport = false; grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL); @@ -975,8 +973,8 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, close_transport = !grpc_chttp2_has_streams(t); } - if (op->set_accept_stream != NULL) { - t->channel_callback.accept_stream = op->set_accept_stream; + if (op->set_accept_stream) { + t->channel_callback.accept_stream = op->set_accept_stream_fn; t->channel_callback.accept_stream_user_data = op->set_accept_stream_user_data; } @@ -997,15 +995,29 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, close_transport_locked(exec_ctx, t); } - unlock(exec_ctx, t); - if (close_transport) { - lock(t); close_transport_locked(exec_ctx, t); - unlock(exec_ctx, t); } } +static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_transport_op *op) { + grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; + + lock(t); + + /* Let's be overly cautious: don't change any state while we're parsing */ + if (t->parsing_active) { + GPR_ASSERT(t->post_parsing_op == NULL); + t->post_parsing_op = gpr_malloc(sizeof(*op)); + memcpy(t->post_parsing_op, op, sizeof(*op)); + } else { + perform_transport_op_locked(exec_ctx, t, op); + } + + unlock(exec_ctx, t); +} + /******************************************************************************* * INPUT PROCESSING */ @@ -1401,6 +1413,13 @@ static void recv_data(grpc_exec_ctx *exec_ctx, void *tp, bool success) { /* handle higher level things */ grpc_chttp2_publish_reads(exec_ctx, transport_global, transport_parsing); t->parsing_active = 0; + /* handle delayed transport ops (if there is one) */ + if (t->post_parsing_op) { + grpc_transport_op *op = t->post_parsing_op; + t->post_parsing_op = NULL; + perform_transport_op_locked(exec_ctx, t, op); + gpr_free(op); + } /* if a stream is in the stream map, and gets cancelled, we need to ensure * we are not parsing before continuing the cancellation to keep things in * a sane state */ diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h index 8902c5d2f65..7a007ef777b 100644 --- a/src/core/transport/transport.h +++ b/src/core/transport/transport.h @@ -139,8 +139,10 @@ typedef struct grpc_transport_op { gpr_slice *goaway_message; /** set the callback for accepting new streams; this is a permanent callback, unlike the other one-shot closures */ - void (*set_accept_stream)(grpc_exec_ctx *exec_ctx, void *user_data, - grpc_transport *transport, const void *server_data); + bool set_accept_stream; + void (*set_accept_stream_fn)(grpc_exec_ctx *exec_ctx, void *user_data, + grpc_transport *transport, + const void *server_data); void *set_accept_stream_user_data; /** add this transport to a pollset */ grpc_pollset *bind_pollset; From 093ff5db8b4d9fa9965b505fc8e33316c3daf2bb Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 3 Mar 2016 12:18:27 -0800 Subject: [PATCH 177/236] Fix copyright --- test/cpp/interop/reconnect_interop_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1f6b352db17..79a60cc8602 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 66a6d014a4b2afec45906ff803e31dd89e82e22a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 3 Mar 2016 12:35:08 -0800 Subject: [PATCH 178/236] run build_ext in build_python step --- tools/run_tests/build_python.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 365abb4d863..f120fc7ed69 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -46,4 +46,5 @@ tox --notest $ROOT/.tox/py27/bin/python $ROOT/setup.py build $ROOT/.tox/py27/bin/python $ROOT/setup.py build_py +$ROOT/.tox/py27/bin/python $ROOT/setup.py build_ext --inplace $ROOT/.tox/py27/bin/python $ROOT/setup.py gather --test From 13ee2f2df3ded81d096a1440a0f273eeece6a2cc Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 3 Mar 2016 13:57:32 -0800 Subject: [PATCH 179/236] Properly integrate async API with server-side cancellations. There is a comment above IsCancelled that says when it is ok to use this. --- .../grpc++/impl/codegen/completion_queue.h | 1 + include/grpc++/impl/codegen/server_context.h | 3 + src/cpp/server/server_context.cc | 26 ++- test/cpp/end2end/async_end2end_test.cc | 218 +++++++++++++----- 4 files changed, 187 insertions(+), 61 deletions(-) diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h index 102831e1c9b..928ab2db317 100644 --- a/include/grpc++/impl/codegen/completion_queue.h +++ b/include/grpc++/impl/codegen/completion_queue.h @@ -184,6 +184,7 @@ class CompletionQueue : private GrpcLibrary { bool Pluck(CompletionQueueTag* tag); /// Performs a single polling pluck on \a tag. + /// \warning Must not be mixed with calls to \a Next. void TryPluck(CompletionQueueTag* tag); grpc_completion_queue* cq_; // owned diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h index ad08b8210d6..91ebe574b14 100644 --- a/include/grpc++/impl/codegen/server_context.h +++ b/include/grpc++/impl/codegen/server_context.h @@ -103,6 +103,9 @@ class ServerContext { void AddInitialMetadata(const grpc::string& key, const grpc::string& value); void AddTrailingMetadata(const grpc::string& key, const grpc::string& value); + // IsCancelled is always safe to call when using sync API + // When using async API, it is only safe to call IsCancelled after + // the AsyncNotifyWhenDone tag has been delivered bool IsCancelled() const; // Cancel the Call from the server. This is a best-effort API and depending on diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index e205a1969b3..eb49b210379 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -62,7 +62,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void FillOps(grpc_op* ops, size_t* nops) GRPC_OVERRIDE; bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE; - bool CheckCancelled(CompletionQueue* cq); + bool CheckCancelled(CompletionQueue* cq) { + cq->TryPluck(this); + return CheckCancelledNoPluck(); + } + bool CheckCancelledAsync() { return CheckCancelledNoPluck(); } void set_tag(void* tag) { has_tag_ = true; @@ -72,6 +76,11 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void Unref(); private: + bool CheckCancelledNoPluck() { + grpc::lock_guard g(mu_); + return finalized_ ? (cancelled_ != 0) : false; + } + bool has_tag_; void* tag_; grpc::mutex mu_; @@ -88,12 +97,6 @@ void ServerContext::CompletionOp::Unref() { } } -bool ServerContext::CompletionOp::CheckCancelled(CompletionQueue* cq) { - cq->TryPluck(this); - grpc::lock_guard g(mu_); - return finalized_ ? cancelled_ != 0 : false; -} - void ServerContext::CompletionOp::FillOps(grpc_op* ops, size_t* nops) { ops->op = GRPC_OP_RECV_CLOSE_ON_SERVER; ops->data.recv_close_on_server.cancelled = &cancelled_; @@ -182,7 +185,14 @@ void ServerContext::TryCancel() const { } bool ServerContext::IsCancelled() const { - return completion_op_ && completion_op_->CheckCancelled(cq_); + if (has_notify_when_done_tag_) { + // when using async API, but the result is only valid + // if the tag has already been delivered at the completion queue + return completion_op_ && completion_op_->CheckCancelledAsync(); + } else { + // when using sync API + return completion_op_ && completion_op_->CheckCancelled(cq_); + } } void ServerContext::set_compression_level(grpc_compression_level level) { diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 9ca3bf98f85..09e973f28d5 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -68,6 +68,7 @@ namespace testing { namespace { void* tag(int i) { return (void*)(intptr_t)i; } +int detag(void* p) { return static_cast(reinterpret_cast(p)); } #ifdef GPR_POSIX_SOCKET static int maybe_assert_non_blocking_poll(struct pollfd* pfds, nfds_t nfds, @@ -111,30 +112,35 @@ class Verifier { return *this; } + int Next(CompletionQueue* cq, bool ignore_ok) { + bool ok; + void* got_tag; + if (spin_) { + for (;;) { + auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); + if (r == CompletionQueue::TIMEOUT) continue; + if (r == CompletionQueue::GOT_EVENT) break; + gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); + abort(); + } + } else { + EXPECT_TRUE(cq->Next(&got_tag, &ok)); + } + auto it = expectations_.find(got_tag); + EXPECT_TRUE(it != expectations_.end()); + if (!ignore_ok) { + EXPECT_EQ(it->second, ok); + } + expectations_.erase(it); + return detag(got_tag); + } + void Verify(CompletionQueue* cq) { Verify(cq, false); } void Verify(CompletionQueue* cq, bool ignore_ok) { GPR_ASSERT(!expectations_.empty()); while (!expectations_.empty()) { - bool ok; - void* got_tag; - if (spin_) { - for (;;) { - auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); - if (r == CompletionQueue::TIMEOUT) continue; - if (r == CompletionQueue::GOT_EVENT) break; - gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); - abort(); - } - } else { - EXPECT_TRUE(cq->Next(&got_tag, &ok)); - } - auto it = expectations_.find(got_tag); - EXPECT_TRUE(it != expectations_.end()); - if (!ignore_ok) { - EXPECT_EQ(it->second, ok); - } - expectations_.erase(it); + Next(cq, ignore_ok); } } void Verify(CompletionQueue* cq, @@ -793,7 +799,8 @@ TEST_P(AsyncEnd2endTest, UnimplementedRpc) { } // This class is for testing scenarios where RPCs are cancelled on the server -// by calling ServerContext::TryCancel() +// by calling ServerContext::TryCancel(). Server uses AsyncNotifyWhenDone +// API to check for cancellation class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { protected: typedef enum { @@ -803,13 +810,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { CANCEL_AFTER_PROCESSING } ServerTryCancelRequestPhase; - void ServerTryCancel(ServerContext* context) { - EXPECT_FALSE(context->IsCancelled()); - context->TryCancel(); - gpr_log(GPR_INFO, "Server called TryCancel()"); - EXPECT_TRUE(context->IsCancelled()); - } - // Helper for testing client-streaming RPCs which are cancelled on the server. // Depending on the value of server_try_cancel parameter, this will test one // of the following three scenarios: @@ -843,6 +843,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of 'RequestStream' calls // and receive the 'RequestStream' call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestRequestStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -858,9 +859,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_server_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // Since cancellation is done before server reads any results, we know // for sure that all cq results will return false from this point forward @@ -868,22 +872,39 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + &ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while reading the // requests from the client. Since the cancellation can happen at anytime, // some of the cq results (i.e those until cancellation) might be true but // its non deterministic. So better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } // Server reads 3 messages (tags 6, 7 and 8) + // But if want_done_tag is true, we might also see tag 11 for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { srv_stream.Read(&recv_request, tag(tag_idx)); - Verifier(GetParam()) - .Expect(tag_idx, expected_server_cq_result) - .Verify(cq_.get(), ignore_cq_result); + // Note that we'll add something to the verifier and verify that + // something was seen, but it might be tag 11 and not what we + // just added + int got_tag = verif.Expect(tag_idx, expected_server_cq_result) + .Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); + } } if (server_try_cancel_thd != NULL) { @@ -892,7 +913,15 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); + } + + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; } // The RPC has been cancelled at this point for sure (i.e irrespective of @@ -945,6 +974,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { Verifier(GetParam()).Expect(1, true).Verify(cq_.get()); // On the server, request to be notified of 'ResponseStream' calls and // receive the call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -952,9 +982,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -962,24 +995,41 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + &ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while writing responses // to the client. Since the cancellation can happen at anytime, some of // the cq results (i.e those until cancellation) might be true but it is // non deterministic. So better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } // Server sends three messages (tags 3, 4 and 5) + // But if want_done tag is true, we might also see tag 11 for (int tag_idx = 3; tag_idx <= 5; tag_idx++) { send_response.set_message("Pong " + std::to_string(tag_idx)); srv_stream.Write(send_response, tag(tag_idx)); - Verifier(GetParam()) - .Expect(tag_idx, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + // Note that we'll add something to the verifier and verify that + // something was seen, but it might be tag 11 and not what we + // just added + int got_tag = verif.Expect(tag_idx, expected_cq_result) + .Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); + } } if (server_try_cancel_thd != NULL) { @@ -988,13 +1038,21 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); // Client reads may fail bacause it is notified that the stream is // cancelled. ignore_cq_result = true; } + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + } + // Client attemts to read the three messages from the server for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { cli_stream->Read(&recv_response, tag(tag_idx)); @@ -1052,6 +1110,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of the 'BidiStream' call and // receive the call just made by the client + srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestBidiStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -1063,9 +1122,12 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; + bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -1073,42 +1135,84 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; + + auto verif = Verifier(GetParam()); + if (server_try_cancel == CANCEL_DURING_PROCESSING) { server_try_cancel_thd = new std::thread( - &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); + &ServerContext::TryCancel, &srv_ctx); // Since server is going to cancel the RPC in a parallel thread, some of // the cq results (i.e those until the cancellation) might be true. Since // that number is non-deterministic, it is better to ignore the cq results ignore_cq_result = true; + // Expect that we might possibly see the done tag that + // indicates cancellation completion in this case + want_done_tag = true; + verif.Expect(11, true); } + int got_tag; srv_stream.Read(&recv_request, tag(4)); - Verifier(GetParam()) - .Expect(4, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(4, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 4) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 4); + } send_response.set_message("Pong"); srv_stream.Write(send_response, tag(5)); - Verifier(GetParam()) - .Expect(5, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(5, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 5) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 5); + } cli_stream->Read(&recv_response, tag(6)); - Verifier(GetParam()) - .Expect(6, expected_cq_result) - .Verify(cq_.get(), ignore_cq_result); + verif.Expect(6, expected_cq_result); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 6) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 6); + } // This is expected to succeed in all cases cli_stream->WritesDone(tag(7)); - Verifier(GetParam()).Expect(7, true).Verify(cq_.get()); + verif.Expect(7, true); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 7) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 7); + } // This is expected to fail in all cases i.e for all values of // server_try_cancel. This is because at this point, either there are no // more msgs from the client (because client called WritesDone) or the RPC // is cancelled on the server srv_stream.Read(&recv_request, tag(8)); - Verifier(GetParam()).Expect(8, false).Verify(cq_.get()); + verif.Expect(8, false); + got_tag = verif.Next(cq_.get(), ignore_cq_result); + GPR_ASSERT((got_tag == 8) || (got_tag == 11 && want_done_tag)); + if (got_tag == 11) { + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; + // Now get the other entry that we were waiting on + EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 8); + } if (server_try_cancel_thd != NULL) { server_try_cancel_thd->join(); @@ -1116,7 +1220,15 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - ServerTryCancel(&srv_ctx); + srv_ctx.TryCancel(); + want_done_tag = true; + verif.Expect(11, true); + } + + if (want_done_tag) { + verif.Verify(cq_.get()); + EXPECT_TRUE(srv_ctx.IsCancelled()); + want_done_tag = false; } // The RPC has been cancelled at this point for sure (i.e irrespective of From 2e729387f7943971de2591d0db09a0a87026cfd7 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 3 Mar 2016 14:22:12 -0800 Subject: [PATCH 180/236] clang-format --- test/cpp/end2end/async_end2end_test.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 09e973f28d5..30028dd4aa6 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -876,8 +876,8 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { auto verif = Verifier(GetParam()); if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while reading the // requests from the client. Since the cancellation can happen at anytime, // some of the cq results (i.e those until cancellation) might be true but @@ -897,7 +897,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // something was seen, but it might be tag 11 and not what we // just added int got_tag = verif.Expect(tag_idx, expected_server_cq_result) - .Next(cq_.get(), ignore_cq_result); + .Next(cq_.get(), ignore_cq_result); GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); if (got_tag == 11) { EXPECT_TRUE(srv_ctx.IsCancelled()); @@ -999,8 +999,8 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { auto verif = Verifier(GetParam()); if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Server will cancel the RPC in a parallel thread while writing responses // to the client. Since the cancellation can happen at anytime, some of @@ -1022,7 +1022,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // something was seen, but it might be tag 11 and not what we // just added int got_tag = verif.Expect(tag_idx, expected_cq_result) - .Next(cq_.get(), ignore_cq_result); + .Next(cq_.get(), ignore_cq_result); GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); if (got_tag == 11) { EXPECT_TRUE(srv_ctx.IsCancelled()); @@ -1139,8 +1139,8 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { auto verif = Verifier(GetParam()); if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = new std::thread( - &ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = + new std::thread(&ServerContext::TryCancel, &srv_ctx); // Since server is going to cancel the RPC in a parallel thread, some of // the cq results (i.e those until the cancellation) might be true. Since From 6dd3c707c2dd921288a82ec9813d41b567ff07b6 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 13:53:49 -0800 Subject: [PATCH 181/236] Update base INSTALL to markdown and remove outdated content --- CONTRIBUTING.md | 2 +- INSTALL | 217 ------------------------------ INSTALL.md | 45 +++++++ README.md | 2 +- examples/cpp/README.md | 2 +- examples/cpp/cpptutorial.md | 2 +- examples/cpp/helloworld/README.md | 2 +- 7 files changed, 50 insertions(+), 222 deletions(-) delete mode 100644 INSTALL create mode 100644 INSTALL.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9423c46547a..03d2e276a82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ In order to protect both you and ourselves, you will need to sign the ### Technical requirements You will need several tools to work with this repository. In addition to all of -the packages described in the [INSTALL](INSTALL) file, you will also need +the packages described in the [INSTALL](INSTALL.md) file, you will also need python, and the mako template renderer. To install the latter, using pip, one should simply be able to do `pip install mako`. diff --git a/INSTALL b/INSTALL deleted file mode 100644 index e33f8970a95..00000000000 --- a/INSTALL +++ /dev/null @@ -1,217 +0,0 @@ -These instructions only cover building grpc C and C++ libraries under -typical unix systems. If you need more information, please try grpc's -wiki pages: - - https://github.com/google/grpc/wiki - - -************************* -* If you are in a hurry * -************************* - -On Linux (Debian): - - Note: you will need to add the Debian 'jessie-backports' distribution to your sources - file first. - - Add the following line to your `/etc/apt/sources.list` file: - - deb http://http.debian.net/debian jessie-backports main - - Install the gRPC library: - - $ [sudo] apt-get install libgrpc-dev - -OR - - $ git clone https://github.com/grpc/grpc.git - $ cd grpc - $ git submodule update --init - $ make - $ [sudo] make install - -You don't need anything else than GNU Make, gcc and autotools. Under a Debian -or Ubuntu system, this should boil down to the following packages: - - $ [sudo] apt-get install build-essential autoconf libtool - -Building the python wrapper requires the following: - - $ [sudo] apt-get install python-all-dev python-virtualenv - -If you want to install in a different directory than the default /usr/lib, you can -override it on the command line: - - $ [sudo] make install prefix=/opt - - -******************************* -* More detailled instructions * -******************************* - -Setting up dependencies -======================= - -Dependencies to compile the libraries -------------------------------------- - -grpc libraries have few external dependencies. If you need to compile and -install them, they are present in the third_party directory if you have -cloned the github repository recursively. If you didn't clone recursively, -you can still get them later by running the following command: - - $ git submodule update --init - -Note that the Makefile makes it much easier for you to compile from sources -if you were to clone recursively our git repository: it will automatically -compile zlib and OpenSSL, which are core requirements for grpc. Note this -creates grpc libraries that will have zlib and OpenSSL built-in inside of them, -which significantly increases the libraries' size. - -In order to decrease that size, you can manually install zlib and OpenSSL on -your system, so that the Makefile can use them instead. - -Under a Debian or Ubuntu system, one can acquire the development package -for zlib this way: - - # apt-get install zlib1g-dev - -To the best of our knowledge, no distribution has an OpenSSL package that -supports ALPN yet, so you would still have to depend on installing from source -for that particular dependency if you want to reduce the libraries' size. - -The recommended version of OpenSSL that provides ALPN support is available -at this URL: - - https://www.openssl.org/source/openssl-1.0.2.tar.gz - - -Dependencies to compile and run the tests ------------------------------------------ - -Compiling and running grpc plain-C tests dont't require any more dependency. - - -Compiling and running grpc C++ tests depend on protobuf 3.0.0, gtest and -gflags. Although gflags is provided in third_party, you will need to manually -install that dependency on your system to run these tests. - -Under a Debian or Ubuntu system, you can install the gtests and gflags packages -using apt-get: - - # apt-get install libgflags-dev libgtest-dev - -However, protobuf 3.0.0 isn't in a debian package yet, but the Makefile will -automatically try and compile the one present in third_party if you cloned the -repository recursively, and that it detects your system is lacking it. - -Compiling and installing protobuf 3.0.0 requires a few more dependencies in -itself, notably the autoconf suite. If you have apt-get, you can install -these dependencies this way: - - # apt-get install autoconf libtool - -If you want to run the tests using one of the sanitized configurations, you -will need clang and its instrumented libc++: - - # apt-get install clang libc++-dev - -Mac-specific notes: -------------------- - -For a Mac system, git is not available by default. You will first need to -install Xcode from the Mac AppStore and then run the following command from a -terminal: - - $ sudo xcode-select --install - -You should also install "port" following the instructions at -https://www.macports.org . This will reside in /opt/local/bin/port for -most Mac installations. Do the "git submodule" command listed above. - -Then execute the following for all the needed build dependencies - - $ sudo /opt/local/bin/port install autoconf automake libtool gflags cmake - $ mkdir ~/gtest-svn - $ svn checkout http://googletest.googlecode.com/svn/trunk/ gtest-svn - $ mkdir mybuild - $ cd mybuild - $ cmake ../gtest-svn - $ make - $ make gtest.a gtest_main.a - $ sudo cp libgtest.a libgtest_main.a /opt/local/lib - $ sudo mkdir /opt/local/include/gtest - $ sudo cp -pr ../gtest-svn/include/gtest /opt/local/include/gtest - -If you are going to make changes and need to regenerate the projects file, -you will need to install certain modules for python. - - $ sudo easy_install simplejson mako - -Mingw-specific notes: ---------------------- - -While gRPC compiles properly under mingw, some more preparation work is needed. -The recommendation is to use msys2. The installation instructions are available -at that address: http://msys2.github.io/ - -Once this is installed, make sure you are using the following: MinGW-w64 Win64. -You'll be required to install a few more packages: - - $ pacman -S make mingw-w64-x86_64-gcc mingw-w64-x86_64-zlib autoconf automake libtool - -Please also install OpenSSL from that website: - - http://slproweb.com/products/Win32OpenSSL.html - -The package Win64 OpenSSL v1.0.2a should do. At that point you should be able -to compile gRPC with the following: - - $ export LDFLAGS="-L/mingw64/lib -L/c/OpenSSL-Win64" - $ export CPPFLAGS="-I/mingw64/include -I/c/OpenSSL-Win64/include" - $ make - -A word on OpenSSL ------------------ - -Secure HTTP2 requires the TLS extension ALPN (see rfc 7301 and -http://http2.github.io/http2-spec/ section 3.3). Our HTTP2 implementation -relies on OpenSSL's implementation. OpenSSL 1.0.2 is the first released version -of OpenSSL that has ALPN support, and this explains our dependency on it. - -Note that the Makefile supports compiling only the unsecure elements of grpc, -and if you do not have OpenSSL and do not want it, you can still proceed -with installing only the elements you require. However, we strongly recommend -the use of encryption for all network traffic, and discourage the use of grpc -without TLS. - - -Compiling -========= - -If you have all the dependencies mentioned above, you should simply be able -to go ahead and run "make" to compile grpc's C and C++ libraries: - - $ make - - -Testing -======= - -To build and run the tests, you can run the command: - - $ make test - -If you want to be able to run them in parallel, and get better output, you can -also use the python tool we have written: - - $ ./tools/run_tests/run_tests.py - - -Installing -========== - -Once everything is compiled, you should be able to install grpc C and C++ -libraries and headers: - - # make install diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 00000000000..aa809f23c8f --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,45 @@ +#If you are in a hurry + +For language-specific installation instructions, please refer to these +documents + + * [C++](examples/cpp) + * [C#](src/csharp): NuGet package `Grpc` + * [Go](https://github.com/grpc/grpc-go): `go get google.golang.org/grpc` + * [Java](https://github.com/grpc/grpc-java) + * [Node](src/node): `npm install grpc` + * [Objective-C](src/objective-c) + * [PHP](src/php): `pecl install grpc-beta` + * [Python](src/python/grpcio): `pip install grpcio` + * [Ruby](src/ruby): `gem install grpc` + + +#Pre-requisites + +##Linux + +```sh + $ [sudo] apt-get install build-essential autoconf libtool +``` + +##Mac OSX + +For a Mac system, git is not available by default. You will first need to +install Xcode from the Mac AppStore and then run the following command from a +terminal: + +```sh + $ [sudo] xcode-select --install +``` + +#Build from Source + +This is for compiling just the gRPC C Core library. + +```sh + $ git clone https://github.com/grpc/grpc.git + $ cd grpc + $ git submodule update --init + $ make + $ [sudo] make install +``` diff --git a/README.md b/README.md index 033e09b91b7..abb4905392c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You can find more detailed documentation and examples in the [doc](doc) and [exa #Installation -See [grpc/INSTALL](INSTALL) for installation instructions for various platforms. +See [INSTALL](INSTALL.md) for installation instructions for various platforms. #Repository Structure & Status diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 85c495099b7..979ba55dbb5 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -2,7 +2,7 @@ ## Installation -To install gRPC on your system, follow the instructions [here](../../INSTALL). +To install gRPC on your system, follow the instructions [here](../../INSTALL.md). ## Hello C++ gRPC! diff --git a/examples/cpp/cpptutorial.md b/examples/cpp/cpptutorial.md index cd1cddb1115..ef9ca99c0fe 100644 --- a/examples/cpp/cpptutorial.md +++ b/examples/cpp/cpptutorial.md @@ -91,7 +91,7 @@ message Point { Next we need to generate the gRPC client and server interfaces from our .proto service definition. We do this using the protocol buffer compiler `protoc` with a special gRPC C++ plugin. -For simplicity, we've provided a [makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL) first): +For simplicity, we've provided a [makefile](route_guide/Makefile) that runs `protoc` for you with the appropriate plugin, input, and output (if you want to run this yourself, make sure you've installed protoc and followed the gRPC code [installation instructions](../../INSTALL.md) first): ```shell $ make route_guide.grpc.pb.cc route_guide.pb.cc diff --git a/examples/cpp/helloworld/README.md b/examples/cpp/helloworld/README.md index 90f3d6b1475..8e11f7cdf3e 100644 --- a/examples/cpp/helloworld/README.md +++ b/examples/cpp/helloworld/README.md @@ -2,7 +2,7 @@ ### Install gRPC Make sure you have installed gRPC on your system. Follow the instructions here: -[https://github.com/grpc/grpc/blob/master/INSTALL](../../../INSTALL). +[https://github.com/grpc/grpc/blob/master/INSTALL](../../../INSTALL.md). ### Get the tutorial source code From a22a50d61795c38522fe8b032603c63fcef7f93f Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 14:49:03 -0800 Subject: [PATCH 182/236] bit of text change --- INSTALL.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index aa809f23c8f..d9411db021d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,7 +1,7 @@ #If you are in a hurry -For language-specific installation instructions, please refer to these -documents +For language-specific installation instructions for gRPC runtime, please +refer to these documents * [C++](examples/cpp) * [C#](src/csharp): NuGet package `Grpc` @@ -34,7 +34,8 @@ terminal: #Build from Source -This is for compiling just the gRPC C Core library. +For developers who are interested to contribute, here is how to compile the +gRPC C Core library. ```sh $ git clone https://github.com/grpc/grpc.git From 19f703dbb7805b95bdc64d166a4e5fb36e2dd192 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 3 Mar 2016 15:33:29 -0800 Subject: [PATCH 183/236] increase timeout for interop tests --- tools/run_tests/run_interop_tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index df3ab90a839..1dc772a8565 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -60,6 +60,8 @@ _SKIP_COMPRESSION = ['large_compressed_unary', _SKIP_ADVANCED = ['custom_metadata', 'status_code_and_message', 'unimplemented_method'] +_TEST_TIMEOUT = 3*60 + class CXXLanguage: def __init__(self): @@ -459,7 +461,7 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name, environ=environ, shortname='%s:%s:%s:%s' % (suite_name, server_host_name, language, test_case), - timeout_seconds=90, + timeout_seconds=_TEST_TIMEOUT, flake_retries=5 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, kill_handler=_job_kill_handler) @@ -495,7 +497,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, environ=environ, shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name, test_case), - timeout_seconds=90, + timeout_seconds=_TEST_TIMEOUT, flake_retries=5 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, kill_handler=_job_kill_handler) From dbf47fabd406b51e8d3d03c5eb06c5fe6f7d0d5d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 3 Mar 2016 16:25:06 -0800 Subject: [PATCH 184/236] Better comments. --- test/cpp/end2end/async_end2end_test.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 30028dd4aa6..dc8c2bb6e5b 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -107,11 +107,14 @@ class PollingOverrider { class Verifier { public: explicit Verifier(bool spin) : spin_(spin) {} + // Expect sets the expected ok value for a specific tag Verifier& Expect(int i, bool expect_ok) { expectations_[tag(i)] = expect_ok; return *this; } + // Next waits for 1 async tag to complete, checks its + // expectations, and returns the tag int Next(CompletionQueue* cq, bool ignore_ok) { bool ok; void* got_tag; @@ -135,14 +138,19 @@ class Verifier { return detag(got_tag); } + // Verify keeps calling Next until all currently set + // expected tags are complete void Verify(CompletionQueue* cq) { Verify(cq, false); } + // This version of Verify allows optionally ignoring the + // outcome of the expectation void Verify(CompletionQueue* cq, bool ignore_ok) { GPR_ASSERT(!expectations_.empty()); while (!expectations_.empty()) { Next(cq, ignore_ok); } } + // This version of Verify stops after a certain deadline void Verify(CompletionQueue* cq, std::chrono::system_clock::time_point deadline) { if (expectations_.empty()) { From 79a0f0611c6235fbf636c3e754860e6de28b4e3c Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 3 Mar 2016 17:18:48 -0800 Subject: [PATCH 185/236] Properly delete Node OpenSSL headers after downloading them --- tools/run_tests/build_node.bat | 10 +++++++++- tools/run_tests/pre_build_node.bat | 9 +-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tools/run_tests/build_node.bat b/tools/run_tests/build_node.bat index 6896bc1d1bb..3ddd0e73cf7 100644 --- a/tools/run_tests/build_node.bat +++ b/tools/run_tests/build_node.bat @@ -27,4 +27,12 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -npm install --build-from-source \ No newline at end of file +call npm install --build-from-source + +@rem delete the redundant openssl headers +for /f "delims=v" %%v in ('node --version') do ( + rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q +) + +@rem rebuild, because it probably failed the first time +call npm install --build-from-source \ No newline at end of file diff --git a/tools/run_tests/pre_build_node.bat b/tools/run_tests/pre_build_node.bat index 6e7cbe5d420..ffb4a09f15a 100644 --- a/tools/run_tests/pre_build_node.bat +++ b/tools/run_tests/pre_build_node.bat @@ -28,12 +28,5 @@ @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @rem Expire cache after 1 week -npm update --cache-min 604800 +call npm update --cache-min 604800 -npm install node-gyp-install -.\node_modules\.bin\node-gyp-install.cmd - -@rem delete the redundant openssl headers -for /f "delims=v" %%v in ('node --version') do ( - rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q -) \ No newline at end of file From 4913ea06eb4093a2229b3009dedb02828af1edc2 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 3 Mar 2016 17:22:53 -0800 Subject: [PATCH 186/236] Ensure node and npm are in the path when running tests --- tools/run_tests/build_node.bat | 4 ++++ tools/run_tests/pre_build_node.bat | 2 ++ tools/run_tests/run_node.bat | 1 + 3 files changed, 7 insertions(+) diff --git a/tools/run_tests/build_node.bat b/tools/run_tests/build_node.bat index 3ddd0e73cf7..886af0610ff 100644 --- a/tools/run_tests/build_node.bat +++ b/tools/run_tests/build_node.bat @@ -27,6 +27,10 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm + +del /f /q BUILD || rmdir build /s /q + call npm install --build-from-source @rem delete the redundant openssl headers diff --git a/tools/run_tests/pre_build_node.bat b/tools/run_tests/pre_build_node.bat index ffb4a09f15a..a29456f9ed9 100644 --- a/tools/run_tests/pre_build_node.bat +++ b/tools/run_tests/pre_build_node.bat @@ -27,6 +27,8 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm + @rem Expire cache after 1 week call npm update --cache-min 604800 diff --git a/tools/run_tests/run_node.bat b/tools/run_tests/run_node.bat index 41777363568..0987fbee559 100644 --- a/tools/run_tests/run_node.bat +++ b/tools/run_tests/run_node.bat @@ -27,6 +27,7 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm set JUNIT_REPORT_PATH=src\node\report.xml set JUNIT_REPORT_STACK=1 .\node_modules\.bin\mocha.cmd --reporter mocha-jenkins-reporter --timeout 8000 src\node\test \ No newline at end of file From 77fae2784ed7f894dc557ee2004364f2bb1b6b74 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 19:39:56 -0800 Subject: [PATCH 187/236] fix broken links --- doc/interop-test-descriptions.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md index e618e967ee0..3beb1d11f4a 100644 --- a/doc/interop-test-descriptions.md +++ b/doc/interop-test-descriptions.md @@ -2,9 +2,8 @@ Interoperability Test Case Descriptions ======================================= Client and server use -[test.proto](https://github.com/grpc/grpc/blob/master/test/proto/test.proto) -and the [gRPC over HTTP/2 v2 -protocol](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md). +[test.proto](../src/proto/grpc/testing/test.proto) +and the [gRPC over HTTP/2 v2 protocol](./PROTOCOL-HTTP2.md). Client ------ From 3f5c5c7e18f6b09d85e39f310f214e90316cc0ef Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 19:46:13 -0800 Subject: [PATCH 188/236] fix broken link --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 84ec80057e1..cd85417ca20 100644 --- a/examples/README.md +++ b/examples/README.md @@ -447,4 +447,4 @@ $ greeter_client ## Read more! - You can find links to language-specific tutorials, examples, and other docs in each language's [quick start](#quickstart). -- [gRPC Authentication Support](doc/grpc-auth-support.md) introduces authentication support in gRPC with supported mechanisms and examples. +- [gRPC Authentication Support](http://www.grpc.io/docs/guides/auth.html) introduces authentication support in gRPC with supported mechanisms and examples. From 5821a444c703757d4bacdf2d58a3045c6cfbb838 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 19:46:58 -0800 Subject: [PATCH 189/236] remove stray grpc_common reference --- examples/node/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/node/README.md b/examples/node/README.md index 7a2bc9794f3..c1ef6b05ab0 100644 --- a/examples/node/README.md +++ b/examples/node/README.md @@ -20,7 +20,7 @@ TRY IT! - Run the server ```sh - $ # from this directory (grpc_common/node). + $ # from this directory $ node ./greeter_server.js & ``` From f9946d579bc5b258261177b5ff87df22e7d49a07 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 3 Mar 2016 20:06:20 -0800 Subject: [PATCH 190/236] remove old stuff from tools/README.md --- tools/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/README.md b/tools/README.md index a0c41eb79ff..df4c6ef7d25 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,7 +1,5 @@ buildgen: template renderer for our build system. -distpackages: script to generate debian packages. - distrib: scripts to distribute language-specific packages. dockerfile: Docker files to test gRPC. @@ -12,6 +10,4 @@ gce: scripts to help setup testing infrastructure on GCE. jenkins: support for running tests on Jenkins. -profile_analyzer: pretty printer for gRPC profiling data. - run_tests: scripts to run gRPC tests in parallel. From 389297dedfe384246d4831a4fc33e7af7b0200e5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 3 Mar 2016 21:02:54 -0800 Subject: [PATCH 191/236] Document some things --- src/core/transport/transport.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h index 7a007ef777b..5a90bd6d387 100644 --- a/src/core/transport/transport.h +++ b/src/core/transport/transport.h @@ -123,7 +123,8 @@ typedef struct grpc_transport_stream_op { /** Transport op: a set of operations to perform on a transport as a whole */ typedef struct grpc_transport_op { - /** called when processing of this op is done */ + /** Called when processing of this op is done. + Only one transport_op is allowed to be outstanding at any time. */ grpc_closure *on_consumed; /** connectivity monitoring - set connectivity_state to NULL to unsubscribe */ grpc_closure *on_connectivity_state_change; @@ -138,7 +139,9 @@ typedef struct grpc_transport_op { grpc_status_code goaway_status; gpr_slice *goaway_message; /** set the callback for accepting new streams; - this is a permanent callback, unlike the other one-shot closures */ + this is a permanent callback, unlike the other one-shot closures. + If true, the callback is set to set_accept_stream_fn, with its + user_data argument set to set_accept_stream_user_data */ bool set_accept_stream; void (*set_accept_stream_fn)(grpc_exec_ctx *exec_ctx, void *user_data, grpc_transport *transport, From 1bd9ea407fd673f6c795a524c6454bc5db339a85 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 3 Mar 2016 21:09:31 -0800 Subject: [PATCH 192/236] Refine condition --- src/core/transport/chttp2_transport.c | 5 +++-- src/core/transport/transport.h | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 369ff0ad7f6..cf1dbfa0e59 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -1006,8 +1006,9 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, lock(t); - /* Let's be overly cautious: don't change any state while we're parsing */ - if (t->parsing_active) { + /* If there's a set_accept_stream ensure that we're not parsing + to avoid changing things out from underneath */ + if (t->parsing_active && t->set_accept_stream) { GPR_ASSERT(t->post_parsing_op == NULL); t->post_parsing_op = gpr_malloc(sizeof(*op)); memcpy(t->post_parsing_op, op, sizeof(*op)); diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h index 5a90bd6d387..ed6e121c9cb 100644 --- a/src/core/transport/transport.h +++ b/src/core/transport/transport.h @@ -123,8 +123,7 @@ typedef struct grpc_transport_stream_op { /** Transport op: a set of operations to perform on a transport as a whole */ typedef struct grpc_transport_op { - /** Called when processing of this op is done. - Only one transport_op is allowed to be outstanding at any time. */ + /** Called when processing of this op is done. */ grpc_closure *on_consumed; /** connectivity monitoring - set connectivity_state to NULL to unsubscribe */ grpc_closure *on_connectivity_state_change; From 687bf6295f5e3a02f249aced69868e497c1bcc30 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 3 Mar 2016 21:18:48 -0800 Subject: [PATCH 193/236] Fix typo --- src/core/transport/chttp2_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index cf1dbfa0e59..643e3feeb97 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -1008,7 +1008,7 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, /* If there's a set_accept_stream ensure that we're not parsing to avoid changing things out from underneath */ - if (t->parsing_active && t->set_accept_stream) { + if (t->parsing_active && op->set_accept_stream) { GPR_ASSERT(t->post_parsing_op == NULL); t->post_parsing_op = gpr_malloc(sizeof(*op)); memcpy(t->post_parsing_op, op, sizeof(*op)); From af22087a43ffe7e69cb269419f046635adfc46e6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 10:00:31 -0800 Subject: [PATCH 194/236] Fix leak: dont become writable if writes are closed --- src/core/transport/chttp2_transport.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 643e3feeb97..ce54f7e6e82 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -617,6 +617,7 @@ static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { if (!TRANSPORT_FROM_GLOBAL(transport_global)->closed && + !stream_global->write_closed && grpc_chttp2_list_add_writable_stream(transport_global, stream_global)) { GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); } From ad23ae196702cc3889a8c04a21273aa6a43aad75 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 10:50:56 -0800 Subject: [PATCH 195/236] Unref writable streams on transport close to avoid leaks --- src/core/transport/chttp2_transport.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index ce54f7e6e82..19265252ca3 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -432,6 +432,14 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, if (t->ep) { allow_endpoint_shutdown_locked(exec_ctx, t); } + + /* flush writable stream list to avoid dangling references */ + grpc_chttp2_stream_global *stream_global; + grpc_chttp2_stream_writing *stream_writing; + while (grpc_chttp2_list_pop_writable_stream( + &t->global, &t->writing, &stream_global, &stream_writing)) { + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing"); + } } } @@ -617,7 +625,6 @@ static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { void grpc_chttp2_become_writable(grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { if (!TRANSPORT_FROM_GLOBAL(transport_global)->closed && - !stream_global->write_closed && grpc_chttp2_list_add_writable_stream(transport_global, stream_global)) { GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); } From 34fcf43d374d312290d05408fbd554c3138a1d7d Mon Sep 17 00:00:00 2001 From: vjpai Date: Fri, 4 Mar 2016 11:02:05 -0800 Subject: [PATCH 196/236] Restart workers in each scenario --- tools/jenkins/run_performance.sh | 33 ++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index fbc078330f7..abad97358cc 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -42,15 +42,10 @@ config=opt make CONFIG=$config qps_worker qps_driver -j8 -bins/$config/qps_worker -driver_port 10000 & -PID1=$! -bins/$config/qps_worker -driver_port 10010 & -PID2=$! - # # Put a timeout on these tests # -((sleep 900; kill $$ && killall qps_worker && rm -f /tmp/qps-test.$$ )&) +((sleep 900; killall qps_worker && rm -f /tmp/qps-test.$$ && kill $$)&) export QPS_WORKERS="localhost:10000,localhost:10010" @@ -71,38 +66,52 @@ halfcores=`expr $cores / 2` for secure in true false; do # Scenario 1: generic async streaming ping-pong (contentionless latency) + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - + bins/$config/qps_driver --quit=true + # Scenario 2: generic async streaming "unconstrained" (QPS) + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 \ --server_core_limit=$halfcores --client_core_limit=0 |& tee /tmp/qps-test.$$ + bins/$config/qps_driver --quit=true # Scenario 2b: QPS with a single server core + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 --server_core_limit=1 --client_core_limit=0 + bins/$config/qps_driver --quit=true # Scenario 2c: protobuf-based QPS + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --simple_req_size=0 --simple_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 \ --server_core_limit=$halfcores --client_core_limit=0 + bins/$config/qps_driver --quit=true # Scenario 3: Latency at sub-peak load (all clients equally loaded) for loadfactor in 0.7; do + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -110,27 +119,31 @@ for secure in true false; do --num_servers=1 --num_clients=0 --poisson_load=`awk -v lf=$loadfactor \ '$5 == "QPS:" {print int(lf * $6); exit}' /tmp/qps-test.$$` \ --server_core_limit=$halfcores --client_core_limit=0 + bins/$config/qps_driver --quit=true done rm /tmp/qps-test.$$ # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM). + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 + bins/$config/qps_driver --quit=true # Scenario 5: Sync unary ping-pong with protobufs + bins/$config/qps_worker -driver_port 10000 & + bins/$config/qps_worker -driver_port 10010 & bins/$config/qps_driver --rpc_type=UNARY --client_type=SYNC_CLIENT \ --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \ --secure_test=$secure --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - + bins/$config/qps_driver --quit=true done -bins/$config/qps_driver --quit=true - wait From 2d9476898b119ead1045eb31894bcd10176344fd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 11:05:42 -0800 Subject: [PATCH 197/236] Revert "Ensure that no #includes are inside of a namespace." --- test/cpp/interop/reconnect_interop_client.cc | 10 +++++----- test/cpp/interop/reconnect_interop_server.cc | 12 +++++------- test/cpp/qps/limit_cores.cc | 7 +++---- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index c668edaceb0..79a60cc8602 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -34,16 +34,16 @@ #include #include +#include +#include #include #include #include -#include -#include -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" -#include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/util/create_test_channel.h" #include "test/cpp/util/test_config.h" +#include "src/proto/grpc/testing/test.grpc.pb.h" +#include "src/proto/grpc/testing/empty.grpc.pb.h" +#include "src/proto/grpc/testing/messages.grpc.pb.h" DEFINE_int32(server_control_port, 0, "Server port for control rpcs."); DEFINE_int32(server_retry_port, 0, "Server port for testing reconnection."); diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 1f9147d0efa..3602b8c2b05 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -31,8 +31,6 @@ * */ -// Test description at doc/connection-backoff-interop-test-description.md - #include #include @@ -42,17 +40,17 @@ #include #include +#include +#include #include #include #include -#include -#include -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" -#include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/core/util/reconnect_server.h" #include "test/cpp/util/test_config.h" +#include "src/proto/grpc/testing/test.grpc.pb.h" +#include "src/proto/grpc/testing/empty.grpc.pb.h" +#include "src/proto/grpc/testing/messages.grpc.pb.h" DEFINE_int32(control_port, 0, "Server port for controlling the server."); DEFINE_int32(retry_port, 0, diff --git a/test/cpp/qps/limit_cores.cc b/test/cpp/qps/limit_cores.cc index 1fb2d628f6d..fad9a323afd 100644 --- a/test/cpp/qps/limit_cores.cc +++ b/test/cpp/qps/limit_cores.cc @@ -37,15 +37,14 @@ #include #include +namespace grpc { +namespace testing { + #ifdef GPR_CPU_LINUX #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include - -namespace grpc { -namespace testing { - int LimitCores(const int* cores, int cores_size) { const int num_cores = gpr_cpu_num_cores(); int cores_set = 0; From 180f6dfc8d16f16c607f0b5e51f1efe1596f3212 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 4 Mar 2016 11:23:48 -0800 Subject: [PATCH 198/236] Keep changing the ports each scenario also --- tools/jenkins/run_performance.sh | 58 ++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index abad97358cc..b06fce7acb9 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -47,8 +47,6 @@ make CONFIG=$config qps_worker qps_driver -j8 # ((sleep 900; killall qps_worker && rm -f /tmp/qps-test.$$ && kill $$)&) -export QPS_WORKERS="localhost:10000,localhost:10010" - # big is the size in bytes of large messages (0 is the size otherwise) big=65536 @@ -64,10 +62,18 @@ deep=100 cores=`grep -c ^processor /proc/cpuinfo` halfcores=`expr $cores / 2` +# +# Keep changing the ports +port1=10000 +port2=10100 + for secure in true false; do # Scenario 1: generic async streaming ping-pong (contentionless latency) - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -75,10 +81,13 @@ for secure in true false; do --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 bins/$config/qps_driver --quit=true - + # Scenario 2: generic async streaming "unconstrained" (QPS) - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -88,8 +97,11 @@ for secure in true false; do bins/$config/qps_driver --quit=true # Scenario 2b: QPS with a single server core - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -98,8 +110,11 @@ for secure in true false; do bins/$config/qps_driver --quit=true # Scenario 2c: protobuf-based QPS - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --simple_req_size=0 --simple_resp_size=0 \ @@ -110,8 +125,11 @@ for secure in true false; do # Scenario 3: Latency at sub-peak load (all clients equally loaded) for loadfactor in 0.7; do - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -125,8 +143,11 @@ for secure in true false; do rm /tmp/qps-test.$$ # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM). - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \ @@ -136,8 +157,11 @@ for secure in true false; do bins/$config/qps_driver --quit=true # Scenario 5: Sync unary ping-pong with protobufs - bins/$config/qps_worker -driver_port 10000 & - bins/$config/qps_worker -driver_port 10010 & + bins/$config/qps_worker -driver_port $port1 & + bins/$config/qps_worker -driver_port $port2 & + export QPS_WORKERS="localhost:$port1,localhost:$port2" + port1=`expr $port1 + 1` + port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=UNARY --client_type=SYNC_CLIENT \ --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \ From edd96e4926373644bac1c169431c213d361bed21 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 4 Mar 2016 14:32:50 -0500 Subject: [PATCH 199/236] Revert "Properly integrate async API with server-side cancellations." --- .../grpc++/impl/codegen/completion_queue.h | 1 - include/grpc++/impl/codegen/server_context.h | 3 - src/cpp/server/server_context.cc | 26 +- test/cpp/end2end/async_end2end_test.cc | 232 +++++------------- 4 files changed, 64 insertions(+), 198 deletions(-) diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h index 928ab2db317..102831e1c9b 100644 --- a/include/grpc++/impl/codegen/completion_queue.h +++ b/include/grpc++/impl/codegen/completion_queue.h @@ -184,7 +184,6 @@ class CompletionQueue : private GrpcLibrary { bool Pluck(CompletionQueueTag* tag); /// Performs a single polling pluck on \a tag. - /// \warning Must not be mixed with calls to \a Next. void TryPluck(CompletionQueueTag* tag); grpc_completion_queue* cq_; // owned diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h index 91ebe574b14..ad08b8210d6 100644 --- a/include/grpc++/impl/codegen/server_context.h +++ b/include/grpc++/impl/codegen/server_context.h @@ -103,9 +103,6 @@ class ServerContext { void AddInitialMetadata(const grpc::string& key, const grpc::string& value); void AddTrailingMetadata(const grpc::string& key, const grpc::string& value); - // IsCancelled is always safe to call when using sync API - // When using async API, it is only safe to call IsCancelled after - // the AsyncNotifyWhenDone tag has been delivered bool IsCancelled() const; // Cancel the Call from the server. This is a best-effort API and depending on diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index eb49b210379..e205a1969b3 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -62,11 +62,7 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void FillOps(grpc_op* ops, size_t* nops) GRPC_OVERRIDE; bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE; - bool CheckCancelled(CompletionQueue* cq) { - cq->TryPluck(this); - return CheckCancelledNoPluck(); - } - bool CheckCancelledAsync() { return CheckCancelledNoPluck(); } + bool CheckCancelled(CompletionQueue* cq); void set_tag(void* tag) { has_tag_ = true; @@ -76,11 +72,6 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface { void Unref(); private: - bool CheckCancelledNoPluck() { - grpc::lock_guard g(mu_); - return finalized_ ? (cancelled_ != 0) : false; - } - bool has_tag_; void* tag_; grpc::mutex mu_; @@ -97,6 +88,12 @@ void ServerContext::CompletionOp::Unref() { } } +bool ServerContext::CompletionOp::CheckCancelled(CompletionQueue* cq) { + cq->TryPluck(this); + grpc::lock_guard g(mu_); + return finalized_ ? cancelled_ != 0 : false; +} + void ServerContext::CompletionOp::FillOps(grpc_op* ops, size_t* nops) { ops->op = GRPC_OP_RECV_CLOSE_ON_SERVER; ops->data.recv_close_on_server.cancelled = &cancelled_; @@ -185,14 +182,7 @@ void ServerContext::TryCancel() const { } bool ServerContext::IsCancelled() const { - if (has_notify_when_done_tag_) { - // when using async API, but the result is only valid - // if the tag has already been delivered at the completion queue - return completion_op_ && completion_op_->CheckCancelledAsync(); - } else { - // when using sync API - return completion_op_ && completion_op_->CheckCancelled(cq_); - } + return completion_op_ && completion_op_->CheckCancelled(cq_); } void ServerContext::set_compression_level(grpc_compression_level level) { diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index dc8c2bb6e5b..9ca3bf98f85 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -68,7 +68,6 @@ namespace testing { namespace { void* tag(int i) { return (void*)(intptr_t)i; } -int detag(void* p) { return static_cast(reinterpret_cast(p)); } #ifdef GPR_POSIX_SOCKET static int maybe_assert_non_blocking_poll(struct pollfd* pfds, nfds_t nfds, @@ -107,50 +106,37 @@ class PollingOverrider { class Verifier { public: explicit Verifier(bool spin) : spin_(spin) {} - // Expect sets the expected ok value for a specific tag Verifier& Expect(int i, bool expect_ok) { expectations_[tag(i)] = expect_ok; return *this; } - // Next waits for 1 async tag to complete, checks its - // expectations, and returns the tag - int Next(CompletionQueue* cq, bool ignore_ok) { - bool ok; - void* got_tag; - if (spin_) { - for (;;) { - auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); - if (r == CompletionQueue::TIMEOUT) continue; - if (r == CompletionQueue::GOT_EVENT) break; - gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); - abort(); - } - } else { - EXPECT_TRUE(cq->Next(&got_tag, &ok)); - } - auto it = expectations_.find(got_tag); - EXPECT_TRUE(it != expectations_.end()); - if (!ignore_ok) { - EXPECT_EQ(it->second, ok); - } - expectations_.erase(it); - return detag(got_tag); - } - - // Verify keeps calling Next until all currently set - // expected tags are complete void Verify(CompletionQueue* cq) { Verify(cq, false); } - // This version of Verify allows optionally ignoring the - // outcome of the expectation void Verify(CompletionQueue* cq, bool ignore_ok) { GPR_ASSERT(!expectations_.empty()); while (!expectations_.empty()) { - Next(cq, ignore_ok); + bool ok; + void* got_tag; + if (spin_) { + for (;;) { + auto r = cq->AsyncNext(&got_tag, &ok, gpr_time_0(GPR_CLOCK_REALTIME)); + if (r == CompletionQueue::TIMEOUT) continue; + if (r == CompletionQueue::GOT_EVENT) break; + gpr_log(GPR_ERROR, "unexpected result from AsyncNext"); + abort(); + } + } else { + EXPECT_TRUE(cq->Next(&got_tag, &ok)); + } + auto it = expectations_.find(got_tag); + EXPECT_TRUE(it != expectations_.end()); + if (!ignore_ok) { + EXPECT_EQ(it->second, ok); + } + expectations_.erase(it); } } - // This version of Verify stops after a certain deadline void Verify(CompletionQueue* cq, std::chrono::system_clock::time_point deadline) { if (expectations_.empty()) { @@ -807,8 +793,7 @@ TEST_P(AsyncEnd2endTest, UnimplementedRpc) { } // This class is for testing scenarios where RPCs are cancelled on the server -// by calling ServerContext::TryCancel(). Server uses AsyncNotifyWhenDone -// API to check for cancellation +// by calling ServerContext::TryCancel() class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { protected: typedef enum { @@ -818,6 +803,13 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { CANCEL_AFTER_PROCESSING } ServerTryCancelRequestPhase; + void ServerTryCancel(ServerContext* context) { + EXPECT_FALSE(context->IsCancelled()); + context->TryCancel(); + gpr_log(GPR_INFO, "Server called TryCancel()"); + EXPECT_TRUE(context->IsCancelled()); + } + // Helper for testing client-streaming RPCs which are cancelled on the server. // Depending on the value of server_try_cancel parameter, this will test one // of the following three scenarios: @@ -851,7 +843,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of 'RequestStream' calls // and receive the 'RequestStream' call just made by the client - srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestRequestStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -867,12 +858,9 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_server_cq_result = true; bool ignore_cq_result = false; - bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - srv_ctx.TryCancel(); - Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); + ServerTryCancel(&srv_ctx); // Since cancellation is done before server reads any results, we know // for sure that all cq results will return false from this point forward @@ -880,39 +868,22 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; - - auto verif = Verifier(GetParam()); - if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = - new std::thread(&ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = new std::thread( + &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); // Server will cancel the RPC in a parallel thread while reading the // requests from the client. Since the cancellation can happen at anytime, // some of the cq results (i.e those until cancellation) might be true but // its non deterministic. So better to ignore the cq results ignore_cq_result = true; - // Expect that we might possibly see the done tag that - // indicates cancellation completion in this case - want_done_tag = true; - verif.Expect(11, true); } // Server reads 3 messages (tags 6, 7 and 8) - // But if want_done_tag is true, we might also see tag 11 for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { srv_stream.Read(&recv_request, tag(tag_idx)); - // Note that we'll add something to the verifier and verify that - // something was seen, but it might be tag 11 and not what we - // just added - int got_tag = verif.Expect(tag_idx, expected_server_cq_result) - .Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); - } + Verifier(GetParam()) + .Expect(tag_idx, expected_server_cq_result) + .Verify(cq_.get(), ignore_cq_result); } if (server_try_cancel_thd != NULL) { @@ -921,15 +892,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - srv_ctx.TryCancel(); - want_done_tag = true; - verif.Expect(11, true); - } - - if (want_done_tag) { - verif.Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; + ServerTryCancel(&srv_ctx); } // The RPC has been cancelled at this point for sure (i.e irrespective of @@ -982,7 +945,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { Verifier(GetParam()).Expect(1, true).Verify(cq_.get()); // On the server, request to be notified of 'ResponseStream' calls and // receive the call just made by the client - srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -990,12 +952,9 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; - bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - srv_ctx.TryCancel(); - Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); + ServerTryCancel(&srv_ctx); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -1003,41 +962,24 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; - - auto verif = Verifier(GetParam()); - if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = - new std::thread(&ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = new std::thread( + &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); // Server will cancel the RPC in a parallel thread while writing responses // to the client. Since the cancellation can happen at anytime, some of // the cq results (i.e those until cancellation) might be true but it is // non deterministic. So better to ignore the cq results ignore_cq_result = true; - // Expect that we might possibly see the done tag that - // indicates cancellation completion in this case - want_done_tag = true; - verif.Expect(11, true); } // Server sends three messages (tags 3, 4 and 5) - // But if want_done tag is true, we might also see tag 11 for (int tag_idx = 3; tag_idx <= 5; tag_idx++) { send_response.set_message("Pong " + std::to_string(tag_idx)); srv_stream.Write(send_response, tag(tag_idx)); - // Note that we'll add something to the verifier and verify that - // something was seen, but it might be tag 11 and not what we - // just added - int got_tag = verif.Expect(tag_idx, expected_cq_result) - .Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == tag_idx) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), tag_idx); - } + Verifier(GetParam()) + .Expect(tag_idx, expected_cq_result) + .Verify(cq_.get(), ignore_cq_result); } if (server_try_cancel_thd != NULL) { @@ -1046,21 +988,13 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - srv_ctx.TryCancel(); - want_done_tag = true; - verif.Expect(11, true); + ServerTryCancel(&srv_ctx); // Client reads may fail bacause it is notified that the stream is // cancelled. ignore_cq_result = true; } - if (want_done_tag) { - verif.Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - } - // Client attemts to read the three messages from the server for (int tag_idx = 6; tag_idx <= 8; tag_idx++) { cli_stream->Read(&recv_response, tag(tag_idx)); @@ -1118,7 +1052,6 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { // On the server, request to be notified of the 'BidiStream' call and // receive the call just made by the client - srv_ctx.AsyncNotifyWhenDone(tag(11)); service_.RequestBidiStream(&srv_ctx, &srv_stream, cq_.get(), cq_.get(), tag(2)); Verifier(GetParam()).Expect(2, true).Verify(cq_.get()); @@ -1130,12 +1063,9 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { bool expected_cq_result = true; bool ignore_cq_result = false; - bool want_done_tag = false; if (server_try_cancel == CANCEL_BEFORE_PROCESSING) { - srv_ctx.TryCancel(); - Verifier(GetParam()).Expect(11, true).Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); + ServerTryCancel(&srv_ctx); // We know for sure that all cq results will be false from this point // since the server cancelled the RPC @@ -1143,84 +1073,42 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } std::thread* server_try_cancel_thd = NULL; - - auto verif = Verifier(GetParam()); - if (server_try_cancel == CANCEL_DURING_PROCESSING) { - server_try_cancel_thd = - new std::thread(&ServerContext::TryCancel, &srv_ctx); + server_try_cancel_thd = new std::thread( + &AsyncEnd2endServerTryCancelTest::ServerTryCancel, this, &srv_ctx); // Since server is going to cancel the RPC in a parallel thread, some of // the cq results (i.e those until the cancellation) might be true. Since // that number is non-deterministic, it is better to ignore the cq results ignore_cq_result = true; - // Expect that we might possibly see the done tag that - // indicates cancellation completion in this case - want_done_tag = true; - verif.Expect(11, true); } - int got_tag; srv_stream.Read(&recv_request, tag(4)); - verif.Expect(4, expected_cq_result); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 4) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 4); - } + Verifier(GetParam()) + .Expect(4, expected_cq_result) + .Verify(cq_.get(), ignore_cq_result); send_response.set_message("Pong"); srv_stream.Write(send_response, tag(5)); - verif.Expect(5, expected_cq_result); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 5) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 5); - } + Verifier(GetParam()) + .Expect(5, expected_cq_result) + .Verify(cq_.get(), ignore_cq_result); cli_stream->Read(&recv_response, tag(6)); - verif.Expect(6, expected_cq_result); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 6) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 6); - } + Verifier(GetParam()) + .Expect(6, expected_cq_result) + .Verify(cq_.get(), ignore_cq_result); // This is expected to succeed in all cases cli_stream->WritesDone(tag(7)); - verif.Expect(7, true); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 7) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 7); - } + Verifier(GetParam()).Expect(7, true).Verify(cq_.get()); // This is expected to fail in all cases i.e for all values of // server_try_cancel. This is because at this point, either there are no // more msgs from the client (because client called WritesDone) or the RPC // is cancelled on the server srv_stream.Read(&recv_request, tag(8)); - verif.Expect(8, false); - got_tag = verif.Next(cq_.get(), ignore_cq_result); - GPR_ASSERT((got_tag == 8) || (got_tag == 11 && want_done_tag)); - if (got_tag == 11) { - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; - // Now get the other entry that we were waiting on - EXPECT_EQ(verif.Next(cq_.get(), ignore_cq_result), 8); - } + Verifier(GetParam()).Expect(8, false).Verify(cq_.get()); if (server_try_cancel_thd != NULL) { server_try_cancel_thd->join(); @@ -1228,15 +1116,7 @@ class AsyncEnd2endServerTryCancelTest : public AsyncEnd2endTest { } if (server_try_cancel == CANCEL_AFTER_PROCESSING) { - srv_ctx.TryCancel(); - want_done_tag = true; - verif.Expect(11, true); - } - - if (want_done_tag) { - verif.Verify(cq_.get()); - EXPECT_TRUE(srv_ctx.IsCancelled()); - want_done_tag = false; + ServerTryCancel(&srv_ctx); } // The RPC has been cancelled at this point for sure (i.e irrespective of From 4cbf32ff66b92162da3553e254ea7a8ca0d82057 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Fri, 4 Mar 2016 12:21:05 -0800 Subject: [PATCH 200/236] mention how to get protoc compiler in base INSTALL.md --- INSTALL.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index d9411db021d..ee4bc2b73d1 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -32,6 +32,16 @@ terminal: $ [sudo] xcode-select --install ``` +##Protoc + +By default gRPC uses [protocol buffers](https://github.com/google/protobuf), +you will need the `protoc` compiler to generate stub server and client code. + +If you compile from source, see below, the Makefile will automatically try +and compile the one present in third_party if you cloned the repository +recursively, and that it detects your system is lacking it. + + #Build from Source For developers who are interested to contribute, here is how to compile the From 60d38cbc1d4c3d5a460ab3e718e7d9acc34c3fba Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Fri, 4 Mar 2016 13:05:04 -0800 Subject: [PATCH 201/236] Simplify Node Windows tests slightly Simplify command that removes OpenSSL headers and remove now-extraneous post-test script. --- tools/run_tests/build_node.bat | 2 +- tools/run_tests/post_test_node.bat | 30 ------------------------------ tools/run_tests/run_tests.py | 5 +---- 3 files changed, 2 insertions(+), 35 deletions(-) delete mode 100644 tools/run_tests/post_test_node.bat diff --git a/tools/run_tests/build_node.bat b/tools/run_tests/build_node.bat index 886af0610ff..82e82083486 100644 --- a/tools/run_tests/build_node.bat +++ b/tools/run_tests/build_node.bat @@ -35,7 +35,7 @@ call npm install --build-from-source @rem delete the redundant openssl headers for /f "delims=v" %%v in ('node --version') do ( - rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q + rmdir "%USERPROFILE%\.node-gyp\%%v\include\node\openssl" /S /Q ) @rem rebuild, because it probably failed the first time diff --git a/tools/run_tests/post_test_node.bat b/tools/run_tests/post_test_node.bat deleted file mode 100644 index 1a2a5491fa7..00000000000 --- a/tools/run_tests/post_test_node.bat +++ /dev/null @@ -1,30 +0,0 @@ -@rem Copyright 2016, Google Inc. -@rem All rights reserved. -@rem -@rem Redistribution and use in source and binary forms, with or without -@rem modification, are permitted provided that the following conditions are -@rem met: -@rem -@rem * Redistributions of source code must retain the above copyright -@rem notice, this list of conditions and the following disclaimer. -@rem * Redistributions in binary form must reproduce the above -@rem copyright notice, this list of conditions and the following disclaimer -@rem in the documentation and/or other materials provided with the -@rem distribution. -@rem * Neither the name of Google Inc. nor the names of its -@rem contributors may be used to endorse or promote products derived from -@rem this software without specific prior written permission. -@rem -@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -rmdir node_modules /S /Q \ No newline at end of file diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index c55d1fbe633..cc004f38d7f 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -290,10 +290,7 @@ class NodeLanguage(object): return [['tools/run_tests/build_node.sh', self.node_version]] def post_tests_steps(self): - if self.platform == 'windows': - return [['tools\\run_tests\\post_test_node.bat']] - else: - return [] + return [] def makefile_name(self): return 'Makefile' From aaf66a9981a658b62be005ac5e04d704545ab2da Mon Sep 17 00:00:00 2001 From: Greg Haines Date: Fri, 4 Mar 2016 13:10:26 -0800 Subject: [PATCH 202/236] Add comments and feature flag. --- src/objective-c/GRPCClient/private/GRPCCompletionQueue.m | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index ffbb14374d1..b7d5af67d9b 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -69,7 +69,11 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ - gpr_timespec deadline = gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); + // Using a non-infinite deadline to re-enter grpc_completion_queue_next() + // alleviates https://github.com/grpc/grpc/issues/5593 + gpr_timespec deadline = (timeoutSecs < 0) + ? gpr_inf_future(GPR_CLOCK_REALTIME) + : gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); while (YES) { // The following call blocks until an event is available or the deadline elapses. grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); From 11ce4ffc75fef5a790ae55285457d787cbb50cea Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Fri, 4 Mar 2016 13:47:32 -0800 Subject: [PATCH 203/236] update min node version --- src/node/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/README.md b/src/node/README.md index b46b9862432..3501b54a665 100644 --- a/src/node/README.md +++ b/src/node/README.md @@ -5,7 +5,7 @@ Beta ## PREREQUISITES -- `node`: This requires `node` to be installed. If you instead have the `nodejs` executable on Debian, you should install the [`nodejs-legacy`](https://packages.debian.org/sid/nodejs-legacy) package. +- `node`: This requires `node` to be installed, version `0.12` or above. If you instead have the `nodejs` executable on Debian, you should install the [`nodejs-legacy`](https://packages.debian.org/sid/nodejs-legacy) package. ## INSTALLATION From 459edd97123b522de40e03302eb8741841aad8ec Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 4 Mar 2016 14:32:11 -0800 Subject: [PATCH 204/236] Revert "Pass a non-infinite deadline to grpc_completion_queue_next() to prevent queues from blocking indefinitely in poll()" --- .../GRPCClient/private/GRPCCompletionQueue.h | 7 ------ .../GRPCClient/private/GRPCCompletionQueue.m | 24 ++++--------------- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h index 03fd2e0d955..fe3b8f39d12 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h @@ -36,8 +36,6 @@ typedef void(^GRPCQueueCompletionHandler)(bool success); -extern const int64_t kGRPCCompletionQueueDefaultTimeoutSecs; - /** * This class lets one more easily use |grpc_completion_queue|. To use it, pass the value of the * |unmanagedQueue| property of an instance of this class to |grpc_channel_create_call|. Then for @@ -51,11 +49,6 @@ extern const int64_t kGRPCCompletionQueueDefaultTimeoutSecs; */ @interface GRPCCompletionQueue : NSObject @property(nonatomic, readonly) grpc_completion_queue *unmanagedQueue; -@property(nonatomic, readonly) int64_t timeoutSecs; + (instancetype)completionQueue; - -- (instancetype)init; -- (instancetype)initWithTimeout:(int64_t)timeoutSecs NS_DESIGNATED_INITIALIZER; - @end diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index b7d5af67d9b..ea2b01ee1d7 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -35,9 +35,6 @@ #import - -const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; - @implementation GRPCCompletionQueue + (instancetype)completionQueue { @@ -45,13 +42,8 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; } - (instancetype)init { - return [self initWithTimeout:kGRPCCompletionQueueDefaultTimeoutSecs]; -} - -- (instancetype)initWithTimeout:(int64_t)timeoutSecs { if ((self = [super init])) { _unmanagedQueue = grpc_completion_queue_create(NULL); - _timeoutSecs = timeoutSecs; // This is for the following block to capture the pointer by value (instead // of retaining self and doing self->_unmanagedQueue). This is essential @@ -69,28 +61,22 @@ const int64_t kGRPCCompletionQueueDefaultTimeoutSecs = 60; gDefaultConcurrentQueue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); }); dispatch_async(gDefaultConcurrentQueue, ^{ - // Using a non-infinite deadline to re-enter grpc_completion_queue_next() - // alleviates https://github.com/grpc/grpc/issues/5593 - gpr_timespec deadline = (timeoutSecs < 0) - ? gpr_inf_future(GPR_CLOCK_REALTIME) - : gpr_time_from_seconds(timeoutSecs, GPR_CLOCK_REALTIME); while (YES) { - // The following call blocks until an event is available or the deadline elapses. - grpc_event event = grpc_completion_queue_next(unmanagedQueue, deadline, NULL); + // The following call blocks until an event is available. + grpc_event event = grpc_completion_queue_next(unmanagedQueue, + gpr_inf_future(GPR_CLOCK_REALTIME), + NULL); GRPCQueueCompletionHandler handler; switch (event.type) { case GRPC_OP_COMPLETE: handler = (__bridge_transfer GRPCQueueCompletionHandler)event.tag; handler(event.success); break; - case GRPC_QUEUE_TIMEOUT: - // Nothing to do here - break; case GRPC_QUEUE_SHUTDOWN: grpc_completion_queue_destroy(unmanagedQueue); return; default: - [NSException raise:@"Unrecognized completion type" format:@"type=%d", event.type]; + [NSException raise:@"Unrecognized completion type" format:@""]; } }; }); From f5dca1089639ab124e9cbc0db6893f6f0ada0fc6 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 4 Mar 2016 14:45:39 -0800 Subject: [PATCH 205/236] Revert "Improve perf smoke test stability" --- tools/jenkins/run_performance.sh | 59 ++++++-------------------------- 1 file changed, 11 insertions(+), 48 deletions(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index b06fce7acb9..fbc078330f7 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -42,10 +42,17 @@ config=opt make CONFIG=$config qps_worker qps_driver -j8 +bins/$config/qps_worker -driver_port 10000 & +PID1=$! +bins/$config/qps_worker -driver_port 10010 & +PID2=$! + # # Put a timeout on these tests # -((sleep 900; killall qps_worker && rm -f /tmp/qps-test.$$ && kill $$)&) +((sleep 900; kill $$ && killall qps_worker && rm -f /tmp/qps-test.$$ )&) + +export QPS_WORKERS="localhost:10000,localhost:10010" # big is the size in bytes of large messages (0 is the size otherwise) big=65536 @@ -62,74 +69,40 @@ deep=100 cores=`grep -c ^processor /proc/cpuinfo` halfcores=`expr $cores / 2` -# -# Keep changing the ports -port1=10000 -port2=10100 - for secure in true false; do # Scenario 1: generic async streaming ping-pong (contentionless latency) - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true # Scenario 2: generic async streaming "unconstrained" (QPS) - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 \ --server_core_limit=$halfcores --client_core_limit=0 |& tee /tmp/qps-test.$$ - bins/$config/qps_driver --quit=true # Scenario 2b: QPS with a single server core - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 --server_core_limit=1 --client_core_limit=0 - bins/$config/qps_driver --quit=true # Scenario 2c: protobuf-based QPS - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --simple_req_size=0 --simple_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true # Scenario 3: Latency at sub-peak load (all clients equally loaded) for loadfactor in 0.7; do - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ @@ -137,37 +110,27 @@ for secure in true false; do --num_servers=1 --num_clients=0 --poisson_load=`awk -v lf=$loadfactor \ '$5 == "QPS:" {print int(lf * $6); exit}' /tmp/qps-test.$$` \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true done rm /tmp/qps-test.$$ # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM). - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true # Scenario 5: Sync unary ping-pong with protobufs - bins/$config/qps_worker -driver_port $port1 & - bins/$config/qps_worker -driver_port $port2 & - export QPS_WORKERS="localhost:$port1,localhost:$port2" - port1=`expr $port1 + 1` - port2=`expr $port2 + 1` bins/$config/qps_driver --rpc_type=UNARY --client_type=SYNC_CLIENT \ --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \ --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \ --secure_test=$secure --num_servers=1 --num_clients=1 \ --server_core_limit=$halfcores --client_core_limit=0 - bins/$config/qps_driver --quit=true + done +bins/$config/qps_driver --quit=true + wait From b3320fc7c387c0105aeb13911288ec33e9f9470e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 14:52:06 -0800 Subject: [PATCH 206/236] Fix race in poll() based pollset It was possible for entries in watchers[] to be deleted by another thread before we called begin_poll. I'd like to do something nicer than this eventually... but that'll be easier once we've got the polling loop cleaned up (which is currently WIP) --- src/core/iomgr/pollset_multipoller_with_poll_posix.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 4dddfff2302..0a447ba96c1 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -122,6 +122,7 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( } else { h->fds[fd_count++] = h->fds[i]; watchers[pfd_count].fd = h->fds[i]; + GRPC_FD_REF(watchers[pfd_count].fd, "multipoller_start"); pfds[pfd_count].fd = h->fds[i]->fd; pfds[pfd_count].revents = 0; pfd_count++; @@ -135,8 +136,10 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( gpr_mu_unlock(&pollset->mu); for (i = 2; i < pfd_count; i++) { - pfds[i].events = (short)grpc_fd_begin_poll(watchers[i].fd, pollset, worker, + grpc_fd *fd = watchers[i].fd; + pfds[i].events = (short)grpc_fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, &watchers[i]); + GRPC_FD_UNREF(fd, "multipoller_start"); } /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid From 9adecb06e0bc66ffc98aafa087f946ee42473c01 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 4 Mar 2016 14:54:10 -0800 Subject: [PATCH 207/236] Fix race between parsing messages and receiving status in Node client --- src/node/interop/interop_client.js | 3 + src/node/src/client.js | 114 +++++++++++++++++++++-------- src/node/test/surface_test.js | 8 ++ 3 files changed, 95 insertions(+), 30 deletions(-) diff --git a/src/node/interop/interop_client.js b/src/node/interop/interop_client.js index db383e4d000..5602011a8e0 100644 --- a/src/node/interop/interop_client.js +++ b/src/node/interop/interop_client.js @@ -290,6 +290,7 @@ function timeoutOnSleepingServer(client, done) { call.write({ payload: {body: zeroBuffer(27182)} }); + call.on('data', function() {}); call.on('error', function(error) { assert(error.code === grpc.status.DEADLINE_EXCEEDED || @@ -336,6 +337,7 @@ function customMetadata(client, done) { ['test_initial_metadata_value']); done(); }); + stream.on('data', function() {}); stream.on('status', function(status) { var echo_trailer = status.metadata.get(ECHO_TRAILING_KEY); assert(echo_trailer.length > 0); @@ -361,6 +363,7 @@ function statusCodeAndMessage(client, done) { done(); }); var duplex = client.fullDuplexCall(); + duplex.on('data', function() {}); duplex.on('status', function(status) { assert(status); assert.strictEqual(status.code, 2); diff --git a/src/node/src/client.js b/src/node/src/client.js index c65dd736503..9acf51bd98b 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -131,8 +131,68 @@ function ClientReadableStream(call, deserialize) { this.finished = false; this.reading = false; this.deserialize = common.wrapIgnoreNull(deserialize); + /* Status generated from reading messages from the server. Overrides the + * status from the server if not OK */ + this.read_status = null; + /* Status received from the server. */ + this.received_status = null; } +/** + * Called when all messages from the server have been processed. The status + * parameter indicates that the call should end with that status. status + * defaults to OK if not provided. + * @param {Object!} status The status that the call should end with + */ +function _readsDone(status) { + /* jshint validthis: true */ + if (!status) { + status = {code: grpc.status.OK, details: 'OK'}; + } + this.finished = true; + this.read_status = status; + this._emitStatusIfDone(); +} + +ClientReadableStream.prototype._readsDone = _readsDone; + +/** + * Called to indicate that we have received a status from the server. + */ +function _receiveStatus(status) { + /* jshint validthis: true */ + this.received_status = status; + this._emitStatusIfDone(); +} + +ClientReadableStream.prototype._receiveStatus = _receiveStatus; + +/** + * If we have both processed all incoming messages and received the status from + * the server, emit the status. Otherwise, do nothing. + */ +function _emitStatusIfDone() { + /* jshint validthis: true */ + var status; + if (this.read_status && this.received_status) { + if (this.read_status.code !== grpc.status.OK) { + status = this.read_status; + } else { + status = this.received_status; + } + this.emit('status', status); + if (status.code !== grpc.status.OK) { + var error = new Error(status.details); + error.code = status.code; + error.metadata = status.metadata; + this.emit('error', error); + return; + } + } +} + +ClientReadableStream.prototype._emitStatusIfDone = _emitStatusIfDone; + /** * Read the next object from the stream. * @access private @@ -150,6 +210,7 @@ function _read(size) { if (err) { // Something has gone wrong. Stop reading and wait for status self.finished = true; + self._readsDone(); return; } var data = event.read; @@ -157,8 +218,11 @@ function _read(size) { try { deserialized = self.deserialize(data); } catch (e) { - self.call.cancelWithStatus(grpc.status.INTERNAL, - 'Failed to parse server response'); + self._readsDone({code: grpc.status.INTERNAL, + details: 'Failed to parse server response'}); + } + if (data === null) { + self._readsDone(); } if (self.push(deserialized) && data !== null) { var read_batch = {}; @@ -198,6 +262,11 @@ function ClientDuplexStream(call, serialize, deserialize) { this.serialize = common.wrapIgnoreNull(serialize); this.deserialize = common.wrapIgnoreNull(deserialize); this.call = call; + /* Status generated from reading messages from the server. Overrides the + * status from the server if not OK */ + this.read_status = null; + /* Status received from the server. */ + this.received_status = null; this.on('finish', function() { var batch = {}; batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; @@ -205,6 +274,9 @@ function ClientDuplexStream(call, serialize, deserialize) { }); } +ClientDuplexStream.prototype._readsDone = _readsDone; +ClientDuplexStream.prototype._receiveStatus = _receiveStatus; +ClientDuplexStream.prototype._emitStatusIfDone = _emitStatusIfDone; ClientDuplexStream.prototype._read = _read; ClientDuplexStream.prototype._write = _write; @@ -487,22 +559,13 @@ function makeServerStreamRequestFunction(method, serialize, deserialize) { var status_batch = {}; status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; call.startBatch(status_batch, function(err, response) { - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - stream.emit('status', response.status); - if (response.status.code !== grpc.status.OK) { - var error = new Error(response.status.details); - error.code = response.status.code; - error.metadata = response.status.metadata; - stream.emit('error', error); + if (err) { + stream.emit('error', err); return; - } else { - if (err) { - // Got a batch error, but OK status. Something went wrong - stream.emit('error', err); - return; - } } + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + stream._receiveStatus(response.status); }); return stream; } @@ -552,22 +615,13 @@ function makeBidiStreamRequestFunction(method, serialize, deserialize) { var status_batch = {}; status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; call.startBatch(status_batch, function(err, response) { - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - stream.emit('status', response.status); - if (response.status.code !== grpc.status.OK) { - var error = new Error(response.status.details); - error.code = response.status.code; - error.metadata = response.status.metadata; - stream.emit('error', error); + if (err) { + stream.emit('error', err); return; - } else { - if (err) { - // Got a batch error, but OK status. Something went wrong - stream.emit('error', err); - return; - } } + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + stream._receiveStatus(response.status); }); return stream; } diff --git a/src/node/test/surface_test.js b/src/node/test/surface_test.js index 530f1f77494..8a232d6fc44 100644 --- a/src/node/test/surface_test.js +++ b/src/node/test/surface_test.js @@ -1000,6 +1000,7 @@ describe('Call propagation', function() { proxy_impl.serverStream = function(parent) { var child = client.serverStream(parent.request, null, {parent: parent}); + child.on('data', function() {}); child.on('error', function(err) { assert(err); assert.strictEqual(err.code, grpc.status.CANCELLED); @@ -1013,6 +1014,7 @@ describe('Call propagation', function() { var proxy_client = new Client('localhost:' + proxy_port, grpc.credentials.createInsecure()); call = proxy_client.serverStream({}); + call.on('data', function() {}); call.on('error', function(err) { done(); }); @@ -1022,6 +1024,7 @@ describe('Call propagation', function() { var call; proxy_impl.bidiStream = function(parent) { var child = client.bidiStream(null, {parent: parent}); + child.on('data', function() {}); child.on('error', function(err) { assert(err); assert.strictEqual(err.code, grpc.status.CANCELLED); @@ -1035,6 +1038,7 @@ describe('Call propagation', function() { var proxy_client = new Client('localhost:' + proxy_port, grpc.credentials.createInsecure()); call = proxy_client.bidiStream(); + call.on('data', function() {}); call.on('error', function(err) { done(); }); @@ -1074,6 +1078,7 @@ describe('Call propagation', function() { proxy_impl.bidiStream = function(parent) { var child = client.bidiStream( null, {parent: parent, propagate_flags: deadline_flags}); + child.on('data', function() {}); child.on('error', function(err) { assert(err); assert(err.code === grpc.status.DEADLINE_EXCEEDED || @@ -1089,6 +1094,7 @@ describe('Call propagation', function() { var deadline = new Date(); deadline.setSeconds(deadline.getSeconds() + 1); var call = proxy_client.bidiStream(null, {deadline: deadline}); + call.on('data', function() {}); call.on('error', function(err) { done(); }); @@ -1130,6 +1136,7 @@ describe('Cancelling surface client', function() { }); it('Should correctly cancel a server stream call', function(done) { var call = client.fib({'limit': 5}); + call.on('data', function() {}); call.on('error', function(error) { assert.strictEqual(error.code, surface_client.status.CANCELLED); done(); @@ -1138,6 +1145,7 @@ describe('Cancelling surface client', function() { }); it('Should correctly cancel a bidi stream call', function(done) { var call = client.divMany(); + call.on('data', function() {}); call.on('error', function(error) { assert.strictEqual(error.code, surface_client.status.CANCELLED); done(); From f1b6d61965adf0cf264c826b0a08b5a8b32e57c3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 15:00:02 -0800 Subject: [PATCH 208/236] Fix formatting --- src/core/iomgr/pollset_multipoller_with_poll_posix.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 0a447ba96c1..92d6fb72414 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -137,8 +137,8 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( for (i = 2; i < pfd_count; i++) { grpc_fd *fd = watchers[i].fd; - pfds[i].events = (short)grpc_fd_begin_poll(fd, pollset, worker, - POLLIN, POLLOUT, &watchers[i]); + pfds[i].events = (short)grpc_fd_begin_poll(fd, pollset, worker, POLLIN, + POLLOUT, &watchers[i]); GRPC_FD_UNREF(fd, "multipoller_start"); } From c1edf224797367b62cd88ecff37dae23d333d308 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 4 Mar 2016 15:01:03 -0800 Subject: [PATCH 209/236] Fix sanity --- src/objective-c/GRPCClient/private/GRPCCompletionQueue.h | 2 +- src/objective-c/GRPCClient/private/GRPCCompletionQueue.m | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h index fe3b8f39d12..7b66cd4c329 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index ea2b01ee1d7..ff3031678c5 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 208fd6b339364f9677bbcdfc3bb955ca908fbe2e Mon Sep 17 00:00:00 2001 From: VcamX Date: Mon, 7 Mar 2016 13:08:38 +0800 Subject: [PATCH 210/236] Set grace=0 to server stop in route_guide Fix #5619 --- examples/python/route_guide/route_guide_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python/route_guide/route_guide_server.py b/examples/python/route_guide/route_guide_server.py index f23b98bf367..24f948c42c5 100644 --- a/examples/python/route_guide/route_guide_server.py +++ b/examples/python/route_guide/route_guide_server.py @@ -128,7 +128,7 @@ def serve(): while True: time.sleep(_ONE_DAY_IN_SECONDS) except KeyboardInterrupt: - server.stop() + server.stop(0) if __name__ == '__main__': serve() From 0211098e935165801a63a866d19006bd8c49cccf Mon Sep 17 00:00:00 2001 From: Lisa Carey Date: Mon, 7 Mar 2016 17:41:51 +0000 Subject: [PATCH 211/236] Removes confusing/unnecessary section on generating code. --- examples/cpp/README.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 979ba55dbb5..e4b0eb698bf 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -23,21 +23,6 @@ Change your current directory to examples/cpp/helloworld $ cd examples/cpp/helloworld/ ``` - -### Generating gRPC code - -To generate the client and server side interfaces: - -```sh -$ make helloworld.grpc.pb.cc helloworld.pb.cc -``` -Which internally invokes the proto-compiler as: - -```sh -$ protoc -I ../../protos/ --grpc_out=. --plugin=protoc-gen-grpc=grpc_cpp_plugin ../../protos/helloworld.proto -$ protoc -I ../../protos/ --cpp_out=. ../../protos/helloworld.proto -``` - ### Client and server implementations The client implementation is at [greeter_client.cc](helloworld/greeter_client.cc). From f880e62e04319c92caf361bc6d6a308209a56938 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 10:14:48 -0800 Subject: [PATCH 212/236] Potential fix for race condition exposed by Node --- src/core/channel/subchannel_call_holder.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c index 81297c8d449..0f765c218b4 100644 --- a/src/core/channel/subchannel_call_holder.c +++ b/src/core/channel/subchannel_call_holder.c @@ -168,21 +168,24 @@ retry: static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) { grpc_subchannel_call_holder *holder = arg; - grpc_subchannel_call *call; gpr_mu_lock(&holder->mu); GPR_ASSERT(holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL); - call = GET_CALL(holder); - GPR_ASSERT(call == NULL || call == CANCELLED_CALL); holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; if (holder->connected_subchannel == NULL) { fail_locked(exec_ctx, holder); } else { - gpr_atm_rel_store( + if (!gpr_atm_rel_cas( &holder->subchannel_call, + 0, (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( - exec_ctx, holder->connected_subchannel, holder->pollset)); - retry_waiting_locked(exec_ctx, holder); + exec_ctx, holder->connected_subchannel, holder->pollset))) { + GPR_ASSERT(gpr_atm_acq_load(&holder->subchannel_call) == 1); + /* if this cas fails, the call was cancelled before the pick completed */ + fail_locked(exec_ctx, holder); + } else { + retry_waiting_locked(exec_ctx, holder); + } } gpr_mu_unlock(&holder->mu); GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel"); From 84774b6c4a25ee38a1778778e00cdd81203d874f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 10:26:31 -0800 Subject: [PATCH 213/236] clang-format --- src/core/channel/subchannel_call_holder.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c index 0f765c218b4..8f46885a043 100644 --- a/src/core/channel/subchannel_call_holder.c +++ b/src/core/channel/subchannel_call_holder.c @@ -176,10 +176,9 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) { fail_locked(exec_ctx, holder); } else { if (!gpr_atm_rel_cas( - &holder->subchannel_call, - 0, - (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( - exec_ctx, holder->connected_subchannel, holder->pollset))) { + &holder->subchannel_call, 0, + (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( + exec_ctx, holder->connected_subchannel, holder->pollset))) { GPR_ASSERT(gpr_atm_acq_load(&holder->subchannel_call) == 1); /* if this cas fails, the call was cancelled before the pick completed */ fail_locked(exec_ctx, holder); From de14e50a914c9db99d90812cbaec0110b3e1a52d Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 7 Mar 2016 12:11:47 -0800 Subject: [PATCH 214/236] text update --- INSTALL.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index ee4bc2b73d1..3c694ba5822 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -37,9 +37,10 @@ terminal: By default gRPC uses [protocol buffers](https://github.com/google/protobuf), you will need the `protoc` compiler to generate stub server and client code. -If you compile from source, see below, the Makefile will automatically try -and compile the one present in third_party if you cloned the repository -recursively, and that it detects your system is lacking it. +If you compile gRPC from source, as described below, the Makefile will +automatically try and compile the `protoc` in third_party if you cloned the +repository recursively and it detects that you don't already have it +installed. #Build from Source From e6cc9c7a253464e17abfd0fbd7dd5f1a52df36da Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 7 Mar 2016 12:36:38 -0800 Subject: [PATCH 215/236] custom test target for parallel test running --- setup.py | 1 + src/python/grpcio/commands.py | 35 +++++++++++++++++++++++++++++++++++ tools/run_tests/run_python.sh | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cfb578e2158..e1abe7fd8ff 100644 --- a/setup.py +++ b/setup.py @@ -165,6 +165,7 @@ COMMAND_CLASS = { 'build_tagged_ext': precompiled.BuildTaggedExt, 'gather': commands.Gather, 'run_interop': commands.RunInterop, + 'test_lite': commands.TestLite } # Ensure that package data is copied over before any commands have been run: diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index aa29c728f25..ec116f58abe 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -264,6 +264,41 @@ class Gather(setuptools.Command): self.distribution.fetch_build_eggs(self.distribution.tests_require) +class TestLite(setuptools.Command): + """Command to run tests without fetching or building anything.""" + + description = 'run tests without fetching or building anything.' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + # distutils requires this override. + pass + + def run(self): + self._add_eggs_to_path() + + import tests + loader = tests.Loader() + loader.loadTestsFromNames(['tests']) + runner = tests.Runner() + result = runner.run(loader.suite) + if not result.wasSuccessful(): + sys.exit(1) + + def _add_eggs_to_path(self): + """Adds all egg files under .eggs to sys.path""" + import pkg_resources + eggs_dir = os.path.join(PYTHON_STEM, '../../../.eggs') + eggs = [os.path.join(eggs_dir, filename) + for filename in os.listdir(eggs_dir) + if filename.endswith('.egg')] + for egg in eggs: + sys.path.insert(0, pkg_resources.normalize_path(egg)) + + class RunInterop(test.test): description = 'run interop test client/server' diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index beb747a6169..ace49e15142 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -46,7 +46,7 @@ if [ "$CONFIG" = "gcov" ] then tox else - $ROOT/.tox/py27/bin/python $ROOT/setup.py test + $ROOT/.tox/py27/bin/python $ROOT/setup.py test_lite fi mkdir -p $ROOT/reports From 8395bfc0b6b443830291b76b961548fc28682f3d Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 7 Mar 2016 14:07:56 -0800 Subject: [PATCH 216/236] Put back the gflags/gtest requirement to doc --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9423c46547a..6fb9a846098 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,13 @@ In order to run all of the tests we provide, you will need valgrind and clang. More specifically, under debian, you will need the package libc++-dev to properly run all the tests. +Compiling and running grpc C++ tests depend on protobuf 3.0.0, gtest and gflags. +Although gflags is provided in third_party, you will need to manually install +that dependency on your system to run these tests. Under a Debian or Ubuntu +system, you can install the gtests and gflags packages using apt-get: + +`apt-get install libgflags-dev libgtest-dev` + If you are planning to work on any of the languages other than C and C++, you will also need their appropriate development environments. From 3a84f46a29a0a4db44a70a0026f726327e074877 Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 7 Mar 2016 14:44:46 -0800 Subject: [PATCH 217/236] resolve comments --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6fb9a846098..d549c5d2084 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,9 @@ Although gflags is provided in third_party, you will need to manually install that dependency on your system to run these tests. Under a Debian or Ubuntu system, you can install the gtests and gflags packages using apt-get: -`apt-get install libgflags-dev libgtest-dev` +```sh + $ [sudo] apt-get install libgflags-dev libgtest-dev +``` If you are planning to work on any of the languages other than C and C++, you will also need their appropriate development environments. From 17908c168def9cba95c548786c39bfe49de00a3f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 7 Mar 2016 15:04:17 -0800 Subject: [PATCH 218/236] Address comments and add a TODO. --- src/python/grpcio/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index ec116f58abe..3207ea3052e 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -286,10 +286,11 @@ class TestLite(setuptools.Command): runner = tests.Runner() result = runner.run(loader.suite) if not result.wasSuccessful(): - sys.exit(1) + sys.exit('Test failure') def _add_eggs_to_path(self): """Adds all egg files under .eggs to sys.path""" + # TODO(jtattemusch): there has to be a cleaner way to do this import pkg_resources eggs_dir = os.path.join(PYTHON_STEM, '../../../.eggs') eggs = [os.path.join(eggs_dir, filename) From ac731e051e755b4392ffa272804b7ada0d0bb8b3 Mon Sep 17 00:00:00 2001 From: vjpai Date: Mon, 7 Mar 2016 15:07:33 -0800 Subject: [PATCH 219/236] @a11r asked me to respond to some BSD folks, so I figured that I should also be a potential mentor if needed. --- summerofcode/ideas.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/summerofcode/ideas.md b/summerofcode/ideas.md index c7f368e7cc2..83f2cecd48c 100644 --- a/summerofcode/ideas.md +++ b/summerofcode/ideas.md @@ -19,7 +19,9 @@ gRPC C Core: 1. Port gRPC to one of the major BSD platforms ([FreeBSD](https://freebsd.org), [NetBSD](https://netbsd.org), and [OpenBSD](https://openbsd.org)) and create packages for them. Add [kqueue](https://www.freebsd.org/cgi/man.cgi?query=kqueue) support in the process. * **Required skills:** C programming language, BSD operating system. - * **Likely mentors:** [Craig Tiller](https://github.com/ctiller), [Nicolas Noble](https://github.com/nicolasnoble). + * **Likely mentors:** [Craig Tiller](https://github.com/ctiller), + [Nicolas Noble](https://github.com/nicolasnoble), + [Vijay Pai](https://github.com/vjpai). 1. Fix gRPC C-core's URI parser. The current parser does not qualify as a standard parser according to [RFC3986]( https://tools.ietf.org/html/rfc3986). Write test suites to verify this and make changes necessary to make the URI parser compliant. * **Required skills:** C programming language, HTTP standard compliance. * **Likely mentors:** [Craig Tiller](https://github.com/ctiller). From a41a6775d4487cce3eea5084928da39ceff923fd Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 7 Mar 2016 17:10:07 -0800 Subject: [PATCH 220/236] php: update readme --- examples/php/README.md | 10 ++-- src/php/README.md | 115 +++++++++++++++++++++++++++++------------ 2 files changed, 87 insertions(+), 38 deletions(-) diff --git a/examples/php/README.md b/examples/php/README.md index 8fb060863a3..ea9ccb67908 100644 --- a/examples/php/README.md +++ b/examples/php/README.md @@ -4,16 +4,15 @@ gRPC in 3 minutes (PHP) PREREQUISITES ------------- -This requires PHP 5.5 or greater. +This requires `php` >=5.5, `phpize`, `pecl`, `phpunit` INSTALL ------- - - On Mac OS X, install [homebrew][]. Run the following command to install gRPC. + - Install the gRPC PHP extension ```sh - $ curl -fsSL https://goo.gl/getgrpc | bash -s php + $ [sudo] pecl install grpc-beta ``` - This will download and run the [gRPC install script][] and compile the gRPC PHP extension. - Clone this repository @@ -37,6 +36,7 @@ TRY IT! Please follow the instruction in [Node][] to run the server ``` $ cd examples/node + $ npm install $ nodejs greeter_server.js ``` @@ -58,7 +58,5 @@ TUTORIAL You can find a more detailed tutorial in [gRPC Basics: PHP][] -[homebrew]:http://brew.sh -[gRPC install script]:https://raw.githubusercontent.com/grpc/homebrew-grpc/master/scripts/install [Node]:https://github.com/grpc/grpc/tree/master/examples/node [gRPC Basics: PHP]:http://www.grpc.io/docs/tutorials/basic/php.html diff --git a/src/php/README.md b/src/php/README.md index b368482f068..c97c3796cda 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -9,14 +9,21 @@ Beta ## Environment -Prerequisite: PHP 5.5 or later, `phpunit`, `pecl` +Prerequisite: `php` >=5.5, `phpize`, `pecl`, `phpunit` -**Linux:** +**Linux (Debian):** ```sh $ sudo apt-get install php5 php5-dev php-pear ``` +**Linux (CentOS):** + +```sh +$ yum install php55w +$ yum --enablerepo=remi,remi-php55 install php-devel php-pear +``` + **Mac OS X:** ```sh @@ -24,11 +31,11 @@ $ curl -O http://pear.php.net/go-pear.phar $ sudo php -d detect_unicode=0 go-pear.phar ``` -**PHPUnit: (Both Linux and Mac OS X)** +**PHPUnit:** ```sh -$ curl https://phar.phpunit.de/phpunit.phar -o phpunit.phar -$ chmod +x phpunit.phar -$ sudo mv phpunit.phar /usr/local/bin/phpunit +$ wget https://phar.phpunit.de/phpunit-old.phar +$ chmod +x phpunit-old.phar +$ sudo mv phpunit-old.phar /usr/bin/phpunit ``` ## Quick Install @@ -39,15 +46,22 @@ Install the gRPC PHP extension sudo pecl install grpc-beta ``` +This will compile and install the gRPC PHP extension into the standard PHP extension directory. You should be able to run the [unit tests](#unit-tests), with the PHP extension installed. + +To run tests with generated stub code from `.proto` files, you will need the `composer`, `protoc` and `protoc-gen-php` binaries additionally. See sections [below](#generated-code-tests). + ## Build from Source + +### gRPC C core library + Clone this repository ```sh $ git clone https://github.com/grpc/grpc.git ``` -Build and install the gRPC C core libraries +Build and install the gRPC C core library ```sh $ cd grpc @@ -56,20 +70,15 @@ $ make $ sudo make install ``` -Note: you may encounter a warning about the Protobuf compiler `protoc` 3.0.0+ not being installed. The following might help, and will be useful later on when we need to compile the `protoc-gen-php` tool. +### gRPC PHP extension -```sh -$ cd grpc/third_party/protobuf -$ sudo make install # 'make' should have been run by core grpc -``` - -Install the gRPC PHP extension +Install the gRPC PHP extension from PECL ```sh $ sudo pecl install grpc-beta ``` -OR +Or, compile from source ```sh $ cd grpc/src/php/ext/grpc @@ -79,58 +88,96 @@ $ make $ sudo make install ``` +### Update php.ini + Add this line to your `php.ini` file, e.g. `/etc/php5/cli/php.ini` ```sh extension=grpc.so ``` -Install Composer +## Unit Tests + +You will need the source code to run tests + +```sh +$ git clone https://github.com/grpc/grpc.git +$ cd grpc +$ git pull --recurse-submodules && git submodule update --init --recursive +``` + +Run unit tests ```sh $ cd grpc/src/php +$ ./bin/run_tests.sh +``` + +## Generated Code Tests + +### Composer + +You need to install `composer`, to pull in some runtime dependencies based on the `composer.json` file. + +```sh $ curl -sS https://getcomposer.org/installer | php $ sudo mv composer.phar /usr/local/bin/composer + +$ cd grpc/src/php $ composer install ``` -## Unit Tests +### Protobuf compiler -Run unit tests +You need the install the protobuf compiler, `protoc`, 3.0.0+. + +If you had compiled the gRPC C core library from source above, the `protoc` binary should have been installed as well. In the case it wasn't, you can run the following commands to install it. ```sh -$ cd grpc/src/php -$ ./bin/run_tests.sh +$ cd grpc/third_party/protobuf +$ sudo make install # 'make' should have been run by core grpc ``` -## Generated Code Tests +Or you can download a `protoc` binaries from [here](https://github.com/google/protobuf/releases). + -Install `protoc-gen-php` +### PHP protobuf compiler + +You need to install `protoc-gen-php`, so that you can generate stub classes `.php` files from service definition `.proto` files. ```sh -$ cd grpc/src/php/vendor/datto/protobuf-php +$ cd grpc/src/php/vendor/datto/protobuf-php # if you had run `composer install` in the previous step + +OR + +$ git clone https://github.com/stanley-cheung/Protobuf-PHP # clone from github repo + $ gem install rake ronn $ rake pear:package version=1.0 $ sudo pear install Protobuf-1.0.tgz ``` -Generate client stub code +### Client Stub + +Generate client stub classes from `.proto` files ```sh $ cd grpc/src/php $ ./bin/generate_proto_php.sh ``` -Run a local server serving the math services +### Run test server - - Please see [Node][] on how to run an example server +Run a local server serving the math services. Please see [Node][] on how to run an example server ```sh -$ cd grpc/src/node +$ cd grpc $ npm install -$ nodejs examples/math_server.js +$ nodejs src/node/test/math/math_server.js ``` +### Run test client + Run the generated code tests ```sh @@ -161,13 +208,15 @@ $ sudo service apache2 restart Make sure the Node math server is still running, as above. ```sh -$ cd grpc/src/node -$ nodejs examples/math_server.js +$ cd grpc +$ npm install +$ nodejs src/node/test/math/math_server.js ``` Make sure you have run `composer install` to generate the `vendor/autoload.php` file ```sh +$ cd grpc/src/php $ composer install ``` @@ -229,13 +278,15 @@ $ sudo service php5-fpm restart Make sure the Node math server is still running, as above. ```sh -$ cd grpc/src/node -$ nodejs examples/math_server.js +$ cd grpc +$ npm install +$ nodejs src/node/test/math/math_server.js ``` Make sure you have run `composer install` to generate the `vendor/autoload.php` file ```sh +$ cd grpc/src/php $ composer install ``` From 710d58cfae07c1815cc4791977069beb47aedf78 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 20:00:50 -0800 Subject: [PATCH 221/236] Disable profiling in qps_worker for now --- test/cpp/qps/qps_worker.cc | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index 9442017ddf5..ce4b773d92b 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -101,6 +101,19 @@ static std::unique_ptr CreateServer(const ServerConfig& config) { abort(); } +class ScopedProfile GRPC_FINAL { + public: + ScopedProfile(const char *filename, bool enable) : enable_(enable) { + if (enable_) grpc_profiler_start(filename); + } + ~ScopedProfile() { + if (enable_) grpc_profiler_stop(); + } + + private: + const bool enable_; +}; + class WorkerServiceImpl GRPC_FINAL : public WorkerService::Service { public: WorkerServiceImpl(int server_port, QpsWorker* worker) @@ -114,9 +127,8 @@ class WorkerServiceImpl GRPC_FINAL : public WorkerService::Service { return Status(StatusCode::RESOURCE_EXHAUSTED, ""); } - grpc_profiler_start("qps_client.prof"); + ScopedProfile profile("qps_client.prof", false); Status ret = RunClientBody(ctx, stream); - grpc_profiler_stop(); return ret; } @@ -128,9 +140,8 @@ class WorkerServiceImpl GRPC_FINAL : public WorkerService::Service { return Status(StatusCode::RESOURCE_EXHAUSTED, ""); } - grpc_profiler_start("qps_server.prof"); + ScopedProfile profile("qps_server.prof", false); Status ret = RunServerBody(ctx, stream); - grpc_profiler_stop(); return ret; } From 9d0638c03874b99ad3b0d42c50bad660d5f97c77 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 20:23:37 -0800 Subject: [PATCH 222/236] Fix copyright --- examples/python/route_guide/route_guide_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python/route_guide/route_guide_server.py b/examples/python/route_guide/route_guide_server.py index 24f948c42c5..f95b36b0736 100644 --- a/examples/python/route_guide/route_guide_server.py +++ b/examples/python/route_guide/route_guide_server.py @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From baaf65546ace6e8ab44f7ac869d868d7864d7578 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 7 Mar 2016 22:39:11 -0800 Subject: [PATCH 223/236] Fix formatting --- test/cpp/qps/qps_worker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index ce4b773d92b..b83e9d1dd7f 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -103,7 +103,7 @@ static std::unique_ptr CreateServer(const ServerConfig& config) { class ScopedProfile GRPC_FINAL { public: - ScopedProfile(const char *filename, bool enable) : enable_(enable) { + ScopedProfile(const char* filename, bool enable) : enable_(enable) { if (enable_) grpc_profiler_start(filename); } ~ScopedProfile() { From a2c8b20ac785d88e4df9bfe1a832e48173458c3e Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 8 Mar 2016 08:21:56 -0800 Subject: [PATCH 224/236] review changes --- src/php/README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/php/README.md b/src/php/README.md index c97c3796cda..cf8f2c11b0f 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -48,7 +48,7 @@ sudo pecl install grpc-beta This will compile and install the gRPC PHP extension into the standard PHP extension directory. You should be able to run the [unit tests](#unit-tests), with the PHP extension installed. -To run tests with generated stub code from `.proto` files, you will need the `composer`, `protoc` and `protoc-gen-php` binaries additionally. See sections [below](#generated-code-tests). +To run tests with generated stub code from `.proto` files, you will also need the `composer`, `protoc` and `protoc-gen-php` binaries. You can find out how to get these [below](#generated-code-tests). ## Build from Source @@ -115,9 +115,11 @@ $ ./bin/run_tests.sh ## Generated Code Tests +This section specifies the prerequisites for running the generated code tests, as well as how to run the tests themselves. + ### Composer -You need to install `composer`, to pull in some runtime dependencies based on the `composer.json` file. +If you don't have it already, install `composer` to pull in some runtime dependencies based on the `composer.json` file. ```sh $ curl -sS https://getcomposer.org/installer | php @@ -129,21 +131,21 @@ $ composer install ### Protobuf compiler -You need the install the protobuf compiler, `protoc`, 3.0.0+. +Again if you don't have it already, you need to install the protobuf compiler `protoc`, version 3.0.0+. -If you had compiled the gRPC C core library from source above, the `protoc` binary should have been installed as well. In the case it wasn't, you can run the following commands to install it. +If you compiled the gRPC C core library from source above, the `protoc` binary should have been installed as well. If it hasn't been installed, you can run the following commands to install it. ```sh $ cd grpc/third_party/protobuf $ sudo make install # 'make' should have been run by core grpc ``` -Or you can download a `protoc` binaries from [here](https://github.com/google/protobuf/releases). +Alternatively, you can download `protoc` binaries from [the protocol buffers Github repository](https://github.com/google/protobuf/releases). ### PHP protobuf compiler -You need to install `protoc-gen-php`, so that you can generate stub classes `.php` files from service definition `.proto` files. +You need to install `protoc-gen-php` to generate stub class `.php` files from service definition `.proto` files. ```sh $ cd grpc/src/php/vendor/datto/protobuf-php # if you had run `composer install` in the previous step @@ -168,7 +170,7 @@ $ ./bin/generate_proto_php.sh ### Run test server -Run a local server serving the math services. Please see [Node][] on how to run an example server +Run a local server serving the math services. Please see [Node][] for how to run an example server. ```sh $ cd grpc From 3be6801b72e91b0e3a1c6b497db02e144e32f7af Mon Sep 17 00:00:00 2001 From: Lisa Carey Date: Tue, 8 Mar 2016 17:47:14 +0000 Subject: [PATCH 225/236] Clarifying how to install for gRPC C++ --- INSTALL.md | 2 +- examples/cpp/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 3c694ba5822..454a8b7b2fd 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -3,7 +3,7 @@ For language-specific installation instructions for gRPC runtime, please refer to these documents - * [C++](examples/cpp) + * [C++](examples/cpp): Currently to install gRPC for C++, you need to build from source as described below. * [C#](src/csharp): NuGet package `Grpc` * [Go](https://github.com/grpc/grpc-go): `go get google.golang.org/grpc` * [Java](https://github.com/grpc/grpc-java) diff --git a/examples/cpp/README.md b/examples/cpp/README.md index e4b0eb698bf..d93cbacf7b2 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -2,7 +2,7 @@ ## Installation -To install gRPC on your system, follow the instructions [here](../../INSTALL.md). +To install gRPC on your system, follow the instructions to build from source [here](../../INSTALL.md). This also installs the protocol buffer compiler `protoc` (if you don't have it already), and the C++ gRPC plugin for `protoc`. ## Hello C++ gRPC! From 9d1476364a002372f2ab375c3aceeb29b7a227cd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 8 Mar 2016 17:00:53 -0800 Subject: [PATCH 226/236] Fix timers --- src/core/iomgr/timer.c | 2 +- src/core/iomgr/timer_heap.c | 14 +++--- test/core/iomgr/timer_heap_test.c | 84 ++----------------------------- 3 files changed, 12 insertions(+), 88 deletions(-) diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 8379fffad02..1bf8387d74b 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -41,7 +41,7 @@ #define INVALID_HEAP_INDEX 0xffffffffu -#define LOG2_NUM_SHARDS 5 +#define LOG2_NUM_SHARDS 2 #define NUM_SHARDS (1 << LOG2_NUM_SHARDS) #define ADD_DEADLINE_SCALE 0.33 #define MIN_QUEUE_WINDOW_DURATION 0.01 diff --git a/src/core/iomgr/timer_heap.c b/src/core/iomgr/timer_heap.c index 9d8be5c1fcb..167bb7c4d11 100644 --- a/src/core/iomgr/timer_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -46,7 +46,7 @@ static void adjust_upwards(grpc_timer **first, uint32_t i, grpc_timer *t) { while (i > 0) { uint32_t parent = (uint32_t)(((int)i - 1) / 2); - if (gpr_time_cmp(first[parent]->deadline, t->deadline) >= 0) break; + if (gpr_time_cmp(first[parent]->deadline, t->deadline) <= 0) break; first[i] = first[parent]; first[i]->heap_index = i; i = parent; @@ -62,16 +62,14 @@ static void adjust_downwards(grpc_timer **first, uint32_t i, uint32_t length, grpc_timer *t) { for (;;) { uint32_t left_child = 1u + 2u * i; - uint32_t right_child; - uint32_t next_i; if (left_child >= length) break; - right_child = left_child + 1; - next_i = right_child < length && + uint32_t right_child = left_child + 1; + uint32_t next_i = right_child < length && gpr_time_cmp(first[left_child]->deadline, - first[right_child]->deadline) < 0 + first[right_child]->deadline) > 0 ? right_child : left_child; - if (gpr_time_cmp(t->deadline, first[next_i]->deadline) >= 0) break; + if (gpr_time_cmp(t->deadline, first[next_i]->deadline) <= 0) break; first[i] = first[next_i]; first[i]->heap_index = i; i = next_i; @@ -95,7 +93,7 @@ static void maybe_shrink(grpc_timer_heap *heap) { static void note_changed_priority(grpc_timer_heap *heap, grpc_timer *timer) { uint32_t i = timer->heap_index; uint32_t parent = (uint32_t)(((int)i - 1) / 2); - if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) < 0) { + if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) > 0) { adjust_upwards(heap->timers, i, timer); } else { adjust_downwards(heap->timers, i, heap->timer_count, timer); diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 077a9fd6bd8..9b51b912003 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -57,79 +57,6 @@ static grpc_timer *create_test_elements(size_t num_elements) { return elems; } -static int cmp_elem(const void *a, const void *b) { - int i = *(const int *)a; - int j = *(const int *)b; - return i - j; -} - -static size_t *all_top(grpc_timer_heap *pq, size_t *n) { - size_t *vec = NULL; - size_t *need_to_check_children; - size_t num_need_to_check_children = 0; - - *n = 0; - if (pq->timer_count == 0) return vec; - need_to_check_children = - gpr_malloc(pq->timer_count * sizeof(*need_to_check_children)); - need_to_check_children[num_need_to_check_children++] = 0; - vec = gpr_malloc(pq->timer_count * sizeof(*vec)); - while (num_need_to_check_children > 0) { - size_t ind = need_to_check_children[0]; - size_t leftchild, rightchild; - num_need_to_check_children--; - memmove(need_to_check_children, need_to_check_children + 1, - num_need_to_check_children * sizeof(*need_to_check_children)); - vec[(*n)++] = ind; - leftchild = 1u + 2u * ind; - if (leftchild < pq->timer_count) { - if (gpr_time_cmp(pq->timers[leftchild]->deadline, - pq->timers[ind]->deadline) >= 0) { - need_to_check_children[num_need_to_check_children++] = leftchild; - } - rightchild = leftchild + 1; - if (rightchild < pq->timer_count && - gpr_time_cmp(pq->timers[rightchild]->deadline, - pq->timers[ind]->deadline) >= 0) { - need_to_check_children[num_need_to_check_children++] = rightchild; - } - } - } - - gpr_free(need_to_check_children); - - return vec; -} - -static void check_pq_top(grpc_timer *elements, grpc_timer_heap *pq, - uint8_t *inpq, size_t num_elements) { - gpr_timespec max_deadline = gpr_inf_past(GPR_CLOCK_REALTIME); - size_t *max_deadline_indices = - gpr_malloc(num_elements * sizeof(*max_deadline_indices)); - size_t *top_elements; - size_t num_max_deadline_indices = 0; - size_t num_top_elements; - size_t i; - for (i = 0; i < num_elements; ++i) { - if (inpq[i] && gpr_time_cmp(elements[i].deadline, max_deadline) >= 0) { - if (gpr_time_cmp(elements[i].deadline, max_deadline) > 0) { - num_max_deadline_indices = 0; - max_deadline = elements[i].deadline; - } - max_deadline_indices[num_max_deadline_indices++] = elements[i].heap_index; - } - } - qsort(max_deadline_indices, num_max_deadline_indices, - sizeof(*max_deadline_indices), cmp_elem); - top_elements = all_top(pq, &num_top_elements); - GPR_ASSERT(num_top_elements == num_max_deadline_indices); - for (i = 0; i < num_top_elements; i++) { - GPR_ASSERT(max_deadline_indices[i] == top_elements[i]); - } - gpr_free(max_deadline_indices); - gpr_free(top_elements); -} - static int contains(grpc_timer_heap *pq, grpc_timer *el) { size_t i; for (i = 0; i < pq->timer_count; i++) { @@ -145,11 +72,11 @@ static void check_valid(grpc_timer_heap *pq) { size_t right_child = left_child + 1u; if (left_child < pq->timer_count) { GPR_ASSERT(gpr_time_cmp(pq->timers[i]->deadline, - pq->timers[left_child]->deadline) >= 0); + pq->timers[left_child]->deadline) <= 0); } if (right_child < pq->timer_count) { GPR_ASSERT(gpr_time_cmp(pq->timers[i]->deadline, - pq->timers[right_child]->deadline) >= 0); + pq->timers[right_child]->deadline) <= 0); } } } @@ -162,6 +89,8 @@ static void test1(void) { grpc_timer *test_elements = create_test_elements(num_test_elements); uint8_t *inpq = gpr_malloc(num_test_elements); + gpr_log(GPR_DEBUG, "******************************************* test1"); + grpc_timer_heap_init(&pq); memset(inpq, 0, num_test_elements); GPR_ASSERT(grpc_timer_heap_is_empty(&pq)); @@ -172,7 +101,6 @@ static void test1(void) { check_valid(&pq); GPR_ASSERT(contains(&pq, &test_elements[i])); inpq[i] = 1; - check_pq_top(test_elements, &pq, inpq, num_test_elements); } for (i = 0; i < num_test_elements; ++i) { /* Test that check still succeeds even for element that wasn't just @@ -182,7 +110,7 @@ static void test1(void) { GPR_ASSERT(pq.timer_count == num_test_elements); - check_pq_top(test_elements, &pq, inpq, num_test_elements); + check_valid(&pq); for (i = 0; i < num_test_operations; ++i) { size_t elem_num = (size_t)rand() % num_test_elements; @@ -193,14 +121,12 @@ static void test1(void) { grpc_timer_heap_add(&pq, el); GPR_ASSERT(contains(&pq, el)); inpq[elem_num] = 1; - check_pq_top(test_elements, &pq, inpq, num_test_elements); check_valid(&pq); } else { GPR_ASSERT(contains(&pq, el)); grpc_timer_heap_remove(&pq, el); GPR_ASSERT(!contains(&pq, el)); inpq[elem_num] = 0; - check_pq_top(test_elements, &pq, inpq, num_test_elements); check_valid(&pq); } } From 08e0c191beaa9fc818ae2363c7684307fde084bf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 8 Mar 2016 17:04:50 -0800 Subject: [PATCH 227/236] Revert mistaken change --- src/core/iomgr/timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/iomgr/timer.c b/src/core/iomgr/timer.c index 1bf8387d74b..8379fffad02 100644 --- a/src/core/iomgr/timer.c +++ b/src/core/iomgr/timer.c @@ -41,7 +41,7 @@ #define INVALID_HEAP_INDEX 0xffffffffu -#define LOG2_NUM_SHARDS 2 +#define LOG2_NUM_SHARDS 5 #define NUM_SHARDS (1 << LOG2_NUM_SHARDS) #define ADD_DEADLINE_SCALE 0.33 #define MIN_QUEUE_WINDOW_DURATION 0.01 From 1b393da4332ad97aac62ee8ecb422d6d0fe108f3 Mon Sep 17 00:00:00 2001 From: Lisa Carey Date: Wed, 9 Mar 2016 15:16:56 +0000 Subject: [PATCH 228/236] Updated main examples README to link to main docs and quickstarts --- examples/README.md | 447 ++------------------------------------------- 1 file changed, 12 insertions(+), 435 deletions(-) diff --git a/examples/README.md b/examples/README.md index cd85417ca20..287a80266c9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,450 +1,27 @@ +# Examples -# Getting started +This directory contains code examples for all the C-based gRPC implementations: C++, Node.js, Python, Ruby, Objective-C, PHP, and C#. You can find examples and instructions specific to your +favourite language in the relevant subdirectory. + +Examples for Go and Java gRPC live in their own repositories: -Welcome to the developer documentation for gRPC, a language-neutral, -platform-neutral remote procedure call (RPC) system developed at Google. +* [Java](https://github.com/grpc/grpc-java/tree/master/examples) +* [Android Java](https://github.com/grpc/grpc-java/tree/master/examples/android) +* [Go](https://github.com/grpc/grpc-go/tree/master/examples) -This document introduces you to gRPC with a quick overview and a simple -Hello World example. You'll find more tutorials and reference docs in this repository - more documentation is coming soon! +For more comprehensive documentation, including an [overview](http://www.grpc.io/docs/) and tutorials that use this example code, visit [grpc.io](http://www.grpc.io/docs/). - ## Quick start -You can find quick start guides for each language, including installation instructions, examples, and tutorials here: + +Each example directory has quick start instructions for the appropriate language, including installation instructions and how to run our simplest Hello World example: + * [C++](cpp) -* [Java](https://github.com/grpc/grpc-java/tree/master/examples) -* [Go](https://github.com/grpc/grpc-go/tree/master/examples) * [Ruby](ruby) * [Node.js](node) -* [Android Java](https://github.com/grpc/grpc-java/tree/master/examples/android) * [Python](python/helloworld) * [C#](csharp) * [Objective-C](objective-c/helloworld) * [PHP](php) -## What's in this repository? - -The `examples` directory contains documentation, resources, and examples -for all gRPC users. You can find examples and instructions specific to your -favourite language in the relevant subdirectory. - -You can find out about the gRPC source code repositories in -[`grpc`](https://github.com/grpc/grpc). Each repository provides instructions -for building the appropriate libraries for your language. - - -## What is gRPC? - -In gRPC a *client* application can directly call -methods on a *server* application on a different machine as if it was a -local object, making it easier for you to create distributed applications and -services. As in many RPC systems, gRPC is based around the idea of defining -a *service*, specifying the methods that can be called remotely with their -parameters and return types. On the server side, the server implements this -interface and runs a gRPC server to handle client calls. On the client side, -the client has a *stub* that provides exactly the same methods as the server. - - - -gRPC clients and servers can run and talk to each other in a variety of -environments - from servers inside Google to your own desktop - and can -be written in any of gRPC's [supported languages](#quickstart). So, for -example, you can easily create a gRPC server in Java with clients in Go, -Python, or Ruby. In addition, the latest Google APIs will have gRPC versions -of their interfaces, letting you easily build Google functionality into -your applications. - - -### Working with protocol buffers - -By default gRPC uses *protocol buffers*, Google’s -mature open source mechanism for serializing structured data (although it -can be used with other data formats such as JSON). As you'll -see in our example below, you define gRPC services using *proto files*, -with method parameters and return types specified as protocol buffer message -types. You -can find out lots more about protocol buffers in the [Protocol Buffers -documentation](https://developers.google.com/protocol-buffers/docs/overview). - -#### Protocol buffer versions - -While protocol buffers have been available for open source users for some -time, our examples use a new flavour of protocol buffers called proto3, -which has a slightly simplified syntax, some useful new features, and supports -lots more languages. This is currently available as an alpha release in -Java, C++, Java_nano (Android Java), Python, and Ruby from [the protocol buffers Github -repo](https://github.com/google/protobuf/releases), as well as a Go language -generator from [the golang/protobuf Github repo](https://github.com/golang/protobuf), with more languages in development. You can find out more in the [proto3 language guide](https://developers.google.com/protocol-buffers/docs/proto3), and see -the major differences from the current default version in the [release notes](https://github.com/google/protobuf/releases). More proto3 documentation is coming soon. - -In general, while you *can* use proto2 (the current default protocol buffers version), we recommend that you use proto3 with gRPC as it lets you use the full range of gRPC-supported languages, as well as avoiding compatibility -issues with proto2 clients talking to proto3 servers and vice versa. - - -## Hello gRPC! - -Now that you know a bit more about gRPC, the easiest way to see how it -works is to look at a simple example. Our Hello World walks you through the -construction of a simple gRPC client-server application, showing you how to: - -- Create a protocol buffers schema that defines a simple RPC service with -a single -Hello World method. -- Create a Java server that implements this interface. -- Create a Java client that accesses the Java server. -- Create a Go client that accesses -the same Java server. - -The complete code for the example is available in the `examples` -directory. We use the Git versioning system for source code management: -however, you don't need to know anything about Git to follow along other -than how to install and run a few git commands. - -This is an introductory example rather than a comprehensive tutorial, so -don't worry if you're not a Go or -Java developer - the concepts are similar for all languages, and you can -find more implementations of our Hello World example in other languages (and full tutorials where available) in -the [language-specific folders](#quickstart) in this repository. Complete tutorials and -reference documentation for all gRPC languages are coming soon. - - -### Setup - -This section explains how to set up your local machine to work with -the example code. If you just want to read the example, you can go straight -to the [next step](#servicedef). - -#### Install Git - -You can download and install Git from http://git-scm.com/download. Once -installed you should have access to the git command line tool. The main -commands that you will need to use are: - -- git clone ... : clone a remote repository onto your local machine -- git checkout ... : check out a particular branch or a tagged version of -the code to hack on - -#### Install gRPC - -To build and install gRPC plugins and related tools: -- For Java, see the [Java quick start](https://github.com/grpc/grpc-java). -- For Go, see the [Go quick start](https://github.com/grpc/grpc-go). - -#### Get the source code - -The example code for our Java example lives in the `grpc-java` -GitHub repository. Clone this repository to your local machine by running the -following command: - - -``` -git clone https://github.com/grpc/grpc-java.git -``` - -Change your current directory to grpc-java/examples - -``` -cd grpc-java/examples -``` - - - - -### Defining a service - -The first step in creating our example is to define a *service*: an RPC -service specifies the methods that can be called remotely with their parameters -and return types. As you saw in the -[overview](#protocolbuffers) above, gRPC does this using [protocol -buffers](https://developers.google.com/protocol-buffers/docs/overview). We -use the protocol buffers interface definition language (IDL) to define our -service methods, and define the parameters and return -types as protocol buffer message types. Both the client and the -server use interface code generated from the service definition. - -Here's our example service definition, defined using protocol buffers IDL in -[helloworld.proto](https://github.com/grpc/grpc-java/tree/master/examples/src/main/proto). The `Greeter` -service has one method, `SayHello`, that lets the server receive a single -`HelloRequest` -message from the remote client containing the user's name, then send back -a greeting in a single `HelloReply`. This is the simplest type of RPC you -can specify in gRPC - you can find out about other types in the tutorial for your chosen language. - -```proto -syntax = "proto3"; - -option java_package = "io.grpc.examples"; - -package helloworld; - -// The greeter service definition. -service Greeter { - // Sends a greeting - rpc SayHello (HelloRequest) returns (HelloReply) {} -} - -// The request message containing the user's name. -message HelloRequest { - string name = 1; -} - -// The response message containing the greetings -message HelloReply { - string message = 1; -} - -``` - - -### Generating gRPC code - -Once we've defined our service, we use the protocol buffer compiler -`protoc` to generate the special client and server code we need to create -our application - right now we're going to generate Java code, though you -can generate gRPC code in any gRPC-supported language (as you'll see later -in this example). The generated code contains both stub code for clients to -use and an abstract interface for servers to implement, both with the method -defined in our `Greeter` service. - -(If you didn't install the gRPC plugins and protoc on your system and are just reading along with -the example, you can skip this step and move -onto the next one where we examine the generated code.) - -For simplicity, we've provided a [Gradle build file](https://github.com/grpc/grpc-java/blob/master/examples/build.gradle) with our Java examples that runs `protoc` for you with the appropriate plugin, input, and output: - -```shell -../gradlew build -``` - -This generates the following classes from our .proto, which contain all the generated code -we need to create our example: - -- `Helloworld.java`, which -has all the protocol buffer code to populate, serialize, and retrieve our -`HelloRequest` and `HelloReply` message types -- `GreeterGrpc.java`, which contains (along with some other useful code): - - an interface for `Greeter` servers to implement - - ```java - public static interface Greeter { - public void sayHello(io.grpc.examples.Helloworld.HelloRequest request, - io.grpc.stub.StreamObserver responseObserver); - } - ``` - - - _stub_ classes that clients can use to talk to a `Greeter` server. As you can see, they also implement the `Greeter` interface. - - ```java - public static class GreeterStub extends - io.grpc.stub.AbstractStub - implements Greeter { - ... - } - ``` - - -### Writing a server - -Now let's write some code! First we'll create a server application to implement -our service. Note that we're not going to go into a lot of detail about how -to create a server in this section. More detailed information will be in the -tutorial for your chosen language: check if there's one available yet in the relevant [quick start](#quickstart). - -Our server application has two classes: - -- a main server class that hosts the service implementation and allows access over the -network: [HelloWorldServer.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java). - - -- a simple service implementation class [GreeterImpl.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java#L51). - - -#### Service implementation - -[GreeterImpl.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java#L51) -actually implements our `Greeter` service's required behaviour. - -As you can see, the class `GreeterImpl` implements the interface -`GreeterGrpc.Greeter` that we [generated](#generating) from our proto -[IDL](https://github.com/grpc/grpc-java/tree/master/examples/src/main/proto) by implementing the method `sayHello`: - -```java - @Override - public void sayHello(HelloRequest req, StreamObserver responseObserver) { - HelloReply reply = HelloReply.newBuilder().setMessage("Hello " + req.getName()).build(); - responseObserver.onValue(reply); - responseObserver.onCompleted(); - } -``` -- `sayHello` takes two parameters: - - `HelloRequest`: the request - - `StreamObserver`: a response observer, which is - a special interface for the server to call with its response - -To return our response to the client and complete the call: - -1. We construct and populate a `HelloReply` response object with our exciting -message, as specified in our interface definition. -2. We return the `HelloReply` to the client and then specify that we've finished dealing with the RPC. - - -#### Server implementation - -[HelloWorldServer.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java) -shows the other main feature required to provide a gRPC service; making the service -implementation available from the network. - -```java - /* The port on which the server should run */ - private int port = 50051; - private ServerImpl server; - - private void start() throws Exception { - server = NettyServerBuilder.forPort(port) - .addService(GreeterGrpc.bindService(new GreeterImpl())) - .build().start(); - logger.info("Server started, listening on " + port); - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - // Use stderr here since the logger may have been reset by its JVM shutdown hook. - System.err.println("*** shutting down gRPC server since JVM is shutting down"); - HelloWorldServer.this.stop(); - System.err.println("*** server shut down"); - } - }); - } - -``` - -Here we create an appropriate gRPC server, binding the `Greeter` service -implementation that we created to a port. Then we start the server running: the server is now ready to receive -requests from `Greeter` service clients on our specified port. We'll cover -how all this works in a bit more detail in our language-specific documentation. - - -### Writing a client - -Client-side gRPC is pretty simple. In this step, we'll use the generated code -to write a simple client that can access the `Greeter` server we created -in the [previous section](#server). You can see the complete client code in -[HelloWorldClient.java](https://github.com/grpc/grpc-java/blob/master/examples/src/main/java/io/grpc/examples/helloworld/HelloWorldClient.java). - -Again, we're not going to go into much detail about how to implement a client; -we'll leave that for the tutorial. - -#### Connecting to the service - -First let's look at how we connect to the `Greeter` server. First we need -to create a gRPC channel, specifying the hostname and port of the server we -want to connect to. Then we use the channel to construct the stub instance. - - -```java - private final ChannelImpl channel; - private final GreeterGrpc.GreeterBlockingStub blockingStub; - - public HelloWorldClient(String host, int port) { - channel = - NettyChannelBuilder.forAddress(host, port).negotiationType(NegotiationType.PLAINTEXT) - .build(); - blockingStub = GreeterGrpc.newBlockingStub(channel); - } - -``` - -In this case, we create a blocking stub. This means that the RPC call waits -for the server to respond, and will either return a response or raise an -exception. gRPC Java has other kinds of stubs that make non-blocking calls -to the server, where the response is returned asynchronously. - -#### Calling an RPC - -Now we can contact the service and obtain a greeting: - -1. We construct and fill in a `HelloRequest` to send to the service. -2. We call the stub's `hello()` RPC with our request and get a `HelloReply` -back, from which we can get our greeting. - - -```java - HelloRequest req = HelloRequest.newBuilder().setName(name).build(); - HelloReply reply = blockingStub.sayHello(req); - -``` - - -### Try it out! - -Our [Gradle build file](https://github.com/grpc/grpc-java/blob/master/examples/build.gradle) simplifies building and running the examples. - -You can build and run the server from the `grpc-java` root folder with: - -```sh -$ ./gradlew :grpc-examples:helloWorldServer -``` - -and in another terminal window confirm that it receives a message. - -```sh -$ ./gradlew :grpc-examples:helloWorldClient -``` - -### Adding another client - -Finally, let's look at one of gRPC's most useful features - interoperability -between code in different languages. So far, we've just looked at Java code -generated from and implementing our `Greeter` service definition. However, -as you'll see if you look at the language-specific subdirectories -in this repository, we've also generated and implemented `Greeter` -in some of gRPC's other supported languages. Each service -and client uses interface code generated from the same proto -that we used for the Java example. - -So, for example, if we visit the [`go` example -directory](https://github.com/grpc/grpc-go/tree/master/examples) and look at the -[`greeter_client`](https://github.com/grpc/grpc-go/blob/master/examples/greeter_client/main.go), -we can see that like the Java client, it connects to a `Greeter` service -at `localhost:50051` and uses a stub to call the `SayHello` method with a -`HelloRequest`: - -```go -const ( - address = "localhost:50051" - defaultName = "world" -) - -func main() { - // Set up a connection to the server. - conn, err := grpc.Dial(address) - if err != nil { - log.Fatalf("did not connect: %v", err) - } - defer conn.Close() - c := pb.NewGreeterClient(conn) - - // Contact the server and print out its response. - name := defaultName - if len(os.Args) > 1 { - name = os.Args[1] - } - r, err := c.SayHello(context.Background(), &pb.HelloRequest{Name: - name}) - if err != nil { - log.Fatalf("could not greet: %v", err) - } - log.Printf("Greeting: %s", r.Message) -} -``` - - -If we run the Java server from earlier in another terminal window, we can -run the Go client and connect to it just like the Java client, even though -it's written in a different language. -``` -$ greeter_client -``` -## Read more! -- You can find links to language-specific tutorials, examples, and other docs in each language's [quick start](#quickstart). -- [gRPC Authentication Support](http://www.grpc.io/docs/guides/auth.html) introduces authentication support in gRPC with supported mechanisms and examples. From 0e6e34e4fae218bbff1d586fe42f2f4151615282 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:21:43 -0800 Subject: [PATCH 229/236] Add an additional test --- test/core/iomgr/timer_heap_test.c | 108 +++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 9b51b912003..3c78a6c1171 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -38,6 +38,8 @@ #include #include +#include + #include "test/core/util/test_config.h" static gpr_timespec random_deadline(void) { @@ -81,6 +83,10 @@ static void check_valid(grpc_timer_heap *pq) { } } +/******************************************************************************* + * test1 + */ + static void test1(void) { grpc_timer_heap pq; const size_t num_test_elements = 200; @@ -89,7 +95,7 @@ static void test1(void) { grpc_timer *test_elements = create_test_elements(num_test_elements); uint8_t *inpq = gpr_malloc(num_test_elements); - gpr_log(GPR_DEBUG, "******************************************* test1"); + gpr_log(GPR_INFO, "test1"); grpc_timer_heap_init(&pq); memset(inpq, 0, num_test_elements); @@ -136,7 +142,106 @@ static void test1(void) { gpr_free(inpq); } +/******************************************************************************* + * test2 + */ + +typedef struct { + grpc_timer elem; + bool inserted; +} elem_struct; + +static elem_struct *search_elems(elem_struct *elems, size_t count, bool inserted) { + size_t *search_order = gpr_malloc(count * sizeof(*search_order)); + for (size_t i = 0; i < count; i++) { + search_order[i] = i; + } + for (size_t i = 0; i < count * 2; i++) { + size_t a = (size_t)rand() % count; + size_t b = (size_t)rand() % count; + GPR_SWAP(size_t, search_order[a], search_order[b]); + } + elem_struct *out = NULL; + for (size_t i = 0; out == NULL && i < count; i++) { + if (elems[search_order[i]].inserted == inserted) { + out = &elems[search_order[i]]; + } + } + gpr_free(search_order); + return out; +} + +static void test2(void) { +gpr_log(GPR_INFO, "test2"); + + grpc_timer_heap pq; + + elem_struct elems[1000]; + size_t num_inserted = 0; + + grpc_timer_heap_init(&pq); + memset(elems, 0, sizeof(elems)); + + for (size_t round = 0; round < 10000; round++) { + int r = (size_t)rand() % 1000; + if (r <= 550) { + /* 55% of the time we try to add something */ + elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), false); + if (el != NULL) { + el->elem.deadline = random_deadline(); + grpc_timer_heap_add(&pq, &el->elem); + el->inserted = true; + num_inserted++; + check_valid(&pq); + } + } else if (r <= 650) { + /* 10% of the time we try to remove something */ + elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), true); + if (el != NULL) { + grpc_timer_heap_remove(&pq, &el->elem); + el->inserted = false; + num_inserted--; + check_valid(&pq); + } + } else { + /* the remaining times we pop */ + if (num_inserted > 0) { + grpc_timer *top = grpc_timer_heap_top(&pq); + grpc_timer_heap_pop(&pq); + for (size_t i = 0; i < GPR_ARRAY_SIZE(elems); i++) { + if (top == &elems[i].elem) { + GPR_ASSERT(elems[i].inserted); + elems[i].inserted = false; + } + } + num_inserted--; + check_valid(&pq); + } + } + + if (num_inserted) { + gpr_timespec *min_deadline = NULL; + for (size_t i = 0; i < GPR_ARRAY_SIZE(elems); i++) { + if (elems[i].inserted) { + if (min_deadline == NULL) { + min_deadline = &elems[i].elem.deadline; + } else { + if (gpr_time_cmp(elems[i].elem.deadline, *min_deadline) < 0) { + min_deadline = &elems[i].elem.deadline; + } + } + } + } + GPR_ASSERT(0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline)); + } + } + + grpc_timer_heap_destroy(&pq); +} + static void shrink_test(void) { + gpr_log(GPR_INFO, "shrink_test"); + grpc_timer_heap pq; size_t i; size_t expected_size; @@ -200,6 +305,7 @@ int main(int argc, char **argv) { for (i = 0; i < 5; i++) { test1(); + test2(); shrink_test(); } From 3a951bd975cb14ba7fe2b59e7c6b598932998a73 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:23:57 -0800 Subject: [PATCH 230/236] clang-format --- src/core/iomgr/timer_heap.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/iomgr/timer_heap.c b/src/core/iomgr/timer_heap.c index 167bb7c4d11..9d4aaf3eee7 100644 --- a/src/core/iomgr/timer_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -65,10 +65,10 @@ static void adjust_downwards(grpc_timer **first, uint32_t i, uint32_t length, if (left_child >= length) break; uint32_t right_child = left_child + 1; uint32_t next_i = right_child < length && - gpr_time_cmp(first[left_child]->deadline, - first[right_child]->deadline) > 0 - ? right_child - : left_child; + gpr_time_cmp(first[left_child]->deadline, + first[right_child]->deadline) > 0 + ? right_child + : left_child; if (gpr_time_cmp(t->deadline, first[next_i]->deadline) <= 0) break; first[i] = first[next_i]; first[i]->heap_index = i; From a39c1995623ed5ef48c49fc5212f2140b3390462 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:26:52 -0800 Subject: [PATCH 231/236] clang-format --- test/core/iomgr/timer_heap_test.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 3c78a6c1171..4510d51bd70 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -151,7 +151,8 @@ typedef struct { bool inserted; } elem_struct; -static elem_struct *search_elems(elem_struct *elems, size_t count, bool inserted) { +static elem_struct *search_elems(elem_struct *elems, size_t count, + bool inserted) { size_t *search_order = gpr_malloc(count * sizeof(*search_order)); for (size_t i = 0; i < count; i++) { search_order[i] = i; @@ -172,7 +173,7 @@ static elem_struct *search_elems(elem_struct *elems, size_t count, bool inserted } static void test2(void) { -gpr_log(GPR_INFO, "test2"); + gpr_log(GPR_INFO, "test2"); grpc_timer_heap pq; @@ -232,7 +233,8 @@ gpr_log(GPR_INFO, "test2"); } } } - GPR_ASSERT(0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline)); + GPR_ASSERT( + 0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline)); } } From 5f3ba8c834b829254f85fedee6441fc82f14bcfc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 07:44:51 -0800 Subject: [PATCH 232/236] Copyright, casting fixes --- src/core/iomgr/timer_heap.c | 2 +- test/core/iomgr/timer_heap_test.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/timer_heap.c b/src/core/iomgr/timer_heap.c index 9d4aaf3eee7..b5df566c453 100644 --- a/src/core/iomgr/timer_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/core/iomgr/timer_heap_test.c b/test/core/iomgr/timer_heap_test.c index 4510d51bd70..cd34696f7dd 100644 --- a/test/core/iomgr/timer_heap_test.c +++ b/test/core/iomgr/timer_heap_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -184,7 +184,7 @@ static void test2(void) { memset(elems, 0, sizeof(elems)); for (size_t round = 0; round < 10000; round++) { - int r = (size_t)rand() % 1000; + int r = rand() % 1000; if (r <= 550) { /* 55% of the time we try to add something */ elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), false); From 76a5c0e433eb0a4a941e0bf9172e55c6f8058ed4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 09:05:30 -0800 Subject: [PATCH 233/236] Fix memory leak: keep all resolver calls under the same lock --- src/core/channel/client_channel.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index d4ba9508185..f021a8ae329 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -165,7 +165,6 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, channel_data *chand = arg; grpc_lb_policy *lb_policy = NULL; grpc_lb_policy *old_lb_policy; - grpc_resolver *old_resolver; grpc_connectivity_state state = GRPC_CHANNEL_TRANSIENT_FAILURE; int exit_idle = 0; @@ -201,28 +200,25 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg, } if (iomgr_success && chand->resolver) { - grpc_resolver *resolver = chand->resolver; - GRPC_RESOLVER_REF(resolver, "channel-next"); grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, state, "new_lb+resolver"); if (lb_policy != NULL) { watch_lb_policy(exec_ctx, chand, lb_policy, state); } - gpr_mu_unlock(&chand->mu_config); GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver"); - grpc_resolver_next(exec_ctx, resolver, &chand->incoming_configuration, + grpc_resolver_next(exec_ctx, chand->resolver, + &chand->incoming_configuration, &chand->on_config_changed); - GRPC_RESOLVER_UNREF(exec_ctx, resolver, "channel-next"); + gpr_mu_unlock(&chand->mu_config); } else { - old_resolver = chand->resolver; - chand->resolver = NULL; + if (chand->resolver != NULL) { + grpc_resolver_shutdown(exec_ctx, chand->resolver); + GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel"); + chand->resolver = NULL; + } grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, GRPC_CHANNEL_FATAL_FAILURE, "resolver_gone"); gpr_mu_unlock(&chand->mu_config); - if (old_resolver != NULL) { - grpc_resolver_shutdown(exec_ctx, old_resolver); - GRPC_RESOLVER_UNREF(exec_ctx, old_resolver, "channel"); - } } if (exit_idle) { @@ -247,7 +243,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_transport_op *op) { channel_data *chand = elem->channel_data; - grpc_resolver *destroy_resolver = NULL; grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL); @@ -279,7 +274,8 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, if (op->disconnect && chand->resolver != NULL) { grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, GRPC_CHANNEL_FATAL_FAILURE, "disconnect"); - destroy_resolver = chand->resolver; + grpc_resolver_shutdown(exec_ctx, chand->resolver); + GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel"); chand->resolver = NULL; if (chand->lb_policy != NULL) { grpc_pollset_set_del_pollset_set(exec_ctx, @@ -290,11 +286,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, } } gpr_mu_unlock(&chand->mu_config); - - if (destroy_resolver) { - grpc_resolver_shutdown(exec_ctx, destroy_resolver); - GRPC_RESOLVER_UNREF(exec_ctx, destroy_resolver, "channel"); - } } typedef struct { From d426d9dfdbd1712b1b241a17452ed15441b502a1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 9 Mar 2016 09:30:18 -0800 Subject: [PATCH 234/236] Fix memory leak if call is already cancelled Obvious in hindsight, if the cas failed, we still created the subchannel call object and then left it dangling. Fixes a leak observed on master in the past 48 hours. --- src/core/channel/subchannel_call_holder.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c index 8f46885a043..9c087dc2a1d 100644 --- a/src/core/channel/subchannel_call_holder.c +++ b/src/core/channel/subchannel_call_holder.c @@ -174,17 +174,15 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) { holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; if (holder->connected_subchannel == NULL) { fail_locked(exec_ctx, holder); + } else if (1 == gpr_atm_acq_load(&holder->subchannel_call)) { + /* already cancelled before subchannel became ready */ + fail_locked(exec_ctx, holder); } else { - if (!gpr_atm_rel_cas( - &holder->subchannel_call, 0, - (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( - exec_ctx, holder->connected_subchannel, holder->pollset))) { - GPR_ASSERT(gpr_atm_acq_load(&holder->subchannel_call) == 1); - /* if this cas fails, the call was cancelled before the pick completed */ - fail_locked(exec_ctx, holder); - } else { - retry_waiting_locked(exec_ctx, holder); - } + gpr_atm_rel_store( + &holder->subchannel_call, + (gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call( + exec_ctx, holder->connected_subchannel, holder->pollset)); + retry_waiting_locked(exec_ctx, holder); } gpr_mu_unlock(&holder->mu); GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel"); From ec49e156fe62c4c451cd65686efaa07999178166 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 11:51:23 -0800 Subject: [PATCH 235/236] Fix typo --- src/python/grpcio/tests/_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py index 38a5432e791..b0dbd92a495 100644 --- a/src/python/grpcio/tests/_runner.py +++ b/src/python/grpcio/tests/_runner.py @@ -144,7 +144,7 @@ class Runner(object): def run(self, suite): """See setuptools' test_runner setup argument for information.""" # only run test cases with id starting with given prefix - testcase_filter = os.getenv('GPRC_PYTHON_TESTRUNNER_FILTER') + testcase_filter = os.getenv('GRPC_PYTHON_TESTRUNNER_FILTER') filtered_cases = [] for case in _loader.iterate_suite_cases(suite): if not testcase_filter or case.id().startswith(testcase_filter): From 1f646dc8873db8ce9bc1aac189cc2cc471e25bb1 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 9 Mar 2016 14:46:45 -0800 Subject: [PATCH 236/236] Release GIL in queue __dealloc__ --- .../grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index c139147114b..e11138b1cd8 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -140,7 +140,8 @@ cdef class CompletionQueue: grpc_completion_queue_shutdown(self.c_completion_queue) # Pump the queue while not self.is_shutdown: - event = grpc_completion_queue_next( - self.c_completion_queue, c_deadline, NULL) + with nogil: + event = grpc_completion_queue_next( + self.c_completion_queue, c_deadline, NULL) self._interpret_event(event) grpc_completion_queue_destroy(self.c_completion_queue)